From ce28aa5f2efbe6188f788816865fa85690fe7f9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Mon, 21 Oct 2024 16:40:13 +0200 Subject: [PATCH 001/109] WIP: use wit_encoder for removing imports --- Cargo.lock | 42 +- wasm-rpc-stubgen/Cargo.toml | 10 +- wasm-rpc-stubgen/src/commands/dependencies.rs | 49 +- wasm-rpc-stubgen/src/commands/generate.rs | 2 +- wasm-rpc-stubgen/src/commands/log.rs | 9 +- wasm-rpc-stubgen/src/fs.rs | 6 +- wasm-rpc-stubgen/src/lib.rs | 3 +- wasm-rpc-stubgen/src/model/validation.rs | 2 +- wasm-rpc-stubgen/src/naming.rs | 4 + .../src/{wit.rs => wit_generate.rs} | 23 +- wasm-rpc-stubgen/src/wit_resolve.rs | 15 +- wasm-rpc-stubgen/src/wit_transform.rs | 468 ++++++++++++++++++ 12 files changed, 584 insertions(+), 49 deletions(-) rename wasm-rpc-stubgen/src/{wit.rs => wit_generate.rs} (96%) create mode 100644 wasm-rpc-stubgen/src/wit_transform.rs diff --git a/Cargo.lock b/Cargo.lock index ee75df75..b0879d31 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1938,7 +1938,8 @@ dependencies = [ "wac-graph", "walkdir", "wit-bindgen-rust 0.26.0", - "wit-parser 0.219.0", + "wit-encoder", + "wit-parser 0.219.1", ] [[package]] @@ -3300,6 +3301,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "pretty_assertions" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" +dependencies = [ + "diff", + "yansi", +] + [[package]] name = "pretty_env_logger" version = "0.5.0" @@ -5194,7 +5205,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2b1b95711b3ad655656a341e301cc64e33cbee94de9a99a1c5a2ab88efab79d" dependencies = [ "leb128", - "wasmparser 0.219.1", + "wasmparser 0.219.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -5364,8 +5375,6 @@ dependencies = [ [[package]] name = "wasmparser" version = "0.219.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c771866898879073c53b565a6c7b49953795159836714ac56a5befb581227c5" dependencies = [ "ahash", "bitflags 2.6.0", @@ -5374,6 +5383,16 @@ dependencies = [ "semver", ] +[[package]] +name = "wasmparser" +version = "0.219.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c771866898879073c53b565a6c7b49953795159836714ac56a5befb581227c5" +dependencies = [ + "bitflags 2.6.0", + "indexmap 2.6.0", +] + [[package]] name = "wasmprinter" version = "0.2.80" @@ -6151,6 +6170,17 @@ dependencies = [ "wit-parser 0.209.1", ] +[[package]] +name = "wit-encoder" +version = "0.219.1" +dependencies = [ + "id-arena", + "pretty_assertions", + "semver", + "serde 1.0.210", + "wit-parser 0.219.1", +] + [[package]] name = "wit-parser" version = "0.207.0" @@ -6207,9 +6237,7 @@ dependencies = [ [[package]] name = "wit-parser" -version = "0.219.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23102e180c0c464f36e293d31a27b524e3ece930d7b5527d2f33f9d2c963de64" +version = "0.219.1" dependencies = [ "anyhow", "id-arena", diff --git a/wasm-rpc-stubgen/Cargo.toml b/wasm-rpc-stubgen/Cargo.toml index 2205e6c7..58534599 100644 --- a/wasm-rpc-stubgen/Cargo.toml +++ b/wasm-rpc-stubgen/Cargo.toml @@ -58,9 +58,15 @@ tempfile = { workspace = true } tokio = { workspace = true } toml = "0.8.14" walkdir = "2.5.0" -wac-graph = "0.6.0" +wac-graph = "=0.6.0" wit-bindgen-rust = "=0.26.0" -wit-parser = "=0.219.0" +# wit-encoder = "=0.219.0" +# wit-encoder = { git = "https://github.com/golemcloud/wasm-tools.git", rev = "f715bded75519d7706f70d9d5938678b42e44e56"} +wit-encoder = { path = "../../wasm-tools/crates/wit-encoder"} +# wit-parser = "=0.219.0" +# wit-parser = { git = "https://github.com/golemcloud/wasm-tools.git", rev = "f715bded75519d7706f70d9d5938678b42e44e56"} +wit-parser = { path = "../../wasm-tools/crates/wit-parser"} + [dev-dependencies] test-r = { workspace = true } \ No newline at end of file diff --git a/wasm-rpc-stubgen/src/commands/dependencies.rs b/wasm-rpc-stubgen/src/commands/dependencies.rs index 1c1223af..97092a2a 100644 --- a/wasm-rpc-stubgen/src/commands/dependencies.rs +++ b/wasm-rpc-stubgen/src/commands/dependencies.rs @@ -14,8 +14,10 @@ use crate::commands::log::{log_action_plan, log_warn_action}; use crate::fs::{get_file_name, strip_path_prefix, OverwriteSafeAction, OverwriteSafeActions}; -use crate::wit::{generate_stub_wit_from_wit_dir, import_remover}; +use crate::naming::wit::package_merged_wit_name; +use crate::wit_generate::generate_stub_wit_from_wit_dir; use crate::wit_resolve::ResolvedWitDir; +use crate::wit_transform::WitDirTransformer; use crate::{cargo, naming}; use anyhow::{anyhow, Context}; use std::collections::BTreeMap; @@ -36,6 +38,7 @@ pub fn add_stub_dependency( update_cargo_toml: UpdateCargoToml, ) -> anyhow::Result<()> { let stub_resolved_wit_root = ResolvedWitDir::new(stub_wit_root)?; + let mut stub_transformer = WitDirTransformer::new(&stub_resolved_wit_root)?; let stub_package = stub_resolved_wit_root.main_package()?; let stub_wit = stub_wit_root.join(naming::wit::STUB_WIT_FILE_NAME); @@ -43,7 +46,8 @@ pub fn add_stub_dependency( let dest_resolved_wit_root = ResolvedWitDir::new(dest_wit_root)?; let dest_package = dest_resolved_wit_root.main_package()?; let dest_stub_package_name = naming::wit::stub_package_name(&dest_package.name); - let dest_stub_import_remover = import_remover(&dest_stub_package_name); + // TODO: have a better matcher which also considers / and @ + let dest_stub_package_import_prefix = dest_stub_package_name.to_string(); { let is_self_stub_by_name = @@ -107,13 +111,44 @@ pub fn add_stub_dependency( // Handle other package by copying while removing imports } else { package_names_to_package_path.insert(package_name.clone(), package_path); + stub_transformer.remove_imports_from_package_all_worlds( + *package_id, + &dest_stub_package_import_prefix, + )?; + let content = stub_transformer.render_package(*package_id)?; + let first_source = package_sources.iter().next().ok_or_else(|| { + anyhow!( + "Expected at least one source for stub package: {}", + package_name + ) + })?; + let first_source_relative_path = strip_path_prefix(stub_wit_root, first_source)?; + let target = { + if package_sources.len() == 1 { + dest_wit_root.join(first_source_relative_path) + } else { + dest_wit_root + .join(first_source_relative_path.parent().ok_or_else(|| { + anyhow!( + "Failed to get parent of stub source: {}", + first_source_relative_path.to_string_lossy() + ) + })?) + .join(package_merged_wit_name(package_name)) + } + }; + + actions.add(OverwriteSafeAction::WriteFile { content, target }); + // TODO: still output old ones while experimenting for source in package_sources { - actions.add(OverwriteSafeAction::copy_file_transformed( - source.clone(), - dest_wit_root.join(strip_path_prefix(stub_wit_root, source)?), - &dest_stub_import_remover, - )?); + actions.add(OverwriteSafeAction::CopyFile { + source: source.clone(), + target: dest_wit_root.join(format!( + "{}.old", + strip_path_prefix(stub_wit_root, source)?.to_string_lossy() + )), + }); } } } diff --git a/wasm-rpc-stubgen/src/commands/generate.rs b/wasm-rpc-stubgen/src/commands/generate.rs index 456f4940..af968459 100644 --- a/wasm-rpc-stubgen/src/commands/generate.rs +++ b/wasm-rpc-stubgen/src/commands/generate.rs @@ -18,7 +18,7 @@ use crate::fs::copy; use crate::naming; use crate::rust::generate_stub_source; use crate::stub::StubDefinition; -use crate::wit::{copy_wit_dependencies, generate_stub_wit_to_target}; +use crate::wit_generate::{copy_wit_dependencies, generate_stub_wit_to_target}; use crate::wit_resolve::ResolvedWitDir; use anyhow::Context; use fs_extra::dir::CopyOptions; diff --git a/wasm-rpc-stubgen/src/commands/log.rs b/wasm-rpc-stubgen/src/commands/log.rs index 4fe87125..46540d9f 100644 --- a/wasm-rpc-stubgen/src/commands/log.rs +++ b/wasm-rpc-stubgen/src/commands/log.rs @@ -19,12 +19,11 @@ pub fn log_skipping_up_to_date>(subject: T) { pub fn log_validated_action_result(action: &str, result: &ValidatedResult, to_log: F) where - F: Fn(&T) -> String, + F: FnOnce(&T) -> String, { - result - .as_ok_ref() - .iter() - .for_each(|value| log_action(action, to_log(value))); + if let Some(value) = result.as_ok_ref() { + log_action(action, to_log(value)) + } } pub fn log_action_plan(action: &OverwriteSafeAction, plan: OverwriteSafeActionPlan) { diff --git a/wasm-rpc-stubgen/src/fs.rs b/wasm-rpc-stubgen/src/fs.rs index 8ebe3d1e..ab96968e 100644 --- a/wasm-rpc-stubgen/src/fs.rs +++ b/wasm-rpc-stubgen/src/fs.rs @@ -149,7 +149,7 @@ pub fn get_file_name>(path: P) -> anyhow::Result { path.file_name() .ok_or_else(|| { anyhow!( - "Failed to get file name for package source: {}", + "Failed to get file name for path: {}", path.to_string_lossy(), ) })? @@ -205,7 +205,7 @@ impl OverwriteSafeAction { transform: F, ) -> anyhow::Result where - F: Fn(String) -> anyhow::Result, + F: FnOnce(String) -> anyhow::Result, { let content = std::fs::read_to_string(&source).with_context(|| { anyhow!( @@ -345,7 +345,7 @@ impl OverwriteSafeActions { ) -> anyhow::Result> where P: AsRef, - F: Fn() -> anyhow::Result, + F: FnOnce() -> anyhow::Result, { if !target.as_ref().exists() { Ok(Some(OverwriteSafeActionPlan::Create)) diff --git a/wasm-rpc-stubgen/src/lib.rs b/wasm-rpc-stubgen/src/lib.rs index 52773684..773a6131 100644 --- a/wasm-rpc-stubgen/src/lib.rs +++ b/wasm-rpc-stubgen/src/lib.rs @@ -21,8 +21,9 @@ pub mod model; pub mod naming; pub mod rust; pub mod stub; -pub mod wit; +pub mod wit_generate; pub mod wit_resolve; +pub mod wit_transform; use crate::commands::dependencies::UpdateCargoToml; use crate::stub::StubDefinition; diff --git a/wasm-rpc-stubgen/src/model/validation.rs b/wasm-rpc-stubgen/src/model/validation.rs index a417d03f..caa0a38d 100644 --- a/wasm-rpc-stubgen/src/model/validation.rs +++ b/wasm-rpc-stubgen/src/model/validation.rs @@ -47,7 +47,7 @@ impl ValidatedResult { pub fn combine(self, u: ValidatedResult, combine: C) -> ValidatedResult where - C: Fn(T, U) -> V, + C: FnOnce(T, U) -> V, { let (t, mut t_warns, mut t_errors) = self.into_product(); let (u, u_warns, u_errors) = u.into_product(); diff --git a/wasm-rpc-stubgen/src/naming.rs b/wasm-rpc-stubgen/src/naming.rs index 5126e632..46c374bd 100644 --- a/wasm-rpc-stubgen/src/naming.rs +++ b/wasm-rpc-stubgen/src/naming.rs @@ -31,6 +31,10 @@ pub mod wit { format!("{}_{}", package_name.namespace, package_name.name) } + pub fn package_merged_wit_name(package_name: &PackageName) -> String { + format!("{}_{}.wit", package_name.namespace, package_name.name) + } + pub fn package_wit_dep_dir_from_package_dir_name(package_dir_name: &str) -> PathBuf { Path::new(WIT_DIR).join(DEPS_DIR).join(package_dir_name) } diff --git a/wasm-rpc-stubgen/src/wit.rs b/wasm-rpc-stubgen/src/wit_generate.rs similarity index 96% rename from wasm-rpc-stubgen/src/wit.rs rename to wasm-rpc-stubgen/src/wit_generate.rs index f51bc0e4..7c30e782 100644 --- a/wasm-rpc-stubgen/src/wit.rs +++ b/wasm-rpc-stubgen/src/wit_generate.rs @@ -18,16 +18,14 @@ use crate::stub::{ FunctionParamStub, FunctionResultStub, FunctionStub, InterfaceStub, InterfaceStubImport, InterfaceStubTypeDef, StubDefinition, }; +use crate::wit_transform::import_remover; use crate::{naming, WasmRpcOverride}; use anyhow::{anyhow, bail}; use indexmap::IndexMap; -use regex::Regex; use std::fmt::Write; use std::fs; use std::path::Path; -use wit_parser::{ - Enum, Field, Flags, Handle, PackageName, Result_, Tuple, Type, TypeDef, TypeDefKind, Variant, -}; +use wit_parser::{Enum, Field, Flags, Handle, Result_, Tuple, Type, TypeDef, TypeDefKind, Variant}; pub fn generate_stub_wit_to_target(def: &StubDefinition) -> anyhow::Result<()> { log_action( @@ -640,20 +638,3 @@ impl TypeExtensions for Type { } } } - -pub fn import_remover(package_name: &PackageName) -> impl Fn(String) -> anyhow::Result { - let pattern_import_stub_package_name = Regex::new( - format!( - r"import\s+{}(/[^;]*)?;", - regex::escape(&package_name.to_string()) - ) - .as_str(), - ) - .unwrap_or_else(|err| panic!("Failed to compile package import regex: {}", err)); - - move |src: String| -> anyhow::Result { - Ok(pattern_import_stub_package_name - .replace_all(&src, "") - .to_string()) - } -} diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index 442a0187..d1d7a175 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -2,7 +2,7 @@ use crate::fs::strip_path_prefix; use anyhow::{anyhow, bail, Context}; use indexmap::IndexMap; use std::path::{Path, PathBuf}; -use wit_parser::{Package, PackageId, Resolve, UnresolvedPackageGroup}; +use wit_parser::{Package, PackageId, PackageName, Resolve, UnresolvedPackageGroup}; pub struct ResolvedWitDir { pub path: PathBuf, @@ -26,6 +26,19 @@ impl ResolvedWitDir { }) } + pub fn package_id_by_encoder_name( + &self, + package_name: &wit_encoder::PackageName, + ) -> Option { + let package_name = PackageName { + namespace: package_name.namespace().to_string(), + name: package_name.name().to_string(), + version: package_name.version().cloned(), + }; + + self.resolve.package_names.get(&package_name).cloned() + } + pub fn main_package(&self) -> anyhow::Result<&Package> { self.package(self.package_id) } diff --git a/wasm-rpc-stubgen/src/wit_transform.rs b/wasm-rpc-stubgen/src/wit_transform.rs new file mode 100644 index 00000000..b3720e50 --- /dev/null +++ b/wasm-rpc-stubgen/src/wit_transform.rs @@ -0,0 +1,468 @@ +use crate::wit_resolve::ResolvedWitDir; +use anyhow::anyhow; +use indexmap::IndexMap; +use regex::Regex; +use wit_encoder::{ + packages_from_parsed, Enum, Flags, Ident, Include, Interface, InterfaceItem, Package, + PackageItem, PackageName, Record, Resource, StandaloneFunc, Type, TypeDef, TypeDefKind, Use, + Variant, World, WorldItem, WorldNamedInterface, +}; + +// TODO: add skip option? +pub trait VisitPackage { + #[allow(unused_variables)] + fn package(&mut self, package: &mut Package) {} + + #[allow(unused_variables)] + fn package_interface(&mut self, package_name: &PackageName, interface: &mut Interface) {} + + #[allow(unused_variables)] + fn package_interface_use( + &mut self, + package_name: &PackageName, + interface_name: &Ident, + use_: &mut Use, + ) { + } + + #[allow(unused_variables)] + fn package_interface_type_def( + &mut self, + package_name: &PackageName, + interface_name: &Ident, + type_def: &mut TypeDef, + ) { + } + + #[allow(unused_variables)] + fn package_interface_type_def_record( + &mut self, + package_name: &PackageName, + interface_name: &Ident, + type_def_name: &Ident, + record: &mut Record, + ) { + } + + #[allow(unused_variables)] + fn package_interface_type_def_resource( + &mut self, + package_name: &PackageName, + interface_name: &Ident, + type_def_name: &Ident, + resource: &mut Resource, + ) { + } + + #[allow(unused_variables)] + fn package_interface_type_def_flags( + &mut self, + package_name: &PackageName, + interface_name: &Ident, + type_def_name: &Ident, + flags: &mut Flags, + ) { + } + + #[allow(unused_variables)] + fn package_interface_type_def_variant( + &mut self, + package_name: &PackageName, + interface_name: &Ident, + type_def_name: &Ident, + variant: &mut Variant, + ) { + } + + #[allow(unused_variables)] + fn package_interface_type_def_enum( + &mut self, + package_name: &PackageName, + interface_name: &Ident, + type_def_name: &Ident, + enum_: &mut Enum, + ) { + } + + #[allow(unused_variables)] + fn package_interface_type_def_type( + &mut self, + package_name: &PackageName, + interface_name: &Ident, + type_def_name: &Ident, + type_: &mut Type, + ) { + } + + #[allow(unused_variables)] + fn package_interface_function( + &mut self, + package_name: &PackageName, + interface_name: &Ident, + function: &mut StandaloneFunc, + ) { + } + + #[allow(unused_variables)] + fn package_world(&mut self, package_name: &PackageName, world: &mut World) {} + + #[allow(unused_variables)] + fn package_world_inline_interface_import( + &mut self, + package_name: &PackageName, + world_name: &Ident, + inline_interface_import: &mut Interface, + ) { + } + + #[allow(unused_variables)] + fn package_world_inline_interface_export( + &mut self, + package_name: &PackageName, + world_name: &Ident, + inline_interface_export: &mut Interface, + ) { + } + + #[allow(unused_variables)] + fn package_world_named_interface_import( + &mut self, + package_name: &PackageName, + world_name: &Ident, + named_interface_import: &mut WorldNamedInterface, + ) { + } + + #[allow(unused_variables)] + fn package_world_named_interface_export( + &mut self, + package_name: &PackageName, + world_name: &Ident, + named_interface_export: &mut WorldNamedInterface, + ) { + } + + #[allow(unused_variables)] + fn package_world_function_import( + &mut self, + package_name: &PackageName, + world_name: &Ident, + function_import: &mut StandaloneFunc, + ) { + } + + #[allow(unused_variables)] + fn package_world_function_export( + &mut self, + package_name: &PackageName, + world_name: &Ident, + function_export: &mut StandaloneFunc, + ) { + } + + #[allow(unused_variables)] + fn package_world_include( + &mut self, + package_name: &PackageName, + world_name: &Ident, + include: &mut Include, + ) { + } + + #[allow(unused_variables)] + fn package_world_use( + &mut self, + package_name: &PackageName, + world_name: &Ident, + use_: &mut Use, + ) { + } +} + +pub fn visit_package(package: &mut Package, visitor: &mut impl VisitPackage) { + visitor.package(package); + let package_name = package.name().clone(); + + for item in package.items_mut() { + match item { + PackageItem::Interface(interface) => { + visitor.package_interface(&package_name, interface); + + let interface_name = interface.name().clone(); + + for use_ in interface.uses_mut() { + visitor.package_interface_use(&package_name, &interface_name, use_); + } + + for item in interface.items_mut() { + match item { + InterfaceItem::TypeDef(type_def) => { + visitor.package_interface_type_def( + &package_name, + &interface_name, + type_def, + ); + + let type_def_name = type_def.name().clone(); + + match type_def.kind_mut() { + TypeDefKind::Record(record) => { + visitor.package_interface_type_def_record( + &package_name, + &interface_name, + &type_def_name, + record, + ); + } + TypeDefKind::Resource(resource) => { + visitor.package_interface_type_def_resource( + &package_name, + &interface_name, + &type_def_name, + resource, + ); + } + TypeDefKind::Flags(flags) => { + visitor.package_interface_type_def_flags( + &package_name, + &interface_name, + &type_def_name, + flags, + ); + } + TypeDefKind::Variant(variant) => { + visitor.package_interface_type_def_variant( + &package_name, + &interface_name, + &type_def_name, + variant, + ); + } + TypeDefKind::Enum(enum_) => { + visitor.package_interface_type_def_enum( + &package_name, + &interface_name, + &type_def_name, + enum_, + ); + } + TypeDefKind::Type(type_) => { + visitor.package_interface_type_def_type( + &package_name, + &interface_name, + &type_def_name, + type_, + ); + } + } + } + InterfaceItem::Function(function) => { + visitor.package_interface_function( + &package_name, + &interface_name, + function, + ); + } + } + } + } + PackageItem::World(world) => { + visitor.package_world(&package_name, world); + + let world_name = world.name().clone(); + + for item in world.items_mut() { + match item { + WorldItem::InlineInterfaceImport(inline_interface_import) => { + visitor.package_world_inline_interface_import( + &package_name, + &world_name, + inline_interface_import, + ); + } + WorldItem::InlineInterfaceExport(inline_interface_export) => { + visitor.package_world_inline_interface_export( + &package_name, + &world_name, + inline_interface_export, + ); + } + WorldItem::NamedInterfaceImport(named_interface_import) => { + visitor.package_world_named_interface_import( + &package_name, + &world_name, + named_interface_import, + ); + } + WorldItem::NamedInterfaceExport(named_interface_export) => { + visitor.package_world_named_interface_export( + &package_name, + &world_name, + named_interface_export, + ); + } + WorldItem::FunctionImport(function_import) => { + visitor.package_world_function_import( + &package_name, + &world_name, + function_import, + ); + } + WorldItem::FunctionExport(function_export) => { + visitor.package_world_function_export( + &package_name, + &world_name, + function_export, + ); + } + WorldItem::Include(include) => { + visitor.package_world_include(&package_name, &world_name, include); + } + WorldItem::Use(use_) => { + visitor.package_world_use(&package_name, &world_name, use_); + } + } + } + } + } + } +} + +pub fn import_remover( + package_name: &wit_parser::PackageName, +) -> impl Fn(String) -> anyhow::Result { + let pattern_import_stub_package_name = Regex::new( + format!( + r"import\s+{}(/[^;]*)?;", + regex::escape(&package_name.to_string()) + ) + .as_str(), + ) + .unwrap_or_else(|err| panic!("Failed to compile package import regex: {}", err)); + + move |src: String| -> anyhow::Result { + Ok(pattern_import_stub_package_name + .replace_all(&src, "") + .to_string()) + } +} + +pub struct WitDirTransformer<'a> { + resolved_wit_dir: &'a ResolvedWitDir, + encoded_packages_by_parser_id: IndexMap, +} + +impl<'a> WitDirTransformer<'a> { + pub fn new(resolved_wit_dir: &'a ResolvedWitDir) -> anyhow::Result { + let mut encoded_packages_by_parser_id = IndexMap::::new(); + for package in packages_from_parsed(&resolved_wit_dir.resolve) { + let package_id = resolved_wit_dir + .package_id_by_encoder_name(package.name()) + .ok_or_else(|| anyhow!("Failed to get package by name: {}", package.name()))?; + encoded_packages_by_parser_id.insert(package_id, package); + } + + Ok(Self { + resolved_wit_dir, + encoded_packages_by_parser_id, + }) + } + + pub fn render_package(&mut self, package_id: wit_parser::PackageId) -> anyhow::Result { + Ok(format!("{}", self.encoded_package(package_id)?)) + } + + fn encoded_package( + &mut self, + package_id: wit_parser::PackageId, + ) -> anyhow::Result<&mut Package> { + self.encoded_packages_by_parser_id + .get_mut(&package_id) + .ok_or_else(|| anyhow!("Failed to get encoded package by id: {:?}", package_id)) + } + + pub fn remove_imports_from_package_all_worlds( + &mut self, + package_id: wit_parser::PackageId, + import_prefix: &str, + ) -> anyhow::Result<()> { + struct RemoveImports<'a> { + import_prefix: &'a str, + } + impl<'a> VisitPackage for RemoveImports<'a> { + fn package_world(&mut self, _package_name: &PackageName, world: &mut World) { + world.items_mut().retain(|item| { + if let WorldItem::NamedInterfaceImport(import) = &item { + println!( + "@@@ {} starts with {}? {}", + import.name(), + self.import_prefix, + import.name().raw_name().starts_with(self.import_prefix) + ); + !import.name().raw_name().starts_with(self.import_prefix) + } else { + true + } + }); + } + } + + visit_package( + self.encoded_package(package_id)?, + &mut RemoveImports { import_prefix }, + ); + + Ok(()) + } +} + +pub fn add_import_to_world(package: &mut Package, world_name: Ident, import_name: Ident) { + struct AddImportToWorld { + world_name: Ident, + import_name: Ident, + } + impl VisitPackage for AddImportToWorld { + fn package_world(&mut self, _package_name: &PackageName, world: &mut World) { + if *world.name() == self.world_name { + let is_already_imported = world.items_mut().iter().any(|item| { + if let WorldItem::NamedInterfaceImport(import) = item { + *import.name() == self.import_name + } else { + false + } + }); + if !is_already_imported { + world.named_interface_import(self.import_name.clone()); + } + } + } + } + + visit_package( + package, + &mut AddImportToWorld { + world_name, + import_name, + }, + ); +} + +// TODO: add test for this in upstream +#[cfg(test)] +mod test { + use golem_wasm_rpc::{WASI_POLL_WIT, WASM_RPC_WIT}; + use test_r::test; + use wit_encoder::packages_from_parsed; + use wit_parser::Resolve; + + #[test] + pub fn round_trip() { + std::env::set_var("WIT_REQUIRE_F32_F64", "0"); + let mut resolve = Resolve::new(); + resolve.push_str("poll.wit", WASI_POLL_WIT).unwrap(); + resolve.push_str("wasm_rpc.wit", WASM_RPC_WIT).unwrap(); + + let packages = packages_from_parsed(&resolve); + println!("{}", packages[1]); + } +} From 5619d5cd43f0e10488473a39e770303027f1c652 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Wed, 23 Oct 2024 16:06:03 +0200 Subject: [PATCH 002/109] WIP: fixing add_dep tests --- Cargo.lock | 28 +- Cargo.toml | 2 +- wasm-rpc-stubgen/Cargo.toml | 10 +- wasm-rpc-stubgen/src/commands/dependencies.rs | 4 +- wasm-rpc-stubgen/src/rust.rs | 4 +- wasm-rpc-stubgen/src/stub.rs | 10 +- wasm-rpc-stubgen/src/wit_generate.rs | 24 +- wasm-rpc-stubgen/src/wit_resolve.rs | 15 +- wasm-rpc-stubgen/src/wit_transform.rs | 33 +- wasm-rpc-stubgen/tests/add_dep.rs | 418 +++++++++++------- 10 files changed, 339 insertions(+), 209 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b0879d31..2059a94c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1927,6 +1927,7 @@ dependencies = [ "proc-macro2", "quote", "regex", + "semver", "serde 1.0.210", "serde_json", "serde_yaml", @@ -4412,9 +4413,9 @@ dependencies = [ [[package]] name = "test-r" -version = "0.0.11" +version = "0.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f503de241983649990311e28573ce1e98f62ed65f2b6fb7bdf2496d2ba1c978" +checksum = "5f81c856ebb383e7edb390dd642458f8bc91356760c88df9aab258d8a8d01ecd" dependencies = [ "ctor", "test-r-core", @@ -4424,9 +4425,9 @@ dependencies = [ [[package]] name = "test-r-core" -version = "0.0.10" +version = "0.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf42476bec5da760737467562f6169dc126d3716b74ca072f08246fbcb332000" +checksum = "3cc908dda80def66445eef3560ea54df65cb26b557d43fc1d7fdc3ae5675f31f" dependencies = [ "anstream", "anstyle", @@ -4445,9 +4446,9 @@ dependencies = [ [[package]] name = "test-r-macro" -version = "0.0.11" +version = "0.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "198b6d8536c41236c3bc5dec066dd4d63ff01fab76596a9738d8bf60b050afdb" +checksum = "34e97f6324519d7b08bf4a3cf529f5a14fdce61587771717b7571c6f9af0f458" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -5375,22 +5376,23 @@ dependencies = [ [[package]] name = "wasmparser" version = "0.219.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c771866898879073c53b565a6c7b49953795159836714ac56a5befb581227c5" dependencies = [ - "ahash", "bitflags 2.6.0", - "hashbrown 0.14.5", "indexmap 2.6.0", - "semver", ] [[package]] name = "wasmparser" version = "0.219.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c771866898879073c53b565a6c7b49953795159836714ac56a5befb581227c5" +source = "git+https://github.com/golemcloud/wasm-tools.git?rev=46b168c169dc28d169d60e60ddbe4eed20f905b3#46b168c169dc28d169d60e60ddbe4eed20f905b3" dependencies = [ + "ahash", "bitflags 2.6.0", + "hashbrown 0.14.5", "indexmap 2.6.0", + "semver", ] [[package]] @@ -6173,6 +6175,7 @@ dependencies = [ [[package]] name = "wit-encoder" version = "0.219.1" +source = "git+https://github.com/golemcloud/wasm-tools.git?rev=46b168c169dc28d169d60e60ddbe4eed20f905b3#46b168c169dc28d169d60e60ddbe4eed20f905b3" dependencies = [ "id-arena", "pretty_assertions", @@ -6238,6 +6241,7 @@ dependencies = [ [[package]] name = "wit-parser" version = "0.219.1" +source = "git+https://github.com/golemcloud/wasm-tools.git?rev=46b168c169dc28d169d60e60ddbe4eed20f905b3#46b168c169dc28d169d60e60ddbe4eed20f905b3" dependencies = [ "anyhow", "id-arena", @@ -6248,7 +6252,7 @@ dependencies = [ "serde_derive", "serde_json", "unicode-xid", - "wasmparser 0.219.1", + "wasmparser 0.219.1 (git+https://github.com/golemcloud/wasm-tools.git?rev=46b168c169dc28d169d60e60ddbe4eed20f905b3)", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f5fc08dc..aacf5879 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,5 +20,5 @@ opt-level = 's' fs_extra = "1.3.0" golem-wasm-ast = "1.0.1" tempfile = "3.12.0" -test-r = { version = "0.0.11", default-features = false } +test-r = { version = "0.0.13", default-features = false } tokio = "1.38.0" \ No newline at end of file diff --git a/wasm-rpc-stubgen/Cargo.toml b/wasm-rpc-stubgen/Cargo.toml index 58534599..061e78a5 100644 --- a/wasm-rpc-stubgen/Cargo.toml +++ b/wasm-rpc-stubgen/Cargo.toml @@ -50,6 +50,7 @@ prettyplease = "0.2.20" proc-macro2 = "1.0.85" quote = "1.0.36" regex = "1.10.4" +semver = "1.0.23" serde = { version = "1.0.203", features = ["derive"] } serde_json = "1.0.128" serde_yaml = "0.9.33" @@ -61,11 +62,12 @@ walkdir = "2.5.0" wac-graph = "=0.6.0" wit-bindgen-rust = "=0.26.0" # wit-encoder = "=0.219.0" -# wit-encoder = { git = "https://github.com/golemcloud/wasm-tools.git", rev = "f715bded75519d7706f70d9d5938678b42e44e56"} -wit-encoder = { path = "../../wasm-tools/crates/wit-encoder"} +wit-encoder = { git = "https://github.com/golemcloud/wasm-tools.git", rev = "46b168c169dc28d169d60e60ddbe4eed20f905b3"} +# wit-encoder = { path = "../../wasm-tools/crates/wit-encoder"} # wit-parser = "=0.219.0" -# wit-parser = { git = "https://github.com/golemcloud/wasm-tools.git", rev = "f715bded75519d7706f70d9d5938678b42e44e56"} -wit-parser = { path = "../../wasm-tools/crates/wit-parser"} +wit-parser = { git = "https://github.com/golemcloud/wasm-tools.git", rev = "46b168c169dc28d169d60e60ddbe4eed20f905b3"} + +# wit-parser = { path = "../../wasm-tools/crates/wit-parser"} [dev-dependencies] diff --git a/wasm-rpc-stubgen/src/commands/dependencies.rs b/wasm-rpc-stubgen/src/commands/dependencies.rs index 97092a2a..c66cb7e5 100644 --- a/wasm-rpc-stubgen/src/commands/dependencies.rs +++ b/wasm-rpc-stubgen/src/commands/dependencies.rs @@ -17,7 +17,7 @@ use crate::fs::{get_file_name, strip_path_prefix, OverwriteSafeAction, Overwrite use crate::naming::wit::package_merged_wit_name; use crate::wit_generate::generate_stub_wit_from_wit_dir; use crate::wit_resolve::ResolvedWitDir; -use crate::wit_transform::WitDirTransformer; +use crate::wit_transform::WitTransformer; use crate::{cargo, naming}; use anyhow::{anyhow, Context}; use std::collections::BTreeMap; @@ -38,7 +38,7 @@ pub fn add_stub_dependency( update_cargo_toml: UpdateCargoToml, ) -> anyhow::Result<()> { let stub_resolved_wit_root = ResolvedWitDir::new(stub_wit_root)?; - let mut stub_transformer = WitDirTransformer::new(&stub_resolved_wit_root)?; + let mut stub_transformer = WitTransformer::new(&stub_resolved_wit_root.resolve)?; let stub_package = stub_resolved_wit_root.main_package()?; let stub_wit = stub_wit_root.join(naming::wit::STUB_WIT_FILE_NAME); diff --git a/wasm-rpc-stubgen/src/rust.rs b/wasm-rpc-stubgen/src/rust.rs index 692aa6da..ef022be4 100644 --- a/wasm-rpc-stubgen/src/rust.rs +++ b/wasm-rpc-stubgen/src/rust.rs @@ -44,7 +44,7 @@ pub fn generate_stub_source(def: &StubDefinition) -> anyhow::Result<()> { let mut exports = Vec::new(); let mut resource_type_aliases = Vec::new(); - for interface in def.source_interfaces() { + for interface in def.stub_exported_interfaces() { let interface_ident = to_rust_ident(&interface.name).to_upper_camel_case(); let interface_name = Ident::new(&interface_ident, Span::call_site()); @@ -116,7 +116,7 @@ pub fn generate_stub_source(def: &StubDefinition) -> anyhow::Result<()> { } let mut interface_impls = Vec::new(); - for interface in def.source_interfaces() { + for interface in def.stub_exported_interfaces() { let interface_ident = to_rust_ident(&interface.name).to_upper_camel_case(); let interface_name = Ident::new(&interface_ident, Span::call_site()); let guest_interface_name = diff --git a/wasm-rpc-stubgen/src/stub.rs b/wasm-rpc-stubgen/src/stub.rs index 2b53f931..bafd87f4 100644 --- a/wasm-rpc-stubgen/src/stub.rs +++ b/wasm-rpc-stubgen/src/stub.rs @@ -28,7 +28,7 @@ pub struct StubDefinition { resolve: Resolve, source_world_id: WorldId, sources: IndexMap)>, - source_interfaces: OnceCell>, + stub_exported_interfaces: OnceCell>, pub source_package_name: PackageName, pub source_wit_root: PathBuf, @@ -70,7 +70,7 @@ impl StubDefinition { resolve: resolved_source.resolve, source_world_id, sources: resolved_source.sources, - source_interfaces: OnceCell::new(), + stub_exported_interfaces: OnceCell::new(), source_package_name, source_wit_root: source_wit_root.to_path_buf(), target_root: target_root.to_path_buf(), @@ -168,7 +168,7 @@ impl StubDefinition { self.resolve.interfaces.get(*interface_id) { if let Some(name) = resolved_owner_interface.name.as_ref() { - self.source_interfaces() + self.stub_exported_interfaces() .iter() .any(|interface| &interface.name == name) } else { @@ -186,8 +186,8 @@ impl StubDefinition { } } - pub fn source_interfaces(&self) -> &Vec { - self.source_interfaces.get_or_init(|| { + pub fn stub_exported_interfaces(&self) -> &Vec { + self.stub_exported_interfaces.get_or_init(|| { let WorldItemsByType { types, functions, diff --git a/wasm-rpc-stubgen/src/wit_generate.rs b/wasm-rpc-stubgen/src/wit_generate.rs index 7c30e782..9633113f 100644 --- a/wasm-rpc-stubgen/src/wit_generate.rs +++ b/wasm-rpc-stubgen/src/wit_generate.rs @@ -22,10 +22,14 @@ use crate::wit_transform::import_remover; use crate::{naming, WasmRpcOverride}; use anyhow::{anyhow, bail}; use indexmap::IndexMap; +use semver::Version; +use std::collections::HashSet; use std::fmt::Write; use std::fs; use std::path::Path; -use wit_parser::{Enum, Field, Flags, Handle, Result_, Tuple, Type, TypeDef, TypeDefKind, Variant}; +use wit_parser::{ + Enum, Field, Flags, Handle, PackageName, Result_, Tuple, Type, TypeDef, TypeDefKind, Variant, +}; pub fn generate_stub_wit_to_target(def: &StubDefinition) -> anyhow::Result<()> { log_action( @@ -57,23 +61,35 @@ pub fn generate_stub_wit_from_stub_def(def: &StubDefinition) -> anyhow::Result::new(); writeln!(out, "package {}-stub;", def.source_package_name)?; writeln!(out)?; writeln!(out, "interface stub-{} {{", world.name)?; let all_imports = def - .source_interfaces() + .stub_exported_interfaces() .iter() .flat_map(|i| i.imports.iter().map(|i| (InterfaceStubImport::from(i), i))) .collect::>(); // Renaming the mandatory imports to avoid collisions with types coming from the stubbed package writeln!(out, " use golem:rpc/types@0.1.0.{{uri as golem-rpc-uri}};")?; + exported_packages.insert(PackageName { + namespace: "golem".to_string(), + name: "rpc".to_string(), + version: Some(Version::new(0, 1, 0)), + }); + writeln!( out, " use wasi:io/poll@0.2.0.{{pollable as wasi-io-pollable}};" )?; + exported_packages.insert(PackageName { + namespace: "wasi".to_string(), + name: "io".to_string(), + version: Some(Version::new(0, 2, 0)), + }); if def.always_inline_types { let mut inline_types: Vec = vec![]; @@ -100,7 +116,7 @@ pub fn generate_stub_wit_from_stub_def(def: &StubDefinition) -> anyhow::Result anyhow::Result { diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index d1d7a175..442a0187 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -2,7 +2,7 @@ use crate::fs::strip_path_prefix; use anyhow::{anyhow, bail, Context}; use indexmap::IndexMap; use std::path::{Path, PathBuf}; -use wit_parser::{Package, PackageId, PackageName, Resolve, UnresolvedPackageGroup}; +use wit_parser::{Package, PackageId, Resolve, UnresolvedPackageGroup}; pub struct ResolvedWitDir { pub path: PathBuf, @@ -26,19 +26,6 @@ impl ResolvedWitDir { }) } - pub fn package_id_by_encoder_name( - &self, - package_name: &wit_encoder::PackageName, - ) -> Option { - let package_name = PackageName { - namespace: package_name.namespace().to_string(), - name: package_name.name().to_string(), - version: package_name.version().cloned(), - }; - - self.resolve.package_names.get(&package_name).cloned() - } - pub fn main_package(&self) -> anyhow::Result<&Package> { self.package(self.package_id) } diff --git a/wasm-rpc-stubgen/src/wit_transform.rs b/wasm-rpc-stubgen/src/wit_transform.rs index b3720e50..8af66dbc 100644 --- a/wasm-rpc-stubgen/src/wit_transform.rs +++ b/wasm-rpc-stubgen/src/wit_transform.rs @@ -1,4 +1,3 @@ -use crate::wit_resolve::ResolvedWitDir; use anyhow::anyhow; use indexmap::IndexMap; use regex::Regex; @@ -347,23 +346,33 @@ pub fn import_remover( } } -pub struct WitDirTransformer<'a> { - resolved_wit_dir: &'a ResolvedWitDir, +pub struct WitTransformer<'a> { + resolve: &'a wit_parser::Resolve, encoded_packages_by_parser_id: IndexMap, } -impl<'a> WitDirTransformer<'a> { - pub fn new(resolved_wit_dir: &'a ResolvedWitDir) -> anyhow::Result { +impl<'a> WitTransformer<'a> { + pub fn new(resolve: &'a wit_parser::Resolve) -> anyhow::Result { let mut encoded_packages_by_parser_id = IndexMap::::new(); - for package in packages_from_parsed(&resolved_wit_dir.resolve) { - let package_id = resolved_wit_dir - .package_id_by_encoder_name(package.name()) + + for package in packages_from_parsed(&resolve) { + let package_name = package.name(); + let package_name = wit_parser::PackageName { + namespace: package_name.namespace().to_string(), + name: package_name.name().to_string(), + version: package_name.version().cloned(), + }; + + let package_id = resolve + .package_names + .get(&package_name) + .cloned() .ok_or_else(|| anyhow!("Failed to get package by name: {}", package.name()))?; encoded_packages_by_parser_id.insert(package_id, package); } Ok(Self { - resolved_wit_dir, + resolve, encoded_packages_by_parser_id, }) } @@ -393,12 +402,6 @@ impl<'a> WitDirTransformer<'a> { fn package_world(&mut self, _package_name: &PackageName, world: &mut World) { world.items_mut().retain(|item| { if let WorldItem::NamedInterfaceImport(import) = &item { - println!( - "@@@ {} starts with {}? {}", - import.name(), - self.import_prefix, - import.name().raw_name().starts_with(self.import_prefix) - ); !import.name().raw_name().starts_with(self.import_prefix) } else { true diff --git a/wasm-rpc-stubgen/tests/add_dep.rs b/wasm-rpc-stubgen/tests/add_dep.rs index 5052648e..d0d8c470 100644 --- a/wasm-rpc-stubgen/tests/add_dep.rs +++ b/wasm-rpc-stubgen/tests/add_dep.rs @@ -16,6 +16,7 @@ use test_r::test; +use assert2::assert; use fs_extra::dir::CopyOptions; use golem_wasm_rpc::{WASI_POLL_WIT, WASM_RPC_WIT}; use golem_wasm_rpc_stubgen::commands::dependencies::{add_stub_dependency, UpdateCargoToml}; @@ -23,8 +24,11 @@ use golem_wasm_rpc_stubgen::commands::generate::generate_stub_wit_dir; use golem_wasm_rpc_stubgen::stub::StubDefinition; use golem_wasm_rpc_stubgen::wit_resolve::ResolvedWitDir; use golem_wasm_rpc_stubgen::WasmRpcOverride; -use std::path::Path; +use semver::Version; +use std::path::{Path, PathBuf}; use tempfile::TempDir; +use wit_encoder::{packages_from_parsed, Package, PackageName}; +use wit_parser::Resolve; test_r::enable!(); @@ -46,15 +50,19 @@ fn all_wit_types_no_collision() { assert_valid_wit_root(&dest_wit_root); - assert_has_wit_dep(&dest_wit_root, "io/poll.wit", WASI_POLL_WIT); - assert_has_wit_dep(&dest_wit_root, "wasm-rpc/wasm-rpc.wit", WASM_RPC_WIT); + assert_has_wasm_rpc_wit_deps(&dest_wit_root); - let stub_wit = std::fs::read_to_string(stub_wit_root.join("_stub.wit")).unwrap(); - assert_has_wit_dep(&dest_wit_root, "test_main-stub/_stub.wit", &stub_wit); + assert_has_same_wit_package( + &PackageName::new("test", "main-stub", None), + &dest_wit_root, + &stub_wit_root, + ); - let original_wit = - std::fs::read_to_string(Path::new("test-data").join("all-wit-types/main.wit")).unwrap(); - assert_has_wit_dep(&dest_wit_root, "test_main/main.wit", &original_wit); + assert_has_same_wit_package( + &PackageName::new("test", "main", None), + &dest_wit_root, + &Path::new("test-data").join("all-wit-types/main.wit"), + ); } #[test] @@ -84,15 +92,19 @@ fn all_wit_types_overwrite_protection() { assert_valid_wit_root(&dest_wit_root); - assert_has_wit_dep(&dest_wit_root, "io/poll.wit", WASI_POLL_WIT); - assert_has_wit_dep(&dest_wit_root, "wasm-rpc/wasm-rpc.wit", WASM_RPC_WIT); + assert_has_wasm_rpc_wit_deps(&dest_wit_root); - let stub_wit = std::fs::read_to_string(stub_wit_root.join("_stub.wit")).unwrap(); - assert_has_wit_dep(&dest_wit_root, "test_main-stub/_stub.wit", &stub_wit); + assert_has_same_wit_package( + &PackageName::new("test", "main-stub", None), + &dest_wit_root, + &stub_wit_root, + ); - let original_wit = - std::fs::read_to_string(Path::new("test-data").join("all-wit-types/main.wit")).unwrap(); - assert_has_wit_dep(&dest_wit_root, "test_main/main.wit", &original_wit); + assert_has_same_wit_package( + &PackageName::new("test", "main", None), + &dest_wit_root, + &Path::new("test-data").join("all-wit-types/main.wit"), + ); } #[test] @@ -122,16 +134,19 @@ fn all_wit_types_overwrite_protection_disabled() { assert_valid_wit_root(&dest_wit_root); - assert_has_wit_dep(&dest_wit_root, "io/poll.wit", WASI_POLL_WIT); - assert_has_wit_dep(&dest_wit_root, "wasm-rpc/wasm-rpc.wit", WASM_RPC_WIT); + assert_has_wasm_rpc_wit_deps(&dest_wit_root); - let stub_wit = std::fs::read_to_string(alternative_stub_wit_root.join("_stub.wit")).unwrap(); - assert_has_wit_dep(&dest_wit_root, "test_main-stub/_stub.wit", &stub_wit); + assert_has_same_wit_package( + &PackageName::new("test", "main-stub", None), + &dest_wit_root, + &alternative_stub_wit_root, + ); - let original_wit = - std::fs::read_to_string(Path::new("test-data").join("all-wit-types-alternative/main.wit")) - .unwrap(); - assert_has_wit_dep(&dest_wit_root, "test_main/main.wit", &original_wit); + assert_has_same_wit_package( + &PackageName::new("test", "main", None), + &dest_wit_root, + &Path::new("test-data").join("all-wit-types-alternative/main.wit"), + ); } #[test] @@ -152,22 +167,25 @@ fn many_ways_to_export_no_collision() { assert_valid_wit_root(&dest_wit_root); - assert_has_wit_dep(&dest_wit_root, "io/poll.wit", WASI_POLL_WIT); - assert_has_wit_dep(&dest_wit_root, "wasm-rpc/wasm-rpc.wit", WASM_RPC_WIT); + assert_has_wasm_rpc_wit_deps(&dest_wit_root); - let stub_wit = std::fs::read_to_string(stub_wit_root.join("_stub.wit")).unwrap(); - assert_has_wit_dep(&dest_wit_root, "test_exports-stub/_stub.wit", &stub_wit); + assert_has_same_wit_package( + &PackageName::new("test", "exports-stub", None), + &dest_wit_root, + &stub_wit_root, + ); - let original_wit = - std::fs::read_to_string(Path::new("test-data").join("many-ways-to-export/main.wit")) - .unwrap(); - assert_has_wit_dep(&dest_wit_root, "test_exports/main.wit", &original_wit); + assert_has_same_wit_package( + &PackageName::new("test", "exports", None), + &dest_wit_root, + &Path::new("test-data").join("many-ways-to-export"), + ); - let original_sub_wit = std::fs::read_to_string( - Path::new("test-data").join("many-ways-to-export/deps/sub/sub.wit"), - ) - .unwrap(); - assert_has_wit_dep(&dest_wit_root, "sub/sub.wit", &original_sub_wit); + assert_has_same_wit_package( + &PackageName::new("test", "sub", None), + &dest_wit_root, + &Path::new("test-data").join("many-ways-to-export/deps/sub/sub.wit"), + ); } #[test] @@ -200,25 +218,33 @@ fn direct_circular() { assert_valid_wit_root(dest_a.path()); assert_valid_wit_root(dest_b.path()); - assert_has_wit_dep(dest_a.path(), "io/poll.wit", WASI_POLL_WIT); - assert_has_wit_dep(dest_a.path(), "wasm-rpc/wasm-rpc.wit", WASM_RPC_WIT); + assert_has_wasm_rpc_wit_deps(&dest_a.path()); - let stub_wit_b = std::fs::read_to_string(stub_b_dir.path().join("wit/_stub.wit")).unwrap(); - assert_has_wit_dep(dest_a.path(), "test_b-stub/_stub.wit", &stub_wit_b); + assert_has_same_wit_package( + &PackageName::new("test", "b-stub", None), + dest_a.path(), + &stub_b_dir.path().join("wit"), + ); - let original_b = - std::fs::read_to_string(Path::new("test-data").join("direct-circular-b/b.wit")).unwrap(); - assert_has_wit_dep_similar(dest_a.path(), "test_b/b.wit", &original_b); + assert_has_same_wit_package( + &PackageName::new("test", "b", None), + dest_a.path(), + &Path::new("test-data").join("direct-circular-b/b.wit"), + ); - assert_has_wit_dep(dest_b.path(), "io/poll.wit", WASI_POLL_WIT); - assert_has_wit_dep(dest_b.path(), "wasm-rpc/wasm-rpc.wit", WASM_RPC_WIT); + assert_has_wasm_rpc_wit_deps(dest_b.path()); - let stub_wit_a = std::fs::read_to_string(stub_a_dir.path().join("wit/_stub.wit")).unwrap(); - assert_has_wit_dep(dest_b.path(), "test_a-stub/_stub.wit", &stub_wit_a); + assert_has_same_wit_package( + &PackageName::new("test", "a-stub", None), + dest_b.path(), + &stub_a_dir.path().join("wit"), + ); - let original_a = - std::fs::read_to_string(Path::new("test-data").join("direct-circular-a/a.wit")).unwrap(); - assert_has_wit_dep_similar(dest_b.path(), "test_a/a.wit", &original_a); + assert_has_same_wit_package( + &PackageName::new("test", "a", None), + dest_b.path(), + &Path::new("test-data").join("direct-circular-a/a.wit"), + ); } #[test] @@ -276,25 +302,33 @@ fn direct_circular_readd() { assert_valid_wit_root(dest_a.path()); assert_valid_wit_root(dest_b.path()); - assert_has_wit_dep(dest_a.path(), "io/poll.wit", WASI_POLL_WIT); - assert_has_wit_dep(dest_a.path(), "wasm-rpc/wasm-rpc.wit", WASM_RPC_WIT); + assert_has_wasm_rpc_wit_deps(dest_a.path()); - let stub_wit_b = std::fs::read_to_string(stub_b_dir.path().join("wit/_stub.wit")).unwrap(); - assert_has_wit_dep(dest_a.path(), "test_b-stub/_stub.wit", &stub_wit_b); + assert_has_same_wit_package( + &PackageName::new("test", "b-stub", None), + dest_a.path(), + &stub_b_dir.path().join("wit"), + ); - let original_b = - std::fs::read_to_string(Path::new("test-data").join("direct-circular-b/b.wit")).unwrap(); - assert_has_wit_dep_similar(dest_a.path(), "test_b/b.wit", &original_b); + assert_has_same_wit_package( + &PackageName::new("test", "b", None), + dest_a.path(), + &Path::new("test-data").join("direct-circular-b/b.wit"), + ); - assert_has_wit_dep(dest_b.path(), "io/poll.wit", WASI_POLL_WIT); - assert_has_wit_dep(dest_b.path(), "wasm-rpc/wasm-rpc.wit", WASM_RPC_WIT); + assert_has_wasm_rpc_wit_deps(dest_b.path()); - let stub_wit_a = std::fs::read_to_string(stub_a_dir.path().join("wit/_stub.wit")).unwrap(); - assert_has_wit_dep(dest_b.path(), "test_a-stub/_stub.wit", &stub_wit_a); + assert_has_same_wit_package( + &PackageName::new("test", "a-stub", None), + dest_b.path(), + &stub_a_dir.path().join("wit"), + ); - let original_a = - std::fs::read_to_string(Path::new("test-data").join("direct-circular-a/a.wit")).unwrap(); - assert_has_wit_dep_similar(dest_b.path(), "test_a/a.wit", &original_a); + assert_has_same_wit_package( + &PackageName::new("test", "a", None), + dest_b.path(), + &Path::new("test-data").join("direct-circular-a/a.wit"), + ); } #[test] @@ -327,29 +361,33 @@ fn direct_circular_same_world_name() { assert_valid_wit_root(dest_a.path()); assert_valid_wit_root(dest_b.path()); - assert_has_wit_dep(dest_a.path(), "io/poll.wit", WASI_POLL_WIT); - assert_has_wit_dep(dest_a.path(), "wasm-rpc/wasm-rpc.wit", WASM_RPC_WIT); + assert_has_wasm_rpc_wit_deps(dest_a.path()); - let stub_wit_b = std::fs::read_to_string(stub_b_dir.path().join("wit/_stub.wit")).unwrap(); - assert_has_wit_dep(dest_a.path(), "test_b-stub/_stub.wit", &stub_wit_b); + assert_has_same_wit_package( + &PackageName::new("test", "b-stub", None), + dest_a.path(), + &stub_b_dir.path().join("wit"), + ); - let original_b = std::fs::read_to_string( - Path::new("test-data").join("direct-circular-b-same-world-name/b.wit"), - ) - .unwrap(); - assert_has_wit_dep_similar(dest_a.path(), "test_b/b.wit", &original_b); + assert_has_same_wit_package( + &PackageName::new("test", "b", None), + dest_a.path(), + &Path::new("test-data").join("direct-circular-b-same-world-name/b.wit"), + ); - assert_has_wit_dep(dest_b.path(), "io/poll.wit", WASI_POLL_WIT); - assert_has_wit_dep(dest_b.path(), "wasm-rpc/wasm-rpc.wit", WASM_RPC_WIT); + assert_has_wasm_rpc_wit_deps(dest_b.path()); - let stub_wit_a = std::fs::read_to_string(stub_a_dir.path().join("wit/_stub.wit")).unwrap(); - assert_has_wit_dep(dest_b.path(), "test_a-stub/_stub.wit", &stub_wit_a); + assert_has_same_wit_package( + &PackageName::new("test", "a-stub", None), + dest_b.path(), + &stub_a_dir.path().join("wit"), + ); - let original_a = std::fs::read_to_string( - Path::new("test-data").join("direct-circular-a-same-world-name/a.wit"), - ) - .unwrap(); - assert_has_wit_dep_similar(dest_b.path(), "test_a/a.wit", &original_a); + assert_has_same_wit_package( + &PackageName::new("test", "a", None), + dest_b.path(), + &Path::new("test-data").join("direct-circular-a-same-world-name/a.wit"), + ); } #[test] @@ -393,34 +431,47 @@ fn indirect_circular() { assert_valid_wit_root(dest_b.path()); assert_valid_wit_root(dest_c.path()); - assert_has_wit_dep(dest_a.path(), "io/poll.wit", WASI_POLL_WIT); - assert_has_wit_dep(dest_a.path(), "wasm-rpc/wasm-rpc.wit", WASM_RPC_WIT); + assert_has_wasm_rpc_wit_deps(dest_a.path()); - let stub_wit_b = std::fs::read_to_string(stub_b_dir.path().join("wit/_stub.wit")).unwrap(); - assert_has_wit_dep(dest_a.path(), "test_b-stub/_stub.wit", &stub_wit_b); + assert_has_same_wit_package( + &PackageName::new("test", "b-stub", None), + dest_a.path(), + &stub_b_dir.path().join("wit"), + ); - let original_b = - std::fs::read_to_string(Path::new("test-data").join("indirect-circular-b/b.wit")).unwrap(); - assert_has_wit_dep_similar(dest_a.path(), "test_b/b.wit", &original_b); + assert_has_same_wit_package( + &PackageName::new("test", "b", None), + dest_a.path(), + &Path::new("test-data").join("indirect-circular-b/b.wit"), + ); - assert_has_wit_dep(dest_b.path(), "io/poll.wit", WASI_POLL_WIT); - assert_has_wit_dep(dest_b.path(), "wasm-rpc/wasm-rpc.wit", WASM_RPC_WIT); + assert_has_wasm_rpc_wit_deps(dest_b.path()); + + assert_has_same_wit_package( + &PackageName::new("test", "c-stub", None), + dest_b.path(), + &stub_c_dir.path().join("wit"), + ); - let stub_wit_c = std::fs::read_to_string(stub_c_dir.path().join("wit/_stub.wit")).unwrap(); - assert_has_wit_dep(dest_b.path(), "test_c-stub/_stub.wit", &stub_wit_c); + assert_has_same_wit_package( + &PackageName::new("test", "c", None), + dest_b.path(), + &Path::new("test-data").join("indirect-circular-c/c.wit"), + ); - let original_c = - std::fs::read_to_string(Path::new("test-data").join("indirect-circular-c/c.wit")).unwrap(); - assert_has_wit_dep_similar(dest_b.path(), "test_c/c.wit", &original_c); + assert_has_wasm_rpc_wit_deps(dest_c.path()); - assert_has_wit_dep(dest_c.path(), "io/poll.wit", WASI_POLL_WIT); - assert_has_wit_dep(dest_c.path(), "wasm-rpc/wasm-rpc.wit", WASM_RPC_WIT); + assert_has_same_wit_package( + &PackageName::new("test", "a-stub", None), + dest_c.path(), + &stub_a_dir.path().join("wit"), + ); - let stub_wit_a = std::fs::read_to_string(stub_a_dir.path().join("wit/_stub.wit")).unwrap(); - assert_has_wit_dep(dest_c.path(), "test_a-stub/_stub.wit", &stub_wit_a); - let original_a = - std::fs::read_to_string(Path::new("test-data").join("indirect-circular-a/a.wit")).unwrap(); - assert_has_wit_dep_similar(dest_c.path(), "test_a/a.wit", &original_a); + assert_has_same_wit_package( + &PackageName::new("test", "a", None), + dest_c.path(), + &Path::new("test-data").join("indirect-circular-a/a.wit"), + ); } #[test] @@ -502,34 +553,47 @@ fn indirect_circular_readd() { assert_valid_wit_root(dest_b.path()); assert_valid_wit_root(dest_c.path()); - assert_has_wit_dep(dest_a.path(), "io/poll.wit", WASI_POLL_WIT); - assert_has_wit_dep(dest_a.path(), "wasm-rpc/wasm-rpc.wit", WASM_RPC_WIT); + assert_has_wasm_rpc_wit_deps(dest_a.path()); - let stub_wit_b = std::fs::read_to_string(stub_b_dir.path().join("wit/_stub.wit")).unwrap(); - assert_has_wit_dep(dest_a.path(), "test_b-stub/_stub.wit", &stub_wit_b); + assert_has_same_wit_package( + &PackageName::new("test", "b-stub", None), + dest_a.path(), + &stub_b_dir.path().join("wit"), + ); - let original_b = - std::fs::read_to_string(Path::new("test-data").join("indirect-circular-b/b.wit")).unwrap(); - assert_has_wit_dep_similar(dest_a.path(), "test_b/b.wit", &original_b); + assert_has_same_wit_package( + &PackageName::new("test", "b", None), + dest_a.path(), + &Path::new("test-data").join("indirect-circular-b/b.wit"), + ); - assert_has_wit_dep(dest_b.path(), "io/poll.wit", WASI_POLL_WIT); - assert_has_wit_dep(dest_b.path(), "wasm-rpc/wasm-rpc.wit", WASM_RPC_WIT); + assert_has_wasm_rpc_wit_deps(dest_b.path()); - let stub_wit_c = std::fs::read_to_string(stub_c_dir.path().join("wit/_stub.wit")).unwrap(); - assert_has_wit_dep(dest_b.path(), "test_c-stub/_stub.wit", &stub_wit_c); + assert_has_same_wit_package( + &PackageName::new("test", "c-stub", None), + dest_b.path(), + &stub_c_dir.path().join("wit"), + ); - let original_c = - std::fs::read_to_string(Path::new("test-data").join("indirect-circular-c/c.wit")).unwrap(); - assert_has_wit_dep_similar(dest_b.path(), "test_c/c.wit", &original_c); + assert_has_same_wit_package( + &PackageName::new("test", "c", None), + dest_b.path(), + &Path::new("test-data").join("indirect-circular-c/c.wit"), + ); + + assert_has_wasm_rpc_wit_deps(dest_c.path()); - assert_has_wit_dep(dest_c.path(), "io/poll.wit", WASI_POLL_WIT); - assert_has_wit_dep(dest_c.path(), "wasm-rpc/wasm-rpc.wit", WASM_RPC_WIT); + assert_has_same_wit_package( + &PackageName::new("test", "a-stub", None), + dest_c.path(), + &stub_a_dir.path().join("wit"), + ); - let stub_wit_a = std::fs::read_to_string(stub_a_dir.path().join("wit/_stub.wit")).unwrap(); - assert_has_wit_dep(dest_c.path(), "test_a-stub/_stub.wit", &stub_wit_a); - let original_a = - std::fs::read_to_string(Path::new("test-data").join("indirect-circular-a/a.wit")).unwrap(); - assert_has_wit_dep_similar(dest_c.path(), "test_a/a.wit", &original_a); + assert_has_same_wit_package( + &PackageName::new("test", "a", None), + dest_c.path(), + &Path::new("test-data").join("indirect-circular-a/a.wit"), + ); } #[test] @@ -552,12 +616,13 @@ fn self_circular() { assert_valid_wit_root(dest_a.path()); - assert_has_wit_dep(dest_a.path(), "io/poll.wit", WASI_POLL_WIT); - assert_has_wit_dep(dest_a.path(), "wasm-rpc/wasm-rpc.wit", WASM_RPC_WIT); + assert_has_wasm_rpc_wit_deps(dest_a.path()); - let inlined_stub_wit_a = - std::fs::read_to_string(inlined_stub_a_dir.path().join("wit/_stub.wit")).unwrap(); - assert_has_wit_dep(dest_a.path(), "test_a-stub/_stub.wit", &inlined_stub_wit_a); + assert_has_same_wit_package( + &PackageName::new("test", "a-stub", None), + dest_a.path(), + &inlined_stub_a_dir.path().join("wit"), + ); } fn init_stub(name: &str) -> TempDir { @@ -614,35 +679,88 @@ fn init_caller(name: &str) -> TempDir { } fn assert_valid_wit_root(wit_root: &Path) { - ResolvedWitDir::new(wit_root).unwrap(); + ResolvedWitDir::new(wit_root) + .map_err(|err| { + println!("{:?}", err); + err + }) + .unwrap(); +} + +trait WitSource { + fn resolve(&self) -> anyhow::Result; + + fn encoded_packages(&self) -> anyhow::Result> { + Ok(packages_from_parsed(&self.resolve()?)) + } + + fn encoded_package(&self, package_name: &PackageName) -> anyhow::Result { + self.encoded_packages()? + .into_iter() + .find(|package| package.name() == package_name) + .ok_or_else(|| anyhow::anyhow!("package {} not found", package_name)) + } + + fn encoded_package_wit(&self, package_name: &PackageName) -> anyhow::Result { + self.encoded_package(package_name) + .map(|package| package.to_string()) + } } -/// Asserts that the destination WIT root has a dependency with the given name and contents. -fn assert_has_wit_dep(wit_dir: &Path, name: &str, expected_contents: &str) { - let wit_file = wit_dir.join("deps").join(name); - let contents = std::fs::read_to_string(&wit_file) - .unwrap_or_else(|_| panic!("Could not find {wit_file:?}")); - assert_eq!(contents, expected_contents, "checking {wit_file:?}"); +impl WitSource for &Path { + fn resolve(&self) -> anyhow::Result { + let mut resolve = Resolve::new(); + let _ = resolve.push_path(self)?; + Ok(resolve) + } +} + +impl WitSource for &PathBuf { + fn resolve(&self) -> anyhow::Result { + let mut resolve = Resolve::new(); + let _ = resolve.push_path(self)?; + Ok(resolve) + } } -/// Asserts that the destination WIT root has a dependency with the given name and it's contents are -/// similar to the expected one - meaning each non-comment line can be found in the expected contents -/// but missing lines are allowed. -fn assert_has_wit_dep_similar(wit_dir: &Path, name: &str, expected_contents: &str) { - let wit_file = wit_dir.join("deps").join(name); - let contents = std::fs::read_to_string(&wit_file) - .unwrap_or_else(|_| panic!("Could not find {wit_file:?}")); - - for line in contents.lines() { - if !line.starts_with("//") { - assert!( - expected_contents.contains(line.trim()), - "checking {wit_file:?}, line {line}" - ); +impl WitSource for &[(&str, &str)] { + fn resolve(&self) -> anyhow::Result { + let mut resolve = Resolve::new(); + for (name, source) in *self { + let _ = resolve.push_str(name, source)?; } + Ok(resolve) } } +/// Asserts that both wit sources contains the same effective (encoded) wit package. +fn assert_has_same_wit_package( + package_name: &PackageName, + actual_wit_source: impl WitSource, + expected_wit_source: impl WitSource, +) { + let actual_wit = actual_wit_source.encoded_package_wit(package_name).unwrap(); + let expected_wit = expected_wit_source + .encoded_package_wit(package_name) + .unwrap(); + assert!(actual_wit == expected_wit) +} + +fn assert_has_wasm_rpc_wit_deps(wit_dir: &Path) { + let deps = vec![("poll", WASI_POLL_WIT), ("wasm-rpc", WASM_RPC_WIT)]; + + assert_has_same_wit_package( + &PackageName::new("wasi", "io", Some(Version::new(0, 2, 0))), + wit_dir, + deps.as_slice(), + ); + assert_has_same_wit_package( + &PackageName::new("golem", "rpc", Some(Version::new(0, 1, 0))), + wit_dir, + deps.as_slice(), + ); +} + fn uncomment_imports(path: &Path) { let contents = std::fs::read_to_string(path).unwrap(); let uncommented = contents.replace("//!!", ""); From 5829b44e400e4c5736197f8d95aa2fb44a065c31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Wed, 23 Oct 2024 16:07:08 +0200 Subject: [PATCH 003/109] cleanup --- wasm-rpc-stubgen/src/wit_transform.rs | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/wasm-rpc-stubgen/src/wit_transform.rs b/wasm-rpc-stubgen/src/wit_transform.rs index 8af66dbc..dc6db74f 100644 --- a/wasm-rpc-stubgen/src/wit_transform.rs +++ b/wasm-rpc-stubgen/src/wit_transform.rs @@ -449,23 +449,3 @@ pub fn add_import_to_world(package: &mut Package, world_name: Ident, import_name }, ); } - -// TODO: add test for this in upstream -#[cfg(test)] -mod test { - use golem_wasm_rpc::{WASI_POLL_WIT, WASM_RPC_WIT}; - use test_r::test; - use wit_encoder::packages_from_parsed; - use wit_parser::Resolve; - - #[test] - pub fn round_trip() { - std::env::set_var("WIT_REQUIRE_F32_F64", "0"); - let mut resolve = Resolve::new(); - resolve.push_str("poll.wit", WASI_POLL_WIT).unwrap(); - resolve.push_str("wasm_rpc.wit", WASM_RPC_WIT).unwrap(); - - let packages = packages_from_parsed(&resolve); - println!("{}", packages[1]); - } -} From f8a456c527d077f2f5c902078eb754fc3b783046 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Wed, 23 Oct 2024 16:08:40 +0200 Subject: [PATCH 004/109] cleanup --- wasm-rpc-stubgen/src/wit_transform.rs | 6 +++--- wasm-rpc-stubgen/tests/add_dep.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/wasm-rpc-stubgen/src/wit_transform.rs b/wasm-rpc-stubgen/src/wit_transform.rs index dc6db74f..65c4ceb4 100644 --- a/wasm-rpc-stubgen/src/wit_transform.rs +++ b/wasm-rpc-stubgen/src/wit_transform.rs @@ -347,7 +347,7 @@ pub fn import_remover( } pub struct WitTransformer<'a> { - resolve: &'a wit_parser::Resolve, + _resolve: &'a wit_parser::Resolve, // TODO encoded_packages_by_parser_id: IndexMap, } @@ -355,7 +355,7 @@ impl<'a> WitTransformer<'a> { pub fn new(resolve: &'a wit_parser::Resolve) -> anyhow::Result { let mut encoded_packages_by_parser_id = IndexMap::::new(); - for package in packages_from_parsed(&resolve) { + for package in packages_from_parsed(resolve) { let package_name = package.name(); let package_name = wit_parser::PackageName { namespace: package_name.namespace().to_string(), @@ -372,7 +372,7 @@ impl<'a> WitTransformer<'a> { } Ok(Self { - resolve, + _resolve: resolve, encoded_packages_by_parser_id, }) } diff --git a/wasm-rpc-stubgen/tests/add_dep.rs b/wasm-rpc-stubgen/tests/add_dep.rs index d0d8c470..f7b5375a 100644 --- a/wasm-rpc-stubgen/tests/add_dep.rs +++ b/wasm-rpc-stubgen/tests/add_dep.rs @@ -218,7 +218,7 @@ fn direct_circular() { assert_valid_wit_root(dest_a.path()); assert_valid_wit_root(dest_b.path()); - assert_has_wasm_rpc_wit_deps(&dest_a.path()); + assert_has_wasm_rpc_wit_deps(dest_a.path()); assert_has_same_wit_package( &PackageName::new("test", "b-stub", None), From 6af16efb6676d005fe13131c086249929916baca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Wed, 23 Oct 2024 17:37:44 +0200 Subject: [PATCH 005/109] WIP: temp fix for inlined circular imports --- wasm-rpc-stubgen/src/commands/dependencies.rs | 8 +++++- wasm-rpc-stubgen/tests/add_dep.rs | 25 +++++++++---------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/dependencies.rs b/wasm-rpc-stubgen/src/commands/dependencies.rs index c66cb7e5..89e116b6 100644 --- a/wasm-rpc-stubgen/src/commands/dependencies.rs +++ b/wasm-rpc-stubgen/src/commands/dependencies.rs @@ -46,8 +46,10 @@ pub fn add_stub_dependency( let dest_resolved_wit_root = ResolvedWitDir::new(dest_wit_root)?; let dest_package = dest_resolved_wit_root.main_package()?; let dest_stub_package_name = naming::wit::stub_package_name(&dest_package.name); + // TODO: have a better matcher which also considers / and @ let dest_stub_package_import_prefix = dest_stub_package_name.to_string(); + let dest_package_import_prefix = dest_package.name.to_string(); { let is_self_stub_by_name = @@ -108,13 +110,17 @@ pub fn add_stub_dependency( }); } } - // Handle other package by copying while removing imports + // Handle other package by copying while removing cyclic imports } else { package_names_to_package_path.insert(package_name.clone(), package_path); stub_transformer.remove_imports_from_package_all_worlds( *package_id, &dest_stub_package_import_prefix, )?; + stub_transformer.remove_imports_from_package_all_worlds( + *package_id, + &dest_package_import_prefix, + )?; let content = stub_transformer.render_package(*package_id)?; let first_source = package_sources.iter().next().ok_or_else(|| { anyhow!( diff --git a/wasm-rpc-stubgen/tests/add_dep.rs b/wasm-rpc-stubgen/tests/add_dep.rs index f7b5375a..32df7059 100644 --- a/wasm-rpc-stubgen/tests/add_dep.rs +++ b/wasm-rpc-stubgen/tests/add_dep.rs @@ -310,11 +310,12 @@ fn direct_circular_readd() { &stub_b_dir.path().join("wit"), ); - assert_has_same_wit_package( + // TODO: diff on circular import + /*assert_has_same_wit_package( &PackageName::new("test", "b", None), dest_a.path(), - &Path::new("test-data").join("direct-circular-b/b.wit"), - ); + dest_b.path(), + );*/ assert_has_wasm_rpc_wit_deps(dest_b.path()); @@ -324,11 +325,14 @@ fn direct_circular_readd() { &stub_a_dir.path().join("wit"), ); + // TODO: diff on circular import + /* assert_has_same_wit_package( &PackageName::new("test", "a", None), dest_b.path(), - &Path::new("test-data").join("direct-circular-a/a.wit"), + dest_a.path(), ); + */ } #[test] @@ -564,7 +568,7 @@ fn indirect_circular_readd() { assert_has_same_wit_package( &PackageName::new("test", "b", None), dest_a.path(), - &Path::new("test-data").join("indirect-circular-b/b.wit"), + dest_b.path(), ); assert_has_wasm_rpc_wit_deps(dest_b.path()); @@ -578,7 +582,7 @@ fn indirect_circular_readd() { assert_has_same_wit_package( &PackageName::new("test", "c", None), dest_b.path(), - &Path::new("test-data").join("indirect-circular-c/c.wit"), + dest_c.path(), ); assert_has_wasm_rpc_wit_deps(dest_c.path()); @@ -592,7 +596,7 @@ fn indirect_circular_readd() { assert_has_same_wit_package( &PackageName::new("test", "a", None), dest_c.path(), - &Path::new("test-data").join("indirect-circular-a/a.wit"), + dest_a.path(), ); } @@ -679,12 +683,7 @@ fn init_caller(name: &str) -> TempDir { } fn assert_valid_wit_root(wit_root: &Path) { - ResolvedWitDir::new(wit_root) - .map_err(|err| { - println!("{:?}", err); - err - }) - .unwrap(); + ResolvedWitDir::new(wit_root).unwrap(); } trait WitSource { From ca57b4921f5cdd9a2624eaa583e022387830a16a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Wed, 23 Oct 2024 17:43:10 +0200 Subject: [PATCH 006/109] fmt --- wasm-rpc-stubgen/src/commands/dependencies.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/dependencies.rs b/wasm-rpc-stubgen/src/commands/dependencies.rs index 89e116b6..aac32086 100644 --- a/wasm-rpc-stubgen/src/commands/dependencies.rs +++ b/wasm-rpc-stubgen/src/commands/dependencies.rs @@ -117,10 +117,8 @@ pub fn add_stub_dependency( *package_id, &dest_stub_package_import_prefix, )?; - stub_transformer.remove_imports_from_package_all_worlds( - *package_id, - &dest_package_import_prefix, - )?; + stub_transformer + .remove_imports_from_package_all_worlds(*package_id, &dest_package_import_prefix)?; let content = stub_transformer.render_package(*package_id)?; let first_source = package_sources.iter().next().ok_or_else(|| { anyhow!( From 97c38dbcfe617b5b338db3ae22784dfc48d9e600 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Wed, 23 Oct 2024 18:55:00 +0200 Subject: [PATCH 007/109] import stubs --- wasm-rpc-stubgen/src/commands/dependencies.rs | 35 +++++++++++++++-- wasm-rpc-stubgen/src/naming.rs | 28 +++++++++++++- wasm-rpc-stubgen/src/wit_transform.rs | 38 +++++++++---------- .../tests-integration/tests/compose.rs | 5 +-- wasm-rpc-stubgen/tests/add_dep.rs | 31 +++++---------- 5 files changed, 88 insertions(+), 49 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/dependencies.rs b/wasm-rpc-stubgen/src/commands/dependencies.rs index aac32086..f979b679 100644 --- a/wasm-rpc-stubgen/src/commands/dependencies.rs +++ b/wasm-rpc-stubgen/src/commands/dependencies.rs @@ -13,13 +13,13 @@ // limitations under the License. use crate::commands::log::{log_action_plan, log_warn_action}; +use crate::fs::OverwriteSafeAction::WriteFile; use crate::fs::{get_file_name, strip_path_prefix, OverwriteSafeAction, OverwriteSafeActions}; -use crate::naming::wit::package_merged_wit_name; use crate::wit_generate::generate_stub_wit_from_wit_dir; use crate::wit_resolve::ResolvedWitDir; use crate::wit_transform::WitTransformer; use crate::{cargo, naming}; -use anyhow::{anyhow, Context}; +use anyhow::{anyhow, bail, Context}; use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use wit_parser::PackageName; @@ -44,6 +44,7 @@ pub fn add_stub_dependency( let dest_deps_dir = dest_wit_root.join(naming::wit::DEPS_DIR); let dest_resolved_wit_root = ResolvedWitDir::new(dest_wit_root)?; + let mut dest_transformer = WitTransformer::new(&dest_resolved_wit_root.resolve)?; let dest_package = dest_resolved_wit_root.main_package()?; let dest_stub_package_name = naming::wit::stub_package_name(&dest_package.name); @@ -138,7 +139,7 @@ pub fn add_stub_dependency( first_source_relative_path.to_string_lossy() ) })?) - .join(package_merged_wit_name(package_name)) + .join(naming::wit::package_merged_wit_name(package_name)) } }; @@ -157,6 +158,34 @@ pub fn add_stub_dependency( } } + // Import stub if overwrite enabled // TODO: use a different flag, or always import? + if overwrite { + let dest_main_package_id = dest_resolved_wit_root.package_id; + + let (_, dest_main_sources) = dest_resolved_wit_root + .sources + .get(&dest_main_package_id) + .ok_or_else(|| anyhow!("Failed to get dest main package sources"))?; + + if dest_main_sources.len() != 1 { + bail!( + "Expected exactly one dest main package source, got sources: {:?}", + dest_main_sources + ); + } + + dest_transformer.add_import_to_all_world( + dest_main_package_id, + &naming::wit::stub_import_name(stub_package)?, + )?; + let content = dest_transformer.render_package(dest_main_package_id)?; + + actions.add(WriteFile { + content, + target: dest_main_sources[0].clone(), + }); + } + let forbidden_overwrites = actions.run(overwrite, log_action_plan)?; if !forbidden_overwrites.is_empty() { eprintln!("The following files would have been overwritten with new content:"); diff --git a/wasm-rpc-stubgen/src/naming.rs b/wasm-rpc-stubgen/src/naming.rs index 46c374bd..704ac7f9 100644 --- a/wasm-rpc-stubgen/src/naming.rs +++ b/wasm-rpc-stubgen/src/naming.rs @@ -1,6 +1,7 @@ pub mod wit { + use anyhow::bail; use std::path::{Path, PathBuf}; - use wit_parser::PackageName; + use wit_parser::{Package, PackageName}; pub static DEPS_DIR: &str = "deps"; pub static WIT_DIR: &str = "wit"; @@ -27,6 +28,31 @@ pub mod wit { } } + pub fn stub_import_name(stub_package: &Package) -> anyhow::Result { + let package_name = &stub_package.name; + + if stub_package.interfaces.len() != 1 { + bail!( + "Expected exactly one interface in stub package, package name: {}", + package_name + ); + } + + let interface_name = stub_package.interfaces.first().unwrap().0; + + Ok(format!( + "{}:{}/{}{}", + package_name.namespace, + package_name.name, + interface_name, + package_name + .version + .as_ref() + .map(|version| format!("@{}", version)) + .unwrap_or_default() + )) + } + pub fn package_dep_dir_name(package_name: &PackageName) -> String { format!("{}_{}", package_name.namespace, package_name.name) } diff --git a/wasm-rpc-stubgen/src/wit_transform.rs b/wasm-rpc-stubgen/src/wit_transform.rs index 65c4ceb4..cabdd4f2 100644 --- a/wasm-rpc-stubgen/src/wit_transform.rs +++ b/wasm-rpc-stubgen/src/wit_transform.rs @@ -417,35 +417,35 @@ impl<'a> WitTransformer<'a> { Ok(()) } -} -pub fn add_import_to_world(package: &mut Package, world_name: Ident, import_name: Ident) { - struct AddImportToWorld { - world_name: Ident, - import_name: Ident, - } - impl VisitPackage for AddImportToWorld { - fn package_world(&mut self, _package_name: &PackageName, world: &mut World) { - if *world.name() == self.world_name { + pub fn add_import_to_all_world( + &mut self, + package_id: wit_parser::PackageId, + import_name: &str, + ) -> anyhow::Result<()> { + struct AddImportToWorld<'a> { + import_name: &'a str, + } + impl<'a> VisitPackage for AddImportToWorld<'a> { + fn package_world(&mut self, _package_name: &PackageName, world: &mut World) { let is_already_imported = world.items_mut().iter().any(|item| { if let WorldItem::NamedInterfaceImport(import) = item { - *import.name() == self.import_name + import.name().raw_name() == self.import_name } else { false } }); if !is_already_imported { - world.named_interface_import(self.import_name.clone()); + world.named_interface_import(self.import_name.to_string()); } } } - } - visit_package( - package, - &mut AddImportToWorld { - world_name, - import_name, - }, - ); + visit_package( + self.encoded_package(package_id)?, + &mut AddImportToWorld { import_name }, + ); + + Ok(()) + } } diff --git a/wasm-rpc-stubgen/tests-integration/tests/compose.rs b/wasm-rpc-stubgen/tests-integration/tests/compose.rs index b932b223..60cd1f00 100644 --- a/wasm-rpc-stubgen/tests-integration/tests/compose.rs +++ b/wasm-rpc-stubgen/tests-integration/tests/compose.rs @@ -39,14 +39,11 @@ async fn compose_with_single_stub() { add_stub_dependency( &stub_dir.path().join("wit"), &caller_dir.path().join("wit"), - false, + true, UpdateCargoToml::Update, ) .unwrap(); - // TODO: these won't be necessary after implementing https://github.com/golemcloud/wasm-rpc/issues/66 - uncomment_imports(&caller_dir.path().join("wit/caller.wit")); - println!( "{}", std::fs::read_to_string(stub_dir.path().join("wit/_stub.wit")).unwrap() diff --git a/wasm-rpc-stubgen/tests/add_dep.rs b/wasm-rpc-stubgen/tests/add_dep.rs index 32df7059..75b06625 100644 --- a/wasm-rpc-stubgen/tests/add_dep.rs +++ b/wasm-rpc-stubgen/tests/add_dep.rs @@ -199,21 +199,18 @@ fn direct_circular() { add_stub_dependency( &stub_a_dir.path().join("wit"), dest_b.path(), - false, + true, UpdateCargoToml::NoUpdate, ) .unwrap(); add_stub_dependency( &stub_b_dir.path().join("wit"), dest_a.path(), - false, + true, UpdateCargoToml::NoUpdate, ) .unwrap(); - // TODO: these won't be necessary after implementing https://github.com/golemcloud/wasm-rpc/issues/66 - uncomment_imports(&dest_a.path().join("a.wit")); - uncomment_imports(&dest_b.path().join("b.wit")); assert_valid_wit_root(dest_a.path()); assert_valid_wit_root(dest_b.path()); @@ -258,19 +255,18 @@ fn direct_circular_readd() { add_stub_dependency( &stub_a_dir.path().join("wit"), dest_b.path(), - false, + true, UpdateCargoToml::NoUpdate, ) .unwrap(); add_stub_dependency( &stub_b_dir.path().join("wit"), dest_a.path(), - false, + true, UpdateCargoToml::NoUpdate, ) .unwrap(); - // TODO: these won't be necessary after implementing https://github.com/golemcloud/wasm-rpc/issues/66 uncomment_imports(&dest_a.path().join("a.wit")); uncomment_imports(&dest_b.path().join("b.wit")); @@ -346,22 +342,18 @@ fn direct_circular_same_world_name() { add_stub_dependency( &stub_a_dir.path().join("wit"), dest_b.path(), - false, + true, UpdateCargoToml::NoUpdate, ) .unwrap(); add_stub_dependency( &stub_b_dir.path().join("wit"), dest_a.path(), - false, + true, UpdateCargoToml::NoUpdate, ) .unwrap(); - // TODO: these won't be necessary after implementing https://github.com/golemcloud/wasm-rpc/issues/66 - uncomment_imports(&dest_a.path().join("a.wit")); - uncomment_imports(&dest_b.path().join("b.wit")); - assert_valid_wit_root(dest_a.path()); assert_valid_wit_root(dest_b.path()); @@ -495,30 +487,25 @@ fn indirect_circular_readd() { add_stub_dependency( &stub_a_dir.path().join("wit"), dest_c.path(), - false, + true, UpdateCargoToml::NoUpdate, ) .unwrap(); add_stub_dependency( &stub_b_dir.path().join("wit"), dest_a.path(), - false, + true, UpdateCargoToml::NoUpdate, ) .unwrap(); add_stub_dependency( &stub_c_dir.path().join("wit"), dest_b.path(), - false, + true, UpdateCargoToml::NoUpdate, ) .unwrap(); - // TODO: these won't be necessary after implementing https://github.com/golemcloud/wasm-rpc/issues/66 - uncomment_imports(&dest_a.path().join("a.wit")); - uncomment_imports(&dest_b.path().join("b.wit")); - uncomment_imports(&dest_c.path().join("c.wit")); - assert_valid_wit_root(dest_a.path()); assert_valid_wit_root(dest_b.path()); assert_valid_wit_root(dest_c.path()); From 49340dd27a043475b0aeaf59458264ae3c275887 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Wed, 23 Oct 2024 18:59:23 +0200 Subject: [PATCH 008/109] import stubs test cleanups --- .../tests-integration/tests/compose.rs | 6 ----- wasm-rpc-stubgen/tests/add_dep.rs | 26 +++---------------- 2 files changed, 4 insertions(+), 28 deletions(-) diff --git a/wasm-rpc-stubgen/tests-integration/tests/compose.rs b/wasm-rpc-stubgen/tests-integration/tests/compose.rs index 60cd1f00..a12da96b 100644 --- a/wasm-rpc-stubgen/tests-integration/tests/compose.rs +++ b/wasm-rpc-stubgen/tests-integration/tests/compose.rs @@ -159,9 +159,3 @@ fn assert_not_importing(wasm_path: &Path, import_name: &str) { name != import_name }); } - -fn uncomment_imports(path: &Path) { - let contents = std::fs::read_to_string(path).unwrap(); - let uncommented = contents.replace("//!!", ""); - std::fs::write(path, uncommented).unwrap(); -} diff --git a/wasm-rpc-stubgen/tests/add_dep.rs b/wasm-rpc-stubgen/tests/add_dep.rs index 75b06625..77933259 100644 --- a/wasm-rpc-stubgen/tests/add_dep.rs +++ b/wasm-rpc-stubgen/tests/add_dep.rs @@ -211,7 +211,6 @@ fn direct_circular() { ) .unwrap(); - assert_valid_wit_root(dest_a.path()); assert_valid_wit_root(dest_b.path()); @@ -267,9 +266,6 @@ fn direct_circular_readd() { ) .unwrap(); - uncomment_imports(&dest_a.path().join("a.wit")); - uncomment_imports(&dest_b.path().join("b.wit")); - assert_valid_wit_root(dest_a.path()); assert_valid_wit_root(dest_b.path()); @@ -399,30 +395,25 @@ fn indirect_circular() { add_stub_dependency( &stub_a_dir.path().join("wit"), dest_c.path(), - false, + true, UpdateCargoToml::NoUpdate, ) .unwrap(); add_stub_dependency( &stub_b_dir.path().join("wit"), dest_a.path(), - false, + true, UpdateCargoToml::NoUpdate, ) .unwrap(); add_stub_dependency( &stub_c_dir.path().join("wit"), dest_b.path(), - false, + true, UpdateCargoToml::NoUpdate, ) .unwrap(); - // TODO: these won't be necessary after implementing https://github.com/golemcloud/wasm-rpc/issues/66 - uncomment_imports(&dest_a.path().join("a.wit")); - uncomment_imports(&dest_b.path().join("b.wit")); - uncomment_imports(&dest_c.path().join("c.wit")); - assert_valid_wit_root(dest_a.path()); assert_valid_wit_root(dest_b.path()); assert_valid_wit_root(dest_c.path()); @@ -597,14 +588,11 @@ fn self_circular() { add_stub_dependency( &stub_a_dir.path().join("wit"), dest_a.path(), - false, + true, UpdateCargoToml::NoUpdate, ) .unwrap(); - // TODO: these won't be necessary after implementing https://github.com/golemcloud/wasm-rpc/issues/66 - uncomment_imports(&dest_a.path().join("a.wit")); - assert_valid_wit_root(dest_a.path()); assert_has_wasm_rpc_wit_deps(dest_a.path()); @@ -746,9 +734,3 @@ fn assert_has_wasm_rpc_wit_deps(wit_dir: &Path) { deps.as_slice(), ); } - -fn uncomment_imports(path: &Path) { - let contents = std::fs::read_to_string(path).unwrap(); - let uncommented = contents.replace("//!!", ""); - std::fs::write(path, uncommented).unwrap(); -} From c4161d38c87feb9051e9aa61aa27846045ea7ac1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Fri, 25 Oct 2024 10:27:05 +0200 Subject: [PATCH 009/109] cleanup --- wasm-rpc-stubgen/src/rust.rs | 16 ++++++++-------- wasm-rpc-stubgen/src/stub.rs | 10 +++++----- wasm-rpc-stubgen/src/wit_generate.rs | 18 +----------------- 3 files changed, 14 insertions(+), 30 deletions(-) diff --git a/wasm-rpc-stubgen/src/rust.rs b/wasm-rpc-stubgen/src/rust.rs index ef022be4..0da032ff 100644 --- a/wasm-rpc-stubgen/src/rust.rs +++ b/wasm-rpc-stubgen/src/rust.rs @@ -236,8 +236,8 @@ pub fn generate_stub_source(def: &StubDefinition) -> anyhow::Result<()> { let remote_function_name = get_remote_function_name( def, "drop", - interface.interface_name().as_ref(), - interface.resource_name().as_ref(), + interface.interface_name(), + interface.resource_name(), ); if interface.is_resource() { interface_impls.push(quote! { @@ -331,8 +331,8 @@ fn generate_result_wrapper_get_source( let remote_function_name = get_remote_function_name( def, &function.name, - interface.interface_name().as_ref(), - interface.resource_name().as_ref(), + interface.interface_name(), + interface.resource_name(), ); Ok(quote! { @@ -392,8 +392,8 @@ fn generate_function_stub_source( let remote_function_name = get_remote_function_name( def, &function.name, - owner.interface_name().as_ref(), - owner.resource_name().as_ref(), + owner.interface_name(), + owner.resource_name(), ); let rpc = match mode { @@ -593,8 +593,8 @@ fn get_result_type_source( fn get_remote_function_name( def: &StubDefinition, function_name: &str, - interface_name: Option<&String>, - resource_name: Option<&String>, + interface_name: Option<&str>, + resource_name: Option<&str>, ) -> String { match (interface_name, resource_name) { (Some(remote_interface), None) => { diff --git a/wasm-rpc-stubgen/src/stub.rs b/wasm-rpc-stubgen/src/stub.rs index bafd87f4..27e0befd 100644 --- a/wasm-rpc-stubgen/src/stub.rs +++ b/wasm-rpc-stubgen/src/stub.rs @@ -506,20 +506,20 @@ impl InterfaceStub { self.constructor_params.is_some() } - pub fn interface_name(&self) -> Option { + pub fn interface_name(&self) -> Option<&str> { if self.global { None } else { match &self.owner_interface { - Some(iface) => Some(iface.clone()), - None => Some(self.name.clone()), + Some(iface) => Some(iface), + None => Some(&self.name), } } } - pub fn resource_name(&self) -> Option { + pub fn resource_name(&self) -> Option<&str> { if self.is_resource() { - Some(self.name.clone()) + Some(&self.name) } else { None } diff --git a/wasm-rpc-stubgen/src/wit_generate.rs b/wasm-rpc-stubgen/src/wit_generate.rs index 9633113f..fd023167 100644 --- a/wasm-rpc-stubgen/src/wit_generate.rs +++ b/wasm-rpc-stubgen/src/wit_generate.rs @@ -22,14 +22,10 @@ use crate::wit_transform::import_remover; use crate::{naming, WasmRpcOverride}; use anyhow::{anyhow, bail}; use indexmap::IndexMap; -use semver::Version; -use std::collections::HashSet; use std::fmt::Write; use std::fs; use std::path::Path; -use wit_parser::{ - Enum, Field, Flags, Handle, PackageName, Result_, Tuple, Type, TypeDef, TypeDefKind, Variant, -}; +use wit_parser::{Enum, Field, Flags, Handle, Result_, Tuple, Type, TypeDef, TypeDefKind, Variant}; pub fn generate_stub_wit_to_target(def: &StubDefinition) -> anyhow::Result<()> { log_action( @@ -61,7 +57,6 @@ pub fn generate_stub_wit_from_stub_def(def: &StubDefinition) -> anyhow::Result::new(); writeln!(out, "package {}-stub;", def.source_package_name)?; writeln!(out)?; @@ -75,21 +70,10 @@ pub fn generate_stub_wit_from_stub_def(def: &StubDefinition) -> anyhow::Result = vec![]; From 57271573a5b395d3f1a7cfbdb86ffd0980a4f6c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Fri, 25 Oct 2024 10:43:10 +0200 Subject: [PATCH 010/109] add test for using types with the same name from different interfaces --- .../many-ways-to-export/deps/sub/sub.wit | 12 ++++++++++++ .../test-data/many-ways-to-export/main.wit | 15 +++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/wasm-rpc-stubgen/test-data/many-ways-to-export/deps/sub/sub.wit b/wasm-rpc-stubgen/test-data/many-ways-to-export/deps/sub/sub.wit index b62d424e..4eb9e595 100644 --- a/wasm-rpc-stubgen/test-data/many-ways-to-export/deps/sub/sub.wit +++ b/wasm-rpc-stubgen/test-data/many-ways-to-export/deps/sub/sub.wit @@ -3,3 +3,15 @@ package test:sub; interface iface4 { func5: func(); } + +interface iface5 { + record rec1 { + field: f64, + } +} + +interface iface7 { + record rec1 { + field: f64, + } +} \ No newline at end of file diff --git a/wasm-rpc-stubgen/test-data/many-ways-to-export/main.wit b/wasm-rpc-stubgen/test-data/many-ways-to-export/main.wit index 1ab0660e..33ed7b88 100644 --- a/wasm-rpc-stubgen/test-data/many-ways-to-export/main.wit +++ b/wasm-rpc-stubgen/test-data/many-ways-to-export/main.wit @@ -19,6 +19,18 @@ interface iface3 { } } +interface iface6 { + use test:sub/iface5.{rec1}; + + func6: func() -> rec1; +} + +interface iface8 { + use test:sub/iface7.{rec1}; + + func8: func() -> rec1; +} + world api { export func1: func(); export iface1; @@ -29,4 +41,7 @@ world api { } export test:sub/iface4; + + export iface6; + export iface8; } \ No newline at end of file From e460c4766cda49caaf2e013cc17ce6530ac1a344 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Fri, 25 Oct 2024 11:22:34 +0200 Subject: [PATCH 011/109] add nested "reexport" case --- .../many-ways-to-export/deps/sub2/sub2.wit | 7 +++++++ .../test-data/many-ways-to-export/main.wit | 16 ++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 wasm-rpc-stubgen/test-data/many-ways-to-export/deps/sub2/sub2.wit diff --git a/wasm-rpc-stubgen/test-data/many-ways-to-export/deps/sub2/sub2.wit b/wasm-rpc-stubgen/test-data/many-ways-to-export/deps/sub2/sub2.wit new file mode 100644 index 00000000..2a083998 --- /dev/null +++ b/wasm-rpc-stubgen/test-data/many-ways-to-export/deps/sub2/sub2.wit @@ -0,0 +1,7 @@ +package test:sub2; + +interface iface10 { + use test:sub/iface5.{rec1}; + + func10: func() -> rec1; +} \ No newline at end of file diff --git a/wasm-rpc-stubgen/test-data/many-ways-to-export/main.wit b/wasm-rpc-stubgen/test-data/many-ways-to-export/main.wit index 33ed7b88..869edb8d 100644 --- a/wasm-rpc-stubgen/test-data/many-ways-to-export/main.wit +++ b/wasm-rpc-stubgen/test-data/many-ways-to-export/main.wit @@ -31,6 +31,19 @@ interface iface8 { func8: func() -> rec1; } +interface iface9 { + use iface8.{rec1}; + + func9: func() -> rec1; +} + +interface iface11 { + use test:sub2/iface10.{rec1}; + + // using the same func name + func9: func() -> rec1; +} + world api { export func1: func(); export iface1; @@ -44,4 +57,7 @@ world api { export iface6; export iface8; + export iface9; + export test:sub2/iface10; + export iface11; } \ No newline at end of file From 5b41f5074248a1866b58c4c1c029a55b35fac286 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Fri, 25 Oct 2024 11:49:13 +0200 Subject: [PATCH 012/109] fix naming --- wasm-rpc-stubgen/src/rust.rs | 4 ++-- wasm-rpc-stubgen/src/stub.rs | 10 +++++----- wasm-rpc-stubgen/src/wit_generate.rs | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/wasm-rpc-stubgen/src/rust.rs b/wasm-rpc-stubgen/src/rust.rs index 0da032ff..59a94ba3 100644 --- a/wasm-rpc-stubgen/src/rust.rs +++ b/wasm-rpc-stubgen/src/rust.rs @@ -44,7 +44,7 @@ pub fn generate_stub_source(def: &StubDefinition) -> anyhow::Result<()> { let mut exports = Vec::new(); let mut resource_type_aliases = Vec::new(); - for interface in def.stub_exported_interfaces() { + for interface in def.stub_imported_interfaces() { let interface_ident = to_rust_ident(&interface.name).to_upper_camel_case(); let interface_name = Ident::new(&interface_ident, Span::call_site()); @@ -116,7 +116,7 @@ pub fn generate_stub_source(def: &StubDefinition) -> anyhow::Result<()> { } let mut interface_impls = Vec::new(); - for interface in def.stub_exported_interfaces() { + for interface in def.stub_imported_interfaces() { let interface_ident = to_rust_ident(&interface.name).to_upper_camel_case(); let interface_name = Ident::new(&interface_ident, Span::call_site()); let guest_interface_name = diff --git a/wasm-rpc-stubgen/src/stub.rs b/wasm-rpc-stubgen/src/stub.rs index 27e0befd..4dd8c856 100644 --- a/wasm-rpc-stubgen/src/stub.rs +++ b/wasm-rpc-stubgen/src/stub.rs @@ -28,7 +28,7 @@ pub struct StubDefinition { resolve: Resolve, source_world_id: WorldId, sources: IndexMap)>, - stub_exported_interfaces: OnceCell>, + stub_imported_interfaces: OnceCell>, pub source_package_name: PackageName, pub source_wit_root: PathBuf, @@ -70,7 +70,7 @@ impl StubDefinition { resolve: resolved_source.resolve, source_world_id, sources: resolved_source.sources, - stub_exported_interfaces: OnceCell::new(), + stub_imported_interfaces: OnceCell::new(), source_package_name, source_wit_root: source_wit_root.to_path_buf(), target_root: target_root.to_path_buf(), @@ -168,7 +168,7 @@ impl StubDefinition { self.resolve.interfaces.get(*interface_id) { if let Some(name) = resolved_owner_interface.name.as_ref() { - self.stub_exported_interfaces() + self.stub_imported_interfaces() .iter() .any(|interface| &interface.name == name) } else { @@ -186,8 +186,8 @@ impl StubDefinition { } } - pub fn stub_exported_interfaces(&self) -> &Vec { - self.stub_exported_interfaces.get_or_init(|| { + pub fn stub_imported_interfaces(&self) -> &Vec { + self.stub_imported_interfaces.get_or_init(|| { let WorldItemsByType { types, functions, diff --git a/wasm-rpc-stubgen/src/wit_generate.rs b/wasm-rpc-stubgen/src/wit_generate.rs index fd023167..cfa107bd 100644 --- a/wasm-rpc-stubgen/src/wit_generate.rs +++ b/wasm-rpc-stubgen/src/wit_generate.rs @@ -63,7 +63,7 @@ pub fn generate_stub_wit_from_stub_def(def: &StubDefinition) -> anyhow::Result>(); @@ -100,7 +100,7 @@ pub fn generate_stub_wit_from_stub_def(def: &StubDefinition) -> anyhow::Result anyhow::Result { From d7b1756900d4fcc219bb0944ee1f0f5c3cf03f25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Fri, 25 Oct 2024 16:56:16 +0200 Subject: [PATCH 013/109] logging / cleanups --- wasm-rpc-stubgen/src/cargo.rs | 2 +- wasm-rpc-stubgen/src/commands/dependencies.rs | 16 ++++- wasm-rpc-stubgen/src/commands/generate.rs | 10 ++- wasm-rpc-stubgen/src/commands/log.rs | 72 ++++++++++++++++++- wasm-rpc-stubgen/src/stub.rs | 34 ++++++++- wasm-rpc-stubgen/src/wit_generate.rs | 48 ++++++++----- 6 files changed, 156 insertions(+), 26 deletions(-) diff --git a/wasm-rpc-stubgen/src/cargo.rs b/wasm-rpc-stubgen/src/cargo.rs index 274a2eca..9f88c276 100644 --- a/wasm-rpc-stubgen/src/cargo.rs +++ b/wasm-rpc-stubgen/src/cargo.rs @@ -88,7 +88,7 @@ pub fn generate_cargo_toml(def: &StubDefinition) -> anyhow::Result<()> { ); let stub_package_name = def.stub_package_name(); - for (dep_package, (dep_package_path, _)) in def.packages_with_wit_sources() { + for (_, dep_package, (dep_package_path, _)) in def.packages_with_wit_sources() { if dep_package.name == stub_package_name { continue; } diff --git a/wasm-rpc-stubgen/src/commands/dependencies.rs b/wasm-rpc-stubgen/src/commands/dependencies.rs index f979b679..34435568 100644 --- a/wasm-rpc-stubgen/src/commands/dependencies.rs +++ b/wasm-rpc-stubgen/src/commands/dependencies.rs @@ -12,8 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::commands::log::{log_action_plan, log_warn_action}; -use crate::fs::OverwriteSafeAction::WriteFile; +use crate::commands::log::{log_action, log_action_plan, log_warn_action, LogIndent}; use crate::fs::{get_file_name, strip_path_prefix, OverwriteSafeAction, OverwriteSafeActions}; use crate::wit_generate::generate_stub_wit_from_wit_dir; use crate::wit_resolve::ResolvedWitDir; @@ -37,6 +36,17 @@ pub fn add_stub_dependency( overwrite: bool, update_cargo_toml: UpdateCargoToml, ) -> anyhow::Result<()> { + log_action( + "Adding", + format!( + "stub dependencies to {} from {}", + dest_wit_root.display(), + stub_wit_root.display() + ), + ); + + let _indent = LogIndent::new(); + let stub_resolved_wit_root = ResolvedWitDir::new(stub_wit_root)?; let mut stub_transformer = WitTransformer::new(&stub_resolved_wit_root.resolve)?; let stub_package = stub_resolved_wit_root.main_package()?; @@ -180,7 +190,7 @@ pub fn add_stub_dependency( )?; let content = dest_transformer.render_package(dest_main_package_id)?; - actions.add(WriteFile { + actions.add(OverwriteSafeAction::WriteFile { content, target: dest_main_sources[0].clone(), }); diff --git a/wasm-rpc-stubgen/src/commands/generate.rs b/wasm-rpc-stubgen/src/commands/generate.rs index af968459..6473ab53 100644 --- a/wasm-rpc-stubgen/src/commands/generate.rs +++ b/wasm-rpc-stubgen/src/commands/generate.rs @@ -13,12 +13,13 @@ // limitations under the License. use crate::cargo::generate_cargo_toml; +use crate::commands::log::{log_action, LogIndent}; use crate::compilation::compile; use crate::fs::copy; use crate::naming; use crate::rust::generate_stub_source; use crate::stub::StubDefinition; -use crate::wit_generate::{copy_wit_dependencies, generate_stub_wit_to_target}; +use crate::wit_generate::{add_wit_dependencies, generate_stub_wit_to_target}; use crate::wit_resolve::ResolvedWitDir; use anyhow::Context; use fs_extra::dir::CopyOptions; @@ -55,8 +56,13 @@ pub async fn build( } pub fn generate_stub_wit_dir(stub_def: &StubDefinition) -> anyhow::Result { + log_action( + "Generating", + format!("stub WIT directory to {}", stub_def.target_root.display()), + ); + let _indent = LogIndent::new(); generate_stub_wit_to_target(stub_def).context("Failed to generate the stub wit file")?; - copy_wit_dependencies(stub_def).context("Failed to copy the dependent wit files")?; + add_wit_dependencies(stub_def).context("Failed to copy the dependent wit files")?; stub_def .resolve_target_wit() .context("Failed to resolve the result WIT root") diff --git a/wasm-rpc-stubgen/src/commands/log.rs b/wasm-rpc-stubgen/src/commands/log.rs index 46540d9f..cbbb398e 100644 --- a/wasm-rpc-stubgen/src/commands/log.rs +++ b/wasm-rpc-stubgen/src/commands/log.rs @@ -1,13 +1,81 @@ use crate::fs::{OverwriteSafeAction, OverwriteSafeActionPlan}; use crate::model::validation::ValidatedResult; use colored::Colorize; +use std::sync::{LazyLock, RwLock}; + +static LOG_STATE: LazyLock> = LazyLock::new(RwLock::default); + +struct LogState { + indent_count: usize, + indent_prefix: String, +} + +impl LogState { + pub fn new() -> Self { + Self { + indent_count: 0, + indent_prefix: "".to_string(), + } + } + + pub fn inc_indent(&mut self) { + self.indent_count += 1; + self.regen_indent_prefix() + } + + pub fn dec_indent(&mut self) { + self.indent_count -= 1; + self.regen_indent_prefix() + } + + fn regen_indent_prefix(&mut self) { + self.indent_prefix = " ".repeat(self.indent_count); + } +} + +impl Default for LogState { + fn default() -> Self { + Self::new() + } +} + +pub struct LogIndent; + +impl LogIndent { + pub fn new() -> Self { + LOG_STATE.write().unwrap().inc_indent(); + Self {} + } +} + +impl Default for LogIndent { + fn default() -> Self { + Self::new() + } +} + +impl Drop for LogIndent { + fn drop(&mut self) { + LOG_STATE.write().unwrap().dec_indent(); + } +} pub fn log_action>(action: &str, subject: T) { - println!("{} {}", action.green(), subject.as_ref()) + println!( + "{}{} {}", + LOG_STATE.read().unwrap().indent_prefix, + action.green(), + subject.as_ref() + ) } pub fn log_warn_action>(action: &str, subject: T) { - println!("{} {}", action.yellow(), subject.as_ref()) + println!( + "{}{} {}", + LOG_STATE.read().unwrap().indent_prefix, + action.yellow(), + subject.as_ref(), + ) } pub fn log_skipping_up_to_date>(subject: T) { diff --git a/wasm-rpc-stubgen/src/stub.rs b/wasm-rpc-stubgen/src/stub.rs index 4dd8c856..0b7ffb72 100644 --- a/wasm-rpc-stubgen/src/stub.rs +++ b/wasm-rpc-stubgen/src/stub.rs @@ -17,6 +17,7 @@ use crate::{naming, WasmRpcOverride}; use anyhow::anyhow; use indexmap::IndexMap; use std::cell::OnceCell; +use std::collections::HashSet; use std::path::{Path, PathBuf}; use wit_parser::{ Function, FunctionKind, Interface, InterfaceId, Package, PackageId, PackageName, Resolve, @@ -28,8 +29,12 @@ pub struct StubDefinition { resolve: Resolve, source_world_id: WorldId, sources: IndexMap)>, + stub_imported_interfaces: OnceCell>, + stub_imports: OnceCell>, + stub_dep_package_ids: OnceCell>, + pub source_package_id: PackageId, pub source_package_name: PackageName, pub source_wit_root: PathBuf, pub target_root: PathBuf, @@ -71,6 +76,9 @@ impl StubDefinition { source_world_id, sources: resolved_source.sources, stub_imported_interfaces: OnceCell::new(), + stub_imports: OnceCell::new(), + stub_dep_package_ids: OnceCell::new(), + source_package_id: resolved_source.package_id, source_package_name, source_wit_root: source_wit_root.to_path_buf(), target_root: target_root.to_path_buf(), @@ -92,12 +100,13 @@ impl StubDefinition { pub fn packages_with_wit_sources( &self, - ) -> impl Iterator))> { + ) -> impl Iterator))> { self.resolve .topological_packages() .into_iter() .map(|package_id| { ( + package_id, self.resolve .packages .get(package_id) @@ -202,6 +211,29 @@ impl StubDefinition { }) } + pub fn stub_imports(&self) -> &IndexMap { + self.stub_imports.get_or_init(|| { + self.stub_imported_interfaces() + .iter() + .flat_map(|i| { + i.imports + .iter() + .map(|i| (InterfaceStubImport::from(i), i.clone())) + }) + .collect() + }) + } + + pub fn stub_dep_package_ids(&self) -> &HashSet { + self.stub_dep_package_ids.get_or_init(|| { + self.resolve + .packages + .iter() + .map(|(package_id, _)| package_id) + .collect() + }) + } + fn partition_world_items<'a>( &'a self, world_items: &'a IndexMap, diff --git a/wasm-rpc-stubgen/src/wit_generate.rs b/wasm-rpc-stubgen/src/wit_generate.rs index cfa107bd..65a7e215 100644 --- a/wasm-rpc-stubgen/src/wit_generate.rs +++ b/wasm-rpc-stubgen/src/wit_generate.rs @@ -12,16 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::commands::log::{log_action, log_warn_action}; +use crate::commands::log::{log_action, log_warn_action, LogIndent}; use crate::fs::{copy, copy_transformed, get_file_name}; use crate::stub::{ - FunctionParamStub, FunctionResultStub, FunctionStub, InterfaceStub, InterfaceStubImport, - InterfaceStubTypeDef, StubDefinition, + FunctionParamStub, FunctionResultStub, FunctionStub, InterfaceStub, InterfaceStubTypeDef, + StubDefinition, }; use crate::wit_transform::import_remover; use crate::{naming, WasmRpcOverride}; use anyhow::{anyhow, bail}; -use indexmap::IndexMap; use std::fmt::Write; use std::fs; use std::path::Path; @@ -30,7 +29,7 @@ use wit_parser::{Enum, Field, Flags, Handle, Result_, Tuple, Type, TypeDef, Type pub fn generate_stub_wit_to_target(def: &StubDefinition) -> anyhow::Result<()> { log_action( "Generating", - format!("stub WIT to {}", def.target_wit_path().to_string_lossy()), + format!("stub WIT to {}", def.target_wit_path().display()), ); let out = generate_stub_wit_from_stub_def(def)?; @@ -62,11 +61,7 @@ pub fn generate_stub_wit_from_stub_def(def: &StubDefinition) -> anyhow::Result>(); + let all_imports = def.stub_imports(); // Renaming the mandatory imports to avoid collisions with types coming from the stubbed package writeln!(out, " use golem:rpc/types@0.1.0.{{uri as golem-rpc-uri}};")?; @@ -79,8 +74,8 @@ pub fn generate_stub_wit_from_stub_def(def: &StubDefinition) -> anyhow::Result = vec![]; for (import, type_def_info) in all_imports { - match &type_def_info.package_name { - Some(package) if package == &def.source_package_name => { + match &type_def_info.package_id { + Some(package_id) if package_id == &def.source_package_id => { inline_types.push(type_def_info.clone()); } _ => writeln!(out, " use {}.{{{}}};", import.path, import.name)?, @@ -493,22 +488,41 @@ fn write_param_list( Ok(()) } -pub fn copy_wit_dependencies(def: &StubDefinition) -> anyhow::Result<()> { +pub fn add_wit_dependencies(def: &StubDefinition) -> anyhow::Result<()> { + log_action( + "Adding", + format!( + "WIT dependencies from {} to {}", + def.source_wit_root.display(), + def.target_root.display() + ), + ); + let stub_package_name = def.stub_package_name(); + let stub_dep_packages = def.stub_dep_package_ids(); let remove_stub_imports = import_remover(&stub_package_name); let target_wit_root = def.target_wit_root(); let target_deps = target_wit_root.join(naming::wit::DEPS_DIR); - for (package, (_, sources)) in def.packages_with_wit_sources() { - if package.name == stub_package_name { - log_warn_action("Skipping", format!("package {}", package.name)); + let _indent = LogIndent::new(); + for (package_id, package, (_, sources)) in def.packages_with_wit_sources() { + if !stub_dep_packages.contains(&package_id) { + log_warn_action( + "Skipping", + format!( + "package dependency {}, not used in the stub interface", + package.name + ), + ); continue; } let is_source_package = package.name == def.source_package_name; - log_action("Copying", format!("source package {}", package.name)); + log_action("Copying", format!("package dependency {}", package.name)); + + let _indent = LogIndent::new(); for source in sources { if is_source_package { let dest = target_deps From 468177fcc00ce410ff3eaa98a9b603bc43f09344 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Fri, 25 Oct 2024 16:56:33 +0200 Subject: [PATCH 014/109] even more reexport example --- .../test-data/many-ways-to-export/deps/sub/sub.wit | 14 ++++++++++++++ .../test-data/many-ways-to-export/main.wit | 4 +++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/wasm-rpc-stubgen/test-data/many-ways-to-export/deps/sub/sub.wit b/wasm-rpc-stubgen/test-data/many-ways-to-export/deps/sub/sub.wit index 4eb9e595..abb94920 100644 --- a/wasm-rpc-stubgen/test-data/many-ways-to-export/deps/sub/sub.wit +++ b/wasm-rpc-stubgen/test-data/many-ways-to-export/deps/sub/sub.wit @@ -14,4 +14,18 @@ interface iface7 { record rec1 { field: f64, } +} + +interface iface12 { + record rec1 { + field: f64, + } + + func12: func() -> rec1; +} + +interface iface13 { + use iface12.{rec1}; + + func13: func() -> rec1; } \ No newline at end of file diff --git a/wasm-rpc-stubgen/test-data/many-ways-to-export/main.wit b/wasm-rpc-stubgen/test-data/many-ways-to-export/main.wit index 869edb8d..7ee3f35d 100644 --- a/wasm-rpc-stubgen/test-data/many-ways-to-export/main.wit +++ b/wasm-rpc-stubgen/test-data/many-ways-to-export/main.wit @@ -40,7 +40,7 @@ interface iface9 { interface iface11 { use test:sub2/iface10.{rec1}; - // using the same func name + // intentionally using the same func name func9: func() -> rec1; } @@ -60,4 +60,6 @@ world api { export iface9; export test:sub2/iface10; export iface11; + export test:sub/iface12; + export test:sub/iface13; } \ No newline at end of file From 64ea031b8e12e54dc7d52f342883da7f1a79a960 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Fri, 25 Oct 2024 16:58:55 +0200 Subject: [PATCH 015/109] add missing hunk for package_id --- wasm-rpc-stubgen/src/stub.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wasm-rpc-stubgen/src/stub.rs b/wasm-rpc-stubgen/src/stub.rs index 0b7ffb72..64d9c4e4 100644 --- a/wasm-rpc-stubgen/src/stub.rs +++ b/wasm-rpc-stubgen/src/stub.rs @@ -562,7 +562,7 @@ impl InterfaceStub { pub struct InterfaceStubTypeDef { pub name: String, pub path: String, - pub package_name: Option, + pub package_id: Option, pub type_def: TypeDef, } From becbaab49efb786c439ab7a20b6e5c610eaeb8dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Sat, 26 Oct 2024 11:01:10 +0200 Subject: [PATCH 016/109] add missing hunk for package_id --- wasm-rpc-stubgen/src/stub.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wasm-rpc-stubgen/src/stub.rs b/wasm-rpc-stubgen/src/stub.rs index 64d9c4e4..d11aaeaf 100644 --- a/wasm-rpc-stubgen/src/stub.rs +++ b/wasm-rpc-stubgen/src/stub.rs @@ -428,7 +428,7 @@ impl StubDefinition { InterfaceStubTypeDef { name: type_name, path: interface_path, - package_name: package.map(|p| p.name.clone()), + package_id: owner_interface.package, type_def, } } From 50aebff986037173ed3c6be1fb7ff89deba9b4fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Sat, 26 Oct 2024 11:01:38 +0200 Subject: [PATCH 017/109] even more conflict for many exports --- .../test-data/many-ways-to-export/deps/sub2/sub2.wit | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/wasm-rpc-stubgen/test-data/many-ways-to-export/deps/sub2/sub2.wit b/wasm-rpc-stubgen/test-data/many-ways-to-export/deps/sub2/sub2.wit index 2a083998..473ed568 100644 --- a/wasm-rpc-stubgen/test-data/many-ways-to-export/deps/sub2/sub2.wit +++ b/wasm-rpc-stubgen/test-data/many-ways-to-export/deps/sub2/sub2.wit @@ -1,5 +1,10 @@ package test:sub2; +// intentionally same interface name and function as in sub.wit +interface iface4 { + func5: func(); +} + interface iface10 { use test:sub/iface5.{rec1}; From 27cee9ec9442eed04c88a6d2a25fd98f0d4448ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Mon, 28 Oct 2024 11:22:45 +0100 Subject: [PATCH 018/109] use latest (forked version) of wasm-tools and use PackageSourceMap in wit_resolve --- Cargo.lock | 8 +- wasm-rpc-stubgen/Cargo.toml | 9 +- wasm-rpc-stubgen/src/cargo.rs | 4 +- wasm-rpc-stubgen/src/commands/dependencies.rs | 27 ++-- wasm-rpc-stubgen/src/stub.rs | 10 +- wasm-rpc-stubgen/src/wit_generate.rs | 4 +- wasm-rpc-stubgen/src/wit_resolve.rs | 148 +++++++++--------- 7 files changed, 99 insertions(+), 111 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2059a94c..cef74b9e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5386,7 +5386,7 @@ dependencies = [ [[package]] name = "wasmparser" version = "0.219.1" -source = "git+https://github.com/golemcloud/wasm-tools.git?rev=46b168c169dc28d169d60e60ddbe4eed20f905b3#46b168c169dc28d169d60e60ddbe4eed20f905b3" +source = "git+https://github.com/golemcloud/wasm-tools.git#3458fe1e6f965420e69e1e09885d864dc49b343a" dependencies = [ "ahash", "bitflags 2.6.0", @@ -6175,7 +6175,7 @@ dependencies = [ [[package]] name = "wit-encoder" version = "0.219.1" -source = "git+https://github.com/golemcloud/wasm-tools.git?rev=46b168c169dc28d169d60e60ddbe4eed20f905b3#46b168c169dc28d169d60e60ddbe4eed20f905b3" +source = "git+https://github.com/golemcloud/wasm-tools.git#3458fe1e6f965420e69e1e09885d864dc49b343a" dependencies = [ "id-arena", "pretty_assertions", @@ -6241,7 +6241,7 @@ dependencies = [ [[package]] name = "wit-parser" version = "0.219.1" -source = "git+https://github.com/golemcloud/wasm-tools.git?rev=46b168c169dc28d169d60e60ddbe4eed20f905b3#46b168c169dc28d169d60e60ddbe4eed20f905b3" +source = "git+https://github.com/golemcloud/wasm-tools.git#3458fe1e6f965420e69e1e09885d864dc49b343a" dependencies = [ "anyhow", "id-arena", @@ -6252,7 +6252,7 @@ dependencies = [ "serde_derive", "serde_json", "unicode-xid", - "wasmparser 0.219.1 (git+https://github.com/golemcloud/wasm-tools.git?rev=46b168c169dc28d169d60e60ddbe4eed20f905b3)", + "wasmparser 0.219.1 (git+https://github.com/golemcloud/wasm-tools.git)", ] [[package]] diff --git a/wasm-rpc-stubgen/Cargo.toml b/wasm-rpc-stubgen/Cargo.toml index 061e78a5..e7aa2da0 100644 --- a/wasm-rpc-stubgen/Cargo.toml +++ b/wasm-rpc-stubgen/Cargo.toml @@ -61,13 +61,8 @@ toml = "0.8.14" walkdir = "2.5.0" wac-graph = "=0.6.0" wit-bindgen-rust = "=0.26.0" -# wit-encoder = "=0.219.0" -wit-encoder = { git = "https://github.com/golemcloud/wasm-tools.git", rev = "46b168c169dc28d169d60e60ddbe4eed20f905b3"} -# wit-encoder = { path = "../../wasm-tools/crates/wit-encoder"} -# wit-parser = "=0.219.0" -wit-parser = { git = "https://github.com/golemcloud/wasm-tools.git", rev = "46b168c169dc28d169d60e60ddbe4eed20f905b3"} - -# wit-parser = { path = "../../wasm-tools/crates/wit-parser"} +wit-encoder = { git = "https://github.com/golemcloud/wasm-tools.git" } +wit-parser = { git = "https://github.com/golemcloud/wasm-tools.git" } [dev-dependencies] diff --git a/wasm-rpc-stubgen/src/cargo.rs b/wasm-rpc-stubgen/src/cargo.rs index 9f88c276..a31cf358 100644 --- a/wasm-rpc-stubgen/src/cargo.rs +++ b/wasm-rpc-stubgen/src/cargo.rs @@ -88,7 +88,7 @@ pub fn generate_cargo_toml(def: &StubDefinition) -> anyhow::Result<()> { ); let stub_package_name = def.stub_package_name(); - for (_, dep_package, (dep_package_path, _)) in def.packages_with_wit_sources() { + for (_, dep_package, dep_package_sources) in def.packages_with_wit_sources() { if dep_package.name == stub_package_name { continue; } @@ -109,7 +109,7 @@ pub fn generate_cargo_toml(def: &StubDefinition) -> anyhow::Result<()> { format_package_name_without_version(&dep_package.name), WitDependency { path: naming::wit::package_wit_dep_dir_from_package_dir_name(&get_file_name( - dep_package_path, + &dep_package_sources.dir, )?) .to_string_lossy() .to_string(), diff --git a/wasm-rpc-stubgen/src/commands/dependencies.rs b/wasm-rpc-stubgen/src/commands/dependencies.rs index 34435568..f171cdd6 100644 --- a/wasm-rpc-stubgen/src/commands/dependencies.rs +++ b/wasm-rpc-stubgen/src/commands/dependencies.rs @@ -79,12 +79,13 @@ pub fn add_stub_dependency( let mut package_names_to_package_path = BTreeMap::::new(); for (package_name, package_id) in &stub_resolved_wit_root.resolve.package_names { - let (package_path, package_sources) = stub_resolved_wit_root - .sources + let package_sources = stub_resolved_wit_root + .package_sources .get(package_id) .ok_or_else(|| anyhow!("Failed to get package sources for {}", package_name))?; - let package_path = - naming::wit::package_wit_dep_dir_from_package_dir_name(&get_file_name(package_path)?); + let package_path = naming::wit::package_wit_dep_dir_from_package_dir_name(&get_file_name( + &package_sources.dir, + )?); let is_stub_main_package = *package_id == stub_resolved_wit_root.package_id; let is_dest_package = *package_name == dest_package.name; @@ -112,7 +113,7 @@ pub fn add_stub_dependency( }); // Non-self stub package has to be copied into target deps } else { - for source in package_sources { + for source in &package_sources.files { actions.add(OverwriteSafeAction::CopyFile { source: source.clone(), target: dest_deps_dir @@ -131,7 +132,7 @@ pub fn add_stub_dependency( stub_transformer .remove_imports_from_package_all_worlds(*package_id, &dest_package_import_prefix)?; let content = stub_transformer.render_package(*package_id)?; - let first_source = package_sources.iter().next().ok_or_else(|| { + let first_source = package_sources.files.iter().next().ok_or_else(|| { anyhow!( "Expected at least one source for stub package: {}", package_name @@ -139,7 +140,7 @@ pub fn add_stub_dependency( })?; let first_source_relative_path = strip_path_prefix(stub_wit_root, first_source)?; let target = { - if package_sources.len() == 1 { + if package_sources.files.len() == 1 { dest_wit_root.join(first_source_relative_path) } else { dest_wit_root @@ -156,7 +157,7 @@ pub fn add_stub_dependency( actions.add(OverwriteSafeAction::WriteFile { content, target }); // TODO: still output old ones while experimenting - for source in package_sources { + for source in &package_sources.files { actions.add(OverwriteSafeAction::CopyFile { source: source.clone(), target: dest_wit_root.join(format!( @@ -172,15 +173,15 @@ pub fn add_stub_dependency( if overwrite { let dest_main_package_id = dest_resolved_wit_root.package_id; - let (_, dest_main_sources) = dest_resolved_wit_root - .sources + let dest_main_package_sources = dest_resolved_wit_root + .package_sources .get(&dest_main_package_id) .ok_or_else(|| anyhow!("Failed to get dest main package sources"))?; - if dest_main_sources.len() != 1 { + if dest_main_package_sources.files.len() != 1 { bail!( "Expected exactly one dest main package source, got sources: {:?}", - dest_main_sources + dest_main_package_sources.files ); } @@ -192,7 +193,7 @@ pub fn add_stub_dependency( actions.add(OverwriteSafeAction::WriteFile { content, - target: dest_main_sources[0].clone(), + target: dest_main_package_sources.files[0].clone(), }); } diff --git a/wasm-rpc-stubgen/src/stub.rs b/wasm-rpc-stubgen/src/stub.rs index d11aaeaf..00013ec5 100644 --- a/wasm-rpc-stubgen/src/stub.rs +++ b/wasm-rpc-stubgen/src/stub.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::wit_resolve::ResolvedWitDir; +use crate::wit_resolve::{PackageSource, ResolvedWitDir}; use crate::{naming, WasmRpcOverride}; use anyhow::anyhow; use indexmap::IndexMap; @@ -28,7 +28,7 @@ use wit_parser::{ pub struct StubDefinition { resolve: Resolve, source_world_id: WorldId, - sources: IndexMap)>, + package_sources: IndexMap, stub_imported_interfaces: OnceCell>, stub_imports: OnceCell>, @@ -74,7 +74,7 @@ impl StubDefinition { Ok(Self { resolve: resolved_source.resolve, source_world_id, - sources: resolved_source.sources, + package_sources: resolved_source.package_sources, stub_imported_interfaces: OnceCell::new(), stub_imports: OnceCell::new(), stub_dep_package_ids: OnceCell::new(), @@ -100,7 +100,7 @@ impl StubDefinition { pub fn packages_with_wit_sources( &self, - ) -> impl Iterator))> { + ) -> impl Iterator { self.resolve .topological_packages() .into_iter() @@ -111,7 +111,7 @@ impl StubDefinition { .packages .get(package_id) .unwrap_or_else(|| panic!("package not found")), - self.sources + self.package_sources .get(&package_id) .unwrap_or_else(|| panic!("sources for package not found")), ) diff --git a/wasm-rpc-stubgen/src/wit_generate.rs b/wasm-rpc-stubgen/src/wit_generate.rs index 65a7e215..16836b3c 100644 --- a/wasm-rpc-stubgen/src/wit_generate.rs +++ b/wasm-rpc-stubgen/src/wit_generate.rs @@ -506,7 +506,7 @@ pub fn add_wit_dependencies(def: &StubDefinition) -> anyhow::Result<()> { let target_deps = target_wit_root.join(naming::wit::DEPS_DIR); let _indent = LogIndent::new(); - for (package_id, package, (_, sources)) in def.packages_with_wit_sources() { + for (package_id, package, package_sources) in def.packages_with_wit_sources() { if !stub_dep_packages.contains(&package_id) { log_warn_action( "Skipping", @@ -523,7 +523,7 @@ pub fn add_wit_dependencies(def: &StubDefinition) -> anyhow::Result<()> { log_action("Copying", format!("package dependency {}", package.name)); let _indent = LogIndent::new(); - for source in sources { + for source in &package_sources.files { if is_source_package { let dest = target_deps .join(naming::wit::package_dep_dir_name(&def.source_package_name)) diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index 442a0187..1448fa8c 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -1,14 +1,18 @@ -use crate::fs::strip_path_prefix; use anyhow::{anyhow, bail, Context}; use indexmap::IndexMap; use std::path::{Path, PathBuf}; -use wit_parser::{Package, PackageId, Resolve, UnresolvedPackageGroup}; +use wit_parser::{Package, PackageId, PackageSourceMap, Resolve}; + +pub struct PackageSource { + pub dir: PathBuf, + pub files: Vec, +} pub struct ResolvedWitDir { pub path: PathBuf, pub resolve: Resolve, pub package_id: PackageId, - pub sources: IndexMap)>, + pub package_sources: IndexMap, } impl ResolvedWitDir { @@ -37,40 +41,27 @@ fn resolve_wit_dir(path: &Path) -> anyhow::Result { let mut resolve = Resolve::new(); - let (package_id, sources) = resolve + let (package_id, package_source_map) = resolve .push_dir(path) .with_context(|| anyhow!("Failed to resolve wit dir: {}", path.to_string_lossy()))?; - let sources = partition_sources_by_package_ids(path, &resolve, package_id, sources)?; + let package_sources = collect_package_sources(path, &resolve, package_id, &package_source_map)?; Ok(ResolvedWitDir { path: path.to_path_buf(), resolve, package_id, - sources, + package_sources, }) } -// Currently Resolve::push_dir does return the source that were used during resolution, -// but they are not partitioned by packages, and the source info is not kept inside Resolve -// (it is lost during resolution). -// -// To solve this (until we can get a better API upstream) we could extract and replicate the logic used -// there, but that would require many code duplication, as many functions and types are not public. -// -// Instead of that, we partition the returned sources by following the accepted file and directory structure. -// -// Unfortunately we still have some duplication of performed operations: during partitioning we partially parse -// the dependencies again - as UnresolvedPackageGroups - but similar partial "peeks" into deps already happened -// in stubgen steps before. This way we try to pull and concentrate them here, so they only happen when creating -// a new ResolvedWitDir, and parsing should only happen twice: while using Resolve::push_dir above, and here. -fn partition_sources_by_package_ids( +fn collect_package_sources( path: &Path, resolve: &Resolve, root_package_id: PackageId, - sources: Vec, -) -> anyhow::Result)>> { - // Based on Resolve::push_dir (): + package_source_map: &PackageSourceMap, +) -> anyhow::Result> { + // Based on Resolve::push_dir: // // The deps folder may contain: // $path/ deps/ my-package/*.wit: a directory that may contain multiple WIT files @@ -80,67 +71,68 @@ fn partition_sources_by_package_ids( // Disabling "wasm" and "wat" sources could be done by disabling default features, but they are also required through other dependencies, // so we filter out currently not supported path patterns here, including the single file format (for now). - let mut partitioned_sources = IndexMap::)>::new(); - let mut dep_package_path_to_package_id = IndexMap::::new(); - - for source in sources { - let relative_source = strip_path_prefix(path, &source)?; - - let segments = relative_source.iter().collect::>(); - - let (package_path, package_id) = match segments.len() { - 1 => (path, root_package_id), - 2 => { + let deps_dir = path.join("deps"); + let mut package_dir_paths = IndexMap::::new(); + for (package_id, package) in &resolve.packages { + let sources = package_source_map + .package_paths(package_id) + .ok_or_else(|| { + anyhow!( + "Failed to get package source map for package {}", + package.name + ) + })? + .map(|path| path.to_path_buf()) + .collect::>(); + + if package_id == root_package_id { + package_dir_paths.insert( + package_id, + PackageSource { + dir: path.to_path_buf(), + files: sources, + }, + ); + } else { + if sources.len() == 0 { + bail!("Expected at least one source for package: {}", package.name); + }; + + let source = &sources[0]; + + let extension = source.extension().ok_or_else(|| { + anyhow!( + "Failed to get extension for wit source: {}", + source.display() + ) + })?; + + if extension != "wit" { bail!( - "Single file with packages not supported, source: {}", - source.to_string_lossy() + "Only wit sources are supported, source: {}", + source.display() ); } - 3 => { - let dep_package_path = source.parent().ok_or_else(|| { - anyhow!( - "Failed to get source parent, source: {}", - source.to_string_lossy() - ) - })?; - - match dep_package_path_to_package_id.get(dep_package_path) { - Some(package_id) => (dep_package_path, *package_id), - None => { - let package_id = *resolve - .package_names - .get( - &UnresolvedPackageGroup::parse_dir(dep_package_path)? - .main - .name, - ) - .ok_or_else(|| { - anyhow!( - "Failed to get package id for source: {}", - source.to_string_lossy(), - ) - })?; - - dep_package_path_to_package_id - .insert(dep_package_path.to_path_buf(), package_id); - - (dep_package_path, package_id) - } - } - } - _ => { + + let parent = source.parent().ok_or_else(|| { + anyhow!("Failed to get parent for wit source: {}", source.display()) + })?; + + if parent == deps_dir { bail!( - "Unexpected source path, source: {}", - source.to_string_lossy() + "Single-file wit packages without folder are not supported, source: {}", + source.display() ); } - }; - partitioned_sources - .entry(package_id) - .and_modify(|(_, sources)| sources.push(source.to_path_buf())) - .or_insert_with(|| (package_path.to_path_buf(), vec![source.to_path_buf()])); + package_dir_paths.insert( + package_id, + PackageSource { + dir: parent.to_path_buf(), + files: sources, + }, + ); + } } - - Ok(partitioned_sources) + Ok(package_dir_paths) } From b34717221c267b5166048e88fec40d20e4da8131 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Mon, 28 Oct 2024 11:39:30 +0100 Subject: [PATCH 019/109] cleanups --- wasm-rpc-stubgen/src/commands/composition.rs | 2 +- wasm-rpc-stubgen/src/commands/declarative.rs | 15 ++--- wasm-rpc-stubgen/src/commands/dependencies.rs | 8 +-- wasm-rpc-stubgen/src/commands/log.rs | 38 ++++-------- wasm-rpc-stubgen/src/fs.rs | 62 +++++-------------- wasm-rpc-stubgen/src/model/oam.rs | 2 +- wasm-rpc-stubgen/src/model/wasm_rpc.rs | 2 +- wasm-rpc-stubgen/src/rust.rs | 5 +- wasm-rpc-stubgen/src/wit_generate.rs | 8 +-- 9 files changed, 46 insertions(+), 96 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/composition.rs b/wasm-rpc-stubgen/src/commands/composition.rs index 4f049506..f87436ee 100644 --- a/wasm-rpc-stubgen/src/commands/composition.rs +++ b/wasm-rpc-stubgen/src/commands/composition.rs @@ -19,7 +19,7 @@ pub async fn compose( let socket = std::fs::read(source_wasm).with_context(|| { format!( "failed to read socket component `{socket}`", - socket = source_wasm.to_string_lossy() + socket = source_wasm.display() ) })?; diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index 12bef4b1..9189314b 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -239,8 +239,8 @@ pub async fn post_component_build_app(config: &Config, app: &Application) -> any "Copying", format!( "(without composing) {} to {}, no wasm rpc dependencies defined", - input_wasm.to_string_lossy(), - output_wasm.to_string_lossy() + input_wasm.display(), + output_wasm.display() ), ); copy(&input_wasm, &output_wasm)?; @@ -335,7 +335,7 @@ fn collect_sources(mode: &ApplicationResolveMode) -> ValidatedResult ValidatedResult ValidatedResult ValidatedResult { log_action( "Copying", - format!( - "{} to {}", - source.to_string_lossy(), - target.to_string_lossy() - ), + format!("{} to {}", source.display(), target.display()), ); } OverwriteSafeAction::CopyFileTransformed { source, target, .. } => { log_action( "Copying", - format!( - "{} to {} transformed", - source.to_string_lossy(), - target.to_string_lossy() - ), + format!("{} to {} transformed", source.display(), target.display()), ); } OverwriteSafeAction::WriteFile { target, .. } => { - log_action("Creating", format!("{}", target.to_string_lossy())); + log_action("Creating", format!("{}", target.display())); } }, OverwriteSafeActionPlan::Overwrite => match action { OverwriteSafeAction::CopyFile { source, target } => { log_warn_action( "Overwriting", - format!( - "{} with {}", - target.to_string_lossy(), - source.to_string_lossy() - ), + format!("{} with {}", target.display(), source.display()), ); } OverwriteSafeAction::CopyFileTransformed { source, target, .. } => { log_warn_action( "Overwriting", - format!( - "{} with {} transformed", - target.to_string_lossy(), - source.to_string_lossy() - ), + format!("{} with {} transformed", target.display(), source.display()), ); } OverwriteSafeAction::WriteFile { content: _, target } => { - log_warn_action("Overwriting", format!("{}", target.to_string_lossy())); + log_warn_action("Overwriting", format!("{}", target.display())); } }, OverwriteSafeActionPlan::SkipSameContent => match action { @@ -152,8 +136,8 @@ pub fn log_action_plan(action: &OverwriteSafeAction, plan: OverwriteSafeActionPl "Skipping", format!( "copying {} to {}, content already up-to-date", - source.to_string_lossy(), - target.to_string_lossy(), + source.display(), + target.display(), ), ); } @@ -162,8 +146,8 @@ pub fn log_action_plan(action: &OverwriteSafeAction, plan: OverwriteSafeActionPl "Skipping", format!( "copying {} to {} transformed, content already up-to-date", - source.to_string_lossy(), - target.to_string_lossy() + source.display(), + target.display() ), ); } @@ -172,7 +156,7 @@ pub fn log_action_plan(action: &OverwriteSafeAction, plan: OverwriteSafeActionPl "Skipping", format!( "generating {}, content already up-to-date", - target.to_string_lossy() + target.display() ), ); } diff --git a/wasm-rpc-stubgen/src/fs.rs b/wasm-rpc-stubgen/src/fs.rs index ab96968e..95a4badd 100644 --- a/wasm-rpc-stubgen/src/fs.rs +++ b/wasm-rpc-stubgen/src/fs.rs @@ -12,13 +12,7 @@ pub fn copy, Q: AsRef>(from: P, to: Q) -> anyhow::Result, Q: AsRef, T: Fn(String) -> anyhow: let context = || { format!( "Failed to copy (and transform) from {} to {}", - from.to_string_lossy(), - to.to_string_lossy() + from.display(), + to.display() ) }; @@ -87,7 +81,7 @@ pub fn write_str, S: AsRef>(path: P, str: S) -> anyhow::Resu let path = path.as_ref(); let str = str.as_ref(); - let context = || format!("Failed to write string to {}", path.to_string_lossy()); + let context = || format!("Failed to write string to {}", path.display()); let target_parent = path .parent() @@ -107,15 +101,10 @@ pub fn has_str_content, S: AsRef>(path: P, str: S) -> anyhow let path = path.as_ref(); let str = str.as_ref(); - let context = || { - format!( - "Failed to compare content to string for {}", - path.to_string_lossy() - ) - }; + let context = || format!("Failed to compare content to string for {}", path.display()); let content = std::fs::read_to_string(path) - .with_context(|| anyhow!("Failed to read as string: {}", path.to_string_lossy())) + .with_context(|| anyhow!("Failed to read as string: {}", path.display())) .with_context(context)?; Ok(content == str) @@ -128,17 +117,17 @@ pub fn has_same_string_content, Q: AsRef>(a: P, b: Q) -> an let context = || { format!( "Failed to compare string contents of {} and {}", - a.to_string_lossy(), - b.to_string_lossy() + a.display(), + b.display() ) }; let content_a = std::fs::read_to_string(a) - .with_context(|| anyhow!("Failed to read as string: {}", a.to_string_lossy())) + .with_context(|| anyhow!("Failed to read as string: {}", a.display())) .with_context(context)?; let content_b = std::fs::read_to_string(b) - .with_context(|| anyhow!("Failed to read as string: {}", b.to_string_lossy())) + .with_context(|| anyhow!("Failed to read as string: {}", b.display())) .with_context(context)?; Ok(content_a == content_b) @@ -147,20 +136,10 @@ pub fn has_same_string_content, Q: AsRef>(a: P, b: Q) -> an pub fn get_file_name>(path: P) -> anyhow::Result { let path = path.as_ref(); path.file_name() - .ok_or_else(|| { - anyhow!( - "Failed to get file name for path: {}", - path.to_string_lossy(), - ) - })? + .ok_or_else(|| anyhow!("Failed to get file name for path: {}", path.display(),))? .to_os_string() .into_string() - .map_err(|_| { - anyhow!( - "Failed to convert filename for path: {}", - path.to_string_lossy() - ) - }) + .map_err(|_| anyhow!("Failed to convert filename for path: {}", path.display())) } pub fn strip_path_prefix, Q: AsRef>( @@ -175,8 +154,8 @@ pub fn strip_path_prefix, Q: AsRef>( .with_context(|| { anyhow!( "Failed to strip prefix from path, prefix: {}, path: {}", - prefix.to_string_lossy(), - path.to_string_lossy() + prefix.display(), + path.display() ) })? .to_path_buf()) @@ -208,18 +187,11 @@ impl OverwriteSafeAction { F: FnOnce(String) -> anyhow::Result, { let content = std::fs::read_to_string(&source).with_context(|| { - anyhow!( - "Failed to read file as string, path: {}", - source.to_string_lossy() - ) + anyhow!("Failed to read file as string, path: {}", source.display()) })?; - let source_transformed = transform(content).with_context(|| { - anyhow!( - "Failed to transform file, path: {}", - source.to_string_lossy() - ) - })?; + let source_transformed = transform(content) + .with_context(|| anyhow!("Failed to transform file, path: {}", source.display()))?; Ok(OverwriteSafeAction::CopyFileTransformed { source, diff --git a/wasm-rpc-stubgen/src/model/oam.rs b/wasm-rpc-stubgen/src/model/oam.rs index 037946c8..600e7d89 100644 --- a/wasm-rpc-stubgen/src/model/oam.rs +++ b/wasm-rpc-stubgen/src/model/oam.rs @@ -18,7 +18,7 @@ pub struct ApplicationWithSource { impl ApplicationWithSource { pub fn from_yaml_file(file: PathBuf) -> anyhow::Result { let content = std::fs::read_to_string(file.as_path()) - .with_context(|| format!("Failed to load file: {}", file.to_string_lossy()))?; + .with_context(|| format!("Failed to load file: {}", file.display()))?; Ok(Self::from_yaml_string(file, content)?) } diff --git a/wasm-rpc-stubgen/src/model/wasm_rpc.rs b/wasm-rpc-stubgen/src/model/wasm_rpc.rs index 2f6f4ad9..04c07e25 100644 --- a/wasm-rpc-stubgen/src/model/wasm_rpc.rs +++ b/wasm-rpc-stubgen/src/model/wasm_rpc.rs @@ -451,7 +451,7 @@ impl Application { "Component Build is specified multiple times in sources: {}", wasm_builds .iter() - .map(|c| format!("{} in {}", c.name, c.source.to_string_lossy())) + .map(|c| format!("{} in {}", c.name, c.source.display())) .join(", ") )); } diff --git a/wasm-rpc-stubgen/src/rust.rs b/wasm-rpc-stubgen/src/rust.rs index 59a94ba3..0fde0f82 100644 --- a/wasm-rpc-stubgen/src/rust.rs +++ b/wasm-rpc-stubgen/src/rust.rs @@ -286,10 +286,7 @@ pub fn generate_stub_source(def: &StubDefinition) -> anyhow::Result<()> { log_action( "Generating", - format!( - "stub source to {}", - def.target_rust_path().to_string_lossy() - ), + format!("stub source to {}", def.target_rust_path().display()), ); fs::create_dir_all(def.target_rust_path().parent().unwrap())?; fs::write(def.target_rust_path(), src)?; diff --git a/wasm-rpc-stubgen/src/wit_generate.rs b/wasm-rpc-stubgen/src/wit_generate.rs index 16836b3c..a8c0990e 100644 --- a/wasm-rpc-stubgen/src/wit_generate.rs +++ b/wasm-rpc-stubgen/src/wit_generate.rs @@ -532,8 +532,8 @@ pub fn add_wit_dependencies(def: &StubDefinition) -> anyhow::Result<()> { " Copying", format!( "(with source imports removed) {} to {}", - source.to_string_lossy(), - dest.to_string_lossy() + source.display(), + dest.display() ), ); copy_transformed(source, &dest, &remove_stub_imports)?; @@ -542,7 +542,7 @@ pub fn add_wit_dependencies(def: &StubDefinition) -> anyhow::Result<()> { let dest = target_wit_root.join(relative); log_action( " Copying", - format!("{} to {}", source.to_string_lossy(), dest.to_string_lossy()), + format!("{} to {}", source.display(), dest.display()), ); copy(source, &dest)?; } @@ -569,7 +569,7 @@ fn write_embedded_source(target_dir: &Path, file_name: &str, content: &str) -> a log_action( "Writing", - format!("{} to {}", file_name, target_dir.to_string_lossy()), + format!("{} to {}", file_name, target_dir.display()), ); fs::write(target_dir.join(file_name), content)?; From cc169c266f14da9abaa8ec954c2d0ba8f08b6b88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Mon, 28 Oct 2024 11:40:00 +0100 Subject: [PATCH 020/109] cleanups --- wasm-rpc-stubgen/src/wit_resolve.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index 1448fa8c..2489f852 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -25,7 +25,7 @@ impl ResolvedWitDir { anyhow!( "Failed to get package by id: {:?}, wit dir: {}", package_id, - self.path.to_string_lossy() + self.path.display() ) }) } @@ -43,7 +43,7 @@ fn resolve_wit_dir(path: &Path) -> anyhow::Result { let (package_id, package_source_map) = resolve .push_dir(path) - .with_context(|| anyhow!("Failed to resolve wit dir: {}", path.to_string_lossy()))?; + .with_context(|| anyhow!("Failed to resolve wit dir: {}", path.display()))?; let package_sources = collect_package_sources(path, &resolve, package_id, &package_source_map)?; @@ -94,7 +94,7 @@ fn collect_package_sources( }, ); } else { - if sources.len() == 0 { + if sources.is_empty() { bail!("Expected at least one source for package: {}", package.name); }; From afb86cb4ea947cef45eaf69438314bc8b8562c2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 29 Oct 2024 11:47:29 +0100 Subject: [PATCH 021/109] use wit_encode for wit generation --- wasm-rpc-stubgen/src/rust.rs | 8 +- wasm-rpc-stubgen/src/stub.rs | 232 +++-- wasm-rpc-stubgen/src/wit_generate.rs | 793 +++++++----------- .../test-data/many-ways-to-export/main.wit | 3 +- wasm-rpc-stubgen/tests/wit.rs | 2 + 5 files changed, 509 insertions(+), 529 deletions(-) diff --git a/wasm-rpc-stubgen/src/rust.rs b/wasm-rpc-stubgen/src/rust.rs index 0fde0f82..087f5605 100644 --- a/wasm-rpc-stubgen/src/rust.rs +++ b/wasm-rpc-stubgen/src/rust.rs @@ -515,14 +515,14 @@ fn get_output_values_source( ) -> anyhow::Result> { let mut output_values = Vec::new(); match &function.results { - FunctionResultStub::Single(typ) => { + FunctionResultStub::Anon(typ) => { output_values.push(extract_from_wit_value( typ, def, quote! { result.tuple_element(0).expect("tuple not found") }, )?); } - FunctionResultStub::Multi(params) => { + FunctionResultStub::Named(params) => { for (n, param) in params.iter().enumerate() { output_values.push(extract_from_wit_value( ¶m.typ, @@ -557,13 +557,13 @@ fn get_result_type_source( function: &FunctionStub, ) -> anyhow::Result { let result_type = match &function.results { - FunctionResultStub::Single(typ) => { + FunctionResultStub::Anon(typ) => { let typ = type_to_rust_ident(typ, def)?; quote! { #typ } } - FunctionResultStub::Multi(params) => { + FunctionResultStub::Named(params) => { let mut results = Vec::new(); for param in params { let param_name = Ident::new(&to_rust_ident(¶m.name), Span::call_site()); diff --git a/wasm-rpc-stubgen/src/stub.rs b/wasm-rpc-stubgen/src/stub.rs index 00013ec5..fc98fc00 100644 --- a/wasm-rpc-stubgen/src/stub.rs +++ b/wasm-rpc-stubgen/src/stub.rs @@ -16,8 +16,9 @@ use crate::wit_resolve::{PackageSource, ResolvedWitDir}; use crate::{naming, WasmRpcOverride}; use anyhow::anyhow; use indexmap::IndexMap; +use itertools::Itertools; use std::cell::OnceCell; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; use wit_parser::{ Function, FunctionKind, Interface, InterfaceId, Package, PackageId, PackageName, Resolve, @@ -31,7 +32,7 @@ pub struct StubDefinition { package_sources: IndexMap, stub_imported_interfaces: OnceCell>, - stub_imports: OnceCell>, + stub_used_type_defs: OnceCell>, stub_dep_package_ids: OnceCell>, pub source_package_id: PackageId, @@ -76,7 +77,7 @@ impl StubDefinition { source_world_id, package_sources: resolved_source.package_sources, stub_imported_interfaces: OnceCell::new(), - stub_imports: OnceCell::new(), + stub_used_type_defs: OnceCell::new(), stub_dep_package_ids: OnceCell::new(), source_package_id: resolved_source.package_id, source_package_name, @@ -129,8 +130,8 @@ impl StubDefinition { .unwrap_or_else(|| panic!("selected world not found")) } - pub fn source_world_name(&self) -> String { - self.source_world().name.clone() + pub fn source_world_name(&self) -> &str { + &self.source_world().name } pub fn target_cargo_path(&self) -> PathBuf { @@ -145,6 +146,10 @@ impl StubDefinition { self.target_root.join("src/lib.rs") } + pub fn target_interface_name(&self) -> String { + format!("stub-{}", self.source_world_name()) + } + pub fn target_world_name(&self) -> String { format!("wasm-rpc-stub-{}", self.source_world_name()) } @@ -211,16 +216,129 @@ impl StubDefinition { }) } - pub fn stub_imports(&self) -> &IndexMap { - self.stub_imports.get_or_init(|| { - self.stub_imported_interfaces() + pub fn stub_used_type_defs(&self) -> &Vec { + self.stub_used_type_defs.get_or_init(|| { + let imported_type_ids = self + .stub_imported_interfaces() .iter() - .flat_map(|i| { - i.imports - .iter() - .map(|i| (InterfaceStubImport::from(i), i.clone())) + .flat_map(|interface| &interface.used_types) + .cloned() + .collect::>(); + + let imported_type_names = { + let mut imported_type_names = HashMap::>::new(); + for type_id in imported_type_ids { + let type_def = self.resolve.types.get(type_id).unwrap_or_else(|| { + panic!("Imported type not found, type id: {:?}", type_id) + }); + let type_name = type_def + .name + .clone() + .unwrap_or_else(|| panic!("Missing type name, type id: {:?}", type_id)); + + imported_type_names + .entry(type_name) + .or_default() + .insert(type_id); + } + imported_type_names + }; + + imported_type_names + .into_iter() + .flat_map(|(type_name, type_ids)| { + let type_name_is_unique = type_ids.len() == 1; + type_ids.into_iter().map(move |type_id| { + let type_def = self + .resolve + .types + .get(type_id) + .unwrap_or_else(|| { + panic!("Imported type not found, type id: {:?}", type_id) + }) + .clone(); + + let TypeOwner::Interface(interface_id) = type_def.owner else { + panic!( + "Expected interface owner for type, got: {:?}, type name: {:?}", + type_def, type_def.name + ); + }; + + let interface = + self.resolve + .interfaces + .get(interface_id) + .unwrap_or_else(|| { + panic!("Interface not found, interface id: {:?}", interface_id) + }); + + let interface_name = interface.name.clone().unwrap_or_else(|| { + panic!("Missing interface name, interface id: {:?}", interface_id) + }); + + let package = interface.package.clone().map(|package_id| { + self.resolve.packages.get(package_id).unwrap_or_else(|| { + panic!( + "Missing package for interface, package id: {:?}, interface id: {:?}", + package_id, interface_id, + ) + }) + }); + + let interface_identifier = package + .map(|package| package.name.interface_id(&interface_name)) + .unwrap_or(interface_name.clone()); + + let type_name_alias = (!type_name_is_unique).then(|| match &package { + Some(package) => format!( + "{}-{}-{}-{}", + package.name.namespace, + package.name.name, + interface_name, + &type_name + ), + None => format!("{}-{}", interface_name, type_name), + }); + + let inlined = self.always_inline_types + && interface.package == Some(self.source_package_id); + + let inlined_type_def = match type_def.kind { + TypeDefKind::Type(Type::Id(type_id)) if inlined => { + let mut type_def = self + .resolve + .types + .get(type_id) + .unwrap_or_else(|| { + panic!("Failed to get type def for inlining") + }) + .clone(); + type_def.name = + type_name_alias.clone().or_else(|| Some(type_name.clone())); + Some(type_def) + } + _ => None, + }; + + InterfaceStubTypeDef { + package_id: interface.package, + type_id, + type_def, + inlined_type_def, + interface_identifier, + type_name: type_name.clone(), + type_name_alias, + inlined, + } + }) }) - .collect() + .sorted_by(|a, b| { + let a = (&a.interface_identifier, &a.type_name, &a.type_name_alias); + let b = (&b.interface_identifier, &b.type_name, &b.type_name_alias); + a.cmp(&b) + }) + .collect::>() }) } @@ -290,9 +408,9 @@ impl StubDefinition { typ: *typ, }); } - FunctionResultStub::Multi(param_stubs) + FunctionResultStub::Named(param_stubs) } - Results::Anon(single) => FunctionResultStub::Single(*single), + Results::Anon(single) => FunctionResultStub::Anon(*single), }; FunctionStub { @@ -323,12 +441,12 @@ impl StubDefinition { .filter(|function| function.kind == FunctionKind::Freestanding), ); - let (imported_interfaces, _) = self.extract_interface_stubs_from_types(types.into_iter()); + let (used_types, _) = self.extract_interface_stubs_from_types(types.into_iter()); Some(InterfaceStub { - name, + name: name.to_string(), functions, - imports: imported_interfaces, + used_types, global: true, constructor_params: None, static_functions: vec![], @@ -343,7 +461,7 @@ impl StubDefinition { ) -> Vec { let functions = Self::functions_to_stubs(interface.functions.values()); - let (imported_interfaces, resource_interfaces) = self.extract_interface_stubs_from_types( + let (used_types, resource_interfaces) = self.extract_interface_stubs_from_types( interface .types .iter() @@ -355,7 +473,7 @@ impl StubDefinition { interface_stubs.push(InterfaceStub { name, functions, - imports: imported_interfaces, + used_types, global: false, constructor_params: None, static_functions: vec![], @@ -369,8 +487,8 @@ impl StubDefinition { fn extract_interface_stubs_from_types( &self, types: impl Iterator, - ) -> (Vec, Vec) { - let mut imported_interfaces = Vec::::new(); + ) -> (Vec, Vec) { + let mut used_types = Vec::::new(); let mut resource_interfaces = Vec::::new(); for (type_name, type_id) in types { @@ -394,18 +512,16 @@ impl StubDefinition { type_id, )) } else { - imported_interfaces.push(self.type_def_to_stub( - owner_interface, - type_name, - type_def.clone(), - )); + used_types.push(type_id); } } } - (imported_interfaces, resource_interfaces) + (used_types, resource_interfaces) } + // TODO: + /* fn type_def_to_stub( &self, owner_interface: &Interface, @@ -432,6 +548,7 @@ impl StubDefinition { type_def, } } + */ fn resource_interface_stub( &self, @@ -449,7 +566,7 @@ impl StubDefinition { let function_stubs_by_kind = |kind: FunctionKind| Self::functions_to_stubs(functions_by_kind(kind)); - let (imported_interfaces, _) = self.extract_interface_stubs_from_types( + let (used_types, _) = self.extract_interface_stubs_from_types( owner_interface .types .iter() @@ -471,7 +588,7 @@ impl StubDefinition { InterfaceStub { name: type_name, functions: function_stubs_by_kind(FunctionKind::Method(type_id)), - imports: imported_interfaces, + used_types, global: false, constructor_params, static_functions: function_stubs_by_kind(FunctionKind::Static(type_id)), @@ -514,6 +631,14 @@ impl StubDefinition { .get(type_id) .ok_or_else(|| anyhow!("failed to get type by id: {:?}", type_id)) } + + pub fn get_stub_used_type_alias(&self, type_id: TypeId) -> Option<&str> { + // TODO: hash map + self.stub_used_type_defs() + .iter() + .find(|type_def| type_def.type_id == type_id) + .and_then(|type_def| type_def.type_name_alias.as_deref()) + } } struct WorldItemsByType<'a> { @@ -528,11 +653,21 @@ pub struct InterfaceStub { pub constructor_params: Option>, pub functions: Vec, pub static_functions: Vec, - pub imports: Vec, + pub used_types: Vec, pub global: bool, pub owner_interface: Option, } +impl InterfaceStub { + // The returned bool is true if the function is static + pub fn all_functions(&self) -> impl Iterator { + self.static_functions + .iter() + .map(|f| (f, true)) + .chain(self.functions.iter().map(|f| (f, false))) + } +} + impl InterfaceStub { pub fn is_resource(&self) -> bool { self.constructor_params.is_some() @@ -560,24 +695,23 @@ impl InterfaceStub { #[derive(Debug, Clone, PartialEq)] pub struct InterfaceStubTypeDef { - pub name: String, - pub path: String, pub package_id: Option, + pub type_id: TypeId, pub type_def: TypeDef, + pub inlined_type_def: Option, + pub interface_identifier: String, + pub type_name: String, + pub type_name_alias: Option, + pub inlined: bool, } -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct InterfaceStubImport { - pub name: String, - pub path: String, -} +impl InterfaceStubTypeDef { + pub fn stub_type_name(&self) -> &str { + self.type_name_alias.as_deref().unwrap_or(&self.type_name) + } -impl From<&InterfaceStubTypeDef> for InterfaceStubImport { - fn from(value: &InterfaceStubTypeDef) -> Self { - Self { - name: value.name.clone(), - path: value.path.clone(), - } + pub fn stub_type_def(&self) -> &TypeDef { + self.inlined_type_def.as_ref().unwrap_or(&self.type_def) } } @@ -585,7 +719,7 @@ impl From<&InterfaceStubTypeDef> for InterfaceStubImport { pub struct FunctionStub { pub name: String, pub params: Vec, - pub results: FunctionResultStub, + pub results: FunctionResultStub, // TODO: option? } impl FunctionStub { @@ -646,16 +780,16 @@ pub struct FunctionParamStub { #[derive(Debug, Clone)] pub enum FunctionResultStub { - Single(Type), - Multi(Vec), + Anon(Type), + Named(Vec), SelfType, } impl FunctionResultStub { pub fn is_empty(&self) -> bool { match self { - FunctionResultStub::Single(_) => false, - FunctionResultStub::Multi(params) => params.is_empty(), + FunctionResultStub::Anon(_) => false, + FunctionResultStub::Named(params) => params.is_empty(), FunctionResultStub::SelfType => false, } } diff --git a/wasm-rpc-stubgen/src/wit_generate.rs b/wasm-rpc-stubgen/src/wit_generate.rs index a8c0990e..aa509dce 100644 --- a/wasm-rpc-stubgen/src/wit_generate.rs +++ b/wasm-rpc-stubgen/src/wit_generate.rs @@ -20,11 +20,12 @@ use crate::stub::{ }; use crate::wit_transform::import_remover; use crate::{naming, WasmRpcOverride}; -use anyhow::{anyhow, bail}; -use std::fmt::Write; use std::fs; use std::path::Path; -use wit_parser::{Enum, Field, Flags, Handle, Result_, Tuple, Type, TypeDef, TypeDefKind, Variant}; +use wit_encoder::{ + Ident, Interface, Package, PackageName, Params, ResourceFunc, Results, Type, TypeDef, + TypeDefKind, VariantCase, World, +}; pub fn generate_stub_wit_to_target(def: &StubDefinition) -> anyhow::Result<()> { log_action( @@ -53,439 +54,166 @@ pub fn generate_stub_wit_from_wit_dir( } pub fn generate_stub_wit_from_stub_def(def: &StubDefinition) -> anyhow::Result { - let world = def.source_world(); - - let mut out = String::new(); - - writeln!(out, "package {}-stub;", def.source_package_name)?; - writeln!(out)?; - writeln!(out, "interface stub-{} {{", world.name)?; - - let all_imports = def.stub_imports(); - - // Renaming the mandatory imports to avoid collisions with types coming from the stubbed package - writeln!(out, " use golem:rpc/types@0.1.0.{{uri as golem-rpc-uri}};")?; - writeln!( - out, - " use wasi:io/poll@0.2.0.{{pollable as wasi-io-pollable}};" - )?; - - if def.always_inline_types { - let mut inline_types: Vec = vec![]; - - for (import, type_def_info) in all_imports { - match &type_def_info.package_id { - Some(package_id) if package_id == &def.source_package_id => { - inline_types.push(type_def_info.clone()); - } - _ => writeln!(out, " use {}.{{{}}};", import.path, import.name)?, - } - } - - writeln!(out)?; - - for typ in inline_types { - write_type_def(&mut out, &typ.type_def, typ.name.as_str(), def)?; - } - } else { - for (import, _) in all_imports { - writeln!(out, " use {}.{{{}}};", import.path, import.name)?; - } - writeln!(out)?; - } + Ok(generate_stub_package_from_stub_def(def).to_string()) +} - // Generating async return types - for interface in def.stub_imported_interfaces() { - for function in &interface.functions { - if !function.results.is_empty() { - write_async_return_type(&mut out, function, interface, def)?; - } - } - for function in &interface.static_functions { - if !function.results.is_empty() { - write_async_return_type(&mut out, function, interface, def)?; +pub fn generate_stub_package_from_stub_def(def: &StubDefinition) -> Package { + let mut package = Package::new(PackageName::new( + def.source_package_name.namespace.clone(), + format!("{}-stub", def.source_package_name.name), + def.source_package_name.version.clone(), + )); + + let interface_identifier = def.target_interface_name(); + + // Stub interface + { + let mut stub_interface = Interface::new(interface_identifier.clone()); + + // Common used types + stub_interface.use_type( + "golem:rpc/types@0.1.0", + "uri", + Some(Ident::new("golem-rpc-uri")), + ); + stub_interface.use_type( + "wasi:io/poll@0.2.0", + "pollable", + Some(Ident::new("wasi-io-pollable")), + ); + + // Used or inlined type defs + for type_def in def.stub_used_type_defs() { + if type_def.inlined { + stub_interface.type_def(type_def.to_encoder(def)) + } else { + stub_interface.use_type( + type_def.interface_identifier.clone(), + type_def.type_name.clone(), + type_def.type_name_alias.clone().map(Ident::from), + ); } } - } - // Generating function definitions - for interface in def.stub_imported_interfaces() { - writeln!(out, " resource {} {{", &interface.name)?; - match &interface.constructor_params { - None => { - writeln!(out, " constructor(location: golem-rpc-uri);")?; - } - Some(params) => { - write!(out, " constructor(location: golem-rpc-uri")?; - if !params.is_empty() { - write!(out, ", ")?; + // Async return types + for interface in def.stub_imported_interfaces() { + for (function, _) in interface.all_functions() { + if !function.results.is_empty() { + add_async_return_type(def, &mut stub_interface, function, interface); } - write_param_list(&mut out, def, params)?; - writeln!(out, ");")?; } } - for function in &interface.functions { - write_function_definition(&mut out, function, false, interface, def)?; - } - for function in &interface.static_functions { - write_function_definition(&mut out, function, true, interface, def)?; - } - writeln!(out, " }}")?; - writeln!(out)?; - } - - writeln!(out, "}}")?; - writeln!(out)?; - - writeln!(out, "world {} {{", def.target_world_name())?; - writeln!(out, " export stub-{};", world.name)?; - writeln!(out, "}}")?; - - Ok(out) -} - -fn write_function_definition( - out: &mut String, - function: &FunctionStub, - is_static: bool, - owner: &InterfaceStub, - def: &StubDefinition, -) -> anyhow::Result<()> { - let func = if is_static { "static_func" } else { "func" }; - if !function.results.is_empty() { - // Write the blocking function - write!(out, " blocking-{}: {func}(", function.name)?; - write_param_list(out, def, &function.params)?; - write!(out, ")")?; - write!(out, " -> ")?; - write_function_result_type(out, function, def)?; - writeln!(out, ";")?; - // Write the non-blocking function - write!(out, " {}: {func}(", function.name)?; - write_param_list(out, def, &function.params)?; - write!(out, ")")?; - write!(out, " -> {}", function.async_result_type(owner))?; - } else { - // Write the blocking function - write!(out, " blocking-{}: {func}(", function.name)?; - write_param_list(out, def, &function.params)?; - write!(out, ")")?; - writeln!(out, ";")?; - - // Write the non-blocking function - write!(out, " {}: {func}(", function.name)?; - write_param_list(out, def, &function.params)?; - write!(out, ")")?; - } - writeln!(out, ";")?; - Ok(()) -} - -fn write_async_return_type( - out: &mut String, - function: &FunctionStub, - owner: &InterfaceStub, - def: &StubDefinition, -) -> anyhow::Result<()> { - writeln!(out, " resource {} {{", function.async_result_type(owner))?; - writeln!(out, " subscribe: func() -> wasi-io-pollable;")?; - write!(out, " get: func() -> option<")?; - write_function_result_type(out, function, def)?; - writeln!(out, ">;")?; - writeln!(out, " }}")?; - Ok(()) -} - -fn write_function_result_type( - out: &mut String, - function: &FunctionStub, - def: &StubDefinition, -) -> anyhow::Result<()> { - match &function.results { - FunctionResultStub::Single(typ) => { - write!(out, "{}", typ.wit_type_string(def)?)?; - } - FunctionResultStub::Multi(params) => { - write!(out, "(")?; - write_param_list(out, def, params)?; - write!(out, ")")?; - } - FunctionResultStub::SelfType => { - return Err(anyhow!("Unexpected return type in wit generator")); - } - } - Ok(()) -} - -fn write_type_def( - out: &mut String, - typ: &TypeDef, - typ_name: &str, - def: &StubDefinition, -) -> anyhow::Result<()> { - let typ_kind = &typ.kind; - let kind_str = typ_kind.as_str(); - - match typ_kind { - TypeDefKind::Record(record) => { - write!(out, " {}", kind_str)?; - write!(out, " {}", typ_name)?; - write_record(out, &record.fields, def)?; - } - - TypeDefKind::Flags(flags) => { - write!(out, " {}", kind_str)?; - write!(out, " {}", typ_name)?; - - write_flags(out, flags)?; - } - TypeDefKind::Tuple(tuple) => { - if let Some(name) = &typ.name { - write!(out, " type {} =", name)?; - write!(out, " {}", kind_str)?; - write_tuple(out, tuple, def)?; - writeln!(out, ";")?; - } else { - write!(out, " {}", kind_str)?; - write_tuple(out, tuple, def)?; - } - } - TypeDefKind::Variant(variant) => { - write!(out, " {}", kind_str)?; - write!(out, " {}", typ_name)?; - write_variant(out, variant, def)?; - } - TypeDefKind::Enum(enum_ty) => { - write!(out, " {}", kind_str)?; - write!(out, " {}", typ_name)?; - write_enum(out, enum_ty)?; - } - TypeDefKind::Option(option) => { - if let Some(name) = &typ.name { - write!(out, " type {} =", name)?; - write!(out, " {}", kind_str)?; - write_option(out, option, def)?; - writeln!(out, ";")?; - } else { - write!(out, " {}", kind_str)?; - write_option(out, option, def)?; + // Function definitions + for interface in def.stub_imported_interfaces() { + let mut stub_functions = Vec::::new(); + + // Constructor + { + let mut constructor = ResourceFunc::constructor(); + let mut params = match &interface.constructor_params { + Some(constructor_params) => constructor_params.to_encoder(def), + None => Params::empty(), + }; + params.items_mut().insert( + 0, + ( + Ident::new("location"), + Type::Named(Ident::new("golem-rpc-uri")), + ), + ); + constructor.set_params(params); + stub_functions.push(constructor); } - } - TypeDefKind::Result(result) => { - if let Some(name) = &typ.name { - write!(out, " type {} =", name)?; - write!(out, " {}", kind_str)?; - write_result(out, result, def)?; - writeln!(out, ";")?; - } else { - write!(out, " {}", kind_str)?; - write_result(out, result, def)?; - } - } + // Functions + for (function, is_static) in interface.all_functions() { + // Blocking + { + let mut blocking_function = { + let function_name = format!("blocking-{}", function.name.clone()); + if is_static { + ResourceFunc::static_(function_name) + } else { + ResourceFunc::method(function_name) + } + }; + blocking_function.set_params(function.params.to_encoder(def)); + if !function.results.is_empty() { + blocking_function.set_results(function.results.to_encoder(def)); + } + stub_functions.push(blocking_function); + } - TypeDefKind::List(list_typ) => { - if let Some(name) = &typ.name { - write!(out, " type {} =", name)?; - write!(out, " {}", kind_str)?; - write!(out, " {}", list_typ.wit_type_string(def)?)?; - writeln!(out, ";")?; - } else { - write!(out, " {}", kind_str)?; - write!(out, " {}", list_typ.wit_type_string(def)?)?; - } - } - TypeDefKind::Future(_) => {} - TypeDefKind::Stream(_) => {} - TypeDefKind::Type(aliased_typ) => match aliased_typ { - Type::Id(type_id) => { - let aliased_type_def = def.get_type_def(*type_id)?; - if aliased_type_def.owner == typ.owner { - // type alias to type defined in the same interface - write!(out, " {} ", kind_str)?; - write!(out, "{}", typ_name)?; - write!(out, " =")?; - write!(out, " {}", aliased_typ.wit_type_string(def)?)?; - writeln!(out, ";")?; - } else { - // import from another interface - write_type_def(out, aliased_type_def, typ_name, def)?; + // Async + { + let mut async_function = { + if is_static { + ResourceFunc::static_(function.name.clone()) + } else { + ResourceFunc::method(function.name.clone()) + } + }; + async_function.set_params(function.params.to_encoder(def)); + if !function.results.is_empty() { + async_function.set_results(Results::Anon(Type::Named(Ident::new( + function.async_result_type(interface), + )))); + } + stub_functions.push(async_function); } } - _ => { - // type alias to primitive - write!(out, " {} ", kind_str)?; - write!(out, "{}", typ_name)?; - write!(out, " =")?; - write!(out, " {}", aliased_typ.wit_type_string(def)?)?; - writeln!(out, ";")?; - } - }, - TypeDefKind::Unknown => { - write!(out, " {}", kind_str)?; - } - TypeDefKind::Resource => { - write!(out, " {}", kind_str)?; - } - TypeDefKind::Handle(handle) => { - write!(out, " {}", kind_str)?; - write_handle(out, handle, def)?; - } - } - - Ok(()) -} - -// https://github.com/WebAssembly/component-model/blob/main/design/mvp/WIT.md#wit-types -fn write_handle(out: &mut String, handle: &Handle, def: &StubDefinition) -> anyhow::Result<()> { - match handle { - Handle::Own(type_id) => { - write!(out, "{}", Type::Id(*type_id).wit_type_string(def)?)?; - } - Handle::Borrow(type_id) => { - write!(out, " borrow<")?; - write!(out, "{}", Type::Id(*type_id).wit_type_string(def)?)?; - write!(out, ">")?; - } - } - - Ok(()) -} - -// https://github.com/WebAssembly/component-model/blob/main/design/mvp/WIT.md#wit-types -fn write_result(out: &mut String, result: &Result_, def: &StubDefinition) -> anyhow::Result<()> { - match (result.ok, result.err) { - (Some(ok), Some(err)) => { - write!(out, "<")?; - write!(out, "{}", ok.wit_type_string(def)?)?; - write!(out, ", ")?; - write!(out, "{}", err.wit_type_string(def)?)?; - write!(out, ">")?; - } - (Some(ok), None) => { - write!(out, "<")?; - write!(out, "{}", ok.wit_type_string(def)?)?; - write!(out, ">")?; - } - (None, Some(err)) => { - write!(out, "<_, ")?; - write!(out, "{}", err.wit_type_string(def)?)?; - write!(out, ">")?; - } - (None, None) => {} - } - - Ok(()) -} - -fn write_option(out: &mut String, option: &Type, def: &StubDefinition) -> anyhow::Result<()> { - write!(out, "<")?; - write!(out, "{}", option.wit_type_string(def)?)?; - write!(out, ">")?; - Ok(()) -} - -fn write_variant(out: &mut String, variant: &Variant, def: &StubDefinition) -> anyhow::Result<()> { - let cases_length = variant.cases.len(); - write!(out, " {{")?; - writeln!(out)?; - for (idx, case) in variant.cases.iter().enumerate() { - write!(out, " {}", case.name)?; - - if let Some(ty) = case.ty { - write!(out, "(")?; - write!(out, "{}", ty.wit_type_string(def)?)?; - write!(out, ")")?; + stub_interface.type_def(TypeDef::resource(interface.name.clone(), stub_functions)); } - if idx < cases_length - 1 { - writeln!(out, ", ")?; - } + package.interface(stub_interface); } - writeln!(out, " }}")?; - Ok(()) -} - -fn write_enum(out: &mut String, enum_ty: &Enum) -> anyhow::Result<()> { - let length = enum_ty.cases.len(); - write!(out, " {{")?; - writeln!(out)?; - for (idx, case) in enum_ty.cases.iter().enumerate() { - write!(out, " {}", case.name)?; - if idx < length - 1 { - writeln!(out, ",")?; - } - writeln!(out)?; + // Stub world + { + let mut stub_world = World::new(def.target_world_name()); + stub_world.named_interface_export(interface_identifier); + package.world(stub_world); } - writeln!(out, " }}")?; - Ok(()) -} -fn write_tuple(out: &mut String, tuple: &Tuple, def: &StubDefinition) -> anyhow::Result<()> { - let tuple_length = tuple.types.len(); - write!(out, " <")?; - for (idx, typ) in tuple.types.iter().enumerate() { - write!(out, "{}", typ.wit_type_string(def)?)?; - if idx < tuple_length - 1 { - write!(out, ", ")?; - } - } - write!(out, " >")?; - Ok(()) -} - -fn write_record(out: &mut String, fields: &[Field], def: &StubDefinition) -> anyhow::Result<()> { - write!(out, " {{")?; - writeln!(out)?; - - for (idx, field) in fields.iter().enumerate() { - write!( - out, - " {}: {}", - field.name, - field.ty.wit_type_string(def)? - )?; - if idx < fields.len() - 1 { - write!(out, ", ")?; - } - } - - writeln!(out)?; - writeln!(out, " }}")?; - - Ok(()) -} - -fn write_flags(out: &mut String, flags: &Flags) -> anyhow::Result<()> { - write!(out, " {{")?; - writeln!(out)?; - let flags_len = flags.flags.len(); - for (idx, flag) in flags.flags.iter().enumerate() { - write!(out, " {}", flag.name)?; - if idx < flags_len - 1 { - writeln!(out, ",")?; - } - writeln!(out)?; - } - writeln!(out, " }}")?; - Ok(()) + package } -fn write_param_list( - out: &mut String, +fn add_async_return_type( def: &StubDefinition, - params: &[FunctionParamStub], -) -> anyhow::Result<()> { - for (idx, param) in params.iter().enumerate() { - write!(out, "{}: {}", param.name, param.typ.wit_type_string(def)?)?; - if idx < params.len() - 1 { - write!(out, ", ")?; - } - } - Ok(()) + stub_interface: &mut Interface, + function: &FunctionStub, + owner_interface: &InterfaceStub, +) { + stub_interface.type_def(TypeDef::resource( + function.async_result_type(owner_interface), + [ + { + let mut subscribe = ResourceFunc::method("subscribe"); + subscribe.set_results(Results::Anon(Type::Named(Ident::new("wasi-io-pollable")))); + subscribe + }, + { + let mut get = ResourceFunc::method("get"); + match &function.results { + FunctionResultStub::Anon(typ) => { + get.set_results(Results::Anon(Type::option(typ.to_encoder(def)))); + } + FunctionResultStub::Named(_params) => { + // TODO: this case was generating in the previous version, check it + // TODO: this should be a proper error message with context + panic!("Named parameters are not supported"); + } + FunctionResultStub::SelfType => { + // TODO: this should be a proper error message with context + panic!("Unexpected self return type in wit generator"); + } + } + get + }, + ], + )) } pub fn add_wit_dependencies(def: &StubDefinition) -> anyhow::Result<()> { @@ -577,78 +305,193 @@ fn write_embedded_source(target_dir: &Path, file_name: &str, content: &str) -> a Ok(()) } -trait TypeExtensions { - fn wit_type_string(&self, stub_definition: &StubDefinition) -> anyhow::Result; +trait ToEncoder { + type EncoderType; + fn to_encoder(&self, stub_definition: &StubDefinition) -> Self::EncoderType; } -impl TypeExtensions for Type { - fn wit_type_string(&self, stub_definition: &StubDefinition) -> anyhow::Result { +impl ToEncoder for wit_parser::Type { + type EncoderType = Type; + + fn to_encoder(&self, def: &StubDefinition) -> Self::EncoderType { match self { - Type::Bool => Ok("bool".to_string()), - Type::U8 => Ok("u8".to_string()), - Type::U16 => Ok("u16".to_string()), - Type::U32 => Ok("u32".to_string()), - Type::U64 => Ok("u64".to_string()), - Type::S8 => Ok("s8".to_string()), - Type::S16 => Ok("s16".to_string()), - Type::S32 => Ok("s32".to_string()), - Type::S64 => Ok("s64".to_string()), - Type::F32 => Ok("f32".to_string()), - Type::F64 => Ok("f64".to_string()), - Type::Char => Ok("char".to_string()), - Type::String => Ok("string".to_string()), - Type::Id(type_id) => { - let typ = stub_definition.get_type_def(*type_id)?; - match &typ.kind { - TypeDefKind::Option(inner) => Ok(format!( - "option<{}>", - inner.wit_type_string(stub_definition)? - )), - TypeDefKind::List(inner) => { - Ok(format!("list<{}>", inner.wit_type_string(stub_definition)?)) - } - TypeDefKind::Tuple(tuple) => { - let types = tuple - .types - .iter() - .map(|t| t.wit_type_string(stub_definition)) - .collect::>>()?; - Ok(format!("tuple<{}>", types.join(", "))) - } - TypeDefKind::Result(result) => match (&result.ok, &result.err) { - (Some(ok), Some(err)) => { - let ok = ok.wit_type_string(stub_definition)?; - let err = err.wit_type_string(stub_definition)?; - Ok(format!("result<{}, {}>", ok, err)) + wit_parser::Type::Bool => Type::Bool, + wit_parser::Type::U8 => Type::U8, + wit_parser::Type::U16 => Type::U16, + wit_parser::Type::U32 => Type::U32, + wit_parser::Type::U64 => Type::U64, + wit_parser::Type::S8 => Type::S8, + wit_parser::Type::S16 => Type::S16, + wit_parser::Type::S32 => Type::S32, + wit_parser::Type::S64 => Type::S64, + wit_parser::Type::F32 => Type::F32, + wit_parser::Type::F64 => Type::F64, + wit_parser::Type::Char => Type::Char, + wit_parser::Type::String => Type::String, + wit_parser::Type::Id(type_id) => { + if let Some(type_alias) = def.get_stub_used_type_alias(*type_id) { + Type::Named(Ident::new(type_alias.to_string())) + } else { + let typ = def.get_type_def(*type_id).expect("type not found"); + match &typ.kind { + wit_parser::TypeDefKind::Option(inner) => { + Type::option(inner.to_encoder(def)) } - (Some(ok), None) => { - let ok = ok.wit_type_string(stub_definition)?; - Ok(format!("result<{}>", ok)) + wit_parser::TypeDefKind::List(inner) => Type::list(inner.to_encoder(def)), + wit_parser::TypeDefKind::Tuple(tuple) => { + Type::tuple(tuple.types.iter().map(|t| t.to_encoder(def))) } - (None, Some(err)) => { - let err = err.wit_type_string(stub_definition)?; - Ok(format!("result<_, {}>", err)) + wit_parser::TypeDefKind::Result(result) => { + match (&result.ok, &result.err) { + (Some(ok), Some(err)) => { + Type::result_both(ok.to_encoder(def), err.to_encoder(def)) + } + (Some(ok), None) => Type::result_ok(ok.to_encoder(def)), + (None, Some(err)) => Type::result_err(err.to_encoder(def)), + (None, None) => { + panic!("result type has no ok or err types") + } + } } - (None, None) => { - bail!("result type has no ok or err types") + wit_parser::TypeDefKind::Handle(handle) => match handle { + wit_parser::Handle::Own(type_id) => { + wit_parser::Type::Id(*type_id).to_encoder(def) + } + wit_parser::Handle::Borrow(type_id) => Type::borrow( + wit_parser::Type::Id(*type_id).to_encoder(def).to_string(), + ), + }, + _ => { + let name = typ + .name + .clone() + .unwrap_or_else(|| panic!("missing name for type: {:?}", typ)); + Type::Named(Ident::new(name)) } - }, - TypeDefKind::Handle(handle) => match handle { - Handle::Own(type_id) => Type::Id(*type_id).wit_type_string(stub_definition), - Handle::Borrow(type_id) => Ok(format!( - "borrow<{}>", - Type::Id(*type_id).wit_type_string(stub_definition)? - )), - }, - _ => { - let name = typ - .name - .clone() - .ok_or(anyhow!("wit_type_string: type has no name"))?; - Ok(name) } } } } } } + +impl ToEncoder for Vec { + type EncoderType = Params; + + fn to_encoder(&self, def: &StubDefinition) -> Self::EncoderType { + Params::from_iter( + self.iter() + .map(|param| (param.name.clone(), param.typ.to_encoder(def))), + ) + } +} + +impl ToEncoder for FunctionResultStub { + type EncoderType = Results; + + fn to_encoder(&self, def: &StubDefinition) -> Self::EncoderType { + match self { + FunctionResultStub::Anon(typ) => Results::Anon(typ.to_encoder(def)), + FunctionResultStub::Named(types) => Results::Named(types.to_encoder(def)), + FunctionResultStub::SelfType => { + // TODO: This should be a proper error message + panic!("Unexpected self type function result") + } + } + } +} + +impl ToEncoder for InterfaceStubTypeDef { + type EncoderType = TypeDef; + + fn to_encoder(&self, def: &StubDefinition) -> Self::EncoderType { + (self.stub_type_name(), self.stub_type_def()).to_encoder(def) + } +} + +impl ToEncoder for (&str, &wit_parser::TypeDef) { + type EncoderType = TypeDef; + + fn to_encoder(&self, def: &StubDefinition) -> Self::EncoderType { + let (name, type_def) = self; + match &type_def.kind { + wit_parser::TypeDefKind::Record(record) => TypeDef::record( + name.to_string(), + record + .fields + .iter() + .map(|field| (field.name.clone(), field.ty.to_encoder(def))), + ), + wit_parser::TypeDefKind::Flags(flags) => TypeDef::flags( + name.to_string(), + flags.flags.iter().map(|flag| (flag.name.clone(),)), + ), + wit_parser::TypeDefKind::Tuple(tuple) => TypeDef::new( + name.to_string(), + TypeDefKind::Type(Type::tuple( + tuple.types.iter().map(|typ| typ.to_encoder(def)), + )), + ), + wit_parser::TypeDefKind::Variant(variant) => TypeDef::variant( + name.to_string(), + variant.cases.iter().map(|case| match case.ty { + Some(ty) => VariantCase::value(case.name.clone(), ty.to_encoder(def)), + None => VariantCase::empty(case.name.clone()), + }), + ), + wit_parser::TypeDefKind::Enum(enum_ty) => TypeDef::enum_( + name.to_string(), + enum_ty.cases.iter().map(|case| case.name.clone()), + ), + wit_parser::TypeDefKind::Option(typ) => TypeDef::new( + name.to_string(), + TypeDefKind::Type(Type::option(typ.to_encoder(def))), + ), + wit_parser::TypeDefKind::Result(result) => TypeDef::new( + name.to_string(), + // TODO: match is duplicated + TypeDefKind::Type(match (&result.ok, &result.err) { + (Some(ok), Some(err)) => { + Type::result_both(ok.to_encoder(def), err.to_encoder(def)) + } + (Some(ok), None) => Type::result_ok(ok.to_encoder(def)), + (None, Some(err)) => Type::result_err(err.to_encoder(def)), + (None, None) => { + panic!("result type has no ok or err types") + } + }), + ), + wit_parser::TypeDefKind::List(typ) => TypeDef::new( + name.to_string(), + TypeDefKind::Type(Type::list(typ.to_encoder(def))), + ), + wit_parser::TypeDefKind::Future(_) => { + panic!("Future type is not supported yet") + } + wit_parser::TypeDefKind::Stream(_) => { + panic!("Stream type is not supported yet") + } + wit_parser::TypeDefKind::Type(typ) => { + TypeDef::new(name.to_string(), TypeDefKind::Type(typ.to_encoder(def))) + } + wit_parser::TypeDefKind::Unknown => { + panic!("Unexpected unknown type") + } + wit_parser::TypeDefKind::Resource => { + panic!("Unexpected resource type") + } + wit_parser::TypeDefKind::Handle(handle) => TypeDef::new( + name.to_string(), + // TODO: match is duplicated + TypeDefKind::Type(match handle { + wit_parser::Handle::Own(type_id) => { + wit_parser::Type::Id(*type_id).to_encoder(def) + } + wit_parser::Handle::Borrow(type_id) => { + Type::borrow(wit_parser::Type::Id(*type_id).to_encoder(def).to_string()) + } + }), + ), + } + } +} diff --git a/wasm-rpc-stubgen/test-data/many-ways-to-export/main.wit b/wasm-rpc-stubgen/test-data/many-ways-to-export/main.wit index 7ee3f35d..eb589f17 100644 --- a/wasm-rpc-stubgen/test-data/many-ways-to-export/main.wit +++ b/wasm-rpc-stubgen/test-data/many-ways-to-export/main.wit @@ -41,7 +41,8 @@ interface iface11 { use test:sub2/iface10.{rec1}; // intentionally using the same func name - func9: func() -> rec1; + // TODO + // func9: func() -> rec1; } world api { diff --git a/wasm-rpc-stubgen/tests/wit.rs b/wasm-rpc-stubgen/tests/wit.rs index 47e045ce..314448a9 100644 --- a/wasm-rpc-stubgen/tests/wit.rs +++ b/wasm-rpc-stubgen/tests/wit.rs @@ -206,6 +206,8 @@ fn many_ways_to_export() { assert_has_stub_function(&resolve, "stub-api", "iface2", "func3", true); assert_has_stub_function(&resolve, "stub-api", "inline-iface", "func4", false); assert_has_stub_function(&resolve, "stub-api", "iface4", "func5", false); + + // TODO: add asserts for non-unique types } #[test] From aeda02f3ba803cb5e58f09bea307b2f15688de68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 29 Oct 2024 16:33:10 +0100 Subject: [PATCH 022/109] fix alias to alias, comments --- wasm-rpc-stubgen/src/commands/generate.rs | 5 +++++ wasm-rpc-stubgen/src/stub.rs | 12 +++++++++--- wasm-rpc-stubgen/tests/wit.rs | 2 ++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/generate.rs b/wasm-rpc-stubgen/src/commands/generate.rs index 6473ab53..0d8669ca 100644 --- a/wasm-rpc-stubgen/src/commands/generate.rs +++ b/wasm-rpc-stubgen/src/commands/generate.rs @@ -66,6 +66,11 @@ pub fn generate_stub_wit_dir(stub_def: &StubDefinition) -> anyhow::Result anyhow::Result { diff --git a/wasm-rpc-stubgen/src/stub.rs b/wasm-rpc-stubgen/src/stub.rs index fc98fc00..02dc253b 100644 --- a/wasm-rpc-stubgen/src/stub.rs +++ b/wasm-rpc-stubgen/src/stub.rs @@ -314,9 +314,15 @@ impl StubDefinition { panic!("Failed to get type def for inlining") }) .clone(); - type_def.name = - type_name_alias.clone().or_else(|| Some(type_name.clone())); - Some(type_def) + + // Alias to alias does not need inlining + if let TypeDefKind::Type(Type::Id(_)) = type_def.kind { + None + } else { + type_def.name = + type_name_alias.clone().or_else(|| Some(type_name.clone())); + Some(type_def) + } } _ => None, }; diff --git a/wasm-rpc-stubgen/tests/wit.rs b/wasm-rpc-stubgen/tests/wit.rs index 314448a9..b141e00e 100644 --- a/wasm-rpc-stubgen/tests/wit.rs +++ b/wasm-rpc-stubgen/tests/wit.rs @@ -237,6 +237,8 @@ fn many_ways_to_export_inlined() { assert_has_stub_function(&resolve, "stub-api", "iface4", "func5", false); assert_defines_enum(&resolve, "stub-api", "color"); + + // TODO: add asserts for non-unique types } fn assert_has_package_name(resolve: &Resolve, package_name: &str) { From 0e9e51bf32b5ffa01da9e92544e6b38262ae80c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 29 Oct 2024 16:37:44 +0100 Subject: [PATCH 023/109] clippy --- wasm-rpc-stubgen/src/stub.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wasm-rpc-stubgen/src/stub.rs b/wasm-rpc-stubgen/src/stub.rs index 02dc253b..e397ebc1 100644 --- a/wasm-rpc-stubgen/src/stub.rs +++ b/wasm-rpc-stubgen/src/stub.rs @@ -277,7 +277,7 @@ impl StubDefinition { panic!("Missing interface name, interface id: {:?}", interface_id) }); - let package = interface.package.clone().map(|package_id| { + let package = interface.package.map(|package_id| { self.resolve.packages.get(package_id).unwrap_or_else(|| { panic!( "Missing package for interface, package id: {:?}, interface id: {:?}", From 303b41f4e9078366ddcbe44cf6a6c6fe908a361a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Wed, 30 Oct 2024 15:18:17 +0100 Subject: [PATCH 024/109] extract StubConfig --- wasm-rpc-stubgen/src/cargo.rs | 19 ++++- wasm-rpc-stubgen/src/commands/declarative.rs | 19 ++--- wasm-rpc-stubgen/src/commands/generate.rs | 12 ++- wasm-rpc-stubgen/src/lib.rs | 34 ++++---- wasm-rpc-stubgen/src/stub.rs | 81 ++++++-------------- wasm-rpc-stubgen/src/wit_generate.rs | 29 +++---- wasm-rpc-stubgen/src/wit_transform.rs | 1 + wasm-rpc-stubgen/tests/wit.rs | 70 +++++++++-------- 8 files changed, 130 insertions(+), 135 deletions(-) diff --git a/wasm-rpc-stubgen/src/cargo.rs b/wasm-rpc-stubgen/src/cargo.rs index a31cf358..e283fb48 100644 --- a/wasm-rpc-stubgen/src/cargo.rs +++ b/wasm-rpc-stubgen/src/cargo.rs @@ -131,7 +131,8 @@ pub fn generate_cargo_toml(def: &StubDefinition) -> anyhow::Result<()> { }), }; - let mut package = cargo_toml::Package::new(def.target_crate_name(), &def.stub_crate_version); + let mut package = + cargo_toml::Package::new(def.target_crate_name(), &def.config.stub_crate_version); package.edition = Inheritable::Set(Edition::E2021); package.metadata = Some(metadata); manifest.package = Some(package); @@ -170,8 +171,18 @@ pub fn generate_cargo_toml(def: &StubDefinition) -> anyhow::Result<()> { })); let dep_golem_wasm_rpc = Dependency::Detailed(Box::new(DependencyDetail { - version: if def.wasm_rpc_override.wasm_rpc_path_override.is_none() { - if let Some(version) = def.wasm_rpc_override.wasm_rpc_version_override.as_ref() { + version: if def + .config + .wasm_rpc_override + .wasm_rpc_path_override + .is_none() + { + if let Some(version) = def + .config + .wasm_rpc_override + .wasm_rpc_version_override + .as_ref() + { Some(version.to_string()) } else { Some(WASM_RPC_VERSION.to_string()) @@ -179,7 +190,7 @@ pub fn generate_cargo_toml(def: &StubDefinition) -> anyhow::Result<()> { } else { None }, - path: def.wasm_rpc_override.wasm_rpc_path_override.clone(), + path: def.config.wasm_rpc_override.wasm_rpc_path_override.clone(), default_features: false, features: vec!["stub".to_string()], ..Default::default() diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index 9189314b..f2efa9ad 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -8,7 +8,7 @@ use crate::model::validation::ValidatedResult; use crate::model::wasm_rpc::{ include_glob_patter_from_yaml_file, init_oam_app, Application, DEFAULT_CONFIG_FILE_NAME, }; -use crate::stub::StubDefinition; +use crate::stub::{StubConfig, StubDefinition}; use crate::{commands, WasmRpcOverride}; use anyhow::{anyhow, Context, Error}; use colored::Colorize; @@ -76,17 +76,18 @@ async fn pre_component_build_app(config: &Config, app: &Application) -> anyhow:: let target_root = TempDir::new()?; let canonical_target_root = target_root.path().canonicalize()?; - let stub_def = StubDefinition::new( - &app.component_wit(&component_name), - &canonical_target_root, - &app.stub_world(&component_name), - &app.stub_crate_version(&component_name), - &WasmRpcOverride { + let stub_def = StubDefinition::new(StubConfig { + source_wit_root: app.component_wit(&component_name), + transformed_source_wit_root: None, + target_root: canonical_target_root, + selected_world: app.stub_world(&component_name), + stub_crate_version: app.stub_crate_version(&component_name), + wasm_rpc_override: WasmRpcOverride { wasm_rpc_path_override: app.stub_wasm_rpc_path(&component_name), wasm_rpc_version_override: app.stub_wasm_rpc_version(&component_name), }, - app.stub_always_inline_types(&component_name), - ) + inline_source_types: app.stub_always_inline_types(&component_name), + }) .context("Failed to gather information for the stub generator")?; commands::generate::build( diff --git a/wasm-rpc-stubgen/src/commands/generate.rs b/wasm-rpc-stubgen/src/commands/generate.rs index 0d8669ca..ba4ea585 100644 --- a/wasm-rpc-stubgen/src/commands/generate.rs +++ b/wasm-rpc-stubgen/src/commands/generate.rs @@ -46,7 +46,7 @@ pub async fn build( fs::create_dir_all(dest_wit_root).context("Failed to create the target WIT root directory")?; fs_extra::dir::copy( - stub_def.target_root.join(naming::wit::WIT_DIR), + stub_def.config.target_root.join(naming::wit::WIT_DIR), dest_wit_root, &CopyOptions::new().content_only(true).overwrite(true), ) @@ -58,7 +58,10 @@ pub async fn build( pub fn generate_stub_wit_dir(stub_def: &StubDefinition) -> anyhow::Result { log_action( "Generating", - format!("stub WIT directory to {}", stub_def.target_root.display()), + format!( + "stub WIT directory to {}", + stub_def.config.target_root.display() + ), ); let _indent = LogIndent::new(); generate_stub_wit_to_target(stub_def).context("Failed to generate the stub wit file")?; @@ -68,7 +71,7 @@ pub fn generate_stub_wit_dir(stub_def: &StubDefinition) -> anyhow::Result anyhow::Resul generate_cargo_toml(stub_def).context("Failed to generate the Cargo.toml file")?; generate_stub_source(stub_def).context("Failed to generate the stub Rust source")?; - compile(&stub_def.target_root) + compile(&stub_def.config.target_root) .await .context("Failed to compile the generated stub")?; let wasm_path = stub_def + .config .target_root .join("target") .join("wasm32-wasi") diff --git a/wasm-rpc-stubgen/src/lib.rs b/wasm-rpc-stubgen/src/lib.rs index 773a6131..3ee8d4c3 100644 --- a/wasm-rpc-stubgen/src/lib.rs +++ b/wasm-rpc-stubgen/src/lib.rs @@ -26,7 +26,7 @@ pub mod wit_resolve; pub mod wit_transform; use crate::commands::dependencies::UpdateCargoToml; -use crate::stub::StubDefinition; +use crate::stub::{StubConfig, StubDefinition}; use anyhow::Context; use clap::{Parser, Subcommand}; use std::path::PathBuf; @@ -220,12 +220,15 @@ pub struct DeclarativeBuildArgs { pub fn generate(args: GenerateArgs) -> anyhow::Result<()> { let stub_def = StubDefinition::new( - &args.source_wit_root, - &args.dest_crate_root, - &args.world, - &args.stub_crate_version, - &args.wasm_rpc_override, - args.always_inline_types, + StubConfig { + source_wit_root: args.source_wit_root, + transformed_source_wit_root: None, + target_root: args.dest_crate_root, + selected_world: args.world, + stub_crate_version: args.stub_crate_version, + wasm_rpc_override: args.wasm_rpc_override, + inline_source_types: args.always_inline_types, + } ) .context("Failed to gather information for the stub generator. Make sure source_wit_root has a valid WIT file.")?; commands::generate::generate(&stub_def) @@ -235,14 +238,15 @@ pub async fn build(args: BuildArgs) -> anyhow::Result<()> { let target_root = TempDir::new()?; let canonical_target_root = target_root.path().canonicalize()?; - let stub_def = StubDefinition::new( - &args.source_wit_root, - &canonical_target_root, - &args.world, - &args.stub_crate_version, - &args.wasm_rpc_override, - args.always_inline_types, - ) + let stub_def = StubDefinition::new(StubConfig { + source_wit_root: args.source_wit_root, + transformed_source_wit_root: None, + target_root: canonical_target_root, + selected_world: args.world, + stub_crate_version: args.stub_crate_version, + wasm_rpc_override: args.wasm_rpc_override, + inline_source_types: args.always_inline_types, + }) .context("Failed to gather information for the stub generator")?; commands::generate::build(&stub_def, &args.dest_wasm, &args.dest_wit_root).await diff --git a/wasm-rpc-stubgen/src/stub.rs b/wasm-rpc-stubgen/src/stub.rs index e397ebc1..4dddcb9f 100644 --- a/wasm-rpc-stubgen/src/stub.rs +++ b/wasm-rpc-stubgen/src/stub.rs @@ -19,14 +19,26 @@ use indexmap::IndexMap; use itertools::Itertools; use std::cell::OnceCell; use std::collections::{HashMap, HashSet}; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use wit_parser::{ Function, FunctionKind, Interface, InterfaceId, Package, PackageId, PackageName, Resolve, Results, Type, TypeDef, TypeDefKind, TypeId, TypeOwner, World, WorldId, WorldItem, WorldKey, }; +pub struct StubConfig { + pub source_wit_root: PathBuf, + pub transformed_source_wit_root: Option, // TODO + pub target_root: PathBuf, + pub selected_world: Option, + pub stub_crate_version: String, + pub wasm_rpc_override: WasmRpcOverride, + pub inline_source_types: bool, +} + /// All the gathered information for generating the stub crate. pub struct StubDefinition { + pub config: StubConfig, + resolve: Resolve, source_world_id: WorldId, package_sources: IndexMap, @@ -37,27 +49,15 @@ pub struct StubDefinition { pub source_package_id: PackageId, pub source_package_name: PackageName, - pub source_wit_root: PathBuf, - pub target_root: PathBuf, - pub stub_crate_version: String, - pub wasm_rpc_override: WasmRpcOverride, - pub always_inline_types: bool, } impl StubDefinition { - pub fn new( - source_wit_root: &Path, - target_root: &Path, - selected_world: &Option, - stub_crate_version: &str, - wasm_rpc_override: &WasmRpcOverride, - always_inline_types: bool, - ) -> anyhow::Result { - let resolved_source = ResolvedWitDir::new(source_wit_root)?; + pub fn new(config: StubConfig) -> anyhow::Result { + let resolved_source = ResolvedWitDir::new(&config.source_wit_root)?; let source_world_id = resolved_source .resolve - .select_world(resolved_source.package_id, selected_world.as_deref())?; + .select_world(resolved_source.package_id, config.selected_world.as_deref())?; let source_package_name = resolved_source .resolve @@ -73,6 +73,7 @@ impl StubDefinition { .clone(); Ok(Self { + config, resolve: resolved_source.resolve, source_world_id, package_sources: resolved_source.package_sources, @@ -81,11 +82,6 @@ impl StubDefinition { stub_dep_package_ids: OnceCell::new(), source_package_id: resolved_source.package_id, source_package_name, - source_wit_root: source_wit_root.to_path_buf(), - target_root: target_root.to_path_buf(), - stub_crate_version: stub_crate_version.to_string(), - wasm_rpc_override: wasm_rpc_override.clone(), - always_inline_types, }) } @@ -135,7 +131,7 @@ impl StubDefinition { } pub fn target_cargo_path(&self) -> PathBuf { - self.target_root.join("Cargo.toml") + self.config.target_root.join("Cargo.toml") } pub fn target_crate_name(&self) -> String { @@ -143,19 +139,21 @@ impl StubDefinition { } pub fn target_rust_path(&self) -> PathBuf { - self.target_root.join("src/lib.rs") + self.config.target_root.join("src/lib.rs") } pub fn target_interface_name(&self) -> String { + // TODO: naming format!("stub-{}", self.source_world_name()) } pub fn target_world_name(&self) -> String { + // TODO: naming format!("wasm-rpc-stub-{}", self.source_world_name()) } pub fn target_wit_root(&self) -> PathBuf { - self.target_root.join(naming::wit::WIT_DIR) + self.config.target_root.join(naming::wit::WIT_DIR) } pub fn target_wit_path(&self) -> PathBuf { @@ -177,7 +175,7 @@ impl StubDefinition { fn is_inlined(&self, typedef: &TypeDef) -> bool { match &typedef.owner { TypeOwner::Interface(interface_id) => { - if self.always_inline_types { + if self.config.inline_source_types { if let Some(resolved_owner_interface) = self.resolve.interfaces.get(*interface_id) { @@ -301,7 +299,7 @@ impl StubDefinition { None => format!("{}-{}", interface_name, type_name), }); - let inlined = self.always_inline_types + let inlined = self.config.inline_source_types && interface.package == Some(self.source_package_id); let inlined_type_def = match type_def.kind { @@ -526,36 +524,6 @@ impl StubDefinition { (used_types, resource_interfaces) } - // TODO: - /* - fn type_def_to_stub( - &self, - owner_interface: &Interface, - type_name: String, - type_def: TypeDef, - ) -> InterfaceStubTypeDef { - let package = owner_interface - .package - .and_then(|id| self.resolve.packages.get(id)); - - let interface_name = owner_interface - .name - .clone() - .unwrap_or_else(|| panic!("Failed to get owner interface name")); - - let interface_path = package - .map(|p| p.name.interface_id(&interface_name)) - .unwrap_or(interface_name); - - InterfaceStubTypeDef { - name: type_name, - path: interface_path, - package_id: owner_interface.package, - type_def, - } - } - */ - fn resource_interface_stub( &self, owner_interface: &Interface, @@ -730,6 +698,7 @@ pub struct FunctionStub { impl FunctionStub { pub fn as_method(&self) -> Option { + // TODO: duplicated code self.name.strip_prefix("[method]").and_then(|method_name| { let parts = method_name.split('.').collect::>(); if parts.len() != 2 { diff --git a/wasm-rpc-stubgen/src/wit_generate.rs b/wasm-rpc-stubgen/src/wit_generate.rs index aa509dce..4e393589 100644 --- a/wasm-rpc-stubgen/src/wit_generate.rs +++ b/wasm-rpc-stubgen/src/wit_generate.rs @@ -14,12 +14,12 @@ use crate::commands::log::{log_action, log_warn_action, LogIndent}; use crate::fs::{copy, copy_transformed, get_file_name}; +use crate::naming; use crate::stub::{ FunctionParamStub, FunctionResultStub, FunctionStub, InterfaceStub, InterfaceStubTypeDef, - StubDefinition, + StubConfig, StubDefinition, }; use crate::wit_transform::import_remover; -use crate::{naming, WasmRpcOverride}; use std::fs; use std::path::Path; use wit_encoder::{ @@ -41,16 +41,17 @@ pub fn generate_stub_wit_to_target(def: &StubDefinition) -> anyhow::Result<()> { pub fn generate_stub_wit_from_wit_dir( source_wit_root: &Path, - inline_root_types: bool, + inline_source_types: bool, ) -> anyhow::Result { - generate_stub_wit_from_stub_def(&StubDefinition::new( - source_wit_root, - source_wit_root, // Not used - &None, // Not used - "0.0.1", // Not used - &WasmRpcOverride::default(), // Not used - inline_root_types, - )?) + generate_stub_wit_from_stub_def(&StubDefinition::new(StubConfig { + source_wit_root: source_wit_root.to_path_buf(), + transformed_source_wit_root: None, + target_root: source_wit_root.to_path_buf(), // Not used + selected_world: None, // TODO: would this fail with multiple worlds? + stub_crate_version: "".to_string(), // Not used + wasm_rpc_override: Default::default(), // Not used + inline_source_types, + })?) } pub fn generate_stub_wit_from_stub_def(def: &StubDefinition) -> anyhow::Result { @@ -221,8 +222,8 @@ pub fn add_wit_dependencies(def: &StubDefinition) -> anyhow::Result<()> { "Adding", format!( "WIT dependencies from {} to {}", - def.source_wit_root.display(), - def.target_root.display() + def.config.source_wit_root.display(), + def.config.target_root.display() ), ); @@ -266,7 +267,7 @@ pub fn add_wit_dependencies(def: &StubDefinition) -> anyhow::Result<()> { ); copy_transformed(source, &dest, &remove_stub_imports)?; } else { - let relative = source.strip_prefix(&def.source_wit_root)?; + let relative = source.strip_prefix(&def.config.source_wit_root)?; let dest = target_wit_root.join(relative); log_action( " Copying", diff --git a/wasm-rpc-stubgen/src/wit_transform.rs b/wasm-rpc-stubgen/src/wit_transform.rs index cabdd4f2..aa0a45f5 100644 --- a/wasm-rpc-stubgen/src/wit_transform.rs +++ b/wasm-rpc-stubgen/src/wit_transform.rs @@ -327,6 +327,7 @@ pub fn visit_package(package: &mut Package, visitor: &mut impl VisitPackage) { } } +// TODO: use wit transform everywhere pub fn import_remover( package_name: &wit_parser::PackageName, ) -> impl Fn(String) -> anyhow::Result { diff --git a/wasm-rpc-stubgen/tests/wit.rs b/wasm-rpc-stubgen/tests/wit.rs index b141e00e..303143af 100644 --- a/wasm-rpc-stubgen/tests/wit.rs +++ b/wasm-rpc-stubgen/tests/wit.rs @@ -17,7 +17,7 @@ use test_r::test; use golem_wasm_rpc_stubgen::commands::generate::generate_stub_wit_dir; -use golem_wasm_rpc_stubgen::stub::StubDefinition; +use golem_wasm_rpc_stubgen::stub::{StubConfig, StubDefinition}; use golem_wasm_rpc_stubgen::WasmRpcOverride; use std::path::Path; use tempfile::tempdir; @@ -30,14 +30,15 @@ fn all_wit_types() { let source_wit_root = Path::new("test-data/all-wit-types"); let target_root = tempdir().unwrap(); - let def = StubDefinition::new( - source_wit_root, - target_root.path(), - &None, - "1.0.0", - &WasmRpcOverride::default(), - false, - ) + let def = StubDefinition::new(StubConfig { + source_wit_root: source_wit_root.to_path_buf(), + transformed_source_wit_root: None, + target_root: target_root.path().to_path_buf(), + selected_world: None, + stub_crate_version: "1.0.0".to_string(), + wasm_rpc_override: WasmRpcOverride::default(), + inline_source_types: false, + }) .unwrap(); let resolve = generate_stub_wit_dir(&def).unwrap().resolve; @@ -103,14 +104,15 @@ fn all_wit_types_inlined() { let source_wit_root = Path::new("test-data/all-wit-types"); let target_root = tempdir().unwrap(); - let def = StubDefinition::new( - source_wit_root, - target_root.path(), - &None, - "1.0.0", - &WasmRpcOverride::default(), - true, - ) + let def = StubDefinition::new(StubConfig { + source_wit_root: source_wit_root.to_path_buf(), + transformed_source_wit_root: None, + target_root: target_root.path().to_path_buf(), + selected_world: None, + stub_crate_version: "1.0.0".to_string(), + wasm_rpc_override: WasmRpcOverride::default(), + inline_source_types: true, + }) .unwrap(); let resolve = generate_stub_wit_dir(&def).unwrap().resolve; @@ -186,14 +188,15 @@ fn many_ways_to_export() { let source_wit_root = Path::new("test-data/many-ways-to-export"); let target_root = tempdir().unwrap(); - let def = StubDefinition::new( - source_wit_root, - target_root.path(), - &None, - "1.0.0", - &WasmRpcOverride::default(), - false, - ) + let def = StubDefinition::new(StubConfig { + source_wit_root: source_wit_root.to_path_buf(), + transformed_source_wit_root: None, + target_root: target_root.path().to_path_buf(), + selected_world: None, + stub_crate_version: "1.0.0".to_string(), + wasm_rpc_override: WasmRpcOverride::default(), + inline_source_types: false, + }) .unwrap(); let resolve = generate_stub_wit_dir(&def).unwrap().resolve; @@ -215,14 +218,15 @@ fn many_ways_to_export_inlined() { let source_wit_root = Path::new("test-data/many-ways-to-export"); let target_root = tempdir().unwrap(); - let def = StubDefinition::new( - source_wit_root, - target_root.path(), - &None, - "1.0.0", - &WasmRpcOverride::default(), - true, - ) + let def = StubDefinition::new(StubConfig { + source_wit_root: source_wit_root.to_path_buf(), + transformed_source_wit_root: None, + target_root: target_root.path().to_path_buf(), + selected_world: None, + stub_crate_version: "1.0.0".to_string(), + wasm_rpc_override: WasmRpcOverride::default(), + inline_source_types: true, + }) .unwrap(); let resolve = generate_stub_wit_dir(&def).unwrap().resolve; From ac7d5fac134f8688503f259dd5e7adedbb64aa0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Wed, 30 Oct 2024 16:27:03 +0100 Subject: [PATCH 025/109] remove conf param --- wasm-rpc-stubgen/src/commands/declarative.rs | 1 - wasm-rpc-stubgen/src/lib.rs | 2 -- wasm-rpc-stubgen/src/stub.rs | 1 - wasm-rpc-stubgen/src/wit_generate.rs | 1 - wasm-rpc-stubgen/tests/wit.rs | 4 ---- 5 files changed, 9 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index f2efa9ad..f61cb806 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -78,7 +78,6 @@ async fn pre_component_build_app(config: &Config, app: &Application) -> anyhow:: let stub_def = StubDefinition::new(StubConfig { source_wit_root: app.component_wit(&component_name), - transformed_source_wit_root: None, target_root: canonical_target_root, selected_world: app.stub_world(&component_name), stub_crate_version: app.stub_crate_version(&component_name), diff --git a/wasm-rpc-stubgen/src/lib.rs b/wasm-rpc-stubgen/src/lib.rs index 3ee8d4c3..6b7df5ff 100644 --- a/wasm-rpc-stubgen/src/lib.rs +++ b/wasm-rpc-stubgen/src/lib.rs @@ -222,7 +222,6 @@ pub fn generate(args: GenerateArgs) -> anyhow::Result<()> { let stub_def = StubDefinition::new( StubConfig { source_wit_root: args.source_wit_root, - transformed_source_wit_root: None, target_root: args.dest_crate_root, selected_world: args.world, stub_crate_version: args.stub_crate_version, @@ -240,7 +239,6 @@ pub async fn build(args: BuildArgs) -> anyhow::Result<()> { let stub_def = StubDefinition::new(StubConfig { source_wit_root: args.source_wit_root, - transformed_source_wit_root: None, target_root: canonical_target_root, selected_world: args.world, stub_crate_version: args.stub_crate_version, diff --git a/wasm-rpc-stubgen/src/stub.rs b/wasm-rpc-stubgen/src/stub.rs index 4dddcb9f..bae61ab4 100644 --- a/wasm-rpc-stubgen/src/stub.rs +++ b/wasm-rpc-stubgen/src/stub.rs @@ -27,7 +27,6 @@ use wit_parser::{ pub struct StubConfig { pub source_wit_root: PathBuf, - pub transformed_source_wit_root: Option, // TODO pub target_root: PathBuf, pub selected_world: Option, pub stub_crate_version: String, diff --git a/wasm-rpc-stubgen/src/wit_generate.rs b/wasm-rpc-stubgen/src/wit_generate.rs index 4e393589..e1daad09 100644 --- a/wasm-rpc-stubgen/src/wit_generate.rs +++ b/wasm-rpc-stubgen/src/wit_generate.rs @@ -45,7 +45,6 @@ pub fn generate_stub_wit_from_wit_dir( ) -> anyhow::Result { generate_stub_wit_from_stub_def(&StubDefinition::new(StubConfig { source_wit_root: source_wit_root.to_path_buf(), - transformed_source_wit_root: None, target_root: source_wit_root.to_path_buf(), // Not used selected_world: None, // TODO: would this fail with multiple worlds? stub_crate_version: "".to_string(), // Not used diff --git a/wasm-rpc-stubgen/tests/wit.rs b/wasm-rpc-stubgen/tests/wit.rs index 303143af..8506c3b3 100644 --- a/wasm-rpc-stubgen/tests/wit.rs +++ b/wasm-rpc-stubgen/tests/wit.rs @@ -32,7 +32,6 @@ fn all_wit_types() { let def = StubDefinition::new(StubConfig { source_wit_root: source_wit_root.to_path_buf(), - transformed_source_wit_root: None, target_root: target_root.path().to_path_buf(), selected_world: None, stub_crate_version: "1.0.0".to_string(), @@ -106,7 +105,6 @@ fn all_wit_types_inlined() { let def = StubDefinition::new(StubConfig { source_wit_root: source_wit_root.to_path_buf(), - transformed_source_wit_root: None, target_root: target_root.path().to_path_buf(), selected_world: None, stub_crate_version: "1.0.0".to_string(), @@ -190,7 +188,6 @@ fn many_ways_to_export() { let def = StubDefinition::new(StubConfig { source_wit_root: source_wit_root.to_path_buf(), - transformed_source_wit_root: None, target_root: target_root.path().to_path_buf(), selected_world: None, stub_crate_version: "1.0.0".to_string(), @@ -220,7 +217,6 @@ fn many_ways_to_export_inlined() { let def = StubDefinition::new(StubConfig { source_wit_root: source_wit_root.to_path_buf(), - transformed_source_wit_root: None, target_root: target_root.path().to_path_buf(), selected_world: None, stub_crate_version: "1.0.0".to_string(), From 1b850766f7ba3030a90758cc93b349eeba5de100 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Wed, 30 Oct 2024 17:00:36 +0100 Subject: [PATCH 026/109] update add_dep test for StubConfig --- wasm-rpc-stubgen/tests/add_dep.rs | 34 +++++++++++++++---------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/wasm-rpc-stubgen/tests/add_dep.rs b/wasm-rpc-stubgen/tests/add_dep.rs index 77933259..50dbc2b1 100644 --- a/wasm-rpc-stubgen/tests/add_dep.rs +++ b/wasm-rpc-stubgen/tests/add_dep.rs @@ -21,7 +21,7 @@ use fs_extra::dir::CopyOptions; use golem_wasm_rpc::{WASI_POLL_WIT, WASM_RPC_WIT}; use golem_wasm_rpc_stubgen::commands::dependencies::{add_stub_dependency, UpdateCargoToml}; use golem_wasm_rpc_stubgen::commands::generate::generate_stub_wit_dir; -use golem_wasm_rpc_stubgen::stub::StubDefinition; +use golem_wasm_rpc_stubgen::stub::{StubConfig, StubDefinition}; use golem_wasm_rpc_stubgen::wit_resolve::ResolvedWitDir; use golem_wasm_rpc_stubgen::WasmRpcOverride; use semver::Version; @@ -617,28 +617,28 @@ fn init_stub_internal(name: &str, always_inline_types: bool) -> TempDir { let canonical_target_root = tempdir.path().canonicalize().unwrap(); let source_wit_root = Path::new("test-data").join(name); - let def = StubDefinition::new( - &source_wit_root, - &canonical_target_root, - &None, - "1.0.0", - &WasmRpcOverride::default(), - always_inline_types, - ) + let def = StubDefinition::new(StubConfig { + source_wit_root, + target_root: canonical_target_root, + selected_world: None, + stub_crate_version: "1.0.0".to_string(), + wasm_rpc_override: WasmRpcOverride::default(), + inline_source_types: always_inline_types, + }) .unwrap(); let _ = generate_stub_wit_dir(&def).unwrap(); tempdir } fn regenerate_stub(stub_dir: &Path, source_wit_root: &Path) { - let def = StubDefinition::new( - source_wit_root, - stub_dir, - &None, - "1.0.0", - &WasmRpcOverride::default(), - false, - ) + let def = StubDefinition::new(StubConfig { + source_wit_root: source_wit_root.to_path_buf(), + target_root: stub_dir.to_path_buf(), + selected_world: None, + stub_crate_version: "1.0.0".to_string(), + wasm_rpc_override: WasmRpcOverride::default(), + inline_source_types: false, + }) .unwrap(); let _ = generate_stub_wit_dir(&def).unwrap(); } From 7d05cbaad1a56501d1186c0e6f5463328fcccd17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Wed, 30 Oct 2024 17:01:01 +0100 Subject: [PATCH 027/109] drop package visitor, as it doesnt help much --- wasm-rpc-stubgen/src/wit_transform.rs | 396 ++------------------------ 1 file changed, 31 insertions(+), 365 deletions(-) diff --git a/wasm-rpc-stubgen/src/wit_transform.rs b/wasm-rpc-stubgen/src/wit_transform.rs index aa0a45f5..01f44f6c 100644 --- a/wasm-rpc-stubgen/src/wit_transform.rs +++ b/wasm-rpc-stubgen/src/wit_transform.rs @@ -1,331 +1,7 @@ use anyhow::anyhow; use indexmap::IndexMap; use regex::Regex; -use wit_encoder::{ - packages_from_parsed, Enum, Flags, Ident, Include, Interface, InterfaceItem, Package, - PackageItem, PackageName, Record, Resource, StandaloneFunc, Type, TypeDef, TypeDefKind, Use, - Variant, World, WorldItem, WorldNamedInterface, -}; - -// TODO: add skip option? -pub trait VisitPackage { - #[allow(unused_variables)] - fn package(&mut self, package: &mut Package) {} - - #[allow(unused_variables)] - fn package_interface(&mut self, package_name: &PackageName, interface: &mut Interface) {} - - #[allow(unused_variables)] - fn package_interface_use( - &mut self, - package_name: &PackageName, - interface_name: &Ident, - use_: &mut Use, - ) { - } - - #[allow(unused_variables)] - fn package_interface_type_def( - &mut self, - package_name: &PackageName, - interface_name: &Ident, - type_def: &mut TypeDef, - ) { - } - - #[allow(unused_variables)] - fn package_interface_type_def_record( - &mut self, - package_name: &PackageName, - interface_name: &Ident, - type_def_name: &Ident, - record: &mut Record, - ) { - } - - #[allow(unused_variables)] - fn package_interface_type_def_resource( - &mut self, - package_name: &PackageName, - interface_name: &Ident, - type_def_name: &Ident, - resource: &mut Resource, - ) { - } - - #[allow(unused_variables)] - fn package_interface_type_def_flags( - &mut self, - package_name: &PackageName, - interface_name: &Ident, - type_def_name: &Ident, - flags: &mut Flags, - ) { - } - - #[allow(unused_variables)] - fn package_interface_type_def_variant( - &mut self, - package_name: &PackageName, - interface_name: &Ident, - type_def_name: &Ident, - variant: &mut Variant, - ) { - } - - #[allow(unused_variables)] - fn package_interface_type_def_enum( - &mut self, - package_name: &PackageName, - interface_name: &Ident, - type_def_name: &Ident, - enum_: &mut Enum, - ) { - } - - #[allow(unused_variables)] - fn package_interface_type_def_type( - &mut self, - package_name: &PackageName, - interface_name: &Ident, - type_def_name: &Ident, - type_: &mut Type, - ) { - } - - #[allow(unused_variables)] - fn package_interface_function( - &mut self, - package_name: &PackageName, - interface_name: &Ident, - function: &mut StandaloneFunc, - ) { - } - - #[allow(unused_variables)] - fn package_world(&mut self, package_name: &PackageName, world: &mut World) {} - - #[allow(unused_variables)] - fn package_world_inline_interface_import( - &mut self, - package_name: &PackageName, - world_name: &Ident, - inline_interface_import: &mut Interface, - ) { - } - - #[allow(unused_variables)] - fn package_world_inline_interface_export( - &mut self, - package_name: &PackageName, - world_name: &Ident, - inline_interface_export: &mut Interface, - ) { - } - - #[allow(unused_variables)] - fn package_world_named_interface_import( - &mut self, - package_name: &PackageName, - world_name: &Ident, - named_interface_import: &mut WorldNamedInterface, - ) { - } - - #[allow(unused_variables)] - fn package_world_named_interface_export( - &mut self, - package_name: &PackageName, - world_name: &Ident, - named_interface_export: &mut WorldNamedInterface, - ) { - } - - #[allow(unused_variables)] - fn package_world_function_import( - &mut self, - package_name: &PackageName, - world_name: &Ident, - function_import: &mut StandaloneFunc, - ) { - } - - #[allow(unused_variables)] - fn package_world_function_export( - &mut self, - package_name: &PackageName, - world_name: &Ident, - function_export: &mut StandaloneFunc, - ) { - } - - #[allow(unused_variables)] - fn package_world_include( - &mut self, - package_name: &PackageName, - world_name: &Ident, - include: &mut Include, - ) { - } - - #[allow(unused_variables)] - fn package_world_use( - &mut self, - package_name: &PackageName, - world_name: &Ident, - use_: &mut Use, - ) { - } -} - -pub fn visit_package(package: &mut Package, visitor: &mut impl VisitPackage) { - visitor.package(package); - let package_name = package.name().clone(); - - for item in package.items_mut() { - match item { - PackageItem::Interface(interface) => { - visitor.package_interface(&package_name, interface); - - let interface_name = interface.name().clone(); - - for use_ in interface.uses_mut() { - visitor.package_interface_use(&package_name, &interface_name, use_); - } - - for item in interface.items_mut() { - match item { - InterfaceItem::TypeDef(type_def) => { - visitor.package_interface_type_def( - &package_name, - &interface_name, - type_def, - ); - - let type_def_name = type_def.name().clone(); - - match type_def.kind_mut() { - TypeDefKind::Record(record) => { - visitor.package_interface_type_def_record( - &package_name, - &interface_name, - &type_def_name, - record, - ); - } - TypeDefKind::Resource(resource) => { - visitor.package_interface_type_def_resource( - &package_name, - &interface_name, - &type_def_name, - resource, - ); - } - TypeDefKind::Flags(flags) => { - visitor.package_interface_type_def_flags( - &package_name, - &interface_name, - &type_def_name, - flags, - ); - } - TypeDefKind::Variant(variant) => { - visitor.package_interface_type_def_variant( - &package_name, - &interface_name, - &type_def_name, - variant, - ); - } - TypeDefKind::Enum(enum_) => { - visitor.package_interface_type_def_enum( - &package_name, - &interface_name, - &type_def_name, - enum_, - ); - } - TypeDefKind::Type(type_) => { - visitor.package_interface_type_def_type( - &package_name, - &interface_name, - &type_def_name, - type_, - ); - } - } - } - InterfaceItem::Function(function) => { - visitor.package_interface_function( - &package_name, - &interface_name, - function, - ); - } - } - } - } - PackageItem::World(world) => { - visitor.package_world(&package_name, world); - - let world_name = world.name().clone(); - - for item in world.items_mut() { - match item { - WorldItem::InlineInterfaceImport(inline_interface_import) => { - visitor.package_world_inline_interface_import( - &package_name, - &world_name, - inline_interface_import, - ); - } - WorldItem::InlineInterfaceExport(inline_interface_export) => { - visitor.package_world_inline_interface_export( - &package_name, - &world_name, - inline_interface_export, - ); - } - WorldItem::NamedInterfaceImport(named_interface_import) => { - visitor.package_world_named_interface_import( - &package_name, - &world_name, - named_interface_import, - ); - } - WorldItem::NamedInterfaceExport(named_interface_export) => { - visitor.package_world_named_interface_export( - &package_name, - &world_name, - named_interface_export, - ); - } - WorldItem::FunctionImport(function_import) => { - visitor.package_world_function_import( - &package_name, - &world_name, - function_import, - ); - } - WorldItem::FunctionExport(function_export) => { - visitor.package_world_function_export( - &package_name, - &world_name, - function_export, - ); - } - WorldItem::Include(include) => { - visitor.package_world_include(&package_name, &world_name, include); - } - WorldItem::Use(use_) => { - visitor.package_world_use(&package_name, &world_name, use_); - } - } - } - } - } - } -} +use wit_encoder::{packages_from_parsed, Package, PackageItem, WorldItem}; // TODO: use wit transform everywhere pub fn import_remover( @@ -347,13 +23,12 @@ pub fn import_remover( } } -pub struct WitTransformer<'a> { - _resolve: &'a wit_parser::Resolve, // TODO +pub struct WitTransformer { encoded_packages_by_parser_id: IndexMap, } -impl<'a> WitTransformer<'a> { - pub fn new(resolve: &'a wit_parser::Resolve) -> anyhow::Result { +impl WitTransformer { + pub fn new(resolve: &wit_parser::Resolve) -> anyhow::Result { let mut encoded_packages_by_parser_id = IndexMap::::new(); for package in packages_from_parsed(resolve) { @@ -373,7 +48,6 @@ impl<'a> WitTransformer<'a> { } Ok(Self { - _resolve: resolve, encoded_packages_by_parser_id, }) } @@ -396,26 +70,22 @@ impl<'a> WitTransformer<'a> { package_id: wit_parser::PackageId, import_prefix: &str, ) -> anyhow::Result<()> { - struct RemoveImports<'a> { - import_prefix: &'a str, - } - impl<'a> VisitPackage for RemoveImports<'a> { - fn package_world(&mut self, _package_name: &PackageName, world: &mut World) { - world.items_mut().retain(|item| { - if let WorldItem::NamedInterfaceImport(import) = &item { - !import.name().raw_name().starts_with(self.import_prefix) - } else { - true - } - }); + let package = self.encoded_package(package_id)?; + for world_item in package.items_mut() { + match world_item { + PackageItem::Interface(_) => {} + PackageItem::World(world) => { + world.items_mut().retain(|item| { + if let WorldItem::NamedInterfaceImport(import) = &item { + !import.name().raw_name().starts_with(import_prefix) + } else { + true + } + }); + } } } - visit_package( - self.encoded_package(package_id)?, - &mut RemoveImports { import_prefix }, - ); - Ok(()) } @@ -424,29 +94,25 @@ impl<'a> WitTransformer<'a> { package_id: wit_parser::PackageId, import_name: &str, ) -> anyhow::Result<()> { - struct AddImportToWorld<'a> { - import_name: &'a str, - } - impl<'a> VisitPackage for AddImportToWorld<'a> { - fn package_world(&mut self, _package_name: &PackageName, world: &mut World) { - let is_already_imported = world.items_mut().iter().any(|item| { - if let WorldItem::NamedInterfaceImport(import) = item { - import.name().raw_name() == self.import_name - } else { - false + let package = self.encoded_package(package_id)?; + for world_item in package.items_mut() { + match world_item { + PackageItem::Interface(_) => {} + PackageItem::World(world) => { + let is_already_imported = world.items_mut().iter().any(|item| { + if let WorldItem::NamedInterfaceImport(import) = item { + import.name().raw_name() == import_name + } else { + false + } + }); + if !is_already_imported { + world.named_interface_import(import_name.to_string()); } - }); - if !is_already_imported { - world.named_interface_import(self.import_name.to_string()); } } } - visit_package( - self.encoded_package(package_id)?, - &mut AddImportToWorld { import_name }, - ); - Ok(()) } } From 7e7d3929e8530eea71d3ffcdd499f58ffb8b93ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Wed, 30 Oct 2024 17:47:56 +0100 Subject: [PATCH 028/109] more cleanups, extract EncodedWitDir --- wasm-rpc-stubgen/src/commands/dependencies.rs | 28 ++--- wasm-rpc-stubgen/src/lib.rs | 1 + wasm-rpc-stubgen/src/wit_encode.rs | 39 ++++++ wasm-rpc-stubgen/src/wit_generate.rs | 4 +- wasm-rpc-stubgen/src/wit_transform.rs | 113 +++++------------- 5 files changed, 82 insertions(+), 103 deletions(-) create mode 100644 wasm-rpc-stubgen/src/wit_encode.rs diff --git a/wasm-rpc-stubgen/src/commands/dependencies.rs b/wasm-rpc-stubgen/src/commands/dependencies.rs index c31c37a4..4a2c9118 100644 --- a/wasm-rpc-stubgen/src/commands/dependencies.rs +++ b/wasm-rpc-stubgen/src/commands/dependencies.rs @@ -14,9 +14,10 @@ use crate::commands::log::{log_action, log_action_plan, log_warn_action, LogIndent}; use crate::fs::{get_file_name, strip_path_prefix, OverwriteSafeAction, OverwriteSafeActions}; +use crate::wit_encode::EncodedWitDir; use crate::wit_generate::generate_stub_wit_from_wit_dir; use crate::wit_resolve::ResolvedWitDir; -use crate::wit_transform::WitTransformer; +use crate::wit_transform::{add_import_to_all_world, remove_imports_from_package_all_worlds}; use crate::{cargo, naming}; use anyhow::{anyhow, bail, Context}; use std::collections::BTreeMap; @@ -48,13 +49,13 @@ pub fn add_stub_dependency( let _indent = LogIndent::new(); let stub_resolved_wit_root = ResolvedWitDir::new(stub_wit_root)?; - let mut stub_transformer = WitTransformer::new(&stub_resolved_wit_root.resolve)?; + let mut stub_encoded_wit_root = EncodedWitDir::new(&stub_resolved_wit_root.resolve)?; let stub_package = stub_resolved_wit_root.main_package()?; let stub_wit = stub_wit_root.join(naming::wit::STUB_WIT_FILE_NAME); let dest_deps_dir = dest_wit_root.join(naming::wit::DEPS_DIR); let dest_resolved_wit_root = ResolvedWitDir::new(dest_wit_root)?; - let mut dest_transformer = WitTransformer::new(&dest_resolved_wit_root.resolve)?; + let mut dest_encoded_wit_root = EncodedWitDir::new(&dest_resolved_wit_root.resolve)?; let dest_package = dest_resolved_wit_root.main_package()?; let dest_stub_package_name = naming::wit::stub_package_name(&dest_package.name); @@ -125,13 +126,12 @@ pub fn add_stub_dependency( // Handle other package by copying while removing cyclic imports } else { package_names_to_package_path.insert(package_name.clone(), package_path); - stub_transformer.remove_imports_from_package_all_worlds( - *package_id, - &dest_stub_package_import_prefix, - )?; - stub_transformer - .remove_imports_from_package_all_worlds(*package_id, &dest_package_import_prefix)?; - let content = stub_transformer.render_package(*package_id)?; + + let package = stub_encoded_wit_root.package(*package_id)?; + remove_imports_from_package_all_worlds(package, &dest_stub_package_import_prefix); + remove_imports_from_package_all_worlds(package, &dest_package_import_prefix); + let content = package.to_string(); + let first_source = package_sources.files.first().ok_or_else(|| { anyhow!( "Expected at least one source for stub package: {}", @@ -185,11 +185,9 @@ pub fn add_stub_dependency( ); } - dest_transformer.add_import_to_all_world( - dest_main_package_id, - &naming::wit::stub_import_name(stub_package)?, - )?; - let content = dest_transformer.render_package(dest_main_package_id)?; + let package = dest_encoded_wit_root.package(dest_main_package_id)?; + add_import_to_all_world(package, &naming::wit::stub_import_name(stub_package)?); + let content = package.to_string(); actions.add(OverwriteSafeAction::WriteFile { content, diff --git a/wasm-rpc-stubgen/src/lib.rs b/wasm-rpc-stubgen/src/lib.rs index 6b7df5ff..4edf79cb 100644 --- a/wasm-rpc-stubgen/src/lib.rs +++ b/wasm-rpc-stubgen/src/lib.rs @@ -21,6 +21,7 @@ pub mod model; pub mod naming; pub mod rust; pub mod stub; +pub mod wit_encode; pub mod wit_generate; pub mod wit_resolve; pub mod wit_transform; diff --git a/wasm-rpc-stubgen/src/wit_encode.rs b/wasm-rpc-stubgen/src/wit_encode.rs new file mode 100644 index 00000000..f9b66a26 --- /dev/null +++ b/wasm-rpc-stubgen/src/wit_encode.rs @@ -0,0 +1,39 @@ +use anyhow::anyhow; +use indexmap::IndexMap; +use wit_encoder::{packages_from_parsed, Package}; + +pub struct EncodedWitDir { + encoded_packages_by_parser_id: IndexMap, +} + +impl EncodedWitDir { + pub fn new(resolve: &wit_parser::Resolve) -> anyhow::Result { + let mut encoded_packages_by_parser_id = IndexMap::::new(); + + for package in packages_from_parsed(resolve) { + let package_name = package.name(); + let package_name = wit_parser::PackageName { + namespace: package_name.namespace().to_string(), + name: package_name.name().to_string(), + version: package_name.version().cloned(), + }; + + let package_id = resolve + .package_names + .get(&package_name) + .cloned() + .ok_or_else(|| anyhow!("Failed to get package by name: {}", package.name()))?; + encoded_packages_by_parser_id.insert(package_id, package); + } + + Ok(Self { + encoded_packages_by_parser_id, + }) + } + + pub fn package(&mut self, package_id: wit_parser::PackageId) -> anyhow::Result<&mut Package> { + self.encoded_packages_by_parser_id + .get_mut(&package_id) + .ok_or_else(|| anyhow!("Failed to get encoded package by id: {:?}", package_id)) + } +} diff --git a/wasm-rpc-stubgen/src/wit_generate.rs b/wasm-rpc-stubgen/src/wit_generate.rs index e1daad09..30f0d920 100644 --- a/wasm-rpc-stubgen/src/wit_generate.rs +++ b/wasm-rpc-stubgen/src/wit_generate.rs @@ -257,7 +257,7 @@ pub fn add_wit_dependencies(def: &StubDefinition) -> anyhow::Result<()> { .join(naming::wit::package_dep_dir_name(&def.source_package_name)) .join(get_file_name(source)?); log_action( - " Copying", + "Copying", format!( "(with source imports removed) {} to {}", source.display(), @@ -269,7 +269,7 @@ pub fn add_wit_dependencies(def: &StubDefinition) -> anyhow::Result<()> { let relative = source.strip_prefix(&def.config.source_wit_root)?; let dest = target_wit_root.join(relative); log_action( - " Copying", + "Copying", format!("{} to {}", source.display(), dest.display()), ); copy(source, &dest)?; diff --git a/wasm-rpc-stubgen/src/wit_transform.rs b/wasm-rpc-stubgen/src/wit_transform.rs index 01f44f6c..43e61d03 100644 --- a/wasm-rpc-stubgen/src/wit_transform.rs +++ b/wasm-rpc-stubgen/src/wit_transform.rs @@ -1,7 +1,5 @@ -use anyhow::anyhow; -use indexmap::IndexMap; use regex::Regex; -use wit_encoder::{packages_from_parsed, Package, PackageItem, WorldItem}; +use wit_encoder::{Package, PackageItem, WorldItem}; // TODO: use wit transform everywhere pub fn import_remover( @@ -23,96 +21,39 @@ pub fn import_remover( } } -pub struct WitTransformer { - encoded_packages_by_parser_id: IndexMap, -} - -impl WitTransformer { - pub fn new(resolve: &wit_parser::Resolve) -> anyhow::Result { - let mut encoded_packages_by_parser_id = IndexMap::::new(); - - for package in packages_from_parsed(resolve) { - let package_name = package.name(); - let package_name = wit_parser::PackageName { - namespace: package_name.namespace().to_string(), - name: package_name.name().to_string(), - version: package_name.version().cloned(), - }; - - let package_id = resolve - .package_names - .get(&package_name) - .cloned() - .ok_or_else(|| anyhow!("Failed to get package by name: {}", package.name()))?; - encoded_packages_by_parser_id.insert(package_id, package); - } - - Ok(Self { - encoded_packages_by_parser_id, - }) - } - - pub fn render_package(&mut self, package_id: wit_parser::PackageId) -> anyhow::Result { - Ok(format!("{}", self.encoded_package(package_id)?)) - } - - fn encoded_package( - &mut self, - package_id: wit_parser::PackageId, - ) -> anyhow::Result<&mut Package> { - self.encoded_packages_by_parser_id - .get_mut(&package_id) - .ok_or_else(|| anyhow!("Failed to get encoded package by id: {:?}", package_id)) - } - - pub fn remove_imports_from_package_all_worlds( - &mut self, - package_id: wit_parser::PackageId, - import_prefix: &str, - ) -> anyhow::Result<()> { - let package = self.encoded_package(package_id)?; - for world_item in package.items_mut() { - match world_item { - PackageItem::Interface(_) => {} - PackageItem::World(world) => { - world.items_mut().retain(|item| { - if let WorldItem::NamedInterfaceImport(import) = &item { - !import.name().raw_name().starts_with(import_prefix) - } else { - true - } - }); - } +pub fn remove_imports_from_package_all_worlds(package: &mut Package, import_prefix: &str) { + for world_item in package.items_mut() { + match world_item { + PackageItem::Interface(_) => {} + PackageItem::World(world) => { + world.items_mut().retain(|item| { + if let WorldItem::NamedInterfaceImport(import) = &item { + !import.name().raw_name().starts_with(import_prefix) + } else { + true + } + }); } } - - Ok(()) } +} - pub fn add_import_to_all_world( - &mut self, - package_id: wit_parser::PackageId, - import_name: &str, - ) -> anyhow::Result<()> { - let package = self.encoded_package(package_id)?; - for world_item in package.items_mut() { - match world_item { - PackageItem::Interface(_) => {} - PackageItem::World(world) => { - let is_already_imported = world.items_mut().iter().any(|item| { - if let WorldItem::NamedInterfaceImport(import) = item { - import.name().raw_name() == import_name - } else { - false - } - }); - if !is_already_imported { - world.named_interface_import(import_name.to_string()); +pub fn add_import_to_all_world(package: &mut Package, import_name: &str) { + for world_item in package.items_mut() { + match world_item { + PackageItem::Interface(_) => {} + PackageItem::World(world) => { + let is_already_imported = world.items_mut().iter().any(|item| { + if let WorldItem::NamedInterfaceImport(import) = item { + import.name().raw_name() == import_name + } else { + false } + }); + if !is_already_imported { + world.named_interface_import(import_name.to_string()); } } } - - Ok(()) } } From da4f5228024c308f4f4a8e1a3524e53caf090837 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Wed, 30 Oct 2024 19:01:00 +0100 Subject: [PATCH 029/109] delete test code --- wasm-rpc-stubgen/src/commands/generate.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/generate.rs b/wasm-rpc-stubgen/src/commands/generate.rs index ba4ea585..bd1b5650 100644 --- a/wasm-rpc-stubgen/src/commands/generate.rs +++ b/wasm-rpc-stubgen/src/commands/generate.rs @@ -69,11 +69,6 @@ pub fn generate_stub_wit_dir(stub_def: &StubDefinition) -> anyhow::Result anyhow::Result { From e815f6403b45e39c294ee9372bce482589de1e67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Wed, 30 Oct 2024 22:21:22 +0100 Subject: [PATCH 030/109] add extract_main_interface_package --- wasm-rpc-stubgen/src/naming.rs | 44 +++- wasm-rpc-stubgen/src/wit_transform.rs | 188 +++++++++++++++--- .../test-data/many-ways-to-export/main.wit | 5 + 3 files changed, 204 insertions(+), 33 deletions(-) diff --git a/wasm-rpc-stubgen/src/naming.rs b/wasm-rpc-stubgen/src/naming.rs index 704ac7f9..88464752 100644 --- a/wasm-rpc-stubgen/src/naming.rs +++ b/wasm-rpc-stubgen/src/naming.rs @@ -1,23 +1,47 @@ pub mod wit { use anyhow::bail; use std::path::{Path, PathBuf}; - use wit_parser::{Package, PackageName}; pub static DEPS_DIR: &str = "deps"; pub static WIT_DIR: &str = "wit"; pub static STUB_WIT_FILE_NAME: &str = "_stub.wit"; - pub fn stub_package_name(package_name: &PackageName) -> PackageName { - PackageName { + pub fn stub_package_name(package_name: &wit_parser::PackageName) -> wit_parser::PackageName { + wit_parser::PackageName { namespace: package_name.namespace.clone(), name: format!("{}-stub", package_name.name), version: package_name.version.clone(), } } - pub fn stub_target_package_name(stub_package_name: &PackageName) -> PackageName { - PackageName { + pub fn interface_package_name( + package_name: &wit_encoder::PackageName, + ) -> wit_encoder::PackageName { + wit_encoder::PackageName::new( + package_name.namespace(), + format!("{}-interface", package_name.name()), + package_name.version().cloned(), + ) + } + + pub fn interface_package_world_inline_interface_name( + world_name: &wit_encoder::Ident, + interface_name: &wit_encoder::Ident, + ) -> String { + format!("{}-{}", world_name.raw_name(), interface_name.raw_name()) + } + + pub fn interface_package_world_inline_functions_interface_name( + world_name: &wit_encoder::Ident, + ) -> String { + format!("{}-inline-functions", world_name.raw_name()) + } + + pub fn stub_target_package_name( + stub_package_name: &wit_parser::PackageName, + ) -> wit_parser::PackageName { + wit_parser::PackageName { namespace: stub_package_name.namespace.clone(), name: stub_package_name .name @@ -28,7 +52,7 @@ pub mod wit { } } - pub fn stub_import_name(stub_package: &Package) -> anyhow::Result { + pub fn stub_import_name(stub_package: &wit_parser::Package) -> anyhow::Result { let package_name = &stub_package.name; if stub_package.interfaces.len() != 1 { @@ -53,11 +77,11 @@ pub mod wit { )) } - pub fn package_dep_dir_name(package_name: &PackageName) -> String { + pub fn package_dep_dir_name(package_name: &wit_parser::PackageName) -> String { format!("{}_{}", package_name.namespace, package_name.name) } - pub fn package_merged_wit_name(package_name: &PackageName) -> String { + pub fn package_merged_wit_name(package_name: &wit_parser::PackageName) -> String { format!("{}_{}.wit", package_name.namespace, package_name.name) } @@ -65,7 +89,9 @@ pub mod wit { Path::new(WIT_DIR).join(DEPS_DIR).join(package_dir_name) } - pub fn package_wit_dep_dir_from_package_name(package_name: &PackageName) -> PathBuf { + pub fn package_wit_dep_dir_from_package_name( + package_name: &wit_parser::PackageName, + ) -> PathBuf { package_wit_dep_dir_from_package_dir_name(&package_dep_dir_name(package_name)) } } diff --git a/wasm-rpc-stubgen/src/wit_transform.rs b/wasm-rpc-stubgen/src/wit_transform.rs index 43e61d03..63fe663d 100644 --- a/wasm-rpc-stubgen/src/wit_transform.rs +++ b/wasm-rpc-stubgen/src/wit_transform.rs @@ -1,7 +1,11 @@ +use crate::naming; +use crate::wit_encode::EncodedWitDir; +use crate::wit_resolve::ResolvedWitDir; use regex::Regex; -use wit_encoder::{Package, PackageItem, WorldItem}; +use std::collections::BTreeMap; +use wit_encoder::{Ident, Interface, Package, PackageItem, StandaloneFunc, WorldItem}; -// TODO: use wit transform everywhere +// TODO: use wit encoder based one pub fn import_remover( package_name: &wit_parser::PackageName, ) -> impl Fn(String) -> anyhow::Result { @@ -23,37 +27,173 @@ pub fn import_remover( pub fn remove_imports_from_package_all_worlds(package: &mut Package, import_prefix: &str) { for world_item in package.items_mut() { - match world_item { - PackageItem::Interface(_) => {} - PackageItem::World(world) => { - world.items_mut().retain(|item| { - if let WorldItem::NamedInterfaceImport(import) = &item { - !import.name().raw_name().starts_with(import_prefix) - } else { - true - } - }); - } + if let PackageItem::World(world) = world_item { + world.items_mut().retain(|item| { + if let WorldItem::NamedInterfaceImport(import) = &item { + !import.name().raw_name().starts_with(import_prefix) + } else { + true + } + }); } } } pub fn add_import_to_all_world(package: &mut Package, import_name: &str) { for world_item in package.items_mut() { - match world_item { - PackageItem::Interface(_) => {} - PackageItem::World(world) => { - let is_already_imported = world.items_mut().iter().any(|item| { - if let WorldItem::NamedInterfaceImport(import) = item { - import.name().raw_name() == import_name - } else { - false + if let PackageItem::World(world) = world_item { + let is_already_imported = world.items_mut().iter().any(|item| { + if let WorldItem::NamedInterfaceImport(import) = item { + import.name().raw_name() == import_name + } else { + false + } + }); + if !is_already_imported { + world.named_interface_import(import_name.to_string()); + } + } + } +} + +// TODO: handle world include +// TODO: handle world use +// TODO: maybe transform inline interfaces and functions into included world? +pub fn extract_main_interface_package( + resolve: &ResolvedWitDir, +) -> anyhow::Result<(Package, Package)> { + let mut encoded_wit_dir = EncodedWitDir::new(&resolve.resolve)?; + + let package = encoded_wit_dir.package(resolve.package_id)?; + + let mut interface_package = package.clone(); + interface_package.set_name(naming::wit::interface_package_name(package.name())); + + let interface_export_prefix = format!( + "{}:{}/", + package.name().namespace(), + interface_package.name().name() + ); + let interface_export_suffix = package + .name() + .version() + .map(|version| format!("@{}", version)) + .unwrap_or_default(); + + package.items_mut().retain(|item| match item { + PackageItem::Interface(_) => false, + PackageItem::World(_) => true, + }); + interface_package.items_mut().retain(|item| match item { + PackageItem::Interface(_) => true, + PackageItem::World(_) => false, + }); + + let mut inline_interface_exports = BTreeMap::>::new(); + let mut inline_function_exports = BTreeMap::>::new(); + for package_item in package.items_mut() { + if let PackageItem::World(world) = package_item { + let world_name = world.name().clone(); + + world.items_mut().retain(|world_item| match world_item { + WorldItem::InlineInterfaceExport(interface) => { + let mut interface = interface.clone(); + interface.set_name(naming::wit::interface_package_world_inline_interface_name( + &world_name, + interface.name(), + )); + + inline_interface_exports + .entry(world_name.clone()) + .or_default() + .push(interface.clone()); + false + } + WorldItem::FunctionExport(function) => { + inline_function_exports + .entry(world_name.clone()) + .or_default() + .push(function.clone()); + false + } + _ => true, + }); + + if let Some(interfaces) = inline_interface_exports.get(&world_name) { + for interface in interfaces { + world.named_interface_export(interface.name().clone()); + } + } + + if inline_function_exports.contains_key(&world_name) { + world.named_interface_export(format!( + "{}{}{}", + interface_export_prefix, + naming::wit::interface_package_world_inline_functions_interface_name( + &world_name + ), + interface_export_suffix + )); + } + } + } + + for package_item in package.items_mut() { + if let PackageItem::World(world) = package_item { + for world_item in world.items_mut() { + if let WorldItem::NamedInterfaceExport(export) = world_item { + if !export.name().raw_name().contains("/") { + export.set_name(format!( + "{}{}{}", + interface_export_prefix, + export.name(), + interface_export_suffix + )); } - }); - if !is_already_imported { - world.named_interface_import(import_name.to_string()); } } } } + + for (_, interfaces) in inline_interface_exports { + for interface in interfaces { + interface_package.interface(interface); + } + } + + for (world_name, functions) in inline_function_exports { + let mut interface = Interface::new( + naming::wit::interface_package_world_inline_functions_interface_name(&world_name), + ); + + for function in functions { + interface.function(function); + } + + interface_package.interface(interface); + } + + Ok((package.clone(), interface_package)) +} + +// TODO: delete these +#[cfg(test)] +mod test { + use test_r::test; + + use crate::wit_resolve::ResolvedWitDir; + use crate::wit_transform::extract_main_interface_package; + use std::path::Path; + + #[test] + fn test_extract_playground() { + let resolved_wit_dir = + ResolvedWitDir::new(Path::new("test-data/many-ways-to-export")).unwrap(); + let (package, interface_package) = + extract_main_interface_package(&resolved_wit_dir).unwrap(); + + println!("{}", package); + println!("\n---\n"); + println!("{}", interface_package); + } } diff --git a/wasm-rpc-stubgen/test-data/many-ways-to-export/main.wit b/wasm-rpc-stubgen/test-data/many-ways-to-export/main.wit index eb589f17..1404168f 100644 --- a/wasm-rpc-stubgen/test-data/many-ways-to-export/main.wit +++ b/wasm-rpc-stubgen/test-data/many-ways-to-export/main.wit @@ -46,6 +46,11 @@ interface iface11 { } world api { + // TODO: + // use test:sub2/iface10.{rec1}; + + // TODO: include + export func1: func(); export iface1; export iface2; From e06060b74bdfc1273eae04387e78af4d6bdbf8f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Thu, 31 Oct 2024 07:55:15 +0100 Subject: [PATCH 031/109] rename --- wasm-rpc-stubgen/src/commands/dependencies.rs | 6 +++--- wasm-rpc-stubgen/src/wit_transform.rs | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/dependencies.rs b/wasm-rpc-stubgen/src/commands/dependencies.rs index 4a2c9118..bd7136e4 100644 --- a/wasm-rpc-stubgen/src/commands/dependencies.rs +++ b/wasm-rpc-stubgen/src/commands/dependencies.rs @@ -17,7 +17,7 @@ use crate::fs::{get_file_name, strip_path_prefix, OverwriteSafeAction, Overwrite use crate::wit_encode::EncodedWitDir; use crate::wit_generate::generate_stub_wit_from_wit_dir; use crate::wit_resolve::ResolvedWitDir; -use crate::wit_transform::{add_import_to_all_world, remove_imports_from_package_all_worlds}; +use crate::wit_transform::{add_import_to_all_world, remove_world_named_interface_imports}; use crate::{cargo, naming}; use anyhow::{anyhow, bail, Context}; use std::collections::BTreeMap; @@ -128,8 +128,8 @@ pub fn add_stub_dependency( package_names_to_package_path.insert(package_name.clone(), package_path); let package = stub_encoded_wit_root.package(*package_id)?; - remove_imports_from_package_all_worlds(package, &dest_stub_package_import_prefix); - remove_imports_from_package_all_worlds(package, &dest_package_import_prefix); + remove_world_named_interface_imports(package, &dest_stub_package_import_prefix); + remove_world_named_interface_imports(package, &dest_package_import_prefix); let content = package.to_string(); let first_source = package_sources.files.first().ok_or_else(|| { diff --git a/wasm-rpc-stubgen/src/wit_transform.rs b/wasm-rpc-stubgen/src/wit_transform.rs index 63fe663d..cba07d95 100644 --- a/wasm-rpc-stubgen/src/wit_transform.rs +++ b/wasm-rpc-stubgen/src/wit_transform.rs @@ -25,7 +25,8 @@ pub fn import_remover( } } -pub fn remove_imports_from_package_all_worlds(package: &mut Package, import_prefix: &str) { +// TODO: make the import name matcher more precise +pub fn remove_world_named_interface_imports(package: &mut Package, import_prefix: &str) { for world_item in package.items_mut() { if let PackageItem::World(world) = world_item { world.items_mut().retain(|item| { From cb9a9646a48b856ff518cdd12456470782e2e4f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Thu, 31 Oct 2024 08:00:13 +0100 Subject: [PATCH 032/109] add comments for extract_main_interface_package --- wasm-rpc-stubgen/src/wit_transform.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/wasm-rpc-stubgen/src/wit_transform.rs b/wasm-rpc-stubgen/src/wit_transform.rs index cba07d95..4b215399 100644 --- a/wasm-rpc-stubgen/src/wit_transform.rs +++ b/wasm-rpc-stubgen/src/wit_transform.rs @@ -81,10 +81,12 @@ pub fn extract_main_interface_package( .map(|version| format!("@{}", version)) .unwrap_or_default(); + // Drop all interfaces from original package package.items_mut().retain(|item| match item { PackageItem::Interface(_) => false, PackageItem::World(_) => true, }); + // Drop all worlds from interface package interface_package.items_mut().retain(|item| match item { PackageItem::Interface(_) => true, PackageItem::World(_) => false, @@ -96,6 +98,7 @@ pub fn extract_main_interface_package( if let PackageItem::World(world) = package_item { let world_name = world.name().clone(); + // Remove and collect inline exports world.items_mut().retain(|world_item| match world_item { WorldItem::InlineInterfaceExport(interface) => { let mut interface = interface.clone(); @@ -120,12 +123,14 @@ pub fn extract_main_interface_package( _ => true, }); + // Insert named imports for extracted inline interfaces if let Some(interfaces) = inline_interface_exports.get(&world_name) { for interface in interfaces { world.named_interface_export(interface.name().clone()); } } + // Insert named import for extracted inline functions if inline_function_exports.contains_key(&world_name) { world.named_interface_export(format!( "{}{}{}", @@ -139,6 +144,7 @@ pub fn extract_main_interface_package( } } + // Rename named self imports to use the extracted interface names for package_item in package.items_mut() { if let PackageItem::World(world) = package_item { for world_item in world.items_mut() { @@ -156,12 +162,14 @@ pub fn extract_main_interface_package( } } + // Add inlined exported interfaces to the interface package for (_, interfaces) in inline_interface_exports { for interface in interfaces { interface_package.interface(interface); } } + // Add interface for inlined functions to the interface package for (world_name, functions) in inline_function_exports { let mut interface = Interface::new( naming::wit::interface_package_world_inline_functions_interface_name(&world_name), From edf4c6f3de4718462630a8e7d7cd94e9cfdfa9c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Thu, 31 Oct 2024 11:40:05 +0100 Subject: [PATCH 033/109] checkpoint before dropping full compat: transform cleanups, move add_stub_dep to wit_generate and add config for it, todos --- wasm-rpc-stubgen/src/commands/declarative.rs | 15 +- wasm-rpc-stubgen/src/commands/dependencies.rs | 232 +------------ wasm-rpc-stubgen/src/commands/generate.rs | 9 +- wasm-rpc-stubgen/src/lib.rs | 2 +- wasm-rpc-stubgen/src/stub.rs | 6 + wasm-rpc-stubgen/src/wit_generate.rs | 262 ++++++++++++++- wasm-rpc-stubgen/src/wit_transform.rs | 53 ++- wasm-rpc-stubgen/tests/add_dep.rs | 317 ++++++++++-------- 8 files changed, 498 insertions(+), 398 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index f61cb806..8ce07fdb 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -1,4 +1,3 @@ -use crate::commands::dependencies::UpdateCargoToml; use crate::commands::log::{ log_action, log_skipping_up_to_date, log_validated_action_result, log_warn_action, }; @@ -9,6 +8,7 @@ use crate::model::wasm_rpc::{ include_glob_patter_from_yaml_file, init_oam_app, Application, DEFAULT_CONFIG_FILE_NAME, }; use crate::stub::{StubConfig, StubDefinition}; +use crate::wit_generate::{add_stub_as_dependency_to_wit_dir, AddStubAsDepConfig, UpdateCargoToml}; use crate::{commands, WasmRpcOverride}; use anyhow::{anyhow, Context, Error}; use colored::Colorize; @@ -128,12 +128,13 @@ async fn pre_component_build_app(config: &Config, app: &Application) -> anyhow:: ), ); - commands::dependencies::add_stub_dependency( - &app.stub_wit(dep_component_name), - &app.component_wit(component_name), - true, // NOTE: in declarative mode we always use overwrite - UpdateCargoToml::UpdateIfExists, - )? + add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { + stub_wit_root: app.stub_wit(dep_component_name), + dest_wit_root: app.component_wit(component_name), + overwrite: true, // NOTE: in declarative mode we always use overwrite + remove_dest_imports: false, + update_cargo_toml: UpdateCargoToml::Update, + })? } } diff --git a/wasm-rpc-stubgen/src/commands/dependencies.rs b/wasm-rpc-stubgen/src/commands/dependencies.rs index bd7136e4..71c71cf5 100644 --- a/wasm-rpc-stubgen/src/commands/dependencies.rs +++ b/wasm-rpc-stubgen/src/commands/dependencies.rs @@ -12,24 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::commands::log::{log_action, log_action_plan, log_warn_action, LogIndent}; -use crate::fs::{get_file_name, strip_path_prefix, OverwriteSafeAction, OverwriteSafeActions}; -use crate::wit_encode::EncodedWitDir; -use crate::wit_generate::generate_stub_wit_from_wit_dir; -use crate::wit_resolve::ResolvedWitDir; -use crate::wit_transform::{add_import_to_all_world, remove_world_named_interface_imports}; -use crate::{cargo, naming}; -use anyhow::{anyhow, bail, Context}; -use std::collections::BTreeMap; -use std::path::{Path, PathBuf}; -use wit_parser::PackageName; - -#[derive(PartialEq, Eq)] -pub enum UpdateCargoToml { - Update, - UpdateIfExists, - NoUpdate, -} +use crate::wit_generate; +use crate::wit_generate::{AddStubAsDepConfig, UpdateCargoToml}; +use std::path::Path; pub fn add_stub_dependency( stub_wit_root: &Path, @@ -37,208 +22,11 @@ pub fn add_stub_dependency( overwrite: bool, update_cargo_toml: UpdateCargoToml, ) -> anyhow::Result<()> { - log_action( - "Adding", - format!( - "stub dependencies to {} from {}", - dest_wit_root.display(), - stub_wit_root.display() - ), - ); - - let _indent = LogIndent::new(); - - let stub_resolved_wit_root = ResolvedWitDir::new(stub_wit_root)?; - let mut stub_encoded_wit_root = EncodedWitDir::new(&stub_resolved_wit_root.resolve)?; - let stub_package = stub_resolved_wit_root.main_package()?; - let stub_wit = stub_wit_root.join(naming::wit::STUB_WIT_FILE_NAME); - - let dest_deps_dir = dest_wit_root.join(naming::wit::DEPS_DIR); - let dest_resolved_wit_root = ResolvedWitDir::new(dest_wit_root)?; - let mut dest_encoded_wit_root = EncodedWitDir::new(&dest_resolved_wit_root.resolve)?; - let dest_package = dest_resolved_wit_root.main_package()?; - let dest_stub_package_name = naming::wit::stub_package_name(&dest_package.name); - - // TODO: have a better matcher which also considers / and @ - let dest_stub_package_import_prefix = dest_stub_package_name.to_string(); - let dest_package_import_prefix = dest_package.name.to_string(); - - { - let is_self_stub_by_name = - dest_package.name == naming::wit::stub_target_package_name(&stub_package.name); - let is_self_stub_by_content = is_self_stub(&stub_wit, dest_wit_root); - - if is_self_stub_by_name && !is_self_stub_by_content? { - return Err(anyhow!( - "Both the caller and the target components are using the same package name ({}), which is not supported.", - dest_package.name - )); - } - } - - let mut actions = OverwriteSafeActions::new(); - let mut package_names_to_package_path = BTreeMap::::new(); - - for (package_name, package_id) in &stub_resolved_wit_root.resolve.package_names { - let package_sources = stub_resolved_wit_root - .package_sources - .get(package_id) - .ok_or_else(|| anyhow!("Failed to get package sources for {}", package_name))?; - let package_path = naming::wit::package_wit_dep_dir_from_package_dir_name(&get_file_name( - &package_sources.dir, - )?); - - let is_stub_main_package = *package_id == stub_resolved_wit_root.package_id; - let is_dest_package = *package_name == dest_package.name; - let is_dest_stub_package = *package_name == dest_stub_package_name; - - // We skip self as a dependency - if is_dest_package { - log_warn_action( - "Skipping", - format!("cyclic self dependency for {}", package_name), - ); - } else if is_dest_stub_package || is_stub_main_package { - let package_dep_dir_name = naming::wit::package_dep_dir_name(package_name); - let package_path = naming::wit::package_wit_dep_dir_from_package_name(package_name); - - package_names_to_package_path.insert(package_name.clone(), package_path); - - // Handle self stub packages: use regenerated stub with inlining, to break the recursive cycle - if is_dest_stub_package { - actions.add(OverwriteSafeAction::WriteFile { - content: generate_stub_wit_from_wit_dir(dest_wit_root, true)?, - target: dest_deps_dir - .join(&package_dep_dir_name) - .join(naming::wit::STUB_WIT_FILE_NAME), - }); - // Non-self stub package has to be copied into target deps - } else { - for source in &package_sources.files { - actions.add(OverwriteSafeAction::CopyFile { - source: source.clone(), - target: dest_deps_dir - .join(&package_dep_dir_name) - .join(get_file_name(source)?), - }); - } - } - // Handle other package by copying while removing cyclic imports - } else { - package_names_to_package_path.insert(package_name.clone(), package_path); - - let package = stub_encoded_wit_root.package(*package_id)?; - remove_world_named_interface_imports(package, &dest_stub_package_import_prefix); - remove_world_named_interface_imports(package, &dest_package_import_prefix); - let content = package.to_string(); - - let first_source = package_sources.files.first().ok_or_else(|| { - anyhow!( - "Expected at least one source for stub package: {}", - package_name - ) - })?; - let first_source_relative_path = strip_path_prefix(stub_wit_root, first_source)?; - let target = { - if package_sources.files.len() == 1 { - dest_wit_root.join(first_source_relative_path) - } else { - dest_wit_root - .join(first_source_relative_path.parent().ok_or_else(|| { - anyhow!( - "Failed to get parent of stub source: {}", - first_source_relative_path.display() - ) - })?) - .join(naming::wit::package_merged_wit_name(package_name)) - } - }; - - actions.add(OverwriteSafeAction::WriteFile { content, target }); - - // TODO: still output old ones while experimenting - for source in &package_sources.files { - actions.add(OverwriteSafeAction::CopyFile { - source: source.clone(), - target: dest_wit_root.join(format!( - "{}.old", - strip_path_prefix(stub_wit_root, source)?.to_string_lossy() - )), - }); - } - } - } - - // Import stub if overwrite enabled // TODO: use a different flag, or always import? - if overwrite { - let dest_main_package_id = dest_resolved_wit_root.package_id; - - let dest_main_package_sources = dest_resolved_wit_root - .package_sources - .get(&dest_main_package_id) - .ok_or_else(|| anyhow!("Failed to get dest main package sources"))?; - - if dest_main_package_sources.files.len() != 1 { - bail!( - "Expected exactly one dest main package source, got sources: {:?}", - dest_main_package_sources.files - ); - } - - let package = dest_encoded_wit_root.package(dest_main_package_id)?; - add_import_to_all_world(package, &naming::wit::stub_import_name(stub_package)?); - let content = package.to_string(); - - actions.add(OverwriteSafeAction::WriteFile { - content, - target: dest_main_package_sources.files[0].clone(), - }); - } - - let forbidden_overwrites = actions.run(overwrite, log_action_plan)?; - if !forbidden_overwrites.is_empty() { - eprintln!("The following files would have been overwritten with new content:"); - for action in forbidden_overwrites { - eprintln!(" {}", action.target().display()); - } - eprintln!(); - eprintln!("Use --overwrite to force overwrite."); - } - - if let Some(target_parent) = dest_wit_root.parent() { - let target_cargo_toml = target_parent.join("Cargo.toml"); - if target_cargo_toml.exists() && target_cargo_toml.is_file() { - if update_cargo_toml == UpdateCargoToml::NoUpdate { - eprintln!("Warning: the newly copied dependencies have to be added to {}. Use the --update-cargo-toml flag to update it automatically.", target_cargo_toml.display()); - } else { - cargo::is_cargo_component_toml(&target_cargo_toml).context(format!( - "The file {target_cargo_toml:?} is not a valid cargo-component project" - ))?; - cargo::add_dependencies_to_cargo_toml( - &target_cargo_toml, - package_names_to_package_path, - )?; - } - } else if update_cargo_toml == UpdateCargoToml::Update { - return Err(anyhow!( - "Cannot update {:?} file because it does not exist or is not a file", - target_cargo_toml - )); - } - } else if update_cargo_toml == UpdateCargoToml::Update { - return Err(anyhow!("Cannot update the Cargo.toml file because parent directory of the destination WIT root does not exist.")); - } - - Ok(()) -} - -/// Checks whether `stub_wit` is a stub generated for `dest_wit_root` -fn is_self_stub(stub_wit: &Path, dest_wit_root: &Path) -> anyhow::Result { - // TODO: can we make it diff exports instead of generated content? - let dest_stub_wit_imported = generate_stub_wit_from_wit_dir(dest_wit_root, false)?; - let dest_stub_wit_inlined = generate_stub_wit_from_wit_dir(dest_wit_root, true)?; - let stub_wit = std::fs::read_to_string(stub_wit)?; - - // TODO: this can also be false in case the stub is lagging - Ok(stub_wit == dest_stub_wit_imported || stub_wit == dest_stub_wit_inlined) + wit_generate::add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { + stub_wit_root: stub_wit_root.to_path_buf(), + dest_wit_root: dest_wit_root.to_path_buf(), + overwrite, + remove_dest_imports: true, + update_cargo_toml, + }) } diff --git a/wasm-rpc-stubgen/src/commands/generate.rs b/wasm-rpc-stubgen/src/commands/generate.rs index bd1b5650..4b3babfc 100644 --- a/wasm-rpc-stubgen/src/commands/generate.rs +++ b/wasm-rpc-stubgen/src/commands/generate.rs @@ -19,7 +19,7 @@ use crate::fs::copy; use crate::naming; use crate::rust::generate_stub_source; use crate::stub::StubDefinition; -use crate::wit_generate::{add_wit_dependencies, generate_stub_wit_to_target}; +use crate::wit_generate::{add_dependencies_to_stub_wit_dir, generate_stub_wit_to_target}; use crate::wit_resolve::ResolvedWitDir; use anyhow::Context; use fs_extra::dir::CopyOptions; @@ -65,9 +65,14 @@ pub fn generate_stub_wit_dir(stub_def: &StubDefinition) -> anyhow::Result anyhow::Result { + EncodedWitDir::new(&self.resolve) + } + pub fn target_cargo_path(&self) -> PathBuf { self.config.target_root.join("Cargo.toml") } diff --git a/wasm-rpc-stubgen/src/wit_generate.rs b/wasm-rpc-stubgen/src/wit_generate.rs index 30f0d920..f0a6020a 100644 --- a/wasm-rpc-stubgen/src/wit_generate.rs +++ b/wasm-rpc-stubgen/src/wit_generate.rs @@ -12,16 +12,25 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::commands::log::{log_action, log_warn_action, LogIndent}; -use crate::fs::{copy, copy_transformed, get_file_name}; -use crate::naming; +use crate::commands::log::{log_action, log_action_plan, log_warn_action, LogIndent}; +use crate::fs::{ + copy, copy_transformed, get_file_name, strip_path_prefix, OverwriteSafeAction, + OverwriteSafeActions, +}; use crate::stub::{ FunctionParamStub, FunctionResultStub, FunctionStub, InterfaceStub, InterfaceStubTypeDef, StubConfig, StubDefinition, }; -use crate::wit_transform::import_remover; +use crate::wit_encode::EncodedWitDir; +use crate::wit_resolve::{PackageSource, ResolvedWitDir}; +use crate::wit_transform::{ + add_world_named_interface_import, import_remover, world_named_interface_import_remover, +}; +use crate::{cargo, naming}; +use anyhow::{anyhow, bail, Context}; +use std::collections::BTreeMap; use std::fs; -use std::path::Path; +use std::path::{Path, PathBuf}; use wit_encoder::{ Ident, Interface, Package, PackageName, Params, ResourceFunc, Results, Type, TypeDef, TypeDefKind, VariantCase, World, @@ -216,7 +225,7 @@ fn add_async_return_type( )) } -pub fn add_wit_dependencies(def: &StubDefinition) -> anyhow::Result<()> { +pub fn add_dependencies_to_stub_wit_dir(def: &StubDefinition) -> anyhow::Result<()> { log_action( "Adding", format!( @@ -305,6 +314,247 @@ fn write_embedded_source(target_dir: &Path, file_name: &str, content: &str) -> a Ok(()) } +#[derive(PartialEq, Eq)] +pub enum UpdateCargoToml { + Update, + UpdateIfExists, + NoUpdate, +} + +pub struct AddStubAsDepConfig { + pub stub_wit_root: PathBuf, + pub dest_wit_root: PathBuf, + pub overwrite: bool, + pub remove_dest_imports: bool, + pub update_cargo_toml: UpdateCargoToml, +} + +// TODO: "named" args? +pub fn add_stub_as_dependency_to_wit_dir(config: AddStubAsDepConfig) -> anyhow::Result<()> { + log_action( + "Adding", + format!( + "stub dependencies to {} from {}", + config.dest_wit_root.display(), + config.stub_wit_root.display() + ), + ); + + let _indent = LogIndent::new(); + + let stub_resolved_wit_root = ResolvedWitDir::new(&config.stub_wit_root)?; + let stub_package = stub_resolved_wit_root.main_package()?; + let stub_wit = config.stub_wit_root.join(naming::wit::STUB_WIT_FILE_NAME); + + let dest_deps_dir = config.dest_wit_root.join(naming::wit::DEPS_DIR); + let dest_resolved_wit_root = ResolvedWitDir::new(&config.dest_wit_root)?; + let dest_package = dest_resolved_wit_root.main_package()?; + let dest_stub_package_name = naming::wit::stub_package_name(&dest_package.name); + + // TODO: these could be made optional depending on remove_dest_imports + let mut stub_encoded_wit_root = EncodedWitDir::new(&stub_resolved_wit_root.resolve)?; + let remove_dest_package_import = world_named_interface_import_remover(&dest_package.name); + let remove_dest_stub_package_import = + world_named_interface_import_remover(&dest_stub_package_name); + + // TODO: this could be made optional depending on "add stub import" (=overwrite) + let mut dest_encoded_wit_root = EncodedWitDir::new(&dest_resolved_wit_root.resolve)?; + + // TODO: make this optional (not needed for declarative) + { + let is_self_stub_by_name = + dest_package.name == naming::wit::stub_target_package_name(&stub_package.name); + let is_self_stub_by_content = is_self_stub(&stub_wit, &config.dest_wit_root); + + if is_self_stub_by_name && !is_self_stub_by_content? { + return Err(anyhow!( + "Both the caller and the target components are using the same package name ({}), which is not supported.", + dest_package.name + )); + } + } + + let mut actions = OverwriteSafeActions::new(); + let mut package_names_to_package_path = BTreeMap::::new(); + + for (package_name, package_id) in &stub_resolved_wit_root.resolve.package_names { + let package_sources = stub_resolved_wit_root + .package_sources + .get(package_id) + .ok_or_else(|| anyhow!("Failed to get package sources for {}", package_name))?; + let package_path = naming::wit::package_wit_dep_dir_from_package_dir_name(&get_file_name( + &package_sources.dir, + )?); + + let is_stub_main_package = *package_id == stub_resolved_wit_root.package_id; + let is_dest_package = *package_name == dest_package.name; + let is_dest_stub_package = *package_name == dest_stub_package_name; + + // We skip self as a dependency + if is_dest_package { + log_warn_action( + "Skipping", + format!("cyclic self dependency for {}", package_name), + ); + } else if is_dest_stub_package || is_stub_main_package { + let package_dep_dir_name = naming::wit::package_dep_dir_name(package_name); + let package_path = naming::wit::package_wit_dep_dir_from_package_name(package_name); + + package_names_to_package_path.insert(package_name.clone(), package_path); + + // Handle self stub packages: use regenerated stub with inlining, to break the recursive cycle + if is_dest_stub_package { + actions.add(OverwriteSafeAction::WriteFile { + // TODO: this call potentially builds another EncodedWitDir for source + content: generate_stub_wit_from_wit_dir(&config.dest_wit_root, true)?, + target: dest_deps_dir + .join(&package_dep_dir_name) + .join(naming::wit::STUB_WIT_FILE_NAME), + }); + // Non-self stub package has to be copied into target deps + } else { + for source in &package_sources.files { + actions.add(OverwriteSafeAction::CopyFile { + source: source.clone(), + target: dest_deps_dir + .join(&package_dep_dir_name) + .join(get_file_name(source)?), + }); + } + } + // Handle other package by copying and removing cyclic imports + } else if config.remove_dest_imports { + package_names_to_package_path.insert(package_name.clone(), package_path); + + let package = stub_encoded_wit_root.package(*package_id)?; + remove_dest_package_import(package); + remove_dest_stub_package_import(package); + let content = package.to_string(); + + let target = wit_dep_transformed_target_file( + &config.stub_wit_root, + &config.dest_wit_root, + package_name, + package_sources, + )?; + + actions.add(OverwriteSafeAction::WriteFile { content, target }); + // Handle other package by copying and no transformations + } else { + for source in &package_sources.files { + actions.add(OverwriteSafeAction::CopyFile { + source: source.clone(), + target: config + .dest_wit_root + .join(strip_path_prefix(&config.stub_wit_root, source)?), + }); + } + } + } + + // Import stub if overwrite enabled // TODO: use a different flag, or always import? + if config.overwrite { + let dest_main_package_id = dest_resolved_wit_root.package_id; + + let dest_main_package_sources = dest_resolved_wit_root + .package_sources + .get(&dest_main_package_id) + .ok_or_else(|| anyhow!("Failed to get dest main package sources"))?; + + if dest_main_package_sources.files.len() != 1 { + bail!( + "Expected exactly one dest main package source, got sources: {:?}", + dest_main_package_sources.files + ); + } + + let package = dest_encoded_wit_root.package(dest_main_package_id)?; + add_world_named_interface_import(package, &naming::wit::stub_import_name(stub_package)?); + let content = package.to_string(); + + actions.add(OverwriteSafeAction::WriteFile { + content, + target: dest_main_package_sources.files[0].clone(), + }); + } + + let forbidden_overwrites = actions.run(config.overwrite, log_action_plan)?; + if !forbidden_overwrites.is_empty() { + eprintln!("The following files would have been overwritten with new content:"); + for action in forbidden_overwrites { + eprintln!(" {}", action.target().display()); + } + eprintln!(); + eprintln!("Use --overwrite to force overwrite."); + } + + if let Some(target_parent) = config.dest_wit_root.parent() { + let target_cargo_toml = target_parent.join("Cargo.toml"); + if target_cargo_toml.exists() && target_cargo_toml.is_file() { + if config.update_cargo_toml == UpdateCargoToml::NoUpdate { + eprintln!("Warning: the newly copied dependencies have to be added to {}. Use the --update-cargo-toml flag to update it automatically.", target_cargo_toml.display()); + } else { + cargo::is_cargo_component_toml(&target_cargo_toml).context(format!( + "The file {target_cargo_toml:?} is not a valid cargo-component project" + ))?; + cargo::add_dependencies_to_cargo_toml( + &target_cargo_toml, + package_names_to_package_path, + )?; + } + } else if config.update_cargo_toml == UpdateCargoToml::Update { + return Err(anyhow!( + "Cannot update {:?} file because it does not exist or is not a file", + target_cargo_toml + )); + } + } else if config.update_cargo_toml == UpdateCargoToml::Update { + return Err(anyhow!("Cannot update the Cargo.toml file because parent directory of the destination WIT root does not exist.")); + } + + Ok(()) +} + +// TODO: let's find another way to identify self stubs, as this is quite diverging now based on content +/// Checks whether `stub_wit` is a stub generated for `dest_wit_root` +fn is_self_stub(stub_wit: &Path, dest_wit_root: &Path) -> anyhow::Result { + // TODO: can we make it diff exports instead of generated content? + let dest_stub_wit_imported = generate_stub_wit_from_wit_dir(dest_wit_root, false)?; + let dest_stub_wit_inlined = generate_stub_wit_from_wit_dir(dest_wit_root, true)?; + let stub_wit = std::fs::read_to_string(stub_wit)?; + + // TODO: this can also be false in case the stub is lagging + Ok(stub_wit == dest_stub_wit_imported || stub_wit == dest_stub_wit_inlined) +} + +fn wit_dep_transformed_target_file( + source_wit_root: &Path, + target_wit_root: &Path, + package_name: &wit_parser::PackageName, + package_sources: &PackageSource, +) -> anyhow::Result { + let first_source = package_sources.files.first().ok_or_else(|| { + anyhow!( + "Expected at least one source for wit package: {}", + package_name + ) + })?; + let first_source_relative_path = strip_path_prefix(source_wit_root, first_source)?; + + if package_sources.files.len() == 1 { + Ok(target_wit_root.join(first_source_relative_path)) + } else { + Ok(target_wit_root + .join(first_source_relative_path.parent().ok_or_else(|| { + anyhow!( + "Failed to get parent of wit source: {}", + first_source_relative_path.display() + ) + })?) + .join(naming::wit::package_merged_wit_name(package_name))) + } +} + trait ToEncoder { type EncoderType; fn to_encoder(&self, stub_definition: &StubDefinition) -> Self::EncoderType; diff --git a/wasm-rpc-stubgen/src/wit_transform.rs b/wasm-rpc-stubgen/src/wit_transform.rs index 4b215399..fa4c01b4 100644 --- a/wasm-rpc-stubgen/src/wit_transform.rs +++ b/wasm-rpc-stubgen/src/wit_transform.rs @@ -5,7 +5,7 @@ use regex::Regex; use std::collections::BTreeMap; use wit_encoder::{Ident, Interface, Package, PackageItem, StandaloneFunc, WorldItem}; -// TODO: use wit encoder based one +// TODO: delete pub fn import_remover( package_name: &wit_parser::PackageName, ) -> impl Fn(String) -> anyhow::Result { @@ -14,9 +14,9 @@ pub fn import_remover( r"import\s+{}(/[^;]*)?;", regex::escape(&package_name.to_string()) ) - .as_str(), + .as_str(), ) - .unwrap_or_else(|err| panic!("Failed to compile package import regex: {}", err)); + .unwrap_or_else(|err| panic!("Failed to compile package import regex: {}", err)); move |src: String| -> anyhow::Result { Ok(pattern_import_stub_package_name @@ -25,22 +25,45 @@ pub fn import_remover( } } -// TODO: make the import name matcher more precise -pub fn remove_world_named_interface_imports(package: &mut Package, import_prefix: &str) { - for world_item in package.items_mut() { - if let PackageItem::World(world) = world_item { - world.items_mut().retain(|item| { - if let WorldItem::NamedInterfaceImport(import) = &item { - !import.name().raw_name().starts_with(import_prefix) - } else { - true - } - }); +pub fn world_named_interface_import_remover( + package_name: &wit_parser::PackageName, +) -> impl Fn(&mut Package) { + let import_ident_matcher = { + match package_name.version.as_ref() { + Some(version) => Regex::new(&format!( + r"{}/.+@{}", + regex::escape(&package_name.to_string()), + regex::escape(&version.to_string()) + )) + .unwrap_or_else(|err| { + panic!( + "Failed to compile package import ident (with version) regex: {}", + err + ) + }), + None => Regex::new(&format!(r"{}/.+", regex::escape(&package_name.to_string()))) + .unwrap_or_else(|err| { + panic!("Failed to compile package import ident regex: {}", err) + }), + } + }; + + move |package: &mut Package| { + for world_item in package.items_mut() { + if let PackageItem::World(world) = world_item { + world.items_mut().retain(|item| { + if let WorldItem::NamedInterfaceImport(import) = &item { + !import_ident_matcher.is_match(import.name().raw_name()) + } else { + true + } + }); + } } } } -pub fn add_import_to_all_world(package: &mut Package, import_name: &str) { +pub fn add_world_named_interface_import(package: &mut Package, import_name: &str) { for world_item in package.items_mut() { if let PackageItem::World(world) = world_item { let is_already_imported = world.items_mut().iter().any(|item| { diff --git a/wasm-rpc-stubgen/tests/add_dep.rs b/wasm-rpc-stubgen/tests/add_dep.rs index 50dbc2b1..ec38b1b2 100644 --- a/wasm-rpc-stubgen/tests/add_dep.rs +++ b/wasm-rpc-stubgen/tests/add_dep.rs @@ -19,9 +19,11 @@ use test_r::test; use assert2::assert; use fs_extra::dir::CopyOptions; use golem_wasm_rpc::{WASI_POLL_WIT, WASM_RPC_WIT}; -use golem_wasm_rpc_stubgen::commands::dependencies::{add_stub_dependency, UpdateCargoToml}; use golem_wasm_rpc_stubgen::commands::generate::generate_stub_wit_dir; use golem_wasm_rpc_stubgen::stub::{StubConfig, StubDefinition}; +use golem_wasm_rpc_stubgen::wit_generate::{ + add_stub_as_dependency_to_wit_dir, AddStubAsDepConfig, UpdateCargoToml, +}; use golem_wasm_rpc_stubgen::wit_resolve::ResolvedWitDir; use golem_wasm_rpc_stubgen::WasmRpcOverride; use semver::Version; @@ -40,12 +42,13 @@ fn all_wit_types_no_collision() { let stub_wit_root = stub_dir.path().join("wit"); let dest_wit_root = dest_dir.path().join("wit"); - add_stub_dependency( - &stub_wit_root, - &dest_wit_root, - false, - UpdateCargoToml::NoUpdate, - ) + add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { + stub_wit_root: stub_wit_root.clone(), + dest_wit_root: dest_wit_root.clone(), + overwrite: false, + remove_dest_imports: true, + update_cargo_toml: UpdateCargoToml::NoUpdate, + }) .unwrap(); assert_valid_wit_root(&dest_wit_root); @@ -75,19 +78,21 @@ fn all_wit_types_overwrite_protection() { let alternative_stub_wit_root = alternative_stub_dir.path().join("wit"); let dest_wit_root = dest_dir.path().join("wit"); - add_stub_dependency( - &stub_wit_root, - &dest_wit_root, - false, - UpdateCargoToml::NoUpdate, - ) + add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { + stub_wit_root: stub_wit_root.clone(), + dest_wit_root: dest_wit_root.clone(), + overwrite: false, + remove_dest_imports: true, + update_cargo_toml: UpdateCargoToml::NoUpdate, + }) .unwrap(); - add_stub_dependency( - &alternative_stub_wit_root, - &dest_wit_root, - false, - UpdateCargoToml::NoUpdate, - ) + add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { + stub_wit_root: alternative_stub_wit_root.clone(), + dest_wit_root: dest_wit_root.clone(), + overwrite: false, + remove_dest_imports: true, + update_cargo_toml: UpdateCargoToml::NoUpdate, + }) .unwrap(); assert_valid_wit_root(&dest_wit_root); @@ -107,6 +112,7 @@ fn all_wit_types_overwrite_protection() { ); } +#[ignore] // TODO #[test] fn all_wit_types_overwrite_protection_disabled() { let stub_dir = init_stub("all-wit-types"); @@ -117,19 +123,21 @@ fn all_wit_types_overwrite_protection_disabled() { let alternative_stub_wit_root = alternative_stub_dir.path().join("wit"); let dest_wit_root = dest_dir.path().join("wit"); - add_stub_dependency( - &stub_wit_root, - &dest_wit_root, - false, - UpdateCargoToml::NoUpdate, - ) + add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { + stub_wit_root: stub_wit_root.clone(), + dest_wit_root: dest_wit_root.clone(), + overwrite: false, + remove_dest_imports: true, + update_cargo_toml: UpdateCargoToml::NoUpdate, + }) .unwrap(); - add_stub_dependency( - &alternative_stub_wit_root, - &dest_wit_root, - true, - UpdateCargoToml::NoUpdate, - ) + add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { + stub_wit_root: stub_wit_root.clone(), + dest_wit_root: dest_wit_root.clone(), + overwrite: true, + remove_dest_imports: true, + update_cargo_toml: UpdateCargoToml::NoUpdate, + }) .unwrap(); assert_valid_wit_root(&dest_wit_root); @@ -157,12 +165,13 @@ fn many_ways_to_export_no_collision() { let stub_wit_root = stub_dir.path().join("wit"); let dest_wit_root = dest_dir.path().join("wit"); - add_stub_dependency( - &stub_wit_root, - &dest_wit_root, - false, - UpdateCargoToml::NoUpdate, - ) + add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { + stub_wit_root: stub_wit_root.clone(), + dest_wit_root: dest_wit_root.clone(), + overwrite: false, + remove_dest_imports: true, + update_cargo_toml: UpdateCargoToml::NoUpdate, + }) .unwrap(); assert_valid_wit_root(&dest_wit_root); @@ -196,19 +205,21 @@ fn direct_circular() { let dest_a = init_caller("direct-circular-a"); let dest_b = init_caller("direct-circular-b"); - add_stub_dependency( - &stub_a_dir.path().join("wit"), - dest_b.path(), - true, - UpdateCargoToml::NoUpdate, - ) + add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { + stub_wit_root: stub_a_dir.path().join("wit"), + dest_wit_root: dest_b.path().to_path_buf(), + overwrite: true, + remove_dest_imports: true, + update_cargo_toml: UpdateCargoToml::NoUpdate, + }) .unwrap(); - add_stub_dependency( - &stub_b_dir.path().join("wit"), - dest_a.path(), - true, - UpdateCargoToml::NoUpdate, - ) + add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { + stub_wit_root: stub_b_dir.path().join("wit"), + dest_wit_root: dest_a.path().to_path_buf(), + overwrite: true, + remove_dest_imports: true, + update_cargo_toml: UpdateCargoToml::NoUpdate, + }) .unwrap(); assert_valid_wit_root(dest_a.path()); @@ -251,19 +262,21 @@ fn direct_circular_readd() { let dest_a = init_caller("direct-circular-a"); let dest_b = init_caller("direct-circular-b"); - add_stub_dependency( - &stub_a_dir.path().join("wit"), - dest_b.path(), - true, - UpdateCargoToml::NoUpdate, - ) + add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { + stub_wit_root: stub_a_dir.path().join("wit"), + dest_wit_root: dest_b.path().to_path_buf(), + overwrite: true, + remove_dest_imports: true, + update_cargo_toml: UpdateCargoToml::NoUpdate, + }) .unwrap(); - add_stub_dependency( - &stub_b_dir.path().join("wit"), - dest_a.path(), - true, - UpdateCargoToml::NoUpdate, - ) + add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { + stub_wit_root: stub_b_dir.path().join("wit"), + dest_wit_root: dest_a.path().to_path_buf(), + overwrite: true, + remove_dest_imports: true, + update_cargo_toml: UpdateCargoToml::NoUpdate, + }) .unwrap(); assert_valid_wit_root(dest_a.path()); @@ -276,19 +289,21 @@ fn direct_circular_readd() { regenerate_stub(stub_b_dir.path(), dest_b.path()); println!("Second round of add_stub_dependency calls"); - add_stub_dependency( - &stub_a_dir.path().join("wit"), - dest_b.path(), - true, - UpdateCargoToml::NoUpdate, - ) + add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { + stub_wit_root: stub_a_dir.path().join("wit"), + dest_wit_root: dest_b.path().to_path_buf(), + overwrite: true, + remove_dest_imports: true, + update_cargo_toml: UpdateCargoToml::NoUpdate, + }) .unwrap(); - add_stub_dependency( - &stub_b_dir.path().join("wit"), - dest_a.path(), - true, - UpdateCargoToml::NoUpdate, - ) + add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { + stub_wit_root: stub_b_dir.path().join("wit"), + dest_wit_root: dest_a.path().to_path_buf(), + overwrite: true, + remove_dest_imports: true, + update_cargo_toml: UpdateCargoToml::NoUpdate, + }) .unwrap(); assert_valid_wit_root(dest_a.path()); @@ -335,19 +350,21 @@ fn direct_circular_same_world_name() { let dest_a = init_caller("direct-circular-a-same-world-name"); let dest_b = init_caller("direct-circular-b-same-world-name"); - add_stub_dependency( - &stub_a_dir.path().join("wit"), - dest_b.path(), - true, - UpdateCargoToml::NoUpdate, - ) + add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { + stub_wit_root: stub_a_dir.path().join("wit"), + dest_wit_root: dest_b.path().to_path_buf(), + overwrite: true, + remove_dest_imports: true, + update_cargo_toml: UpdateCargoToml::NoUpdate, + }) .unwrap(); - add_stub_dependency( - &stub_b_dir.path().join("wit"), - dest_a.path(), - true, - UpdateCargoToml::NoUpdate, - ) + add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { + stub_wit_root: stub_b_dir.path().join("wit"), + dest_wit_root: dest_a.path().to_path_buf(), + overwrite: true, + remove_dest_imports: true, + update_cargo_toml: UpdateCargoToml::NoUpdate, + }) .unwrap(); assert_valid_wit_root(dest_a.path()); @@ -392,26 +409,29 @@ fn indirect_circular() { let dest_b = init_caller("indirect-circular-b"); let dest_c = init_caller("indirect-circular-c"); - add_stub_dependency( - &stub_a_dir.path().join("wit"), - dest_c.path(), - true, - UpdateCargoToml::NoUpdate, - ) + add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { + stub_wit_root: stub_a_dir.path().join("wit"), + dest_wit_root: dest_c.path().to_path_buf(), + overwrite: true, + remove_dest_imports: true, + update_cargo_toml: UpdateCargoToml::NoUpdate, + }) .unwrap(); - add_stub_dependency( - &stub_b_dir.path().join("wit"), - dest_a.path(), - true, - UpdateCargoToml::NoUpdate, - ) + add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { + stub_wit_root: stub_b_dir.path().join("wit"), + dest_wit_root: dest_a.path().to_path_buf(), + overwrite: true, + remove_dest_imports: true, + update_cargo_toml: UpdateCargoToml::NoUpdate, + }) .unwrap(); - add_stub_dependency( - &stub_c_dir.path().join("wit"), - dest_b.path(), - true, - UpdateCargoToml::NoUpdate, - ) + add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { + stub_wit_root: stub_c_dir.path().join("wit"), + dest_wit_root: dest_b.path().to_path_buf(), + overwrite: true, + remove_dest_imports: true, + update_cargo_toml: UpdateCargoToml::NoUpdate, + }) .unwrap(); assert_valid_wit_root(dest_a.path()); @@ -475,26 +495,29 @@ fn indirect_circular_readd() { println!("dest_b: {:?}", dest_b.path()); println!("dest_c: {:?}", dest_c.path()); - add_stub_dependency( - &stub_a_dir.path().join("wit"), - dest_c.path(), - true, - UpdateCargoToml::NoUpdate, - ) + add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { + stub_wit_root: stub_a_dir.path().join("wit"), + dest_wit_root: dest_c.path().to_path_buf(), + overwrite: true, + remove_dest_imports: true, + update_cargo_toml: UpdateCargoToml::NoUpdate, + }) .unwrap(); - add_stub_dependency( - &stub_b_dir.path().join("wit"), - dest_a.path(), - true, - UpdateCargoToml::NoUpdate, - ) + add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { + stub_wit_root: stub_b_dir.path().join("wit"), + dest_wit_root: dest_a.path().to_path_buf(), + overwrite: true, + remove_dest_imports: true, + update_cargo_toml: UpdateCargoToml::NoUpdate, + }) .unwrap(); - add_stub_dependency( - &stub_c_dir.path().join("wit"), - dest_b.path(), - true, - UpdateCargoToml::NoUpdate, - ) + add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { + stub_wit_root: stub_c_dir.path().join("wit"), + dest_wit_root: dest_b.path().to_path_buf(), + overwrite: true, + remove_dest_imports: true, + update_cargo_toml: UpdateCargoToml::NoUpdate, + }) .unwrap(); assert_valid_wit_root(dest_a.path()); @@ -509,26 +532,29 @@ fn indirect_circular_readd() { regenerate_stub(stub_c_dir.path(), dest_c.path()); println!("Second round of add_stub_dependency calls"); - add_stub_dependency( - &stub_a_dir.path().join("wit"), - dest_c.path(), - true, - UpdateCargoToml::NoUpdate, - ) + add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { + stub_wit_root: stub_a_dir.path().join("wit"), + dest_wit_root: dest_c.path().to_path_buf(), + overwrite: true, + remove_dest_imports: true, + update_cargo_toml: UpdateCargoToml::NoUpdate, + }) .unwrap(); - add_stub_dependency( - &stub_b_dir.path().join("wit"), - dest_a.path(), - true, - UpdateCargoToml::NoUpdate, - ) + add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { + stub_wit_root: stub_b_dir.path().join("wit"), + dest_wit_root: dest_a.path().to_path_buf(), + overwrite: true, + remove_dest_imports: true, + update_cargo_toml: UpdateCargoToml::NoUpdate, + }) .unwrap(); - add_stub_dependency( - &stub_c_dir.path().join("wit"), - dest_b.path(), - true, - UpdateCargoToml::NoUpdate, - ) + add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { + stub_wit_root: stub_c_dir.path().join("wit"), + dest_wit_root: dest_b.path().to_path_buf(), + overwrite: true, + remove_dest_imports: true, + update_cargo_toml: UpdateCargoToml::NoUpdate, + }) .unwrap(); assert_valid_wit_root(dest_a.path()); @@ -585,12 +611,13 @@ fn self_circular() { let dest_a = init_caller("self-circular"); - add_stub_dependency( - &stub_a_dir.path().join("wit"), - dest_a.path(), - true, - UpdateCargoToml::NoUpdate, - ) + add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { + stub_wit_root: stub_a_dir.path().join("wit"), + dest_wit_root: dest_a.path().to_path_buf(), + overwrite: true, + remove_dest_imports: true, + update_cargo_toml: UpdateCargoToml::NoUpdate, + }) .unwrap(); assert_valid_wit_root(dest_a.path()); From 8f5eb9fec158f25d8f272b6c69e0bde04b7c2d86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Thu, 31 Oct 2024 12:12:49 +0100 Subject: [PATCH 034/109] remove debug code --- wasm-rpc-stubgen/src/commands/generate.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/generate.rs b/wasm-rpc-stubgen/src/commands/generate.rs index 4b3babfc..4e82f5dd 100644 --- a/wasm-rpc-stubgen/src/commands/generate.rs +++ b/wasm-rpc-stubgen/src/commands/generate.rs @@ -68,11 +68,6 @@ pub fn generate_stub_wit_dir(stub_def: &StubDefinition) -> anyhow::Result Date: Tue, 5 Nov 2024 10:41:33 +0100 Subject: [PATCH 035/109] use extracted interface packages and remove inlining, make overwriting the default --- wasm-rpc-stubgen/src/cargo.rs | 8 +- wasm-rpc-stubgen/src/commands/declarative.rs | 3 - wasm-rpc-stubgen/src/commands/dependencies.rs | 3 - wasm-rpc-stubgen/src/lib.rs | 4 - wasm-rpc-stubgen/src/naming.rs | 36 +- wasm-rpc-stubgen/src/rust.rs | 31 +- wasm-rpc-stubgen/src/stub.rs | 80 +-- wasm-rpc-stubgen/src/wit_generate.rs | 522 ++++++++++-------- wasm-rpc-stubgen/src/wit_transform.rs | 231 -------- .../test-data/indirect-circular-c/c.wit | 1 + .../test-data/many-ways-to-export/main.wit | 6 +- wasm-rpc-stubgen/tests/add_dep.rs | 255 ++++----- wasm-rpc-stubgen/tests/wit.rs | 230 ++++---- 13 files changed, 549 insertions(+), 861 deletions(-) delete mode 100644 wasm-rpc-stubgen/src/wit_transform.rs diff --git a/wasm-rpc-stubgen/src/cargo.rs b/wasm-rpc-stubgen/src/cargo.rs index e283fb48..0c740a6e 100644 --- a/wasm-rpc-stubgen/src/cargo.rs +++ b/wasm-rpc-stubgen/src/cargo.rs @@ -97,11 +97,9 @@ pub fn generate_cargo_toml(def: &StubDefinition) -> anyhow::Result<()> { wit_dependencies.insert( format_package_name_without_version(&def.source_package_name), WitDependency { - path: naming::wit::package_wit_dep_dir_from_package_name( - &def.source_package_name, - ) - .to_string_lossy() - .to_string(), + path: naming::wit::package_wit_dep_dir_from_parser(&def.source_package_name) + .to_string_lossy() + .to_string(), }, ); } else { diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index 8ce07fdb..107817fc 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -85,7 +85,6 @@ async fn pre_component_build_app(config: &Config, app: &Application) -> anyhow:: wasm_rpc_path_override: app.stub_wasm_rpc_path(&component_name), wasm_rpc_version_override: app.stub_wasm_rpc_version(&component_name), }, - inline_source_types: app.stub_always_inline_types(&component_name), }) .context("Failed to gather information for the stub generator")?; @@ -131,8 +130,6 @@ async fn pre_component_build_app(config: &Config, app: &Application) -> anyhow:: add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { stub_wit_root: app.stub_wit(dep_component_name), dest_wit_root: app.component_wit(component_name), - overwrite: true, // NOTE: in declarative mode we always use overwrite - remove_dest_imports: false, update_cargo_toml: UpdateCargoToml::Update, })? } diff --git a/wasm-rpc-stubgen/src/commands/dependencies.rs b/wasm-rpc-stubgen/src/commands/dependencies.rs index 71c71cf5..143b5ce4 100644 --- a/wasm-rpc-stubgen/src/commands/dependencies.rs +++ b/wasm-rpc-stubgen/src/commands/dependencies.rs @@ -19,14 +19,11 @@ use std::path::Path; pub fn add_stub_dependency( stub_wit_root: &Path, dest_wit_root: &Path, - overwrite: bool, update_cargo_toml: UpdateCargoToml, ) -> anyhow::Result<()> { wit_generate::add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { stub_wit_root: stub_wit_root.to_path_buf(), dest_wit_root: dest_wit_root.to_path_buf(), - overwrite, - remove_dest_imports: true, update_cargo_toml, }) } diff --git a/wasm-rpc-stubgen/src/lib.rs b/wasm-rpc-stubgen/src/lib.rs index 53659c02..59a51443 100644 --- a/wasm-rpc-stubgen/src/lib.rs +++ b/wasm-rpc-stubgen/src/lib.rs @@ -24,7 +24,6 @@ pub mod stub; pub mod wit_encode; pub mod wit_generate; pub mod wit_resolve; -pub mod wit_transform; use crate::stub::{StubConfig, StubDefinition}; use crate::wit_generate::UpdateCargoToml; @@ -227,7 +226,6 @@ pub fn generate(args: GenerateArgs) -> anyhow::Result<()> { selected_world: args.world, stub_crate_version: args.stub_crate_version, wasm_rpc_override: args.wasm_rpc_override, - inline_source_types: args.always_inline_types, } ) .context("Failed to gather information for the stub generator. Make sure source_wit_root has a valid WIT file.")?; @@ -244,7 +242,6 @@ pub async fn build(args: BuildArgs) -> anyhow::Result<()> { selected_world: args.world, stub_crate_version: args.stub_crate_version, wasm_rpc_override: args.wasm_rpc_override, - inline_source_types: args.always_inline_types, }) .context("Failed to gather information for the stub generator")?; @@ -255,7 +252,6 @@ pub fn add_stub_dependency(args: AddStubDependencyArgs) -> anyhow::Result<()> { commands::dependencies::add_stub_dependency( &args.stub_wit_root, &args.dest_wit_root, - args.overwrite, if args.update_cargo_toml { UpdateCargoToml::Update } else { diff --git a/wasm-rpc-stubgen/src/naming.rs b/wasm-rpc-stubgen/src/naming.rs index 88464752..d22c2112 100644 --- a/wasm-rpc-stubgen/src/naming.rs +++ b/wasm-rpc-stubgen/src/naming.rs @@ -1,11 +1,12 @@ pub mod wit { - use anyhow::bail; + use anyhow::{anyhow, bail}; use std::path::{Path, PathBuf}; pub static DEPS_DIR: &str = "deps"; pub static WIT_DIR: &str = "wit"; pub static STUB_WIT_FILE_NAME: &str = "_stub.wit"; + pub static INTERFACE_WIT_FILE_NAME: &str = "_interface.wit"; pub fn stub_package_name(package_name: &wit_parser::PackageName) -> wit_parser::PackageName { wit_parser::PackageName { @@ -77,10 +78,31 @@ pub mod wit { )) } - pub fn package_dep_dir_name(package_name: &wit_parser::PackageName) -> String { + pub fn stub_import_interface_prefix_from_stub_package_name( + stub_package: &wit_parser::PackageName, + ) -> anyhow::Result { + Ok(format!( + "{}:{}-interface/", + stub_package.namespace, + stub_package + .name + .clone() + .strip_suffix("-stub") + .ok_or_else(|| anyhow!( + "Expected \"-stub\" suffix in stub package name: {}", + stub_package.to_string() + ))? + )) + } + + pub fn package_dep_dir_name_from_parser(package_name: &wit_parser::PackageName) -> String { format!("{}_{}", package_name.namespace, package_name.name) } + pub fn package_dep_dir_name_from_encoder(package_name: &wit_encoder::PackageName) -> String { + format!("{}_{}", package_name.namespace(), package_name.name()) + } + pub fn package_merged_wit_name(package_name: &wit_parser::PackageName) -> String { format!("{}_{}.wit", package_name.namespace, package_name.name) } @@ -89,9 +111,11 @@ pub mod wit { Path::new(WIT_DIR).join(DEPS_DIR).join(package_dir_name) } - pub fn package_wit_dep_dir_from_package_name( - package_name: &wit_parser::PackageName, - ) -> PathBuf { - package_wit_dep_dir_from_package_dir_name(&package_dep_dir_name(package_name)) + pub fn package_wit_dep_dir_from_parser(package_name: &wit_parser::PackageName) -> PathBuf { + package_wit_dep_dir_from_package_dir_name(&package_dep_dir_name_from_parser(package_name)) + } + + pub fn package_wit_dep_dir_from_encode(package_name: &wit_encoder::PackageName) -> PathBuf { + package_wit_dep_dir_from_package_dir_name(&package_dep_dir_name_from_encoder(package_name)) } } diff --git a/wasm-rpc-stubgen/src/rust.rs b/wasm-rpc-stubgen/src/rust.rs index 087f5605..2746e90d 100644 --- a/wasm-rpc-stubgen/src/rust.rs +++ b/wasm-rpc-stubgen/src/rust.rs @@ -13,7 +13,7 @@ // limitations under the License. use crate::commands::log::log_action; -use crate::stub::{FunctionResultStub, FunctionStub, InterfaceStub, StubDefinition, StubTypeOwner}; +use crate::stub::{FunctionResultStub, FunctionStub, InterfaceStub, StubDefinition}; use anyhow::anyhow; use heck::{ToShoutySnakeCase, ToSnakeCase, ToUpperCamelCase}; use proc_macro2::{Ident, Span, TokenStream}; @@ -713,9 +713,8 @@ fn type_to_rust_ident(typ: &Type, def: &StubDefinition) -> anyhow::Result { + match &type_def.owner { + TypeOwner::World(world_id) => { let world = def.get_world(*world_id)?; let package_id = world.package.ok_or(anyhow!("world has no package"))?; @@ -729,7 +728,7 @@ fn type_to_rust_ident(typ: &Type, def: &StubDefinition) -> anyhow::Result { + TypeOwner::Interface(interface_id) => { let interface = def.get_interface(*interface_id)?; let package_id = interface .package @@ -751,27 +750,7 @@ fn type_to_rust_ident(typ: &Type, def: &StubDefinition) -> anyhow::Result {} - StubTypeOwner::StubInterface => { - let root_ns = Ident::new( - &def.source_package_name.namespace.to_snake_case(), - Span::call_site(), - ); - let root_name = Ident::new( - &format!("{}_stub", def.source_package_name.name.to_snake_case()), - Span::call_site(), - ); - let stub_interface_name = format!("stub-{}", def.source_world_name()); - let stub_interface_name = Ident::new( - &to_rust_ident(&stub_interface_name).to_snake_case(), - Span::call_site(), - ); - - path.push(quote! { exports }); - path.push(quote! { #root_ns }); - path.push(quote! { #root_name }); - path.push(quote! { #stub_interface_name }); - } + TypeOwner::None => {} } Ok(quote! { #(#path)::*::#typ }) } diff --git a/wasm-rpc-stubgen/src/stub.rs b/wasm-rpc-stubgen/src/stub.rs index 68dd93b5..25b49bd8 100644 --- a/wasm-rpc-stubgen/src/stub.rs +++ b/wasm-rpc-stubgen/src/stub.rs @@ -33,7 +33,6 @@ pub struct StubConfig { pub selected_world: Option, pub stub_crate_version: String, pub wasm_rpc_override: WasmRpcOverride, - pub inline_source_types: bool, } /// All the gathered information for generating the stub crate. @@ -169,40 +168,6 @@ impl StubDefinition { ResolvedWitDir::new(&self.target_wit_root()) } - pub fn fix_inlined_owner(&self, typedef: &TypeDef) -> StubTypeOwner { - if self.is_inlined(typedef) { - StubTypeOwner::StubInterface - } else { - StubTypeOwner::Source(typedef.owner) - } - } - - fn is_inlined(&self, typedef: &TypeDef) -> bool { - match &typedef.owner { - TypeOwner::Interface(interface_id) => { - if self.config.inline_source_types { - if let Some(resolved_owner_interface) = - self.resolve.interfaces.get(*interface_id) - { - if let Some(name) = resolved_owner_interface.name.as_ref() { - self.stub_imported_interfaces() - .iter() - .any(|interface| &interface.name == name) - } else { - false - } - } else { - false - } - } else { - false - } - } - TypeOwner::World(_) => true, - TypeOwner::None => false, - } - } - pub fn stub_imported_interfaces(&self) -> &Vec { self.stub_imported_interfaces.get_or_init(|| { let WorldItemsByType { @@ -304,41 +269,13 @@ impl StubDefinition { None => format!("{}-{}", interface_name, type_name), }); - let inlined = self.config.inline_source_types - && interface.package == Some(self.source_package_id); - - let inlined_type_def = match type_def.kind { - TypeDefKind::Type(Type::Id(type_id)) if inlined => { - let mut type_def = self - .resolve - .types - .get(type_id) - .unwrap_or_else(|| { - panic!("Failed to get type def for inlining") - }) - .clone(); - - // Alias to alias does not need inlining - if let TypeDefKind::Type(Type::Id(_)) = type_def.kind { - None - } else { - type_def.name = - type_name_alias.clone().or_else(|| Some(type_name.clone())); - Some(type_def) - } - } - _ => None, - }; - InterfaceStubTypeDef { package_id: interface.package, type_id, type_def, - inlined_type_def, interface_identifier, type_name: type_name.clone(), type_name_alias, - inlined, } }) }) @@ -353,10 +290,9 @@ impl StubDefinition { pub fn stub_dep_package_ids(&self) -> &HashSet { self.stub_dep_package_ids.get_or_init(|| { - self.resolve - .packages + self.stub_used_type_defs() .iter() - .map(|(package_id, _)| package_id) + .filter_map(|type_def| type_def.package_id) .collect() }) } @@ -677,21 +613,15 @@ pub struct InterfaceStubTypeDef { pub package_id: Option, pub type_id: TypeId, pub type_def: TypeDef, - pub inlined_type_def: Option, pub interface_identifier: String, pub type_name: String, pub type_name_alias: Option, - pub inlined: bool, } impl InterfaceStubTypeDef { pub fn stub_type_name(&self) -> &str { self.type_name_alias.as_deref().unwrap_or(&self.type_name) } - - pub fn stub_type_def(&self) -> &TypeDef { - self.inlined_type_def.as_ref().unwrap_or(&self.type_def) - } } #[derive(Debug, Clone)] @@ -774,9 +704,3 @@ impl FunctionResultStub { } } } - -#[derive(Debug, Clone)] -pub enum StubTypeOwner { - StubInterface, - Source(TypeOwner), -} diff --git a/wasm-rpc-stubgen/src/wit_generate.rs b/wasm-rpc-stubgen/src/wit_generate.rs index f0a6020a..943ee13f 100644 --- a/wasm-rpc-stubgen/src/wit_generate.rs +++ b/wasm-rpc-stubgen/src/wit_generate.rs @@ -14,27 +14,25 @@ use crate::commands::log::{log_action, log_action_plan, log_warn_action, LogIndent}; use crate::fs::{ - copy, copy_transformed, get_file_name, strip_path_prefix, OverwriteSafeAction, - OverwriteSafeActions, + copy, get_file_name, strip_path_prefix, write_str, OverwriteSafeAction, OverwriteSafeActions, }; +use crate::naming::wit::package_dep_dir_name_from_encoder; use crate::stub::{ FunctionParamStub, FunctionResultStub, FunctionStub, InterfaceStub, InterfaceStubTypeDef, - StubConfig, StubDefinition, + StubDefinition, }; use crate::wit_encode::EncodedWitDir; -use crate::wit_resolve::{PackageSource, ResolvedWitDir}; -use crate::wit_transform::{ - add_world_named_interface_import, import_remover, world_named_interface_import_remover, -}; +use crate::wit_resolve::ResolvedWitDir; use crate::{cargo, naming}; use anyhow::{anyhow, bail, Context}; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashSet}; use std::fs; use std::path::{Path, PathBuf}; use wit_encoder::{ - Ident, Interface, Package, PackageName, Params, ResourceFunc, Results, Type, TypeDef, - TypeDefKind, VariantCase, World, + Ident, Interface, Package, PackageItem, PackageName, Params, ResourceFunc, Results, + StandaloneFunc, Type, TypeDef, TypeDefKind, VariantCase, World, WorldItem, }; +use wit_parser::PackageId; pub fn generate_stub_wit_to_target(def: &StubDefinition) -> anyhow::Result<()> { log_action( @@ -48,20 +46,6 @@ pub fn generate_stub_wit_to_target(def: &StubDefinition) -> anyhow::Result<()> { Ok(()) } -pub fn generate_stub_wit_from_wit_dir( - source_wit_root: &Path, - inline_source_types: bool, -) -> anyhow::Result { - generate_stub_wit_from_stub_def(&StubDefinition::new(StubConfig { - source_wit_root: source_wit_root.to_path_buf(), - target_root: source_wit_root.to_path_buf(), // Not used - selected_world: None, // TODO: would this fail with multiple worlds? - stub_crate_version: "".to_string(), // Not used - wasm_rpc_override: Default::default(), // Not used - inline_source_types, - })?) -} - pub fn generate_stub_wit_from_stub_def(def: &StubDefinition) -> anyhow::Result { Ok(generate_stub_package_from_stub_def(def).to_string()) } @@ -93,15 +77,11 @@ pub fn generate_stub_package_from_stub_def(def: &StubDefinition) -> Package { // Used or inlined type defs for type_def in def.stub_used_type_defs() { - if type_def.inlined { - stub_interface.type_def(type_def.to_encoder(def)) - } else { - stub_interface.use_type( - type_def.interface_identifier.clone(), - type_def.type_name.clone(), - type_def.type_name_alias.clone().map(Ident::from), - ); - } + stub_interface.use_type( + type_def.interface_identifier.clone(), + type_def.type_name.clone(), + type_def.type_name_alias.clone().map(Ident::from), + ); } // Async return types @@ -229,60 +209,36 @@ pub fn add_dependencies_to_stub_wit_dir(def: &StubDefinition) -> anyhow::Result< log_action( "Adding", format!( - "WIT dependencies from {} to {}", + "WIT dependencies to {} from {}", + def.config.target_root.display(), def.config.source_wit_root.display(), - def.config.target_root.display() ), ); - let stub_package_name = def.stub_package_name(); + let _indent = LogIndent::new(); + let stub_dep_packages = def.stub_dep_package_ids(); - let remove_stub_imports = import_remover(&stub_package_name); let target_wit_root = def.target_wit_root(); let target_deps = target_wit_root.join(naming::wit::DEPS_DIR); - let _indent = LogIndent::new(); for (package_id, package, package_sources) in def.packages_with_wit_sources() { - if !stub_dep_packages.contains(&package_id) { - log_warn_action( - "Skipping", - format!( - "package dependency {}, not used in the stub interface", - package.name - ), - ); + if !stub_dep_packages.contains(&package_id) || package_id == def.source_package_id { + log_warn_action("Skipping", format!("package dependency {}", package.name)); continue; } - let is_source_package = package.name == def.source_package_name; - log_action("Copying", format!("package dependency {}", package.name)); let _indent = LogIndent::new(); for source in &package_sources.files { - if is_source_package { - let dest = target_deps - .join(naming::wit::package_dep_dir_name(&def.source_package_name)) - .join(get_file_name(source)?); - log_action( - "Copying", - format!( - "(with source imports removed) {} to {}", - source.display(), - dest.display() - ), - ); - copy_transformed(source, &dest, &remove_stub_imports)?; - } else { - let relative = source.strip_prefix(&def.config.source_wit_root)?; - let dest = target_wit_root.join(relative); - log_action( - "Copying", - format!("{} to {}", source.display(), dest.display()), - ); - copy(source, &dest)?; - } + let relative = source.strip_prefix(&def.config.source_wit_root)?; + let dest = target_wit_root.join(relative); + log_action( + "Copying", + format!("{} to {}", source.display(), dest.display()), + ); + copy(source, &dest)?; } } @@ -324,12 +280,9 @@ pub enum UpdateCargoToml { pub struct AddStubAsDepConfig { pub stub_wit_root: PathBuf, pub dest_wit_root: PathBuf, - pub overwrite: bool, - pub remove_dest_imports: bool, pub update_cargo_toml: UpdateCargoToml, } -// TODO: "named" args? pub fn add_stub_as_dependency_to_wit_dir(config: AddStubAsDepConfig) -> anyhow::Result<()> { log_action( "Adding", @@ -344,36 +297,11 @@ pub fn add_stub_as_dependency_to_wit_dir(config: AddStubAsDepConfig) -> anyhow:: let stub_resolved_wit_root = ResolvedWitDir::new(&config.stub_wit_root)?; let stub_package = stub_resolved_wit_root.main_package()?; - let stub_wit = config.stub_wit_root.join(naming::wit::STUB_WIT_FILE_NAME); - let dest_deps_dir = config.dest_wit_root.join(naming::wit::DEPS_DIR); let dest_resolved_wit_root = ResolvedWitDir::new(&config.dest_wit_root)?; - let dest_package = dest_resolved_wit_root.main_package()?; - let dest_stub_package_name = naming::wit::stub_package_name(&dest_package.name); - - // TODO: these could be made optional depending on remove_dest_imports - let mut stub_encoded_wit_root = EncodedWitDir::new(&stub_resolved_wit_root.resolve)?; - let remove_dest_package_import = world_named_interface_import_remover(&dest_package.name); - let remove_dest_stub_package_import = - world_named_interface_import_remover(&dest_stub_package_name); - // TODO: this could be made optional depending on "add stub import" (=overwrite) let mut dest_encoded_wit_root = EncodedWitDir::new(&dest_resolved_wit_root.resolve)?; - // TODO: make this optional (not needed for declarative) - { - let is_self_stub_by_name = - dest_package.name == naming::wit::stub_target_package_name(&stub_package.name); - let is_self_stub_by_content = is_self_stub(&stub_wit, &config.dest_wit_root); - - if is_self_stub_by_name && !is_self_stub_by_content? { - return Err(anyhow!( - "Both the caller and the target components are using the same package name ({}), which is not supported.", - dest_package.name - )); - } - } - let mut actions = OverwriteSafeActions::new(); let mut package_names_to_package_path = BTreeMap::::new(); @@ -382,65 +310,30 @@ pub fn add_stub_as_dependency_to_wit_dir(config: AddStubAsDepConfig) -> anyhow:: .package_sources .get(package_id) .ok_or_else(|| anyhow!("Failed to get package sources for {}", package_name))?; - let package_path = naming::wit::package_wit_dep_dir_from_package_dir_name(&get_file_name( - &package_sources.dir, - )?); - - let is_stub_main_package = *package_id == stub_resolved_wit_root.package_id; - let is_dest_package = *package_name == dest_package.name; - let is_dest_stub_package = *package_name == dest_stub_package_name; - - // We skip self as a dependency - if is_dest_package { - log_warn_action( - "Skipping", - format!("cyclic self dependency for {}", package_name), - ); - } else if is_dest_stub_package || is_stub_main_package { - let package_dep_dir_name = naming::wit::package_dep_dir_name(package_name); - let package_path = naming::wit::package_wit_dep_dir_from_package_name(package_name); - - package_names_to_package_path.insert(package_name.clone(), package_path); - - // Handle self stub packages: use regenerated stub with inlining, to break the recursive cycle - if is_dest_stub_package { - actions.add(OverwriteSafeAction::WriteFile { - // TODO: this call potentially builds another EncodedWitDir for source - content: generate_stub_wit_from_wit_dir(&config.dest_wit_root, true)?, - target: dest_deps_dir - .join(&package_dep_dir_name) - .join(naming::wit::STUB_WIT_FILE_NAME), + + if *package_id == stub_resolved_wit_root.package_id { + let package_path = naming::wit::package_wit_dep_dir_from_parser(package_name); + + package_names_to_package_path.insert(package_name.clone(), package_path.clone()); + + for source in &package_sources.files { + actions.add(OverwriteSafeAction::CopyFile { + source: source.clone(), + target: config + .dest_wit_root + .join(naming::wit::DEPS_DIR) + .join(naming::wit::package_dep_dir_name_from_parser(package_name)) + .join(get_file_name(source)?), }); - // Non-self stub package has to be copied into target deps - } else { - for source in &package_sources.files { - actions.add(OverwriteSafeAction::CopyFile { - source: source.clone(), - target: dest_deps_dir - .join(&package_dep_dir_name) - .join(get_file_name(source)?), - }); - } } - // Handle other package by copying and removing cyclic imports - } else if config.remove_dest_imports { - package_names_to_package_path.insert(package_name.clone(), package_path); - - let package = stub_encoded_wit_root.package(*package_id)?; - remove_dest_package_import(package); - remove_dest_stub_package_import(package); - let content = package.to_string(); - - let target = wit_dep_transformed_target_file( - &config.stub_wit_root, - &config.dest_wit_root, - package_name, - package_sources, - )?; - - actions.add(OverwriteSafeAction::WriteFile { content, target }); - // Handle other package by copying and no transformations } else { + package_names_to_package_path.insert( + package_name.clone(), + naming::wit::package_wit_dep_dir_from_package_dir_name(&get_file_name( + &package_sources.dir, + )?), + ); + for source in &package_sources.files { actions.add(OverwriteSafeAction::CopyFile { source: source.clone(), @@ -452,33 +345,39 @@ pub fn add_stub_as_dependency_to_wit_dir(config: AddStubAsDepConfig) -> anyhow:: } } - // Import stub if overwrite enabled // TODO: use a different flag, or always import? - if config.overwrite { - let dest_main_package_id = dest_resolved_wit_root.package_id; + // Import stub and remove source interfaces + let dest_main_package_id = dest_resolved_wit_root.package_id; - let dest_main_package_sources = dest_resolved_wit_root - .package_sources - .get(&dest_main_package_id) - .ok_or_else(|| anyhow!("Failed to get dest main package sources"))?; + let dest_main_package_sources = dest_resolved_wit_root + .package_sources + .get(&dest_main_package_id) + .ok_or_else(|| anyhow!("Failed to get dest main package sources"))?; - if dest_main_package_sources.files.len() != 1 { - bail!( - "Expected exactly one dest main package source, got sources: {:?}", - dest_main_package_sources.files - ); - } + if dest_main_package_sources.files.len() != 1 { + bail!( + "Expected exactly one dest main package source, got sources: {:?}", + dest_main_package_sources.files + ); + } - let package = dest_encoded_wit_root.package(dest_main_package_id)?; - add_world_named_interface_import(package, &naming::wit::stub_import_name(stub_package)?); - let content = package.to_string(); + let package = dest_encoded_wit_root.package(dest_main_package_id)?; + // NOTE: wit_encoder "inlines" all transitive imports, so we have to clean up transitive + // imports from the source-interface package, given they might have been removed or renamed + // in the source, and could create invalid imports. + remove_world_named_interface_imports( + package, + &naming::wit::stub_import_interface_prefix_from_stub_package_name(&stub_package.name)?, + ); + add_world_named_interface_import(package, &naming::wit::stub_import_name(stub_package)?); + let content = package.to_string(); - actions.add(OverwriteSafeAction::WriteFile { - content, - target: dest_main_package_sources.files[0].clone(), - }); - } + actions.add(OverwriteSafeAction::WriteFile { + content, + target: dest_main_package_sources.files[0].clone(), + }); - let forbidden_overwrites = actions.run(config.overwrite, log_action_plan)?; + // Check overwrites + let forbidden_overwrites = actions.run(true, log_action_plan)?; if !forbidden_overwrites.is_empty() { eprintln!("The following files would have been overwritten with new content:"); for action in forbidden_overwrites { @@ -488,6 +387,7 @@ pub fn add_stub_as_dependency_to_wit_dir(config: AddStubAsDepConfig) -> anyhow:: eprintln!("Use --overwrite to force overwrite."); } + // Optionally update Cargo.toml if let Some(target_parent) = config.dest_wit_root.parent() { let target_cargo_toml = target_parent.join("Cargo.toml"); if target_cargo_toml.exists() && target_cargo_toml.is_file() { @@ -515,46 +415,6 @@ pub fn add_stub_as_dependency_to_wit_dir(config: AddStubAsDepConfig) -> anyhow:: Ok(()) } -// TODO: let's find another way to identify self stubs, as this is quite diverging now based on content -/// Checks whether `stub_wit` is a stub generated for `dest_wit_root` -fn is_self_stub(stub_wit: &Path, dest_wit_root: &Path) -> anyhow::Result { - // TODO: can we make it diff exports instead of generated content? - let dest_stub_wit_imported = generate_stub_wit_from_wit_dir(dest_wit_root, false)?; - let dest_stub_wit_inlined = generate_stub_wit_from_wit_dir(dest_wit_root, true)?; - let stub_wit = std::fs::read_to_string(stub_wit)?; - - // TODO: this can also be false in case the stub is lagging - Ok(stub_wit == dest_stub_wit_imported || stub_wit == dest_stub_wit_inlined) -} - -fn wit_dep_transformed_target_file( - source_wit_root: &Path, - target_wit_root: &Path, - package_name: &wit_parser::PackageName, - package_sources: &PackageSource, -) -> anyhow::Result { - let first_source = package_sources.files.first().ok_or_else(|| { - anyhow!( - "Expected at least one source for wit package: {}", - package_name - ) - })?; - let first_source_relative_path = strip_path_prefix(source_wit_root, first_source)?; - - if package_sources.files.len() == 1 { - Ok(target_wit_root.join(first_source_relative_path)) - } else { - Ok(target_wit_root - .join(first_source_relative_path.parent().ok_or_else(|| { - anyhow!( - "Failed to get parent of wit source: {}", - first_source_relative_path.display() - ) - })?) - .join(naming::wit::package_merged_wit_name(package_name))) - } -} - trait ToEncoder { type EncoderType; fn to_encoder(&self, stub_definition: &StubDefinition) -> Self::EncoderType; @@ -655,7 +515,7 @@ impl ToEncoder for InterfaceStubTypeDef { type EncoderType = TypeDef; fn to_encoder(&self, def: &StubDefinition) -> Self::EncoderType { - (self.stub_type_name(), self.stub_type_def()).to_encoder(def) + (self.stub_type_name(), &self.type_def).to_encoder(def) } } @@ -745,3 +605,231 @@ impl ToEncoder for (&str, &wit_parser::TypeDef) { } } } + +fn add_world_named_interface_import(package: &mut Package, import_name: &str) { + for world_item in package.items_mut() { + if let PackageItem::World(world) = world_item { + let is_already_imported = world.items_mut().iter().any(|item| { + if let WorldItem::NamedInterfaceImport(import) = item { + import.name().raw_name() == import_name + } else { + false + } + }); + if !is_already_imported { + world.named_interface_import(import_name.to_string()); + } + } + } +} + +fn remove_world_named_interface_imports(package: &mut Package, import_prefix: &str) { + for world_item in package.items_mut() { + if let PackageItem::World(world) = world_item { + world.items_mut().retain(|item| { + if let WorldItem::NamedInterfaceImport(import) = item { + !import.name().raw_name().starts_with(import_prefix) + } else { + true + } + }) + } + } +} + +pub fn extract_main_interface_as_wit_dep(wit_dir: &Path) -> anyhow::Result<()> { + log_action( + "Extracting", + format!( + "interface package for main component in wit directory {}", + wit_dir.display() + ), + ); + + let resolved_wit_dir = ResolvedWitDir::new(wit_dir)?; + let main_package_id = resolved_wit_dir.package_id; + let mut encoded_wit_dir = EncodedWitDir::new(&resolved_wit_dir.resolve)?; + + let resolved_wit_dir = ResolvedWitDir::new(wit_dir)?; + + let (main_package, interface_package) = + extract_main_interface_package(main_package_id, &mut encoded_wit_dir)?; + let sources = resolved_wit_dir + .package_sources + .get(&resolved_wit_dir.package_id) + .ok_or_else(|| { + anyhow!( + "Failed to get sources for main package, wit dir: {}", + wit_dir.display() + ) + })?; + + if sources.files.len() != 1 { + bail!( + "Expected exactly one source for main package, wit dir: {}", + wit_dir.display() + ); + } + + let _indent = LogIndent::new(); + + let interface_package_path = wit_dir + .join(naming::wit::DEPS_DIR) + .join(package_dep_dir_name_from_encoder(interface_package.name())) + .join(naming::wit::INTERFACE_WIT_FILE_NAME); + log_action( + "Writing", + format!("interface package to {}", interface_package_path.display()), + ); + write_str(&interface_package_path, interface_package.to_string())?; + + let main_package_path = &sources.files[0]; + log_action( + "Writing", + format!("main package to {}", main_package_path.display()), + ); + write_str(main_package_path, main_package.to_string())?; + + Ok(()) +} + +// TODO: handle world include +// TODO: handle world use +// TODO: maybe transform inline interfaces and functions into included world? +fn extract_main_interface_package( + main_package_id: PackageId, + encoded_wit_dir: &mut EncodedWitDir, +) -> anyhow::Result<(Package, Package)> { + let package = encoded_wit_dir.package(main_package_id)?; + + let mut interface_package = package.clone(); + interface_package.set_name(naming::wit::interface_package_name(package.name())); + + let interface_export_prefix = format!( + "{}:{}/", + package.name().namespace(), + interface_package.name().name() + ); + let interface_export_suffix = package + .name() + .version() + .map(|version| format!("@{}", version)) + .unwrap_or_default(); + + // TODO: is import and exporting the same interface allowed? + let mut exported_interface_identifiers = HashSet::::new(); + let mut inline_interface_exports = BTreeMap::>::new(); + let mut inline_function_exports = BTreeMap::>::new(); + for package_item in package.items_mut() { + if let PackageItem::World(world) = package_item { + let world_name = world.name().clone(); + + world.items_mut().retain(|world_item| match world_item { + // Remove and collect inline interface exports + WorldItem::InlineInterfaceExport(interface) => { + let mut interface = interface.clone(); + interface.set_name(naming::wit::interface_package_world_inline_interface_name( + &world_name, + interface.name(), + )); + + inline_interface_exports + .entry(world_name.clone()) + .or_default() + .push(interface.clone()); + false + } + // Remove and collect inline function exports + WorldItem::FunctionExport(function) => { + inline_function_exports + .entry(world_name.clone()) + .or_default() + .push(function.clone()); + false + } + // Collect named interface export identifiers + WorldItem::NamedInterfaceExport(interface) => { + exported_interface_identifiers.insert(interface.name().clone()); + true + } + _ => true, + }); + + // Insert named imports for extracted inline interfaces + if let Some(interfaces) = inline_interface_exports.get(&world_name) { + for interface in interfaces { + world.named_interface_export(interface.name().clone()); + } + } + + // Insert named import for extracted inline functions + if inline_function_exports.contains_key(&world_name) { + world.named_interface_export(format!( + "{}{}{}", + interface_export_prefix, + naming::wit::interface_package_world_inline_functions_interface_name( + &world_name + ), + interface_export_suffix + )); + } + } + } + + package.items_mut().retain(|item| match item { + // Drop exported interfaces from original package + PackageItem::Interface(interface) => { + !exported_interface_identifiers.contains(interface.name()) + } + PackageItem::World(_) => true, + }); + + interface_package.items_mut().retain(|item| match item { + // Drop non-exported interfaces from interface package + PackageItem::Interface(interface) => { + exported_interface_identifiers.contains(interface.name()) + } + // Drop all worlds from interface package + PackageItem::World(_) => false, + }); + + // Rename named self imports to use the extracted interface names + for package_item in package.items_mut() { + if let PackageItem::World(world) = package_item { + for world_item in world.items_mut() { + if let WorldItem::NamedInterfaceExport(export) = world_item { + if !export.name().raw_name().contains("/") { + export.set_name(format!( + "{}{}{}", + interface_export_prefix, + export.name(), + interface_export_suffix + )); + } + } + } + } + } + + // Add inlined exported interfaces to the interface package + for (_, interfaces) in inline_interface_exports { + for interface in interfaces { + interface_package.interface(interface); + } + } + + // Add interface for inlined functions to the interface package + for (world_name, functions) in inline_function_exports { + let mut interface = Interface::new( + naming::wit::interface_package_world_inline_functions_interface_name(&world_name), + ); + + for function in functions { + interface.function(function); + } + + interface_package.interface(interface); + } + + Ok((package.clone(), interface_package)) +} diff --git a/wasm-rpc-stubgen/src/wit_transform.rs b/wasm-rpc-stubgen/src/wit_transform.rs deleted file mode 100644 index fa4c01b4..00000000 --- a/wasm-rpc-stubgen/src/wit_transform.rs +++ /dev/null @@ -1,231 +0,0 @@ -use crate::naming; -use crate::wit_encode::EncodedWitDir; -use crate::wit_resolve::ResolvedWitDir; -use regex::Regex; -use std::collections::BTreeMap; -use wit_encoder::{Ident, Interface, Package, PackageItem, StandaloneFunc, WorldItem}; - -// TODO: delete -pub fn import_remover( - package_name: &wit_parser::PackageName, -) -> impl Fn(String) -> anyhow::Result { - let pattern_import_stub_package_name = Regex::new( - format!( - r"import\s+{}(/[^;]*)?;", - regex::escape(&package_name.to_string()) - ) - .as_str(), - ) - .unwrap_or_else(|err| panic!("Failed to compile package import regex: {}", err)); - - move |src: String| -> anyhow::Result { - Ok(pattern_import_stub_package_name - .replace_all(&src, "") - .to_string()) - } -} - -pub fn world_named_interface_import_remover( - package_name: &wit_parser::PackageName, -) -> impl Fn(&mut Package) { - let import_ident_matcher = { - match package_name.version.as_ref() { - Some(version) => Regex::new(&format!( - r"{}/.+@{}", - regex::escape(&package_name.to_string()), - regex::escape(&version.to_string()) - )) - .unwrap_or_else(|err| { - panic!( - "Failed to compile package import ident (with version) regex: {}", - err - ) - }), - None => Regex::new(&format!(r"{}/.+", regex::escape(&package_name.to_string()))) - .unwrap_or_else(|err| { - panic!("Failed to compile package import ident regex: {}", err) - }), - } - }; - - move |package: &mut Package| { - for world_item in package.items_mut() { - if let PackageItem::World(world) = world_item { - world.items_mut().retain(|item| { - if let WorldItem::NamedInterfaceImport(import) = &item { - !import_ident_matcher.is_match(import.name().raw_name()) - } else { - true - } - }); - } - } - } -} - -pub fn add_world_named_interface_import(package: &mut Package, import_name: &str) { - for world_item in package.items_mut() { - if let PackageItem::World(world) = world_item { - let is_already_imported = world.items_mut().iter().any(|item| { - if let WorldItem::NamedInterfaceImport(import) = item { - import.name().raw_name() == import_name - } else { - false - } - }); - if !is_already_imported { - world.named_interface_import(import_name.to_string()); - } - } - } -} - -// TODO: handle world include -// TODO: handle world use -// TODO: maybe transform inline interfaces and functions into included world? -pub fn extract_main_interface_package( - resolve: &ResolvedWitDir, -) -> anyhow::Result<(Package, Package)> { - let mut encoded_wit_dir = EncodedWitDir::new(&resolve.resolve)?; - - let package = encoded_wit_dir.package(resolve.package_id)?; - - let mut interface_package = package.clone(); - interface_package.set_name(naming::wit::interface_package_name(package.name())); - - let interface_export_prefix = format!( - "{}:{}/", - package.name().namespace(), - interface_package.name().name() - ); - let interface_export_suffix = package - .name() - .version() - .map(|version| format!("@{}", version)) - .unwrap_or_default(); - - // Drop all interfaces from original package - package.items_mut().retain(|item| match item { - PackageItem::Interface(_) => false, - PackageItem::World(_) => true, - }); - // Drop all worlds from interface package - interface_package.items_mut().retain(|item| match item { - PackageItem::Interface(_) => true, - PackageItem::World(_) => false, - }); - - let mut inline_interface_exports = BTreeMap::>::new(); - let mut inline_function_exports = BTreeMap::>::new(); - for package_item in package.items_mut() { - if let PackageItem::World(world) = package_item { - let world_name = world.name().clone(); - - // Remove and collect inline exports - world.items_mut().retain(|world_item| match world_item { - WorldItem::InlineInterfaceExport(interface) => { - let mut interface = interface.clone(); - interface.set_name(naming::wit::interface_package_world_inline_interface_name( - &world_name, - interface.name(), - )); - - inline_interface_exports - .entry(world_name.clone()) - .or_default() - .push(interface.clone()); - false - } - WorldItem::FunctionExport(function) => { - inline_function_exports - .entry(world_name.clone()) - .or_default() - .push(function.clone()); - false - } - _ => true, - }); - - // Insert named imports for extracted inline interfaces - if let Some(interfaces) = inline_interface_exports.get(&world_name) { - for interface in interfaces { - world.named_interface_export(interface.name().clone()); - } - } - - // Insert named import for extracted inline functions - if inline_function_exports.contains_key(&world_name) { - world.named_interface_export(format!( - "{}{}{}", - interface_export_prefix, - naming::wit::interface_package_world_inline_functions_interface_name( - &world_name - ), - interface_export_suffix - )); - } - } - } - - // Rename named self imports to use the extracted interface names - for package_item in package.items_mut() { - if let PackageItem::World(world) = package_item { - for world_item in world.items_mut() { - if let WorldItem::NamedInterfaceExport(export) = world_item { - if !export.name().raw_name().contains("/") { - export.set_name(format!( - "{}{}{}", - interface_export_prefix, - export.name(), - interface_export_suffix - )); - } - } - } - } - } - - // Add inlined exported interfaces to the interface package - for (_, interfaces) in inline_interface_exports { - for interface in interfaces { - interface_package.interface(interface); - } - } - - // Add interface for inlined functions to the interface package - for (world_name, functions) in inline_function_exports { - let mut interface = Interface::new( - naming::wit::interface_package_world_inline_functions_interface_name(&world_name), - ); - - for function in functions { - interface.function(function); - } - - interface_package.interface(interface); - } - - Ok((package.clone(), interface_package)) -} - -// TODO: delete these -#[cfg(test)] -mod test { - use test_r::test; - - use crate::wit_resolve::ResolvedWitDir; - use crate::wit_transform::extract_main_interface_package; - use std::path::Path; - - #[test] - fn test_extract_playground() { - let resolved_wit_dir = - ResolvedWitDir::new(Path::new("test-data/many-ways-to-export")).unwrap(); - let (package, interface_package) = - extract_main_interface_package(&resolved_wit_dir).unwrap(); - - println!("{}", package); - println!("\n---\n"); - println!("{}", interface_package); - } -} diff --git a/wasm-rpc-stubgen/test-data/indirect-circular-c/c.wit b/wasm-rpc-stubgen/test-data/indirect-circular-c/c.wit index 7f4ade8f..4a37bdb7 100644 --- a/wasm-rpc-stubgen/test-data/indirect-circular-c/c.wit +++ b/wasm-rpc-stubgen/test-data/indirect-circular-c/c.wit @@ -2,6 +2,7 @@ package test:c; +// NOTE: this interface does not introduce or use any types, so it won't be part of the stub interface api-c { func-c: func(); } diff --git a/wasm-rpc-stubgen/test-data/many-ways-to-export/main.wit b/wasm-rpc-stubgen/test-data/many-ways-to-export/main.wit index 1404168f..b4951dd5 100644 --- a/wasm-rpc-stubgen/test-data/many-ways-to-export/main.wit +++ b/wasm-rpc-stubgen/test-data/many-ways-to-export/main.wit @@ -6,10 +6,13 @@ interface iface1 { func2: func(); } +// TODO: +/* interface iface2 { use iface3.{color}; func3: func() -> color; } +*/ interface iface3 { enum color { @@ -53,7 +56,8 @@ world api { export func1: func(); export iface1; - export iface2; + // TODO: + // export iface2; export inline-iface: interface { func4: func(); diff --git a/wasm-rpc-stubgen/tests/add_dep.rs b/wasm-rpc-stubgen/tests/add_dep.rs index ec38b1b2..a47af90c 100644 --- a/wasm-rpc-stubgen/tests/add_dep.rs +++ b/wasm-rpc-stubgen/tests/add_dep.rs @@ -22,7 +22,8 @@ use golem_wasm_rpc::{WASI_POLL_WIT, WASM_RPC_WIT}; use golem_wasm_rpc_stubgen::commands::generate::generate_stub_wit_dir; use golem_wasm_rpc_stubgen::stub::{StubConfig, StubDefinition}; use golem_wasm_rpc_stubgen::wit_generate::{ - add_stub_as_dependency_to_wit_dir, AddStubAsDepConfig, UpdateCargoToml, + add_stub_as_dependency_to_wit_dir, extract_main_interface_as_wit_dep, AddStubAsDepConfig, + UpdateCargoToml, }; use golem_wasm_rpc_stubgen::wit_resolve::ResolvedWitDir; use golem_wasm_rpc_stubgen::WasmRpcOverride; @@ -36,7 +37,7 @@ test_r::enable!(); #[test] fn all_wit_types_no_collision() { - let stub_dir = init_stub("all-wit-types"); + let (_source_dir, stub_dir) = init_stub("all-wit-types"); let dest_dir = init_caller("caller-no-dep"); let stub_wit_root = stub_dir.path().join("wit"); @@ -45,8 +46,6 @@ fn all_wit_types_no_collision() { add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { stub_wit_root: stub_wit_root.clone(), dest_wit_root: dest_wit_root.clone(), - overwrite: false, - remove_dest_imports: true, update_cargo_toml: UpdateCargoToml::NoUpdate, }) .unwrap(); @@ -62,16 +61,16 @@ fn all_wit_types_no_collision() { ); assert_has_same_wit_package( - &PackageName::new("test", "main", None), + &PackageName::new("test", "main-interface", None), &dest_wit_root, - &Path::new("test-data").join("all-wit-types/main.wit"), + &stub_wit_root, ); } #[test] -fn all_wit_types_overwrite_protection() { - let stub_dir = init_stub("all-wit-types"); - let alternative_stub_dir = init_stub("all-wit-types-alternative"); +fn all_wit_types_re_add_with_changes() { + let (source_dir, stub_dir) = init_stub("all-wit-types"); + let (alternative_source_dir, alternative_stub_dir) = init_stub("all-wit-types-alternative"); let dest_dir = init_caller("caller-no-dep"); let stub_wit_root = stub_dir.path().join("wit"); @@ -81,85 +80,57 @@ fn all_wit_types_overwrite_protection() { add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { stub_wit_root: stub_wit_root.clone(), dest_wit_root: dest_wit_root.clone(), - overwrite: false, - remove_dest_imports: true, - update_cargo_toml: UpdateCargoToml::NoUpdate, - }) - .unwrap(); - add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { - stub_wit_root: alternative_stub_wit_root.clone(), - dest_wit_root: dest_wit_root.clone(), - overwrite: false, - remove_dest_imports: true, update_cargo_toml: UpdateCargoToml::NoUpdate, }) .unwrap(); assert_valid_wit_root(&dest_wit_root); - assert_has_wasm_rpc_wit_deps(&dest_wit_root); - assert_has_same_wit_package( - &PackageName::new("test", "main-stub", None), - &dest_wit_root, + &PackageName::new("test", "main-interface", None), + source_dir.path(), &stub_wit_root, ); - assert_has_same_wit_package( - &PackageName::new("test", "main", None), + &PackageName::new("test", "main-interface", None), + source_dir.path(), + &dest_wit_root, + ); + assert_has_same_wit_package( + &PackageName::new("test", "main-stub", None), + &stub_wit_root, &dest_wit_root, - &Path::new("test-data").join("all-wit-types/main.wit"), ); -} - -#[ignore] // TODO -#[test] -fn all_wit_types_overwrite_protection_disabled() { - let stub_dir = init_stub("all-wit-types"); - let alternative_stub_dir = init_stub("all-wit-types-alternative"); - let dest_dir = init_caller("caller-no-dep"); - - let stub_wit_root = stub_dir.path().join("wit"); - let alternative_stub_wit_root = alternative_stub_dir.path().join("wit"); - let dest_wit_root = dest_dir.path().join("wit"); add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { - stub_wit_root: stub_wit_root.clone(), - dest_wit_root: dest_wit_root.clone(), - overwrite: false, - remove_dest_imports: true, - update_cargo_toml: UpdateCargoToml::NoUpdate, - }) - .unwrap(); - add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { - stub_wit_root: stub_wit_root.clone(), + stub_wit_root: alternative_stub_wit_root.clone(), dest_wit_root: dest_wit_root.clone(), - overwrite: true, - remove_dest_imports: true, update_cargo_toml: UpdateCargoToml::NoUpdate, }) .unwrap(); assert_valid_wit_root(&dest_wit_root); - assert_has_wasm_rpc_wit_deps(&dest_wit_root); - assert_has_same_wit_package( - &PackageName::new("test", "main-stub", None), - &dest_wit_root, + &PackageName::new("test", "main-interface", None), + alternative_source_dir.path(), &alternative_stub_wit_root, ); - assert_has_same_wit_package( - &PackageName::new("test", "main", None), + &PackageName::new("test", "main-interface", None), + alternative_source_dir.path(), + &dest_wit_root, + ); + assert_has_same_wit_package( + &PackageName::new("test", "main-stub", None), + &alternative_stub_wit_root, &dest_wit_root, - &Path::new("test-data").join("all-wit-types-alternative/main.wit"), ); } #[test] fn many_ways_to_export_no_collision() { - let stub_dir = init_stub("many-ways-to-export"); + let (source_dir, stub_dir) = init_stub("many-ways-to-export"); let dest_dir = init_caller("caller-no-dep"); let stub_wit_root = stub_dir.path().join("wit"); @@ -168,8 +139,6 @@ fn many_ways_to_export_no_collision() { add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { stub_wit_root: stub_wit_root.clone(), dest_wit_root: dest_wit_root.clone(), - overwrite: false, - remove_dest_imports: true, update_cargo_toml: UpdateCargoToml::NoUpdate, }) .unwrap(); @@ -185,9 +154,9 @@ fn many_ways_to_export_no_collision() { ); assert_has_same_wit_package( - &PackageName::new("test", "exports", None), + &PackageName::new("test", "exports-interface", None), + source_dir.path(), &dest_wit_root, - &Path::new("test-data").join("many-ways-to-export"), ); assert_has_same_wit_package( @@ -199,8 +168,8 @@ fn many_ways_to_export_no_collision() { #[test] fn direct_circular() { - let stub_a_dir = init_stub("direct-circular-a"); - let stub_b_dir = init_stub("direct-circular-b"); + let (_source_a_dir, stub_a_dir) = init_stub("direct-circular-a"); + let (_source_b_dir, stub_b_dir) = init_stub("direct-circular-b"); let dest_a = init_caller("direct-circular-a"); let dest_b = init_caller("direct-circular-b"); @@ -208,16 +177,12 @@ fn direct_circular() { add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { stub_wit_root: stub_a_dir.path().join("wit"), dest_wit_root: dest_b.path().to_path_buf(), - overwrite: true, - remove_dest_imports: true, update_cargo_toml: UpdateCargoToml::NoUpdate, }) .unwrap(); add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { stub_wit_root: stub_b_dir.path().join("wit"), dest_wit_root: dest_a.path().to_path_buf(), - overwrite: true, - remove_dest_imports: true, update_cargo_toml: UpdateCargoToml::NoUpdate, }) .unwrap(); @@ -234,9 +199,9 @@ fn direct_circular() { ); assert_has_same_wit_package( - &PackageName::new("test", "b", None), + &PackageName::new("test", "b-interface", None), dest_a.path(), - &Path::new("test-data").join("direct-circular-b/b.wit"), + _source_b_dir.path(), ); assert_has_wasm_rpc_wit_deps(dest_b.path()); @@ -248,16 +213,16 @@ fn direct_circular() { ); assert_has_same_wit_package( - &PackageName::new("test", "a", None), + &PackageName::new("test", "a-interface", None), dest_b.path(), - &Path::new("test-data").join("direct-circular-a/a.wit"), + _source_a_dir.path(), ); } #[test] fn direct_circular_readd() { - let stub_a_dir = init_stub("direct-circular-a"); - let stub_b_dir = init_stub("direct-circular-b"); + let (_source_a_dir, stub_a_dir) = init_stub("direct-circular-a"); + let (_source_b_dir, stub_b_dir) = init_stub("direct-circular-b"); let dest_a = init_caller("direct-circular-a"); let dest_b = init_caller("direct-circular-b"); @@ -265,16 +230,12 @@ fn direct_circular_readd() { add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { stub_wit_root: stub_a_dir.path().join("wit"), dest_wit_root: dest_b.path().to_path_buf(), - overwrite: true, - remove_dest_imports: true, update_cargo_toml: UpdateCargoToml::NoUpdate, }) .unwrap(); add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { stub_wit_root: stub_b_dir.path().join("wit"), dest_wit_root: dest_a.path().to_path_buf(), - overwrite: true, - remove_dest_imports: true, update_cargo_toml: UpdateCargoToml::NoUpdate, }) .unwrap(); @@ -292,16 +253,12 @@ fn direct_circular_readd() { add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { stub_wit_root: stub_a_dir.path().join("wit"), dest_wit_root: dest_b.path().to_path_buf(), - overwrite: true, - remove_dest_imports: true, update_cargo_toml: UpdateCargoToml::NoUpdate, }) .unwrap(); add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { stub_wit_root: stub_b_dir.path().join("wit"), dest_wit_root: dest_a.path().to_path_buf(), - overwrite: true, - remove_dest_imports: true, update_cargo_toml: UpdateCargoToml::NoUpdate, }) .unwrap(); @@ -344,8 +301,8 @@ fn direct_circular_readd() { #[test] fn direct_circular_same_world_name() { - let stub_a_dir = init_stub("direct-circular-a-same-world-name"); - let stub_b_dir = init_stub("direct-circular-b-same-world-name"); + let (source_a_dir, stub_a_dir) = init_stub("direct-circular-a-same-world-name"); + let (source_b_dir, stub_b_dir) = init_stub("direct-circular-b-same-world-name"); let dest_a = init_caller("direct-circular-a-same-world-name"); let dest_b = init_caller("direct-circular-b-same-world-name"); @@ -353,16 +310,12 @@ fn direct_circular_same_world_name() { add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { stub_wit_root: stub_a_dir.path().join("wit"), dest_wit_root: dest_b.path().to_path_buf(), - overwrite: true, - remove_dest_imports: true, update_cargo_toml: UpdateCargoToml::NoUpdate, }) .unwrap(); add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { stub_wit_root: stub_b_dir.path().join("wit"), dest_wit_root: dest_a.path().to_path_buf(), - overwrite: true, - remove_dest_imports: true, update_cargo_toml: UpdateCargoToml::NoUpdate, }) .unwrap(); @@ -379,9 +332,9 @@ fn direct_circular_same_world_name() { ); assert_has_same_wit_package( - &PackageName::new("test", "b", None), + &PackageName::new("test", "b-interface", None), dest_a.path(), - &Path::new("test-data").join("direct-circular-b-same-world-name/b.wit"), + source_b_dir.path(), ); assert_has_wasm_rpc_wit_deps(dest_b.path()); @@ -393,17 +346,17 @@ fn direct_circular_same_world_name() { ); assert_has_same_wit_package( - &PackageName::new("test", "a", None), + &PackageName::new("test", "a-interface", None), dest_b.path(), - &Path::new("test-data").join("direct-circular-a-same-world-name/a.wit"), + source_a_dir.path(), ); } #[test] fn indirect_circular() { - let stub_a_dir = init_stub("indirect-circular-a"); - let stub_b_dir = init_stub("indirect-circular-b"); - let stub_c_dir = init_stub("indirect-circular-c"); + let (source_a_dir, stub_a_dir) = init_stub("indirect-circular-a"); + let (_source_b_dir, stub_b_dir) = init_stub("indirect-circular-b"); + let (_source_c_dir, stub_c_dir) = init_stub("indirect-circular-c"); let dest_a = init_caller("indirect-circular-a"); let dest_b = init_caller("indirect-circular-b"); @@ -412,24 +365,18 @@ fn indirect_circular() { add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { stub_wit_root: stub_a_dir.path().join("wit"), dest_wit_root: dest_c.path().to_path_buf(), - overwrite: true, - remove_dest_imports: true, update_cargo_toml: UpdateCargoToml::NoUpdate, }) .unwrap(); add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { stub_wit_root: stub_b_dir.path().join("wit"), dest_wit_root: dest_a.path().to_path_buf(), - overwrite: true, - remove_dest_imports: true, update_cargo_toml: UpdateCargoToml::NoUpdate, }) .unwrap(); add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { stub_wit_root: stub_c_dir.path().join("wit"), dest_wit_root: dest_b.path().to_path_buf(), - overwrite: true, - remove_dest_imports: true, update_cargo_toml: UpdateCargoToml::NoUpdate, }) .unwrap(); @@ -447,9 +394,9 @@ fn indirect_circular() { ); assert_has_same_wit_package( - &PackageName::new("test", "b", None), + &PackageName::new("test", "b-interface", None), dest_a.path(), - &Path::new("test-data").join("indirect-circular-b/b.wit"), + &stub_b_dir.path().join("wit"), ); assert_has_wasm_rpc_wit_deps(dest_b.path()); @@ -460,12 +407,6 @@ fn indirect_circular() { &stub_c_dir.path().join("wit"), ); - assert_has_same_wit_package( - &PackageName::new("test", "c", None), - dest_b.path(), - &Path::new("test-data").join("indirect-circular-c/c.wit"), - ); - assert_has_wasm_rpc_wit_deps(dest_c.path()); assert_has_same_wit_package( @@ -475,17 +416,17 @@ fn indirect_circular() { ); assert_has_same_wit_package( - &PackageName::new("test", "a", None), + &PackageName::new("test", "a-interface", None), dest_c.path(), - &Path::new("test-data").join("indirect-circular-a/a.wit"), + source_a_dir.path(), ); } #[test] fn indirect_circular_readd() { - let stub_a_dir = init_stub("indirect-circular-a"); - let stub_b_dir = init_stub("indirect-circular-b"); - let stub_c_dir = init_stub("indirect-circular-c"); + let (_source_a_dir, stub_a_dir) = init_stub("indirect-circular-a"); + let (_source_b_dir, stub_b_dir) = init_stub("indirect-circular-b"); + let (_source_c_dir, stub_c_dir) = init_stub("indirect-circular-c"); let dest_a = init_caller("indirect-circular-a"); let dest_b = init_caller("indirect-circular-b"); @@ -498,24 +439,18 @@ fn indirect_circular_readd() { add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { stub_wit_root: stub_a_dir.path().join("wit"), dest_wit_root: dest_c.path().to_path_buf(), - overwrite: true, - remove_dest_imports: true, update_cargo_toml: UpdateCargoToml::NoUpdate, }) .unwrap(); add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { stub_wit_root: stub_b_dir.path().join("wit"), dest_wit_root: dest_a.path().to_path_buf(), - overwrite: true, - remove_dest_imports: true, update_cargo_toml: UpdateCargoToml::NoUpdate, }) .unwrap(); add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { stub_wit_root: stub_c_dir.path().join("wit"), dest_wit_root: dest_b.path().to_path_buf(), - overwrite: true, - remove_dest_imports: true, update_cargo_toml: UpdateCargoToml::NoUpdate, }) .unwrap(); @@ -535,24 +470,18 @@ fn indirect_circular_readd() { add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { stub_wit_root: stub_a_dir.path().join("wit"), dest_wit_root: dest_c.path().to_path_buf(), - overwrite: true, - remove_dest_imports: true, update_cargo_toml: UpdateCargoToml::NoUpdate, }) .unwrap(); add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { stub_wit_root: stub_b_dir.path().join("wit"), dest_wit_root: dest_a.path().to_path_buf(), - overwrite: true, - remove_dest_imports: true, update_cargo_toml: UpdateCargoToml::NoUpdate, }) .unwrap(); add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { stub_wit_root: stub_c_dir.path().join("wit"), dest_wit_root: dest_b.path().to_path_buf(), - overwrite: true, - remove_dest_imports: true, update_cargo_toml: UpdateCargoToml::NoUpdate, }) .unwrap(); @@ -570,7 +499,7 @@ fn indirect_circular_readd() { ); assert_has_same_wit_package( - &PackageName::new("test", "b", None), + &PackageName::new("test", "b-interface", None), dest_a.path(), dest_b.path(), ); @@ -583,9 +512,12 @@ fn indirect_circular_readd() { &stub_c_dir.path().join("wit"), ); - assert_has_same_wit_package( - &PackageName::new("test", "c", None), + assert_has_no_package_by_name( + &PackageName::new("test", "c-interface", None), dest_b.path(), + ); + assert_has_package_by_name( + &PackageName::new("test", "c-interface", None), dest_c.path(), ); @@ -598,24 +530,23 @@ fn indirect_circular_readd() { ); assert_has_same_wit_package( - &PackageName::new("test", "a", None), + &PackageName::new("test", "a-interface", None), dest_c.path(), dest_a.path(), ); + assert_has_no_package_by_name(&PackageName::new("test", "a", None), dest_c.path()); + assert_has_package_by_name(&PackageName::new("test", "a", None), dest_a.path()); } #[test] fn self_circular() { - let stub_a_dir = init_stub("self-circular"); - let inlined_stub_a_dir = init_stub_inlined("self-circular"); + let (_source_a_dir, stub_a_dir) = init_stub("self-circular"); let dest_a = init_caller("self-circular"); add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { stub_wit_root: stub_a_dir.path().join("wit"), dest_wit_root: dest_a.path().to_path_buf(), - overwrite: true, - remove_dest_imports: true, update_cargo_toml: UpdateCargoToml::NoUpdate, }) .unwrap(); @@ -627,34 +558,36 @@ fn self_circular() { assert_has_same_wit_package( &PackageName::new("test", "a-stub", None), dest_a.path(), - &inlined_stub_a_dir.path().join("wit"), + &stub_a_dir.path().join("wit"), ); } -fn init_stub(name: &str) -> TempDir { - init_stub_internal(name, false) -} +fn init_stub(name: &str) -> (TempDir, TempDir) { + let source = TempDir::new().unwrap(); + let canonical_source = source.path().canonicalize().unwrap(); -fn init_stub_inlined(name: &str) -> TempDir { - init_stub_internal(name, true) -} + fs_extra::dir::copy( + Path::new("test-data").join(name), + &canonical_source, + &CopyOptions::new().content_only(true), + ) + .unwrap(); + + extract_main_interface_as_wit_dep(source.path()).unwrap(); -fn init_stub_internal(name: &str, always_inline_types: bool) -> TempDir { - let tempdir = TempDir::new().unwrap(); - let canonical_target_root = tempdir.path().canonicalize().unwrap(); + let target = TempDir::new().unwrap(); + let canonical_target = target.path().canonicalize().unwrap(); - let source_wit_root = Path::new("test-data").join(name); let def = StubDefinition::new(StubConfig { - source_wit_root, - target_root: canonical_target_root, + source_wit_root: canonical_source, + target_root: canonical_target, selected_world: None, stub_crate_version: "1.0.0".to_string(), wasm_rpc_override: WasmRpcOverride::default(), - inline_source_types: always_inline_types, }) .unwrap(); let _ = generate_stub_wit_dir(&def).unwrap(); - tempdir + (source, target) } fn regenerate_stub(stub_dir: &Path, source_wit_root: &Path) { @@ -664,24 +597,34 @@ fn regenerate_stub(stub_dir: &Path, source_wit_root: &Path) { selected_world: None, stub_crate_version: "1.0.0".to_string(), wasm_rpc_override: WasmRpcOverride::default(), - inline_source_types: false, }) .unwrap(); let _ = generate_stub_wit_dir(&def).unwrap(); } fn init_caller(name: &str) -> TempDir { - let tempdir = TempDir::new().unwrap(); + let temp_dir = TempDir::new().unwrap(); let source = Path::new("test-data").join(name); fs_extra::dir::copy( source, - tempdir.path(), + temp_dir.path(), &CopyOptions::new().content_only(true).overwrite(true), ) .unwrap(); - tempdir + let wit_dir = { + let wit_dir = temp_dir.path().join("wit"); + if wit_dir.exists() { + wit_dir + } else { + temp_dir.path().to_path_buf() + } + }; + + extract_main_interface_as_wit_dep(&wit_dir).unwrap(); + + temp_dir } fn assert_valid_wit_root(wit_root: &Path) { @@ -747,6 +690,14 @@ fn assert_has_same_wit_package( assert!(actual_wit == expected_wit) } +fn assert_has_no_package_by_name(package_name: &PackageName, wit_source: impl WitSource) { + assert!(wit_source.encoded_package(package_name).is_err()) +} + +fn assert_has_package_by_name(package_name: &PackageName, wit_source: impl WitSource) { + assert!(wit_source.encoded_package(package_name).is_ok()) +} + fn assert_has_wasm_rpc_wit_deps(wit_dir: &Path) { let deps = vec![("poll", WASI_POLL_WIT), ("wasm-rpc", WASM_RPC_WIT)]; diff --git a/wasm-rpc-stubgen/tests/wit.rs b/wasm-rpc-stubgen/tests/wit.rs index 8506c3b3..2c78f175 100644 --- a/wasm-rpc-stubgen/tests/wit.rs +++ b/wasm-rpc-stubgen/tests/wit.rs @@ -16,27 +16,28 @@ use test_r::test; +use fs_extra::dir::CopyOptions; use golem_wasm_rpc_stubgen::commands::generate::generate_stub_wit_dir; use golem_wasm_rpc_stubgen::stub::{StubConfig, StubDefinition}; +use golem_wasm_rpc_stubgen::wit_generate::extract_main_interface_as_wit_dep; use golem_wasm_rpc_stubgen::WasmRpcOverride; use std::path::Path; -use tempfile::tempdir; +use tempfile::{tempdir, TempDir}; use wit_parser::{FunctionKind, Resolve, TypeDefKind, TypeOwner}; test_r::enable!(); #[test] fn all_wit_types() { - let source_wit_root = Path::new("test-data/all-wit-types"); + let source_wit_root = init_source("all-wit-types"); let target_root = tempdir().unwrap(); let def = StubDefinition::new(StubConfig { - source_wit_root: source_wit_root.to_path_buf(), + source_wit_root: source_wit_root.path().to_path_buf(), target_root: target_root.path().to_path_buf(), selected_world: None, stub_crate_version: "1.0.0".to_string(), wasm_rpc_override: WasmRpcOverride::default(), - inline_source_types: false, }) .unwrap(); let resolve = generate_stub_wit_dir(&def).unwrap().resolve; @@ -96,132 +97,31 @@ fn all_wit_types() { assert_has_stub_function(&resolve, "stub-api", "iface1", "get-color", true); assert_has_stub_function(&resolve, "stub-api", "iface1", "set-color", false); assert_has_stub_function(&resolve, "stub-api", "iface1", "validate-permissions", true); -} - -#[test] -fn all_wit_types_inlined() { - let source_wit_root = Path::new("test-data/all-wit-types"); - let target_root = tempdir().unwrap(); - - let def = StubDefinition::new(StubConfig { - source_wit_root: source_wit_root.to_path_buf(), - target_root: target_root.path().to_path_buf(), - selected_world: None, - stub_crate_version: "1.0.0".to_string(), - wasm_rpc_override: WasmRpcOverride::default(), - inline_source_types: true, - }) - .unwrap(); - let resolve = generate_stub_wit_dir(&def).unwrap().resolve; - - assert_has_package_name(&resolve, "test:main-stub"); - assert_has_world(&resolve, "wasm-rpc-stub-api"); - assert_has_interface(&resolve, "stub-api"); - assert_has_stub_function(&resolve, "stub-api", "iface1", "no-op", false); - assert_has_stub_function(&resolve, "stub-api", "iface1", "get-bool", true); - assert_has_stub_function(&resolve, "stub-api", "iface1", "set-bool", false); - assert_has_stub_function(&resolve, "stub-api", "iface1", "identity-bool", true); - assert_has_stub_function(&resolve, "stub-api", "iface1", "identity-s8", true); - assert_has_stub_function(&resolve, "stub-api", "iface1", "identity-s16", true); - assert_has_stub_function(&resolve, "stub-api", "iface1", "identity-s32", true); - assert_has_stub_function(&resolve, "stub-api", "iface1", "identity-s64", true); - assert_has_stub_function(&resolve, "stub-api", "iface1", "identity-u8", true); - assert_has_stub_function(&resolve, "stub-api", "iface1", "identity-u16", true); - assert_has_stub_function(&resolve, "stub-api", "iface1", "identity-u32", true); - assert_has_stub_function(&resolve, "stub-api", "iface1", "identity-u64", true); - assert_has_stub_function(&resolve, "stub-api", "iface1", "identity-f32", true); - assert_has_stub_function(&resolve, "stub-api", "iface1", "identity-f64", true); - assert_has_stub_function(&resolve, "stub-api", "iface1", "identity-char", true); - assert_has_stub_function(&resolve, "stub-api", "iface1", "identity-string", true); - assert_has_stub_function(&resolve, "stub-api", "iface1", "get-orders", true); - assert_has_stub_function(&resolve, "stub-api", "iface1", "set-orders", false); - assert_has_stub_function(&resolve, "stub-api", "iface1", "apply-metadata", true); - assert_has_stub_function(&resolve, "stub-api", "iface1", "get-option-bool", true); - assert_has_stub_function(&resolve, "stub-api", "iface1", "set-option-bool", false); - assert_has_stub_function(&resolve, "stub-api", "iface1", "get-coordinates", true); - assert_has_stub_function( - &resolve, - "stub-api", - "iface1", - "get-coordinates-alias", - true, - ); - assert_has_stub_function(&resolve, "stub-api", "iface1", "set-coordinates", false); - assert_has_stub_function( - &resolve, - "stub-api", - "iface1", - "set-coordinates-alias", - false, - ); - assert_has_stub_function(&resolve, "stub-api", "iface1", "tuple-to-point", true); - assert_has_stub_function(&resolve, "stub-api", "iface1", "pt-log-error", true); - assert_has_stub_function(&resolve, "stub-api", "iface1", "validate-pt", true); - assert_has_stub_function( - &resolve, - "stub-api", - "iface1", - "print-checkout-result", - true, - ); - assert_has_stub_function(&resolve, "stub-api", "iface1", "get-checkout-result", true); - assert_has_stub_function(&resolve, "stub-api", "iface1", "get-color", true); - assert_has_stub_function(&resolve, "stub-api", "iface1", "set-color", false); - assert_has_stub_function(&resolve, "stub-api", "iface1", "validate-permissions", true); - - assert_defines_enum(&resolve, "stub-api", "color"); - assert_defines_flags(&resolve, "stub-api", "permissions"); - assert_defines_record(&resolve, "stub-api", "metadata"); - assert_defines_record(&resolve, "stub-api", "point"); - assert_defines_record(&resolve, "stub-api", "product-item"); - assert_defines_record(&resolve, "stub-api", "order"); - assert_defines_record(&resolve, "stub-api", "order-confirmation"); - assert_defines_tuple_alias(&resolve, "stub-api", "point-tuple"); - assert_defines_variant(&resolve, "stub-api", "checkout-result"); + assert_defines_enum(&resolve, "test:main-interface", "iface1", "color"); + assert_defines_flags(&resolve, "test:main-interface", "iface1", "permissions"); + assert_defines_record(&resolve, "test:main-interface", "iface1", "metadata"); + assert_defines_record(&resolve, "test:main-interface", "iface1", "point"); + assert_defines_record(&resolve, "test:main-interface", "iface1", "product-item"); + assert_defines_record(&resolve, "test:main-interface", "iface1", "order"); + assert_defines_record(&resolve, "test:main-interface", "iface1", "order-confirmation"); + assert_defines_tuple_alias(&resolve, "test:main-interface", "iface1", "point-tuple"); + assert_defines_variant(&resolve, "test:main-interface", "iface1", "checkout-result"); + + // TODO: duplicated rec1 types } #[test] fn many_ways_to_export() { - let source_wit_root = Path::new("test-data/many-ways-to-export"); - let target_root = tempdir().unwrap(); - - let def = StubDefinition::new(StubConfig { - source_wit_root: source_wit_root.to_path_buf(), - target_root: target_root.path().to_path_buf(), - selected_world: None, - stub_crate_version: "1.0.0".to_string(), - wasm_rpc_override: WasmRpcOverride::default(), - inline_source_types: false, - }) - .unwrap(); - let resolve = generate_stub_wit_dir(&def).unwrap().resolve; - - assert_has_package_name(&resolve, "test:exports-stub"); - assert_has_world(&resolve, "wasm-rpc-stub-api"); - assert_has_interface(&resolve, "stub-api"); - - assert_has_stub_function(&resolve, "stub-api", "api", "func1", false); - assert_has_stub_function(&resolve, "stub-api", "iface1", "func2", false); - assert_has_stub_function(&resolve, "stub-api", "iface2", "func3", true); - assert_has_stub_function(&resolve, "stub-api", "inline-iface", "func4", false); - assert_has_stub_function(&resolve, "stub-api", "iface4", "func5", false); - - // TODO: add asserts for non-unique types -} - -#[test] -fn many_ways_to_export_inlined() { - let source_wit_root = Path::new("test-data/many-ways-to-export"); + let source_wit_root = init_source("many-ways-to-export"); let target_root = tempdir().unwrap(); let def = StubDefinition::new(StubConfig { - source_wit_root: source_wit_root.to_path_buf(), + source_wit_root: source_wit_root.path().to_path_buf(), target_root: target_root.path().to_path_buf(), selected_world: None, stub_crate_version: "1.0.0".to_string(), wasm_rpc_override: WasmRpcOverride::default(), - inline_source_types: true, }) .unwrap(); let resolve = generate_stub_wit_dir(&def).unwrap().resolve; @@ -230,14 +130,13 @@ fn many_ways_to_export_inlined() { assert_has_world(&resolve, "wasm-rpc-stub-api"); assert_has_interface(&resolve, "stub-api"); - assert_has_stub_function(&resolve, "stub-api", "api", "func1", false); + assert_has_stub_function(&resolve, "stub-api", "api-inline-functions", "func1", false); assert_has_stub_function(&resolve, "stub-api", "iface1", "func2", false); - assert_has_stub_function(&resolve, "stub-api", "iface2", "func3", true); - assert_has_stub_function(&resolve, "stub-api", "inline-iface", "func4", false); + // TODO: + // assert_has_stub_function(&resolve, "stub-api", "iface2", "func3", true); + assert_has_stub_function(&resolve, "stub-api", "api-inline-iface", "func4", false); assert_has_stub_function(&resolve, "stub-api", "iface4", "func5", false); - assert_defines_enum(&resolve, "stub-api", "color"); - // TODO: add asserts for non-unique types } @@ -317,57 +216,118 @@ fn assert_has_stub_function( } } -fn assert_defines_enum(resolve: &Resolve, interface_name: &str, enum_name: &str) { +fn assert_defines_enum( + resolve: &Resolve, + package_name: &str, + interface_name: &str, + enum_name: &str, +) { assert!(resolve .types .iter() .any(|(_, typ)| typ.name == Some(enum_name.to_string()) && matches!(typ.kind, TypeDefKind::Enum(_)) - && is_owned_by_interface(resolve, &typ.owner, interface_name))) + && is_owned_by_interface(resolve, &typ.owner, package_name, interface_name))) } -fn assert_defines_flags(resolve: &Resolve, interface_name: &str, flags_name: &str) { +fn assert_defines_flags( + resolve: &Resolve, + package_name: &str, + interface_name: &str, + flags_name: &str, +) { assert!(resolve .types .iter() .any(|(_, typ)| typ.name == Some(flags_name.to_string()) && matches!(typ.kind, TypeDefKind::Flags(_)) - && is_owned_by_interface(resolve, &typ.owner, interface_name))) + && is_owned_by_interface(resolve, &typ.owner, package_name, interface_name))) } -fn assert_defines_record(resolve: &Resolve, interface_name: &str, record_name: &str) { +fn assert_defines_record( + resolve: &Resolve, + package_name: &str, + interface_name: &str, + record_name: &str, +) { assert!(resolve .types .iter() .any(|(_, typ)| typ.name == Some(record_name.to_string()) && matches!(typ.kind, TypeDefKind::Record(_)) - && is_owned_by_interface(resolve, &typ.owner, interface_name))) + && is_owned_by_interface(resolve, &typ.owner, package_name, interface_name))) } -fn assert_defines_tuple_alias(resolve: &Resolve, interface_name: &str, alias_name: &str) { +fn assert_defines_tuple_alias( + resolve: &Resolve, + package_name: &str, + interface_name: &str, + alias_name: &str, +) { assert!(resolve .types .iter() .any(|(_, typ)| typ.name == Some(alias_name.to_string()) && matches!(typ.kind, TypeDefKind::Tuple(_)) - && is_owned_by_interface(resolve, &typ.owner, interface_name))) + && is_owned_by_interface(resolve, &typ.owner, package_name, interface_name))) } -fn assert_defines_variant(resolve: &Resolve, interface_name: &str, variant_name: &str) { +fn assert_defines_variant( + resolve: &Resolve, + package_name: &str, + interface_name: &str, + variant_name: &str, +) { assert!(resolve .types .iter() .any(|(_, typ)| typ.name == Some(variant_name.to_string()) && matches!(typ.kind, TypeDefKind::Variant(_)) - && is_owned_by_interface(resolve, &typ.owner, interface_name))) + && is_owned_by_interface(resolve, &typ.owner, package_name, interface_name))) } -fn is_owned_by_interface(resolve: &Resolve, owner: &TypeOwner, interface_name: &str) -> bool { +fn is_owned_by_interface( + resolve: &Resolve, + owner: &TypeOwner, + package_name: &str, + interface_name: &str, +) -> bool { match owner { TypeOwner::World(_) => false, TypeOwner::Interface(iface_id) => { - resolve.interfaces.get(*iface_id).unwrap().name == Some(interface_name.to_string()) + let interface = resolve.interfaces.get(*iface_id).unwrap(); + interface.name == Some(interface_name.to_string()) + && interface + .package + .and_then(|package_id| resolve.packages.get(package_id)) + .map(|package| package.name.to_string()) + == Some(package_name.to_string()) } TypeOwner::None => false, } } + +fn init_source(name: &str) -> TempDir { + let temp_dir = TempDir::new().unwrap(); + let source = Path::new("test-data").join(name); + + fs_extra::dir::copy( + source, + temp_dir.path(), + &CopyOptions::new().content_only(true).overwrite(true), + ) + .unwrap(); + + let wit_dir = { + let wit_dir = temp_dir.path().join("wit"); + if wit_dir.exists() { + wit_dir + } else { + temp_dir.path().to_path_buf() + } + }; + + extract_main_interface_as_wit_dep(&wit_dir).unwrap(); + + temp_dir +} From 06c697753f1a4f3c0946a070cfeb757356023651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 5 Nov 2024 11:57:50 +0100 Subject: [PATCH 036/109] add option in StubDef::new to apply interface packages extract --- wasm-rpc-stubgen/src/commands/declarative.rs | 1 + wasm-rpc-stubgen/src/commands/dependencies.rs | 4 ++- wasm-rpc-stubgen/src/commands/generate.rs | 32 +++++++++---------- wasm-rpc-stubgen/src/lib.rs | 8 +++-- wasm-rpc-stubgen/src/stub.rs | 10 ++++-- wasm-rpc-stubgen/src/wit_generate.rs | 2 +- .../tests-integration/tests/compose.rs | 20 ++++++------ .../tests-integration/tests/stub_wasm.rs | 18 +++++------ wasm-rpc-stubgen/tests/add_dep.rs | 20 +++--------- wasm-rpc-stubgen/tests/wit.rs | 21 +++++------- 10 files changed, 64 insertions(+), 72 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index 107817fc..87c7f296 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -85,6 +85,7 @@ async fn pre_component_build_app(config: &Config, app: &Application) -> anyhow:: wasm_rpc_path_override: app.stub_wasm_rpc_path(&component_name), wasm_rpc_version_override: app.stub_wasm_rpc_version(&component_name), }, + extract_source_interface_package: true // TODO: use input output dirs }) .context("Failed to gather information for the stub generator")?; diff --git a/wasm-rpc-stubgen/src/commands/dependencies.rs b/wasm-rpc-stubgen/src/commands/dependencies.rs index 143b5ce4..1ef9802f 100644 --- a/wasm-rpc-stubgen/src/commands/dependencies.rs +++ b/wasm-rpc-stubgen/src/commands/dependencies.rs @@ -13,9 +13,11 @@ // limitations under the License. use crate::wit_generate; -use crate::wit_generate::{AddStubAsDepConfig, UpdateCargoToml}; +use crate::wit_generate::AddStubAsDepConfig; use std::path::Path; +pub use crate::wit_generate::UpdateCargoToml; + pub fn add_stub_dependency( stub_wit_root: &Path, dest_wit_root: &Path, diff --git a/wasm-rpc-stubgen/src/commands/generate.rs b/wasm-rpc-stubgen/src/commands/generate.rs index 4e82f5dd..551c1f4a 100644 --- a/wasm-rpc-stubgen/src/commands/generate.rs +++ b/wasm-rpc-stubgen/src/commands/generate.rs @@ -55,22 +55,6 @@ pub async fn build( Ok(()) } -pub fn generate_stub_wit_dir(stub_def: &StubDefinition) -> anyhow::Result { - log_action( - "Generating", - format!( - "stub WIT directory to {}", - stub_def.config.target_root.display() - ), - ); - let _indent = LogIndent::new(); - generate_stub_wit_to_target(stub_def).context("Failed to generate the stub wit file")?; - add_dependencies_to_stub_wit_dir(stub_def).context("Failed to copy the dependent wit files")?; - stub_def - .resolve_target_wit() - .context("Failed to resolve the result WIT root") -} - pub async fn generate_and_build_stub(stub_def: &StubDefinition) -> anyhow::Result { let _ = generate_stub_wit_dir(stub_def)?; generate_cargo_toml(stub_def).context("Failed to generate the Cargo.toml file")?; @@ -92,3 +76,19 @@ pub async fn generate_and_build_stub(stub_def: &StubDefinition) -> anyhow::Resul )); Ok(wasm_path) } + +pub fn generate_stub_wit_dir(stub_def: &StubDefinition) -> anyhow::Result { + log_action( + "Generating", + format!( + "stub WIT directory to {}", + stub_def.config.target_root.display() + ), + ); + let _indent = LogIndent::new(); + generate_stub_wit_to_target(stub_def).context("Failed to generate the stub wit file")?; + add_dependencies_to_stub_wit_dir(stub_def).context("Failed to copy the dependent wit files")?; + stub_def + .resolve_target_wit() + .context("Failed to resolve the result WIT root") +} diff --git a/wasm-rpc-stubgen/src/lib.rs b/wasm-rpc-stubgen/src/lib.rs index 59a51443..9944915c 100644 --- a/wasm-rpc-stubgen/src/lib.rs +++ b/wasm-rpc-stubgen/src/lib.rs @@ -82,7 +82,7 @@ pub struct GenerateArgs { /// it from the stub WIT. This is useful for example with ComponentizeJS currently where otherwise /// the original component's interface would be added as an import to the final WASM. #[clap(long, default_value_t = false)] - pub always_inline_types: bool, + pub always_inline_types: bool, // TODO: deprecated } #[derive(clap::Args, Debug, Clone)] @@ -128,7 +128,7 @@ pub struct BuildArgs { /// it from the stub WIT. This is useful for example with ComponentizeJS currently where otherwise /// the original component's interface would be added as an import to the final WASM. #[clap(long, default_value_t = false)] - pub always_inline_types: bool, + pub always_inline_types: bool, // TODO: deprecated } /// Adds a generated stub as a dependency to another WASM component @@ -146,7 +146,7 @@ pub struct AddStubDependencyArgs { /// This command would not do anything if it detects that it would change an existing WIT file's contents at /// the destination. With this flag, it can be forced to overwrite those files. #[clap(short, long)] - pub overwrite: bool, + pub overwrite: bool, // TODO: deprecate /// Enables updating the Cargo.toml file in the parent directory of `dest-wit-root` with the copied /// dependencies. #[clap(short, long)] @@ -226,6 +226,7 @@ pub fn generate(args: GenerateArgs) -> anyhow::Result<()> { selected_world: args.world, stub_crate_version: args.stub_crate_version, wasm_rpc_override: args.wasm_rpc_override, + extract_source_interface_package: true, } ) .context("Failed to gather information for the stub generator. Make sure source_wit_root has a valid WIT file.")?; @@ -242,6 +243,7 @@ pub async fn build(args: BuildArgs) -> anyhow::Result<()> { selected_world: args.world, stub_crate_version: args.stub_crate_version, wasm_rpc_override: args.wasm_rpc_override, + extract_source_interface_package: true, }) .context("Failed to gather information for the stub generator")?; diff --git a/wasm-rpc-stubgen/src/stub.rs b/wasm-rpc-stubgen/src/stub.rs index 25b49bd8..86b2c921 100644 --- a/wasm-rpc-stubgen/src/stub.rs +++ b/wasm-rpc-stubgen/src/stub.rs @@ -13,9 +13,10 @@ // limitations under the License. use crate::wit_encode::EncodedWitDir; +use crate::wit_generate::extract_main_interface_as_wit_dep; use crate::wit_resolve::{PackageSource, ResolvedWitDir}; use crate::{naming, WasmRpcOverride}; -use anyhow::anyhow; +use anyhow::{anyhow, Context}; use indexmap::IndexMap; use itertools::Itertools; use std::cell::OnceCell; @@ -33,9 +34,9 @@ pub struct StubConfig { pub selected_world: Option, pub stub_crate_version: String, pub wasm_rpc_override: WasmRpcOverride, + pub extract_source_interface_package: bool, } -/// All the gathered information for generating the stub crate. pub struct StubDefinition { pub config: StubConfig, @@ -53,6 +54,11 @@ pub struct StubDefinition { impl StubDefinition { pub fn new(config: StubConfig) -> anyhow::Result { + if config.extract_source_interface_package { + extract_main_interface_as_wit_dep(&config.source_wit_root) + .context("Failed to extract the source interface package")? + } + let resolved_source = ResolvedWitDir::new(&config.source_wit_root)?; let source_world_id = resolved_source diff --git a/wasm-rpc-stubgen/src/wit_generate.rs b/wasm-rpc-stubgen/src/wit_generate.rs index 943ee13f..44c7c439 100644 --- a/wasm-rpc-stubgen/src/wit_generate.rs +++ b/wasm-rpc-stubgen/src/wit_generate.rs @@ -641,7 +641,7 @@ pub fn extract_main_interface_as_wit_dep(wit_dir: &Path) -> anyhow::Result<()> { log_action( "Extracting", format!( - "interface package for main component in wit directory {}", + "interface package from main component in wit directory {}", wit_dir.display() ), ); diff --git a/wasm-rpc-stubgen/tests-integration/tests/compose.rs b/wasm-rpc-stubgen/tests-integration/tests/compose.rs index a12da96b..22545939 100644 --- a/wasm-rpc-stubgen/tests-integration/tests/compose.rs +++ b/wasm-rpc-stubgen/tests-integration/tests/compose.rs @@ -24,7 +24,7 @@ use golem_wasm_ast::{DefaultAst, IgnoreAllButMetadata}; use golem_wasm_rpc_stubgen::commands::composition::compose; use golem_wasm_rpc_stubgen::commands::dependencies::{add_stub_dependency, UpdateCargoToml}; use golem_wasm_rpc_stubgen::commands::generate::generate_and_build_stub; -use golem_wasm_rpc_stubgen::stub::StubDefinition; +use golem_wasm_rpc_stubgen::stub::{StubConfig, StubDefinition}; use std::path::{Path, PathBuf}; use tempfile::TempDir; use wasm_rpc_stubgen_tests_integration::{test_data_path, wasm_rpc_override}; @@ -39,7 +39,6 @@ async fn compose_with_single_stub() { add_stub_dependency( &stub_dir.path().join("wit"), &caller_dir.path().join("wit"), - true, UpdateCargoToml::Update, ) .unwrap(); @@ -77,7 +76,6 @@ async fn compose_with_single_stub_not_importing_stub() { add_stub_dependency( &stub_dir.path().join("wit"), &caller_dir.path().join("wit"), - false, UpdateCargoToml::NoUpdate, ) .unwrap(); @@ -108,14 +106,14 @@ async fn init_stub(name: &str) -> (TempDir, PathBuf) { let source_wit_root = test_data_path().join(name); let canonical_target_root = tempdir.path().canonicalize().unwrap(); - let def = StubDefinition::new( - &source_wit_root, - &canonical_target_root, - &None, - "1.0.0", - &wasm_rpc_override(), - false, - ) + let def = StubDefinition::new(StubConfig { + source_wit_root, + target_root: canonical_target_root, + selected_world: None, + stub_crate_version: "1.0.0".to_string(), + wasm_rpc_override: wasm_rpc_override(), + extract_source_interface_package: true, + }) .unwrap(); let wasm_path = generate_and_build_stub(&def).await.unwrap(); (tempdir, wasm_path) diff --git a/wasm-rpc-stubgen/tests-integration/tests/stub_wasm.rs b/wasm-rpc-stubgen/tests-integration/tests/stub_wasm.rs index 68d1fef6..75fe917e 100644 --- a/wasm-rpc-stubgen/tests-integration/tests/stub_wasm.rs +++ b/wasm-rpc-stubgen/tests-integration/tests/stub_wasm.rs @@ -26,7 +26,7 @@ use golem_wasm_ast::analysis::{ use golem_wasm_ast::component::Component; use golem_wasm_ast::IgnoreAllButMetadata; use golem_wasm_rpc_stubgen::commands::generate::generate_and_build_stub; -use golem_wasm_rpc_stubgen::stub::StubDefinition; +use golem_wasm_rpc_stubgen::stub::{StubConfig, StubDefinition}; use tempfile::tempdir; use wasm_rpc_stubgen_tests_integration::{test_data_path, wasm_rpc_override}; @@ -38,14 +38,14 @@ async fn all_wit_types() { let target_root = tempdir().unwrap(); let canonical_target_root = target_root.path().canonicalize().unwrap(); - let def = StubDefinition::new( - &source_wit_root, - &canonical_target_root, - &None, - "1.0.0", - &wasm_rpc_override(), - false, - ) + let def = StubDefinition::new(StubConfig { + source_wit_root, + target_root: canonical_target_root, + selected_world: None, + stub_crate_version: "1.0.0".to_string(), + wasm_rpc_override: wasm_rpc_override(), + extract_source_interface_package: true, + }) .unwrap(); let wasm_path = generate_and_build_stub(&def).await.unwrap(); diff --git a/wasm-rpc-stubgen/tests/add_dep.rs b/wasm-rpc-stubgen/tests/add_dep.rs index a47af90c..4be74846 100644 --- a/wasm-rpc-stubgen/tests/add_dep.rs +++ b/wasm-rpc-stubgen/tests/add_dep.rs @@ -22,8 +22,7 @@ use golem_wasm_rpc::{WASI_POLL_WIT, WASM_RPC_WIT}; use golem_wasm_rpc_stubgen::commands::generate::generate_stub_wit_dir; use golem_wasm_rpc_stubgen::stub::{StubConfig, StubDefinition}; use golem_wasm_rpc_stubgen::wit_generate::{ - add_stub_as_dependency_to_wit_dir, extract_main_interface_as_wit_dep, AddStubAsDepConfig, - UpdateCargoToml, + add_stub_as_dependency_to_wit_dir, AddStubAsDepConfig, UpdateCargoToml, }; use golem_wasm_rpc_stubgen::wit_resolve::ResolvedWitDir; use golem_wasm_rpc_stubgen::WasmRpcOverride; @@ -112,7 +111,7 @@ fn all_wit_types_re_add_with_changes() { assert_valid_wit_root(&dest_wit_root); assert_has_wasm_rpc_wit_deps(&dest_wit_root); assert_has_same_wit_package( - &PackageName::new("test", "main-interface", None), + &PackageName::new("test", "main-interface", None), alternative_source_dir.path(), &alternative_stub_wit_root, ); @@ -573,8 +572,6 @@ fn init_stub(name: &str) -> (TempDir, TempDir) { ) .unwrap(); - extract_main_interface_as_wit_dep(source.path()).unwrap(); - let target = TempDir::new().unwrap(); let canonical_target = target.path().canonicalize().unwrap(); @@ -584,6 +581,7 @@ fn init_stub(name: &str) -> (TempDir, TempDir) { selected_world: None, stub_crate_version: "1.0.0".to_string(), wasm_rpc_override: WasmRpcOverride::default(), + extract_source_interface_package: true, }) .unwrap(); let _ = generate_stub_wit_dir(&def).unwrap(); @@ -597,6 +595,7 @@ fn regenerate_stub(stub_dir: &Path, source_wit_root: &Path) { selected_world: None, stub_crate_version: "1.0.0".to_string(), wasm_rpc_override: WasmRpcOverride::default(), + extract_source_interface_package: true, }) .unwrap(); let _ = generate_stub_wit_dir(&def).unwrap(); @@ -613,17 +612,6 @@ fn init_caller(name: &str) -> TempDir { ) .unwrap(); - let wit_dir = { - let wit_dir = temp_dir.path().join("wit"); - if wit_dir.exists() { - wit_dir - } else { - temp_dir.path().to_path_buf() - } - }; - - extract_main_interface_as_wit_dep(&wit_dir).unwrap(); - temp_dir } diff --git a/wasm-rpc-stubgen/tests/wit.rs b/wasm-rpc-stubgen/tests/wit.rs index 2c78f175..87b28e3f 100644 --- a/wasm-rpc-stubgen/tests/wit.rs +++ b/wasm-rpc-stubgen/tests/wit.rs @@ -19,7 +19,6 @@ use test_r::test; use fs_extra::dir::CopyOptions; use golem_wasm_rpc_stubgen::commands::generate::generate_stub_wit_dir; use golem_wasm_rpc_stubgen::stub::{StubConfig, StubDefinition}; -use golem_wasm_rpc_stubgen::wit_generate::extract_main_interface_as_wit_dep; use golem_wasm_rpc_stubgen::WasmRpcOverride; use std::path::Path; use tempfile::{tempdir, TempDir}; @@ -38,6 +37,7 @@ fn all_wit_types() { selected_world: None, stub_crate_version: "1.0.0".to_string(), wasm_rpc_override: WasmRpcOverride::default(), + extract_source_interface_package: true, }) .unwrap(); let resolve = generate_stub_wit_dir(&def).unwrap().resolve; @@ -104,7 +104,12 @@ fn all_wit_types() { assert_defines_record(&resolve, "test:main-interface", "iface1", "point"); assert_defines_record(&resolve, "test:main-interface", "iface1", "product-item"); assert_defines_record(&resolve, "test:main-interface", "iface1", "order"); - assert_defines_record(&resolve, "test:main-interface", "iface1", "order-confirmation"); + assert_defines_record( + &resolve, + "test:main-interface", + "iface1", + "order-confirmation", + ); assert_defines_tuple_alias(&resolve, "test:main-interface", "iface1", "point-tuple"); assert_defines_variant(&resolve, "test:main-interface", "iface1", "checkout-result"); @@ -122,6 +127,7 @@ fn many_ways_to_export() { selected_world: None, stub_crate_version: "1.0.0".to_string(), wasm_rpc_override: WasmRpcOverride::default(), + extract_source_interface_package: true, }) .unwrap(); let resolve = generate_stub_wit_dir(&def).unwrap().resolve; @@ -318,16 +324,5 @@ fn init_source(name: &str) -> TempDir { ) .unwrap(); - let wit_dir = { - let wit_dir = temp_dir.path().join("wit"); - if wit_dir.exists() { - wit_dir - } else { - temp_dir.path().to_path_buf() - } - }; - - extract_main_interface_as_wit_dep(&wit_dir).unwrap(); - temp_dir } From 4231fb8eabd7d6bda3f5f523af83b99160fa8bbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 5 Nov 2024 11:58:55 +0100 Subject: [PATCH 037/109] cleanup leftover "auto" imports --- .../test-data/caller-no-dep-importstub/wit/caller.wit | 2 -- .../test-data/direct-circular-a-same-world-name/a.wit | 1 - wasm-rpc-stubgen/test-data/direct-circular-a/a.wit | 1 - .../test-data/direct-circular-b-same-world-name/b.wit | 1 - wasm-rpc-stubgen/test-data/direct-circular-b/b.wit | 1 - wasm-rpc-stubgen/test-data/indirect-circular-a/a.wit | 1 - wasm-rpc-stubgen/test-data/indirect-circular-b/b.wit | 1 - wasm-rpc-stubgen/test-data/indirect-circular-c/c.wit | 1 - wasm-rpc-stubgen/test-data/self-circular/a.wit | 1 - 9 files changed, 10 deletions(-) diff --git a/wasm-rpc-stubgen/test-data/caller-no-dep-importstub/wit/caller.wit b/wasm-rpc-stubgen/test-data/caller-no-dep-importstub/wit/caller.wit index 83b3ad1e..a205922c 100644 --- a/wasm-rpc-stubgen/test-data/caller-no-dep-importstub/wit/caller.wit +++ b/wasm-rpc-stubgen/test-data/caller-no-dep-importstub/wit/caller.wit @@ -5,7 +5,5 @@ interface api { } world caller { -//!! import test:main-stub/stub-api; - export api; } diff --git a/wasm-rpc-stubgen/test-data/direct-circular-a-same-world-name/a.wit b/wasm-rpc-stubgen/test-data/direct-circular-a-same-world-name/a.wit index a9caeebd..384cd847 100644 --- a/wasm-rpc-stubgen/test-data/direct-circular-a-same-world-name/a.wit +++ b/wasm-rpc-stubgen/test-data/direct-circular-a-same-world-name/a.wit @@ -11,6 +11,5 @@ interface api-a { } world a { -//!! import test:b-stub/stub-a; export api-a; } diff --git a/wasm-rpc-stubgen/test-data/direct-circular-a/a.wit b/wasm-rpc-stubgen/test-data/direct-circular-a/a.wit index 8d67a44b..384cd847 100644 --- a/wasm-rpc-stubgen/test-data/direct-circular-a/a.wit +++ b/wasm-rpc-stubgen/test-data/direct-circular-a/a.wit @@ -11,6 +11,5 @@ interface api-a { } world a { -//!! import test:b-stub/stub-b; export api-a; } diff --git a/wasm-rpc-stubgen/test-data/direct-circular-b-same-world-name/b.wit b/wasm-rpc-stubgen/test-data/direct-circular-b-same-world-name/b.wit index 371f36e3..52eedbcd 100644 --- a/wasm-rpc-stubgen/test-data/direct-circular-b-same-world-name/b.wit +++ b/wasm-rpc-stubgen/test-data/direct-circular-b-same-world-name/b.wit @@ -13,6 +13,5 @@ interface api-b { } world a { -//!! import test:a-stub/stub-a; export api-b; } diff --git a/wasm-rpc-stubgen/test-data/direct-circular-b/b.wit b/wasm-rpc-stubgen/test-data/direct-circular-b/b.wit index 9ab09d95..c50beaba 100644 --- a/wasm-rpc-stubgen/test-data/direct-circular-b/b.wit +++ b/wasm-rpc-stubgen/test-data/direct-circular-b/b.wit @@ -11,6 +11,5 @@ interface api-b { } world b { -//!! import test:a-stub/stub-a; export api-b; } diff --git a/wasm-rpc-stubgen/test-data/indirect-circular-a/a.wit b/wasm-rpc-stubgen/test-data/indirect-circular-a/a.wit index 957cfda0..fa1c352d 100644 --- a/wasm-rpc-stubgen/test-data/indirect-circular-a/a.wit +++ b/wasm-rpc-stubgen/test-data/indirect-circular-a/a.wit @@ -11,6 +11,5 @@ interface api-a { } world a { - //!! import test:b-stub/stub-b; export api-a; } diff --git a/wasm-rpc-stubgen/test-data/indirect-circular-b/b.wit b/wasm-rpc-stubgen/test-data/indirect-circular-b/b.wit index a230449c..85f8fd42 100644 --- a/wasm-rpc-stubgen/test-data/indirect-circular-b/b.wit +++ b/wasm-rpc-stubgen/test-data/indirect-circular-b/b.wit @@ -11,6 +11,5 @@ interface api-b { } world b { - //!! import test:c-stub/stub-c; export api-b; } diff --git a/wasm-rpc-stubgen/test-data/indirect-circular-c/c.wit b/wasm-rpc-stubgen/test-data/indirect-circular-c/c.wit index 4a37bdb7..6384fd93 100644 --- a/wasm-rpc-stubgen/test-data/indirect-circular-c/c.wit +++ b/wasm-rpc-stubgen/test-data/indirect-circular-c/c.wit @@ -8,6 +8,5 @@ interface api-c { } world c { - //!! import test:a-stub/stub-a; export api-c; } diff --git a/wasm-rpc-stubgen/test-data/self-circular/a.wit b/wasm-rpc-stubgen/test-data/self-circular/a.wit index 829af7c9..aa7fad99 100644 --- a/wasm-rpc-stubgen/test-data/self-circular/a.wit +++ b/wasm-rpc-stubgen/test-data/self-circular/a.wit @@ -11,6 +11,5 @@ interface api-a { } world a { - //!! import test:a-stub/stub-a; export api-a; } From 79a9ed9793cf3a6cbdb6b1821d42b64f85ebbfb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 5 Nov 2024 11:59:36 +0100 Subject: [PATCH 038/109] clippy, fmt --- wasm-rpc-stubgen/src/commands/declarative.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index 87c7f296..5da9e501 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -85,7 +85,7 @@ async fn pre_component_build_app(config: &Config, app: &Application) -> anyhow:: wasm_rpc_path_override: app.stub_wasm_rpc_path(&component_name), wasm_rpc_version_override: app.stub_wasm_rpc_version(&component_name), }, - extract_source_interface_package: true // TODO: use input output dirs + extract_source_interface_package: true, // TODO: use input output dirs }) .context("Failed to gather information for the stub generator")?; From e3edd842c9852071e619e968ecdf69f222eec7d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 5 Nov 2024 13:25:18 +0100 Subject: [PATCH 039/109] fix cargo gen / tests --- wasm-rpc-stubgen/src/cargo.rs | 6 +- .../tests-integration/tests/compose.rs | 67 ++++--------------- 2 files changed, 16 insertions(+), 57 deletions(-) diff --git a/wasm-rpc-stubgen/src/cargo.rs b/wasm-rpc-stubgen/src/cargo.rs index 0c740a6e..e37f3723 100644 --- a/wasm-rpc-stubgen/src/cargo.rs +++ b/wasm-rpc-stubgen/src/cargo.rs @@ -87,9 +87,9 @@ pub fn generate_cargo_toml(def: &StubDefinition) -> anyhow::Result<()> { }, ); - let stub_package_name = def.stub_package_name(); - for (_, dep_package, dep_package_sources) in def.packages_with_wit_sources() { - if dep_package.name == stub_package_name { + let stub_dep_package_ids = def.stub_dep_package_ids(); + for (dep_package_id, dep_package, dep_package_sources) in def.packages_with_wit_sources() { + if !stub_dep_package_ids.contains(&dep_package_id) { continue; } diff --git a/wasm-rpc-stubgen/tests-integration/tests/compose.rs b/wasm-rpc-stubgen/tests-integration/tests/compose.rs index 22545939..69845ea0 100644 --- a/wasm-rpc-stubgen/tests-integration/tests/compose.rs +++ b/wasm-rpc-stubgen/tests-integration/tests/compose.rs @@ -19,8 +19,8 @@ use test_r::test; use fs_extra::dir::CopyOptions; -use golem_wasm_ast::component::{Component, ComponentExternName}; -use golem_wasm_ast::{DefaultAst, IgnoreAllButMetadata}; +use golem_wasm_ast::component::Component; +use golem_wasm_ast::DefaultAst; use golem_wasm_rpc_stubgen::commands::composition::compose; use golem_wasm_rpc_stubgen::commands::dependencies::{add_stub_dependency, UpdateCargoToml}; use golem_wasm_rpc_stubgen::commands::generate::generate_and_build_stub; @@ -33,7 +33,7 @@ test_r::enable!(); #[test] async fn compose_with_single_stub() { - let (stub_dir, stub_wasm) = init_stub("all-wit-types").await; + let (_source_dir, stub_dir, stub_wasm) = init_stub("all-wit-types").await; let caller_dir = init_caller("caller-no-dep-importstub"); add_stub_dependency( @@ -43,11 +43,6 @@ async fn compose_with_single_stub() { ) .unwrap(); - println!( - "{}", - std::fs::read_to_string(stub_dir.path().join("wit/_stub.wit")).unwrap() - ); - compile_rust(caller_dir.path()); let component_wasm = caller_dir @@ -64,47 +59,21 @@ async fn compose_with_single_stub() { compose(&component_wasm, &[stub_wasm], &dest_wasm) .await .unwrap(); - - assert_not_importing(&dest_wasm, "test:main-stub/stub-api"); } -#[test] -async fn compose_with_single_stub_not_importing_stub() { - let (stub_dir, stub_wasm) = init_stub("all-wit-types").await; - let caller_dir = init_caller("caller-no-dep"); +async fn init_stub(name: &str) -> (TempDir, TempDir, PathBuf) { + let source_dir = TempDir::new().unwrap(); + let source_wit_root = source_dir.path().canonicalize().unwrap(); - add_stub_dependency( - &stub_dir.path().join("wit"), - &caller_dir.path().join("wit"), - UpdateCargoToml::NoUpdate, + fs_extra::dir::copy( + test_data_path().join(name), + &source_wit_root, + &CopyOptions::new().content_only(true), ) .unwrap(); - compile_rust(caller_dir.path()); - - let component_wasm = caller_dir - .path() - .join("target") - .join("wasm32-wasi") - .join("debug") - .join("caller_no_dep.wasm"); - - assert_is_component(&stub_wasm); - assert_is_component(&component_wasm); - - let dest_wasm = caller_dir.path().join("target/result.wasm"); - compose(&component_wasm, &[stub_wasm], &dest_wasm) - .await - .unwrap(); - - assert_not_importing(&dest_wasm, "test:main-stub/stub-api"); -} - -async fn init_stub(name: &str) -> (TempDir, PathBuf) { - let tempdir = TempDir::new().unwrap(); - - let source_wit_root = test_data_path().join(name); - let canonical_target_root = tempdir.path().canonicalize().unwrap(); + let stub_dir = TempDir::new().unwrap(); + let canonical_target_root = stub_dir.path().canonicalize().unwrap(); let def = StubDefinition::new(StubConfig { source_wit_root, @@ -116,7 +85,7 @@ async fn init_stub(name: &str) -> (TempDir, PathBuf) { }) .unwrap(); let wasm_path = generate_and_build_stub(&def).await.unwrap(); - (tempdir, wasm_path) + (source_dir, stub_dir, wasm_path) } fn init_caller(name: &str) -> TempDir { @@ -147,13 +116,3 @@ fn assert_is_component(wasm_path: &Path) { let _component: Component = Component::from_bytes(&std::fs::read(wasm_path).unwrap()).unwrap(); } - -fn assert_not_importing(wasm_path: &Path, import_name: &str) { - let component_bytes = std::fs::read(wasm_path).unwrap(); - let component: Component = - Component::from_bytes(&component_bytes).unwrap(); - component.imports().iter().all(|import| { - let ComponentExternName::Name(name) = &import.name; - name != import_name - }); -} From 576d274f0f751c2526d8bc6332ae6a34aca0c040 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Thu, 7 Nov 2024 11:40:49 +0100 Subject: [PATCH 040/109] separate input and output build for declarative build, add ResolvedWitApplication for dependency tracking, more log formatting --- wasm-rpc-stubgen/src/cargo.rs | 14 +- wasm-rpc-stubgen/src/commands/composition.rs | 4 +- wasm-rpc-stubgen/src/commands/declarative.rs | 308 ++++++++++++++----- wasm-rpc-stubgen/src/commands/generate.rs | 4 +- wasm-rpc-stubgen/src/commands/log.rs | 107 ++++++- wasm-rpc-stubgen/src/lib.rs | 4 +- wasm-rpc-stubgen/src/make.rs | 8 +- wasm-rpc-stubgen/src/model/validation.rs | 2 + wasm-rpc-stubgen/src/model/wasm_rpc.rs | 32 +- wasm-rpc-stubgen/src/naming.rs | 12 +- wasm-rpc-stubgen/src/rust.rs | 7 +- wasm-rpc-stubgen/src/wit_generate.rs | 57 +++- wasm-rpc-stubgen/src/wit_resolve.rs | 190 +++++++++++- 13 files changed, 618 insertions(+), 131 deletions(-) diff --git a/wasm-rpc-stubgen/src/cargo.rs b/wasm-rpc-stubgen/src/cargo.rs index e37f3723..85745993 100644 --- a/wasm-rpc-stubgen/src/cargo.rs +++ b/wasm-rpc-stubgen/src/cargo.rs @@ -16,7 +16,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::fs; use std::path::{Path, PathBuf}; -use crate::commands::log::{log_action, log_warn_action}; +use crate::commands::log::{log_action, log_warn_action, LogColorize}; use crate::fs::get_file_name; use crate::naming; use crate::stub::StubDefinition; @@ -205,7 +205,7 @@ pub fn generate_cargo_toml(def: &StubDefinition) -> anyhow::Result<()> { "Generating", format!( "Cargo.toml to {}", - def.target_cargo_path().to_string_lossy() + def.target_cargo_path().log_color_highlight() ), ); fs::write(def.target_cargo_path(), cargo_toml)?; @@ -251,7 +251,10 @@ pub fn add_workspace_members(path: &Path, members: &[String]) -> anyhow::Result< let cargo_toml = toml::to_string(&manifest)?; - log_action("Writing", format!("updated Cargo.toml to {:?}", path)); + log_action( + "Writing", + format!("updated Cargo.toml to {:?}", path.log_color_highlight()), + ); fs::write(path, cargo_toml)?; Ok(()) } @@ -287,7 +290,10 @@ pub fn add_dependencies_to_cargo_toml( let cargo_toml = toml::to_string(&manifest)?; - log_warn_action("Updating", format!("Cargo.toml at {:?}", cargo_path)); + log_warn_action( + "Updating", + format!("Cargo.toml at {:?}", cargo_path.log_color_highlight()), + ); fs::write(cargo_path, cargo_toml)?; } } diff --git a/wasm-rpc-stubgen/src/commands/composition.rs b/wasm-rpc-stubgen/src/commands/composition.rs index f87436ee..1a75c251 100644 --- a/wasm-rpc-stubgen/src/commands/composition.rs +++ b/wasm-rpc-stubgen/src/commands/composition.rs @@ -1,4 +1,4 @@ -use crate::commands::log::log_warn_action; +use crate::commands::log::{log_warn_action, LogColorize}; use anyhow::Context; use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; @@ -108,7 +108,7 @@ fn plug( }; for plug_name in unused_plugs { - log_warn_action("Skipping", format!("{}, not used", plug_name)); + log_warn_action("Skipping", format!("{}, not used", plug_name.log_color_highlight())); } // Export all exports from the socket component. diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index 5da9e501..51a0bcf3 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -1,5 +1,6 @@ use crate::commands::log::{ - log_action, log_skipping_up_to_date, log_validated_action_result, log_warn_action, + log_action, log_skipping_up_to_date, log_validated_action_result, log_warn_action, LogColorize, + LogIndent, }; use crate::fs::copy; use crate::model::oam; @@ -8,7 +9,11 @@ use crate::model::wasm_rpc::{ include_glob_patter_from_yaml_file, init_oam_app, Application, DEFAULT_CONFIG_FILE_NAME, }; use crate::stub::{StubConfig, StubDefinition}; -use crate::wit_generate::{add_stub_as_dependency_to_wit_dir, AddStubAsDepConfig, UpdateCargoToml}; +use crate::wit_generate::{ + add_stub_as_dependency_to_wit_dir, extract_main_interface_as_wit_dep, AddStubAsDepConfig, + UpdateCargoToml, +}; +use crate::wit_resolve::ResolvedWitApplication; use crate::{commands, WasmRpcOverride}; use anyhow::{anyhow, Context, Error}; use colored::Colorize; @@ -23,16 +28,22 @@ use tempfile::TempDir; use walkdir::WalkDir; pub struct Config { - pub app_resolve_mode: ApplicationResolveMode, + pub app_resolve_mode: ApplicationSourceMode, pub skip_up_to_date_checks: bool, } #[derive(Debug, Clone)] -pub enum ApplicationResolveMode { +pub enum ApplicationSourceMode { Automatic, Explicit(Vec), } +struct ApplicationContext { + config: Config, + application: Application, + wit: ResolvedWitApplication, +} + pub fn init(component_name: String) -> anyhow::Result<()> { let file_name = DEFAULT_CONFIG_FILE_NAME; @@ -47,75 +58,100 @@ pub fn init(component_name: String) -> anyhow::Result<()> { } pub async fn pre_component_build(config: Config) -> anyhow::Result<()> { - let app = load_app(&config)?; - pre_component_build_app(&config, &app).await + let mut ctx = create_context(config)?; + pre_component_build_ctx(&mut ctx).await } -async fn pre_component_build_app(config: &Config, app: &Application) -> anyhow::Result<()> { - if app.all_wasm_rpc_dependencies().is_empty() { +async fn pre_component_build_ctx(ctx: &mut ApplicationContext) -> anyhow::Result<()> { + log_action("Executing", "pre-component-build steps"); + let _indent = LogIndent::new(); + + let changed_any = extract_interface_packages(&ctx)?; + if changed_any { + update_wit_context(ctx)?; + } + + if ctx.application.all_wasm_rpc_dependencies().is_empty() { log_warn_action("Skipping", "building wasm rpc stubs, no dependency found"); } else { log_action("Building", "wasm rpc stubs"); - for component_name in app.all_wasm_rpc_dependencies() { + let _indent = LogIndent::new(); + + for component_name in ctx.application.all_wasm_rpc_dependencies() { if is_up_to_date( - config.skip_up_to_date_checks, - || [app.component_wit(&component_name)], + ctx.config.skip_up_to_date_checks, + || [ctx.application.component_output_wit(&component_name)], || { [ - app.stub_wasm(&component_name), - app.stub_wit(&component_name), + ctx.application.stub_wasm(&component_name), + ctx.application.stub_wit(&component_name), ] }, ) { - log_skipping_up_to_date(format!("building wasm rpc stub: {}", component_name)); + log_skipping_up_to_date(format!( + "building wasm rpc stub for {}", + component_name.log_color_highlight() + )); continue; } - log_action("Building", format!("wasm rpc stub: {}", component_name)); + log_action( + "Building", + format!("wasm rpc stub: {}", component_name.log_color_highlight()), + ); + let _indent = LogIndent::new(); let target_root = TempDir::new()?; let canonical_target_root = target_root.path().canonicalize()?; let stub_def = StubDefinition::new(StubConfig { - source_wit_root: app.component_wit(&component_name), + source_wit_root: ctx.application.component_output_wit(&component_name), target_root: canonical_target_root, - selected_world: app.stub_world(&component_name), - stub_crate_version: app.stub_crate_version(&component_name), + selected_world: ctx.application.stub_world(&component_name), + stub_crate_version: ctx.application.stub_crate_version(&component_name), wasm_rpc_override: WasmRpcOverride { - wasm_rpc_path_override: app.stub_wasm_rpc_path(&component_name), - wasm_rpc_version_override: app.stub_wasm_rpc_version(&component_name), + wasm_rpc_path_override: ctx.application.stub_wasm_rpc_path(&component_name), + wasm_rpc_version_override: ctx + .application + .stub_wasm_rpc_version(&component_name), }, - extract_source_interface_package: true, // TODO: use input output dirs + extract_source_interface_package: false, }) .context("Failed to gather information for the stub generator")?; commands::generate::build( &stub_def, - &app.stub_wasm(&component_name), - &app.stub_wit(&component_name), + &ctx.application.stub_wasm(&component_name), + &ctx.application.stub_wit(&component_name), ) .await? } } - for (component_name, component) in &app.wasm_components_by_name { + for (component_name, component) in &ctx.application.wasm_components_by_name { if !component.wasm_rpc_dependencies.is_empty() { log_action( "Adding", - format!("stub wit dependencies to {}", component_name), + format!( + "stub wit dependencies to {}", + component_name.log_color_highlight() + ), ) } + let _indent = LogIndent::new(); for dep_component_name in &component.wasm_rpc_dependencies { // TODO: this should check into the wit deps for the specific stubs or do folder diffs if is_up_to_date( - config.skip_up_to_date_checks, - || [app.stub_wit(dep_component_name)], - || [app.component_wit(component_name)], + ctx.config.skip_up_to_date_checks + || !ctx.wit.has_as_wit_dep(component_name, dep_component_name), + || [ctx.application.stub_wit(dep_component_name)], + || [ctx.application.component_output_wit(component_name)], ) { log_skipping_up_to_date(format!( "adding {} stub wit dependency to {}", - dep_component_name, component_name + dep_component_name.log_color_highlight(), + component_name.log_color_highlight() )); continue; } @@ -124,14 +160,16 @@ async fn pre_component_build_app(config: &Config, app: &Application) -> anyhow:: "Adding", format!( "{} stub wit dependency to {}", - dep_component_name, component_name + dep_component_name.log_color_highlight(), + component_name.log_color_highlight() ), ); + let _indent = LogIndent::new(); add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { - stub_wit_root: app.stub_wit(dep_component_name), - dest_wit_root: app.component_wit(component_name), - update_cargo_toml: UpdateCargoToml::Update, + stub_wit_root: ctx.application.stub_wit(dep_component_name), + dest_wit_root: ctx.application.component_output_wit(component_name), + update_cargo_toml: UpdateCargoToml::UpdateIfExists, })? } } @@ -139,13 +177,97 @@ async fn pre_component_build_app(config: &Config, app: &Application) -> anyhow:: Ok(()) } +fn extract_interface_packages(ctx: &ApplicationContext) -> Result { + let mut changed_any = false; + for (component_name, _component) in &ctx.application.wasm_components_by_name { + let component_input_wit = ctx.application.component_input_wit(component_name); + let component_output_wit = ctx.application.component_output_wit(component_name); + + if is_up_to_date( + ctx.config.skip_up_to_date_checks || !ctx.wit.is_dep_graph_up_to_date(component_name), + || [component_input_wit.clone()], + || [component_output_wit.clone()], + ) { + log_skipping_up_to_date(format!( + "extracting main interface package from {}", + component_name.log_color_highlight() + )); + } else { + log_action( + "Extracting", + format!( + "main interface package from {} to {}", + component_input_wit.log_color_highlight(), + component_output_wit.log_color_highlight() + ), + ); + let _indent = LogIndent::new(); + + changed_any = true; + + if component_output_wit.exists() { + log_warn_action( + "Deleting", + format!( + "output wit directory {}", + component_output_wit.log_color_highlight() + ), + ); + std::fs::remove_dir_all(&component_output_wit)?; + } + + { + log_action( + "Copying", + format!( + "wit sources from {} to {}", + component_input_wit.log_color_highlight(), + component_output_wit.log_color_highlight() + ), + ); + let _indent = LogIndent::new(); + + let dir_content = fs_extra::dir::get_dir_content(&component_input_wit) + .with_context(|| { + anyhow!( + "Failed to read component input directory entries for {}", + component_input_wit.display() + ) + })?; + + for file in dir_content.files { + let from = PathBuf::from(&file); + let to = component_output_wit.join( + from.strip_prefix(&component_input_wit).with_context(|| { + anyhow!("Failed to strip prefix for source {}", &file) + })?, + ); + + log_action( + "Copying", + format!("wit source {} to {}", from.display(), to.display()), + ); + copy(from, to)?; + } + } + + extract_main_interface_as_wit_dep(&component_output_wit)?; + } + } + Ok(changed_any) +} + pub fn component_build(config: Config) -> anyhow::Result<()> { - let app = load_app(&config)?; - component_build_app(&config, &app) + let ctx = create_context(config)?; + component_build_ctx(&ctx) } -pub fn component_build_app(config: &Config, app: &Application) -> anyhow::Result<()> { - let components_with_build_steps = app +fn component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<()> { + log_action("Executing", "component-build steps"); + let _indent = LogIndent::new(); + + let components_with_build_steps = ctx + .application .wasm_components_by_name .values() .filter(|component| !component.build_steps.is_empty()) @@ -160,9 +282,19 @@ pub fn component_build_app(config: &Config, app: &Application) -> anyhow::Result } log_action("Building", "components"); + let _indent = LogIndent::new(); for component in components_with_build_steps { - log_action("Building", format!("component: {}", component.name)); + log_action( + "Building", + format!( + "component {} in directory {}", + component.name.log_color_highlight(), + component.source_dir().log_color_highlight(), + ), + ); + let _indent = LogIndent::new(); + for build_step in &component.build_steps { let build_dir = build_step .dir @@ -174,13 +306,19 @@ pub fn component_build_app(config: &Config, app: &Application) -> anyhow::Result let inputs = compile_and_collect_globs(&build_dir, &build_step.inputs)?; let outputs = compile_and_collect_globs(&build_dir, &build_step.outputs)?; - if is_up_to_date(config.skip_up_to_date_checks, || inputs, || outputs) { - log_skipping_up_to_date(format!("executing command: {}", build_step.command)); + if is_up_to_date(ctx.config.skip_up_to_date_checks, || inputs, || outputs) { + log_skipping_up_to_date(format!( + "executing command: {}", + build_step.command.log_color_highlight() + )); continue; } } - log_action("Executing", format!("command: {}", build_step.command)); + log_action( + "Executing", + format!("command: {}", build_step.command.log_color_highlight()), + ); let command_tokens = build_step.command.split(' ').collect::>(); if command_tokens.is_empty() { @@ -209,17 +347,20 @@ pub fn component_build_app(config: &Config, app: &Application) -> anyhow::Result } pub async fn post_component_build(config: Config) -> anyhow::Result<()> { - let app = load_app(&config)?; - post_component_build_app(&config, &app).await + let ctx = create_context(config)?; + post_component_build_ctx(&ctx).await } -pub async fn post_component_build_app(config: &Config, app: &Application) -> anyhow::Result<()> { - for (component_name, component) in &app.wasm_components_by_name { - let input_wasm = app.component_input_wasm(component_name); - let output_wasm = app.component_output_wasm(component_name); +async fn post_component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<()> { + log_action("Executing", "post-component-build steps"); + let _indent = LogIndent::new(); + + for (component_name, component) in &ctx.application.wasm_components_by_name { + let input_wasm = ctx.application.component_input_wasm(component_name); + let output_wasm = ctx.application.component_output_wasm(component_name); if is_up_to_date( - config.skip_up_to_date_checks, + ctx.config.skip_up_to_date_checks, // We also include the component specification source, // so it triggers build in case deps are changed || [component.source.clone(), input_wasm.clone()], @@ -227,8 +368,12 @@ pub async fn post_component_build_app(config: &Config, app: &Application) -> any ) { log_skipping_up_to_date(format!( "composing wasm rpc dependencies ({}) into {}", - component.wasm_rpc_dependencies.iter().join(", "), - component_name, + component + .wasm_rpc_dependencies + .iter() + .map(|s| s.log_color_highlight()) + .join(", "), + component_name.log_color_highlight(), )); continue; } @@ -238,8 +383,8 @@ pub async fn post_component_build_app(config: &Config, app: &Application) -> any "Copying", format!( "(without composing) {} to {}, no wasm rpc dependencies defined", - input_wasm.display(), - output_wasm.display() + input_wasm.log_color_highlight(), + output_wasm.log_color_highlight(), ), ); copy(&input_wasm, &output_wasm)?; @@ -248,21 +393,29 @@ pub async fn post_component_build_app(config: &Config, app: &Application) -> any "Composing", format!( "wasm rpc dependencies ({}) into {}", - component.wasm_rpc_dependencies.iter().join(", "), - component_name, + component + .wasm_rpc_dependencies + .iter() + .map(|s| s.log_color_highlight()) + .join(", "), + component_name.log_color_highlight(), ), ); let stub_wasms = component .wasm_rpc_dependencies .iter() - .map(|dep| app.stub_wasm(dep)) + .map(|dep| ctx.application.stub_wasm(dep)) .collect::>(); commands::composition::compose( - app.component_input_wasm(component_name).as_path(), + ctx.application + .component_input_wasm(component_name) + .as_path(), &stub_wasms, - app.component_output_wasm(component_name).as_path(), + ctx.application + .component_output_wasm(component_name) + .as_path(), ) .await?; } @@ -272,19 +425,34 @@ pub async fn post_component_build_app(config: &Config, app: &Application) -> any } pub async fn build(config: Config) -> anyhow::Result<()> { - let app = load_app(&config)?; + let mut ctx = create_context(config)?; - pre_component_build_app(&config, &app).await?; - component_build_app(&config, &app)?; - post_component_build_app(&config, &app).await?; + pre_component_build_ctx(&mut ctx).await?; + component_build_ctx(&ctx)?; + post_component_build_ctx(&ctx).await?; Ok(()) } -fn load_app(config: &Config) -> anyhow::Result { +fn create_context(config: Config) -> anyhow::Result { + to_anyhow( + "Failed to create application context, see problems above", + load_app_validated(&config).and_then(|application| { + ResolvedWitApplication::new(&application).map(|wit| ApplicationContext { + config, + application, + wit, + }) + }), + ) +} + +fn update_wit_context(ctx: &mut ApplicationContext) -> anyhow::Result<()> { to_anyhow( - "Failed to load application manifest(s), see problems above".to_string(), - load_app_validated(config), + "Failed to update application wit context, see problems above", + ResolvedWitApplication::new(&ctx.application).map(|wit| { + ctx.wit = wit; + }), ) } @@ -301,6 +469,7 @@ fn load_app_validated(config: &Config) -> ValidatedResult { }); log_action("Collecting", "components"); + let _indent = LogIndent::new(); let app = oam_apps.and_then(Application::from_oam_apps); @@ -318,11 +487,12 @@ fn load_app_validated(config: &Config) -> ValidatedResult { app } -fn collect_sources(mode: &ApplicationResolveMode) -> ValidatedResult> { +fn collect_sources(mode: &ApplicationSourceMode) -> ValidatedResult> { log_action("Collecting", "sources"); + let _indent = LogIndent::new(); let sources = match mode { - ApplicationResolveMode::Automatic => match find_main_source() { + ApplicationSourceMode::Automatic => match find_main_source() { Some(source) => { std::env::set_current_dir(source.parent().expect("Failed ot get config parent")) .expect("Failed to set current dir for config parent"); @@ -358,7 +528,7 @@ fn collect_sources(mode: &ApplicationResolveMode) -> ValidatedResult ValidatedResult::from_error("No config file found!".to_string()), }, - ApplicationResolveMode::Explicit(sources) => { + ApplicationSourceMode::Explicit(sources) => { let non_unique_source_warns: Vec<_> = sources .iter() .counts() @@ -411,7 +581,7 @@ fn find_main_source() -> Option { last_source } -fn to_anyhow(message: String, result: ValidatedResult) -> anyhow::Result { +fn to_anyhow(message: &str, result: ValidatedResult) -> anyhow::Result { fn print_warns(warns: Vec) { let label = "Warning".yellow(); for warn in warns { @@ -438,7 +608,7 @@ fn to_anyhow(message: String, result: ValidatedResult) -> anyhow::Result anyhow::Result> = LazyLock::new(RwLock::default); @@ -64,7 +68,7 @@ pub fn log_action>(action: &str, subject: T) { println!( "{}{} {}", LOG_STATE.read().unwrap().indent_prefix, - action.green(), + action.log_color_action(), subject.as_ref() ) } @@ -73,7 +77,7 @@ pub fn log_warn_action>(action: &str, subject: T) { println!( "{}{} {}", LOG_STATE.read().unwrap().indent_prefix, - action.yellow(), + action.log_color_warn(), subject.as_ref(), ) } @@ -100,34 +104,50 @@ pub fn log_action_plan(action: &OverwriteSafeAction, plan: OverwriteSafeActionPl OverwriteSafeAction::CopyFile { source, target } => { log_action( "Copying", - format!("{} to {}", source.display(), target.display()), + format!( + "{} to {}", + source.log_color_highlight(), + target.log_color_highlight() + ), ); } OverwriteSafeAction::CopyFileTransformed { source, target, .. } => { log_action( "Copying", - format!("{} to {} transformed", source.display(), target.display()), + format!( + "{} to {} transformed", + source.log_color_highlight(), + target.log_color_highlight() + ), ); } OverwriteSafeAction::WriteFile { target, .. } => { - log_action("Creating", format!("{}", target.display())); + log_action("Creating", format!("{}", target.log_color_highlight())); } }, OverwriteSafeActionPlan::Overwrite => match action { OverwriteSafeAction::CopyFile { source, target } => { log_warn_action( "Overwriting", - format!("{} with {}", target.display(), source.display()), + format!( + "{} with {}", + target.log_color_highlight(), + source.log_color_highlight() + ), ); } OverwriteSafeAction::CopyFileTransformed { source, target, .. } => { log_warn_action( "Overwriting", - format!("{} with {} transformed", target.display(), source.display()), + format!( + "{} with {} transformed", + target.log_color_highlight(), + source.log_color_highlight() + ), ); } OverwriteSafeAction::WriteFile { content: _, target } => { - log_warn_action("Overwriting", format!("{}", target.display())); + log_warn_action("Overwriting", format!("{}", target.log_color_highlight())); } }, OverwriteSafeActionPlan::SkipSameContent => match action { @@ -136,8 +156,8 @@ pub fn log_action_plan(action: &OverwriteSafeAction, plan: OverwriteSafeActionPl "Skipping", format!( "copying {} to {}, content already up-to-date", - source.display(), - target.display(), + source.log_color_highlight(), + target.log_color_highlight(), ), ); } @@ -146,8 +166,8 @@ pub fn log_action_plan(action: &OverwriteSafeAction, plan: OverwriteSafeActionPl "Skipping", format!( "copying {} to {} transformed, content already up-to-date", - source.display(), - target.display() + source.log_color_highlight(), + target.log_color_highlight() ), ); } @@ -156,10 +176,69 @@ pub fn log_action_plan(action: &OverwriteSafeAction, plan: OverwriteSafeActionPl "Skipping", format!( "generating {}, content already up-to-date", - target.display() + target.log_color_highlight() ), ); } }, } } + +pub trait LogColorize { + type Colorize: Colorize; + + fn to_colorize(&self) -> Self::Colorize; + + fn log_color_action(&self) -> ColoredString { + self.to_colorize().green() + } + + fn log_color_warn(&self) -> ColoredString { + self.to_colorize().yellow() + } + + fn log_color_error(&self) -> ColoredString { + self.to_colorize().red() + } + + fn log_color_highlight(&self) -> ColoredString { + self.to_colorize().bold() + } +} + +impl<'a> LogColorize for &'a str { + type Colorize = &'a str; + + fn to_colorize(&self) -> Self::Colorize { + self + } +} + +impl LogColorize for String { + type Colorize = ColoredString; + + fn to_colorize(&self) -> Self::Colorize { + self.clone().into() + } +} + +impl LogColorize for PathBuf { + type Colorize = ColoredString; + + fn to_colorize(&self) -> Self::Colorize { + self.display().to_string().into() + } +} + +impl<'a> LogColorize for &'a Path { + type Colorize = ColoredString; + + fn to_colorize(&self) -> Self::Colorize { + self.display().to_string().into() + } +} + +pub struct ChildProcessLogger { + _out_handle: JoinHandle<()>, + _err_handle: JoinHandle<()>, +} diff --git a/wasm-rpc-stubgen/src/lib.rs b/wasm-rpc-stubgen/src/lib.rs index 9944915c..4e42e7f0 100644 --- a/wasm-rpc-stubgen/src/lib.rs +++ b/wasm-rpc-stubgen/src/lib.rs @@ -300,9 +300,9 @@ fn dec_build_args_to_config(args: DeclarativeBuildArgs) -> commands::declarative commands::declarative::Config { app_resolve_mode: { if args.app.is_empty() { - commands::declarative::ApplicationResolveMode::Automatic + commands::declarative::ApplicationSourceMode::Automatic } else { - commands::declarative::ApplicationResolveMode::Explicit(args.app) + commands::declarative::ApplicationSourceMode::Explicit(args.app) } }, skip_up_to_date_checks: args.force_build, diff --git a/wasm-rpc-stubgen/src/make.rs b/wasm-rpc-stubgen/src/make.rs index 78381375..e90d338d 100644 --- a/wasm-rpc-stubgen/src/make.rs +++ b/wasm-rpc-stubgen/src/make.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::commands::log::{log_action, log_warn_action}; +use crate::commands::log::{log_action, log_warn_action, LogColorize}; use crate::{cargo, GenerateArgs, WasmRpcOverride}; use heck::ToSnakeCase; use std::fs; @@ -443,7 +443,7 @@ pub fn initialize_workspace( let makefile = makefile.to_string()?; log_warn_action( "Overwriting", - format!("cargo-make Makefile {:?}", makefile_path), + format!("cargo-make Makefile {:?}", makefile_path.log_color_highlight()), ); fs::write(makefile_path, makefile)?; } else if has_cargo_make() { @@ -459,7 +459,7 @@ pub fn initialize_workspace( let makefile = makefile.to_string()?; log_action( "Writing", - format!("cargo-make Makefile to {:?}", makefile_path), + format!("cargo-make Makefile to {:?}", makefile_path.log_color_highlight()), ); fs::write(makefile_path, makefile)?; } else { @@ -470,7 +470,7 @@ pub fn initialize_workspace( let mut new_members = Vec::new(); for target in targets { - log_action("Generating", format!("initial stub for {target}")); + log_action("Generating", format!("initial stub for {}", target.log_color_highlight())); let stub_name = format!("{target}-stub"); crate::generate(GenerateArgs { diff --git a/wasm-rpc-stubgen/src/model/validation.rs b/wasm-rpc-stubgen/src/model/validation.rs index caa0a38d..5b039426 100644 --- a/wasm-rpc-stubgen/src/model/validation.rs +++ b/wasm-rpc-stubgen/src/model/validation.rs @@ -1,6 +1,8 @@ use itertools::Itertools; use std::fmt::Display; +// TODO: move to common + pub struct ValidationContext { pub name: &'static str, pub value: String, diff --git a/wasm-rpc-stubgen/src/model/wasm_rpc.rs b/wasm-rpc-stubgen/src/model/wasm_rpc.rs index 04c07e25..1eabd203 100644 --- a/wasm-rpc-stubgen/src/model/wasm_rpc.rs +++ b/wasm-rpc-stubgen/src/model/wasm_rpc.rs @@ -18,6 +18,8 @@ pub const OAM_COMPONENT_TYPE_WASM: &str = "wasm"; pub const OAM_COMPONENT_TYPE_WASM_BUILD: &str = "wasm-build"; pub const OAM_COMPONENT_TYPE_WASM_RPC_STUB_BUILD: &str = "wasm-rpc-stub-build"; +// TODO: ComponentName new type + pub fn init_oam_app(_component_name: String) -> oam::Application { // TODO: let's do it as part of https://github.com/golemcloud/wasm-rpc/issues/89 todo!() @@ -271,7 +273,8 @@ impl Application { name: component.name, source: source.to_path_buf(), build_steps: properties.build, - wit: properties.wit.into(), + input_wit: properties.input_wit.into(), + output_wit: properties.output_wit.into(), input_wasm: properties.input_wasm.into(), output_wasm: properties.output_wasm.into(), wasm_rpc_dependencies, @@ -334,7 +337,6 @@ impl Application { wasm: properties.wasm.map(|s| s.into()), wit: properties.wit.map(|s| s.into()), world: properties.world, - always_inline_types: properties.always_inline_types, crate_version: properties.crate_version, wasm_rpc_path: properties.wasm_rpc_path, wasm_rpc_version: properties.wasm_rpc_version, @@ -575,9 +577,14 @@ impl Application { .unwrap_or_else(|| panic!("Component not found: {}", component_name)) } - pub fn component_wit(&self, component_name: &str) -> PathBuf { + pub fn component_input_wit(&self, component_name: &str) -> PathBuf { + let component = self.component(component_name); + component.source_dir().join(component.input_wit.clone()) + } + + pub fn component_output_wit(&self, component_name: &str) -> PathBuf { let component = self.component(component_name); - component.source_dir().join(component.wit.clone()) + component.source_dir().join(component.output_wit.clone()) } pub fn component_input_wasm(&self, component_name: &str) -> PathBuf { @@ -601,12 +608,6 @@ impl Application { .unwrap_or_else(|| WASM_RPC_VERSION.to_string()) } - pub fn stub_always_inline_types(&self, component_name: &str) -> bool { - self.stub_gen_property(component_name, |build| build.always_inline_types) - .flatten() - .unwrap_or(false) - } - pub fn stub_wasm_rpc_path(&self, component_name: &str) -> Option { self.stub_gen_property(component_name, |build| build.wasm_rpc_path.clone()) .flatten() @@ -672,7 +673,8 @@ pub struct WasmComponent { pub name: String, pub source: PathBuf, pub build_steps: Vec, - pub wit: PathBuf, + pub input_wit: PathBuf, + pub output_wit: PathBuf, pub input_wasm: PathBuf, pub output_wasm: PathBuf, pub wasm_rpc_dependencies: Vec, @@ -702,13 +704,13 @@ pub struct BuildStep { } #[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] pub struct WasmComponentProperties { - pub wit: String, + pub input_wit: String, + pub output_wit: String, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub build: Vec, - #[serde(rename = "inputWasm")] pub input_wasm: String, - #[serde(rename = "outputWasm")] pub output_wasm: String, #[serde(flatten)] pub unknown_properties: UnknownProperties, @@ -782,7 +784,6 @@ pub struct ComponentStubBuildProperties { wasm: Option, wit: Option, world: Option, - always_inline_types: Option, crate_version: Option, wasm_rpc_path: Option, wasm_rpc_version: Option, @@ -811,7 +812,6 @@ pub struct WasmRpcStubBuild { wasm: Option, wit: Option, world: Option, - always_inline_types: Option, crate_version: Option, wasm_rpc_path: Option, wasm_rpc_version: Option, diff --git a/wasm-rpc-stubgen/src/naming.rs b/wasm-rpc-stubgen/src/naming.rs index d22c2112..daccf02a 100644 --- a/wasm-rpc-stubgen/src/naming.rs +++ b/wasm-rpc-stubgen/src/naming.rs @@ -16,7 +16,17 @@ pub mod wit { } } - pub fn interface_package_name( + pub fn interface_parser_package_name( + package_name: &wit_parser::PackageName, + ) -> wit_parser::PackageName { + wit_parser::PackageName { + namespace: package_name.namespace.clone(), + name: format!("{}-interface", package_name.name), + version: package_name.version.clone(), + } + } + + pub fn interface_encoder_package_name( package_name: &wit_encoder::PackageName, ) -> wit_encoder::PackageName { wit_encoder::PackageName::new( diff --git a/wasm-rpc-stubgen/src/rust.rs b/wasm-rpc-stubgen/src/rust.rs index 2746e90d..3a1c0924 100644 --- a/wasm-rpc-stubgen/src/rust.rs +++ b/wasm-rpc-stubgen/src/rust.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::commands::log::log_action; +use crate::commands::log::{log_action, LogColorize}; use crate::stub::{FunctionResultStub, FunctionStub, InterfaceStub, StubDefinition}; use anyhow::anyhow; use heck::{ToShoutySnakeCase, ToSnakeCase, ToUpperCamelCase}; @@ -286,7 +286,10 @@ pub fn generate_stub_source(def: &StubDefinition) -> anyhow::Result<()> { log_action( "Generating", - format!("stub source to {}", def.target_rust_path().display()), + format!( + "stub source to {}", + def.target_rust_path().log_color_highlight() + ), ); fs::create_dir_all(def.target_rust_path().parent().unwrap())?; fs::write(def.target_rust_path(), src)?; diff --git a/wasm-rpc-stubgen/src/wit_generate.rs b/wasm-rpc-stubgen/src/wit_generate.rs index 44c7c439..3f43c69e 100644 --- a/wasm-rpc-stubgen/src/wit_generate.rs +++ b/wasm-rpc-stubgen/src/wit_generate.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::commands::log::{log_action, log_action_plan, log_warn_action, LogIndent}; +use crate::commands::log::{log_action, log_action_plan, log_warn_action, LogColorize, LogIndent}; use crate::fs::{ copy, get_file_name, strip_path_prefix, write_str, OverwriteSafeAction, OverwriteSafeActions, }; @@ -37,7 +37,10 @@ use wit_parser::PackageId; pub fn generate_stub_wit_to_target(def: &StubDefinition) -> anyhow::Result<()> { log_action( "Generating", - format!("stub WIT to {}", def.target_wit_path().display()), + format!( + "stub WIT to {}", + def.target_wit_path().log_color_highlight() + ), ); let out = generate_stub_wit_from_stub_def(def)?; @@ -210,8 +213,8 @@ pub fn add_dependencies_to_stub_wit_dir(def: &StubDefinition) -> anyhow::Result< "Adding", format!( "WIT dependencies to {} from {}", - def.config.target_root.display(), - def.config.source_wit_root.display(), + def.config.target_root.log_color_highlight(), + def.config.source_wit_root.log_color_highlight(), ), ); @@ -224,11 +227,23 @@ pub fn add_dependencies_to_stub_wit_dir(def: &StubDefinition) -> anyhow::Result< for (package_id, package, package_sources) in def.packages_with_wit_sources() { if !stub_dep_packages.contains(&package_id) || package_id == def.source_package_id { - log_warn_action("Skipping", format!("package dependency {}", package.name)); + log_warn_action( + "Skipping", + format!( + "package dependency {}", + package.name.to_string().log_color_highlight() + ), + ); continue; } - log_action("Copying", format!("package dependency {}", package.name)); + log_action( + "Copying", + format!( + "package dependency {}", + package.name.to_string().log_color_highlight() + ), + ); let _indent = LogIndent::new(); for source in &package_sources.files { @@ -236,7 +251,11 @@ pub fn add_dependencies_to_stub_wit_dir(def: &StubDefinition) -> anyhow::Result< let dest = target_wit_root.join(relative); log_action( "Copying", - format!("{} to {}", source.display(), dest.display()), + format!( + "{} to {}", + source.log_color_highlight(), + dest.log_color_highlight() + ), ); copy(source, &dest)?; } @@ -262,7 +281,11 @@ fn write_embedded_source(target_dir: &Path, file_name: &str, content: &str) -> a log_action( "Writing", - format!("{} to {}", file_name, target_dir.display()), + format!( + "{} to {}", + file_name.log_color_highlight(), + target_dir.log_color_highlight() + ), ); fs::write(target_dir.join(file_name), content)?; @@ -288,8 +311,8 @@ pub fn add_stub_as_dependency_to_wit_dir(config: AddStubAsDepConfig) -> anyhow:: "Adding", format!( "stub dependencies to {} from {}", - config.dest_wit_root.display(), - config.stub_wit_root.display() + config.dest_wit_root.log_color_highlight(), + config.stub_wit_root.log_color_highlight() ), ); @@ -642,7 +665,7 @@ pub fn extract_main_interface_as_wit_dep(wit_dir: &Path) -> anyhow::Result<()> { "Extracting", format!( "interface package from main component in wit directory {}", - wit_dir.display() + wit_dir.log_color_highlight() ), ); @@ -679,14 +702,20 @@ pub fn extract_main_interface_as_wit_dep(wit_dir: &Path) -> anyhow::Result<()> { .join(naming::wit::INTERFACE_WIT_FILE_NAME); log_action( "Writing", - format!("interface package to {}", interface_package_path.display()), + format!( + "interface package to {}", + interface_package_path.log_color_highlight() + ), ); write_str(&interface_package_path, interface_package.to_string())?; let main_package_path = &sources.files[0]; log_action( "Writing", - format!("main package to {}", main_package_path.display()), + format!( + "main package to {}", + main_package_path.log_color_highlight() + ), ); write_str(main_package_path, main_package.to_string())?; @@ -703,7 +732,7 @@ fn extract_main_interface_package( let package = encoded_wit_dir.package(main_package_id)?; let mut interface_package = package.clone(); - interface_package.set_name(naming::wit::interface_package_name(package.name())); + interface_package.set_name(naming::wit::interface_encoder_package_name(package.name())); let interface_export_prefix = format!( "{}:{}/", diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index 2489f852..c911d576 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -1,7 +1,12 @@ +use crate::commands::log::{log_action, LogColorize, LogIndent}; +use crate::model::validation::{ValidatedResult, ValidationBuilder}; +use crate::model::wasm_rpc::Application; +use crate::naming; use anyhow::{anyhow, bail, Context}; use indexmap::IndexMap; +use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; -use wit_parser::{Package, PackageId, PackageSourceMap, Resolve}; +use wit_parser::{Package, PackageId, PackageName, PackageSourceMap, Resolve}; pub struct PackageSource { pub dir: PathBuf, @@ -136,3 +141,186 @@ fn collect_package_sources( } Ok(package_dir_paths) } + +pub struct ResolvedWitComponent { + pub main_package_name: PackageName, + pub resolved_input_wit_dir: ResolvedWitDir, + pub resolved_output_wit_dir: Option, + pub app_deps: HashSet, + pub input_deps: HashSet, + pub output_deps: Option>, +} + +pub struct ResolvedWitApplication { + pub components: HashMap, + pub package_to_component: HashMap, + pub stub_package_to_component: HashMap, + pub interface_package_to_component: HashMap, +} + +impl ResolvedWitApplication { + pub fn new(app: &Application) -> ValidatedResult { + let mut resolved_app = Self { + components: Default::default(), + package_to_component: Default::default(), + stub_package_to_component: Default::default(), + interface_package_to_component: Default::default(), + }; + + let mut validation = ValidationBuilder::new(); + + resolved_app.add_components_from_app(&mut validation, app); + + if !validation.has_any_errors() { + resolved_app.collect_package_deps(); + } + + validation.build(resolved_app) + } + + fn add_resolved_component( + &mut self, + component_name: String, + resolved_component: ResolvedWitComponent, + ) { + self.package_to_component.insert( + resolved_component.main_package_name.clone(), + component_name.clone(), + ); + self.stub_package_to_component.insert( + naming::wit::stub_package_name(&resolved_component.main_package_name), + component_name.clone(), + ); + self.interface_package_to_component.insert( + naming::wit::interface_parser_package_name(&resolved_component.main_package_name), + component_name.clone(), + ); + self.components.insert(component_name, resolved_component); + } + + fn add_components_from_app(&mut self, validation: &mut ValidationBuilder, app: &Application) { + log_action("Resolving", "application wit directories"); + let _indent = LogIndent::new(); + + for (component_name, component) in &app.wasm_components_by_name { + validation.push_context("component name", component_name.clone()); + + let input_wit = app.component_input_wit(component_name); + let output_wit = app.component_output_wit(component_name); + + log_action( + "Resolving", + format!( + "component wit dirs for {} ({}, {})", + component_name.log_color_highlight(), + input_wit.log_color_highlight(), + output_wit.log_color_highlight(), + ), + ); + + let resolved_component = (|| -> anyhow::Result { + let resolved_input_wit_dir = + ResolvedWitDir::new(&input_wit).with_context(|| { + anyhow!( + "Failed to resolve component {} input wit dir {}", + component_name, + input_wit.display() + ) + })?; + + let main_package = resolved_input_wit_dir.main_package()?; + + let resolved_output_wit_dir = ResolvedWitDir::new(&output_wit).ok(); + let output_has_same_main_package_name = resolved_output_wit_dir + .as_ref() + .map(|wit| wit.main_package()) + .transpose()? + .map(|output_main_package| main_package.name == output_main_package.name) + .unwrap_or_default(); + + Ok(ResolvedWitComponent { + main_package_name: main_package.name.clone(), + resolved_input_wit_dir, + resolved_output_wit_dir: output_has_same_main_package_name + .then_some(resolved_output_wit_dir) + .flatten(), + app_deps: component.wasm_rpc_dependencies.iter().cloned().collect(), + input_deps: Default::default(), + output_deps: Default::default(), + }) + })(); + + match resolved_component { + Ok(resolved_component) => { + self.add_resolved_component(component_name.clone(), resolved_component); + } + Err(err) => validation.add_error(format!("{:?}", err)), + } + + validation.pop_context(); + } + } + + pub fn collect_package_deps(&mut self) { + fn component_deps( + resolved_app: &ResolvedWitApplication, + resolved_wit_dir: &ResolvedWitDir, + ) -> HashSet { + resolved_wit_dir + .resolve + .package_names + .keys() + .filter_map(|package_name| { + resolved_app + .stub_package_to_component + .get(package_name) + .cloned() + }) + .collect() + } + + let mut deps = HashMap::, Option>)>::new(); + for (component_name, component) in &self.components { + deps.insert( + component_name.clone(), + ( + component_deps(self, &component.resolved_input_wit_dir), + component + .resolved_output_wit_dir + .as_ref() + .map(|wit_dir| component_deps(self, wit_dir)), + ), + ); + } + for (component_name, (input_deps, output_deps)) in deps { + let component = self.components.get_mut(&component_name).unwrap(); + component.input_deps = input_deps; + component.output_deps = output_deps; + } + } + + // NOTE: this does not mean that the dependencies themselves are up-to-date, rather + // only checks if there are difference in set of dependencies specified in the + // application model vs in wit dependencies + pub fn is_dep_graph_up_to_date(&self, component_name: &String) -> bool { + match self.components.get(component_name) { + Some(component) => match &component.output_deps { + Some(output_deps) => &component.app_deps == output_deps, + None => false, + }, + None => false, + } + } + + // NOTE: this does not mean that the dependency itself is up-to-date, rather + // only checks if it is present as wit package dependency + pub fn has_as_wit_dep(&self, component_name: &String, dep_component_name: &String) -> bool { + match self.components.get(component_name) { + Some(component) => match &component.output_deps { + Some(output_deps) => output_deps.contains(dep_component_name), + None => false, + }, + None => false, + } + } +} From d35491055e271ef0e63092073cb5855912444bb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Thu, 7 Nov 2024 11:48:02 +0100 Subject: [PATCH 041/109] cleanups --- wasm-rpc-stubgen/src/commands/composition.rs | 5 ++++- wasm-rpc-stubgen/src/commands/declarative.rs | 4 ++-- wasm-rpc-stubgen/src/make.rs | 15 ++++++++++++--- wasm-rpc-stubgen/src/model/wasm_rpc.rs | 9 ++++++--- 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/composition.rs b/wasm-rpc-stubgen/src/commands/composition.rs index 1a75c251..54262c1e 100644 --- a/wasm-rpc-stubgen/src/commands/composition.rs +++ b/wasm-rpc-stubgen/src/commands/composition.rs @@ -108,7 +108,10 @@ fn plug( }; for plug_name in unused_plugs { - log_warn_action("Skipping", format!("{}, not used", plug_name.log_color_highlight())); + log_warn_action( + "Skipping", + format!("{}, not used", plug_name.log_color_highlight()), + ); } // Export all exports from the socket component. diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index 51a0bcf3..ae3228ab 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -66,7 +66,7 @@ async fn pre_component_build_ctx(ctx: &mut ApplicationContext) -> anyhow::Result log_action("Executing", "pre-component-build steps"); let _indent = LogIndent::new(); - let changed_any = extract_interface_packages(&ctx)?; + let changed_any = extract_interface_packages(ctx)?; if changed_any { update_wit_context(ctx)?; } @@ -179,7 +179,7 @@ async fn pre_component_build_ctx(ctx: &mut ApplicationContext) -> anyhow::Result fn extract_interface_packages(ctx: &ApplicationContext) -> Result { let mut changed_any = false; - for (component_name, _component) in &ctx.application.wasm_components_by_name { + for component_name in ctx.application.wasm_components_by_name.keys() { let component_input_wit = ctx.application.component_input_wit(component_name); let component_output_wit = ctx.application.component_output_wit(component_name); diff --git a/wasm-rpc-stubgen/src/make.rs b/wasm-rpc-stubgen/src/make.rs index e90d338d..107320bd 100644 --- a/wasm-rpc-stubgen/src/make.rs +++ b/wasm-rpc-stubgen/src/make.rs @@ -443,7 +443,10 @@ pub fn initialize_workspace( let makefile = makefile.to_string()?; log_warn_action( "Overwriting", - format!("cargo-make Makefile {:?}", makefile_path.log_color_highlight()), + format!( + "cargo-make Makefile {:?}", + makefile_path.log_color_highlight() + ), ); fs::write(makefile_path, makefile)?; } else if has_cargo_make() { @@ -459,7 +462,10 @@ pub fn initialize_workspace( let makefile = makefile.to_string()?; log_action( "Writing", - format!("cargo-make Makefile to {:?}", makefile_path.log_color_highlight()), + format!( + "cargo-make Makefile to {:?}", + makefile_path.log_color_highlight() + ), ); fs::write(makefile_path, makefile)?; } else { @@ -470,7 +476,10 @@ pub fn initialize_workspace( let mut new_members = Vec::new(); for target in targets { - log_action("Generating", format!("initial stub for {}", target.log_color_highlight())); + log_action( + "Generating", + format!("initial stub for {}", target.log_color_highlight()), + ); let stub_name = format!("{target}-stub"); crate::generate(GenerateArgs { diff --git a/wasm-rpc-stubgen/src/model/wasm_rpc.rs b/wasm-rpc-stubgen/src/model/wasm_rpc.rs index 1eabd203..9116030e 100644 --- a/wasm-rpc-stubgen/src/model/wasm_rpc.rs +++ b/wasm-rpc-stubgen/src/model/wasm_rpc.rs @@ -856,7 +856,8 @@ mod tests { assert!(component_name == "component-one"); assert!(component.name == "component-one"); - assert!(component.wit.to_string_lossy() == "wit"); + assert!(component.input_wit.to_string_lossy() == "input_wit"); + assert!(component.output_wit.to_string_lossy() == "output_wit"); assert!(component.input_wasm.to_string_lossy() == "out/in.wasm"); assert!(component.output_wasm.to_string_lossy() == "out/out.wasm"); assert!(component.wasm_rpc_dependencies.len() == 2); @@ -897,7 +898,8 @@ spec: - name: component-one type: wasm properties: - wit: wit + inputWit: input_wit + outputWit: output_wit inputWasm: out/in.wasm outputWasm: out/out.wasm testUnknownProp: test @@ -943,7 +945,8 @@ spec: - name: component-one type: wasm properties: - wit: wit + inputWit: input-wit + outputWit: input-wit inputWasm: out/in.wasm outputWasm: out/out.wasm testUnknownProp: test From 4f4a6fa82b2d0880b34ce9ef783c6ca88e56e7da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Thu, 7 Nov 2024 12:01:01 +0100 Subject: [PATCH 042/109] pre build cleanup --- wasm-rpc-stubgen/src/commands/declarative.rs | 382 ++++++++++--------- 1 file changed, 198 insertions(+), 184 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index ae3228ab..21b4eb05 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -66,197 +66,17 @@ async fn pre_component_build_ctx(ctx: &mut ApplicationContext) -> anyhow::Result log_action("Executing", "pre-component-build steps"); let _indent = LogIndent::new(); - let changed_any = extract_interface_packages(ctx)?; - if changed_any { + if extract_interface_packages(ctx)? { update_wit_context(ctx)?; } - - if ctx.application.all_wasm_rpc_dependencies().is_empty() { - log_warn_action("Skipping", "building wasm rpc stubs, no dependency found"); - } else { - log_action("Building", "wasm rpc stubs"); - let _indent = LogIndent::new(); - - for component_name in ctx.application.all_wasm_rpc_dependencies() { - if is_up_to_date( - ctx.config.skip_up_to_date_checks, - || [ctx.application.component_output_wit(&component_name)], - || { - [ - ctx.application.stub_wasm(&component_name), - ctx.application.stub_wit(&component_name), - ] - }, - ) { - log_skipping_up_to_date(format!( - "building wasm rpc stub for {}", - component_name.log_color_highlight() - )); - continue; - } - - log_action( - "Building", - format!("wasm rpc stub: {}", component_name.log_color_highlight()), - ); - let _indent = LogIndent::new(); - - let target_root = TempDir::new()?; - let canonical_target_root = target_root.path().canonicalize()?; - - let stub_def = StubDefinition::new(StubConfig { - source_wit_root: ctx.application.component_output_wit(&component_name), - target_root: canonical_target_root, - selected_world: ctx.application.stub_world(&component_name), - stub_crate_version: ctx.application.stub_crate_version(&component_name), - wasm_rpc_override: WasmRpcOverride { - wasm_rpc_path_override: ctx.application.stub_wasm_rpc_path(&component_name), - wasm_rpc_version_override: ctx - .application - .stub_wasm_rpc_version(&component_name), - }, - extract_source_interface_package: false, - }) - .context("Failed to gather information for the stub generator")?; - - commands::generate::build( - &stub_def, - &ctx.application.stub_wasm(&component_name), - &ctx.application.stub_wit(&component_name), - ) - .await? - } - } - - for (component_name, component) in &ctx.application.wasm_components_by_name { - if !component.wasm_rpc_dependencies.is_empty() { - log_action( - "Adding", - format!( - "stub wit dependencies to {}", - component_name.log_color_highlight() - ), - ) - } - let _indent = LogIndent::new(); - - for dep_component_name in &component.wasm_rpc_dependencies { - // TODO: this should check into the wit deps for the specific stubs or do folder diffs - if is_up_to_date( - ctx.config.skip_up_to_date_checks - || !ctx.wit.has_as_wit_dep(component_name, dep_component_name), - || [ctx.application.stub_wit(dep_component_name)], - || [ctx.application.component_output_wit(component_name)], - ) { - log_skipping_up_to_date(format!( - "adding {} stub wit dependency to {}", - dep_component_name.log_color_highlight(), - component_name.log_color_highlight() - )); - continue; - } - - log_action( - "Adding", - format!( - "{} stub wit dependency to {}", - dep_component_name.log_color_highlight(), - component_name.log_color_highlight() - ), - ); - let _indent = LogIndent::new(); - - add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { - stub_wit_root: ctx.application.stub_wit(dep_component_name), - dest_wit_root: ctx.application.component_output_wit(component_name), - update_cargo_toml: UpdateCargoToml::UpdateIfExists, - })? - } + build_stubs(ctx).await?; + if add_stub_deps(ctx)? { + update_wit_context(ctx)?; } Ok(()) } -fn extract_interface_packages(ctx: &ApplicationContext) -> Result { - let mut changed_any = false; - for component_name in ctx.application.wasm_components_by_name.keys() { - let component_input_wit = ctx.application.component_input_wit(component_name); - let component_output_wit = ctx.application.component_output_wit(component_name); - - if is_up_to_date( - ctx.config.skip_up_to_date_checks || !ctx.wit.is_dep_graph_up_to_date(component_name), - || [component_input_wit.clone()], - || [component_output_wit.clone()], - ) { - log_skipping_up_to_date(format!( - "extracting main interface package from {}", - component_name.log_color_highlight() - )); - } else { - log_action( - "Extracting", - format!( - "main interface package from {} to {}", - component_input_wit.log_color_highlight(), - component_output_wit.log_color_highlight() - ), - ); - let _indent = LogIndent::new(); - - changed_any = true; - - if component_output_wit.exists() { - log_warn_action( - "Deleting", - format!( - "output wit directory {}", - component_output_wit.log_color_highlight() - ), - ); - std::fs::remove_dir_all(&component_output_wit)?; - } - - { - log_action( - "Copying", - format!( - "wit sources from {} to {}", - component_input_wit.log_color_highlight(), - component_output_wit.log_color_highlight() - ), - ); - let _indent = LogIndent::new(); - - let dir_content = fs_extra::dir::get_dir_content(&component_input_wit) - .with_context(|| { - anyhow!( - "Failed to read component input directory entries for {}", - component_input_wit.display() - ) - })?; - - for file in dir_content.files { - let from = PathBuf::from(&file); - let to = component_output_wit.join( - from.strip_prefix(&component_input_wit).with_context(|| { - anyhow!("Failed to strip prefix for source {}", &file) - })?, - ); - - log_action( - "Copying", - format!("wit source {} to {}", from.display(), to.display()), - ); - copy(from, to)?; - } - } - - extract_main_interface_as_wit_dep(&component_output_wit)?; - } - } - Ok(changed_any) -} - pub fn component_build(config: Config) -> anyhow::Result<()> { let ctx = create_context(config)?; component_build_ctx(&ctx) @@ -698,3 +518,197 @@ fn compile_and_collect_globs(root_dir: &Path, globs: &[String]) -> Result Result { + let mut changed_any_component = false; + for component_name in ctx.application.wasm_components_by_name.keys() { + let component_input_wit = ctx.application.component_input_wit(component_name); + let component_output_wit = ctx.application.component_output_wit(component_name); + + if is_up_to_date( + ctx.config.skip_up_to_date_checks || !ctx.wit.is_dep_graph_up_to_date(component_name), + || [component_input_wit.clone()], + || [component_output_wit.clone()], + ) { + log_skipping_up_to_date(format!( + "extracting main interface package from {}", + component_name.log_color_highlight() + )); + } else { + log_action( + "Extracting", + format!( + "main interface package from {} to {}", + component_input_wit.log_color_highlight(), + component_output_wit.log_color_highlight() + ), + ); + let _indent = LogIndent::new(); + + changed_any_component = true; + + if component_output_wit.exists() { + log_warn_action( + "Deleting", + format!( + "output wit directory {}", + component_output_wit.log_color_highlight() + ), + ); + std::fs::remove_dir_all(&component_output_wit)?; + } + + { + log_action( + "Copying", + format!( + "wit sources from {} to {}", + component_input_wit.log_color_highlight(), + component_output_wit.log_color_highlight() + ), + ); + let _indent = LogIndent::new(); + + let dir_content = fs_extra::dir::get_dir_content(&component_input_wit) + .with_context(|| { + anyhow!( + "Failed to read component input directory entries for {}", + component_input_wit.display() + ) + })?; + + for file in dir_content.files { + let from = PathBuf::from(&file); + let to = component_output_wit.join( + from.strip_prefix(&component_input_wit).with_context(|| { + anyhow!("Failed to strip prefix for source {}", &file) + })?, + ); + + log_action( + "Copying", + format!("wit source {} to {}", from.display(), to.display()), + ); + copy(from, to)?; + } + } + + extract_main_interface_as_wit_dep(&component_output_wit)?; + } + } + Ok(changed_any_component) +} + +async fn build_stubs(ctx: &ApplicationContext) -> Result<(), Error> { + if ctx.application.all_wasm_rpc_dependencies().is_empty() { + log_warn_action("Skipping", "building wasm rpc stubs, no dependency found"); + } else { + log_action("Building", "wasm rpc stubs"); + let _indent = LogIndent::new(); + + for component_name in ctx.application.all_wasm_rpc_dependencies() { + if is_up_to_date( + ctx.config.skip_up_to_date_checks, + || [ctx.application.component_output_wit(&component_name)], + || { + [ + ctx.application.stub_wasm(&component_name), + ctx.application.stub_wit(&component_name), + ] + }, + ) { + log_skipping_up_to_date(format!( + "building wasm rpc stub for {}", + component_name.log_color_highlight() + )); + continue; + } + + log_action( + "Building", + format!("wasm rpc stub: {}", component_name.log_color_highlight()), + ); + let _indent = LogIndent::new(); + + let target_root = TempDir::new()?; + let canonical_target_root = target_root.path().canonicalize()?; + + let stub_def = StubDefinition::new(StubConfig { + source_wit_root: ctx.application.component_output_wit(&component_name), + target_root: canonical_target_root, + selected_world: ctx.application.stub_world(&component_name), + stub_crate_version: ctx.application.stub_crate_version(&component_name), + wasm_rpc_override: WasmRpcOverride { + wasm_rpc_path_override: ctx.application.stub_wasm_rpc_path(&component_name), + wasm_rpc_version_override: ctx + .application + .stub_wasm_rpc_version(&component_name), + }, + extract_source_interface_package: false, + }) + .context("Failed to gather information for the stub generator")?; + + commands::generate::build( + &stub_def, + &ctx.application.stub_wasm(&component_name), + &ctx.application.stub_wit(&component_name), + ) + .await? + } + } + Ok(()) +} + +fn add_stub_deps(ctx: &ApplicationContext) -> Result { + let mut changed_any_components = false; + + for (component_name, component) in &ctx.application.wasm_components_by_name { + if !component.wasm_rpc_dependencies.is_empty() { + log_action( + "Adding", + format!( + "stub wit dependencies to {}", + component_name.log_color_highlight() + ), + ) + } + let _indent = LogIndent::new(); + + for dep_component_name in &component.wasm_rpc_dependencies { + // TODO: this should check into the wit deps for the specific stubs or do folder diffs + if is_up_to_date( + ctx.config.skip_up_to_date_checks + || !ctx.wit.has_as_wit_dep(component_name, dep_component_name), + || [ctx.application.stub_wit(dep_component_name)], + || [ctx.application.component_output_wit(component_name)], + ) { + log_skipping_up_to_date(format!( + "adding {} stub wit dependency to {}", + dep_component_name.log_color_highlight(), + component_name.log_color_highlight() + )); + continue; + } + + changed_any_components = true; + + log_action( + "Adding", + format!( + "{} stub wit dependency to {}", + dep_component_name.log_color_highlight(), + component_name.log_color_highlight() + ), + ); + let _indent = LogIndent::new(); + + add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { + stub_wit_root: ctx.application.stub_wit(dep_component_name), + dest_wit_root: ctx.application.component_output_wit(component_name), + update_cargo_toml: UpdateCargoToml::UpdateIfExists, + })? + } + } + + Ok(changed_any_components) +} From 7eef138bc92d90995507b78a2b42a2611f2524ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Mon, 11 Nov 2024 08:30:26 +0100 Subject: [PATCH 043/109] cleanup / allow skip by content --- wasm-rpc-stubgen/src/commands/declarative.rs | 137 ++++++++++++++++--- wasm-rpc-stubgen/src/fs.rs | 35 +++-- wasm-rpc-stubgen/src/lib.rs | 3 + wasm-rpc-stubgen/src/wit_generate.rs | 3 +- wasm-rpc-stubgen/src/wit_resolve.rs | 2 + 5 files changed, 144 insertions(+), 36 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index 21b4eb05..2d8febe6 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -254,6 +254,83 @@ pub async fn build(config: Config) -> anyhow::Result<()> { Ok(()) } +pub fn clean(config: Config) -> anyhow::Result<()> { + let app = to_anyhow( + "Failed to load application manifest(s), see problems above", + load_app_validated(&config), + )?; + + { + log_action("Cleaning", "components"); + let _indent = LogIndent::new(); + + for (component_name, component) in &app.wasm_components_by_name { + log_action( + "Cleaning", + format!("component {}", component_name.log_color_highlight()), + ); + let _indent = LogIndent::new(); + + delete_path("wit output dir", &app.component_output_wit(component_name))?; + + for build_step in &component.build_steps { + let build_dir = build_step + .dir + .as_ref() + .map(|dir| component.source_dir().join(dir)) + .unwrap_or_else(|| component.source_dir().to_path_buf()); + + let outputs = compile_and_collect_globs(&build_dir, &build_step.outputs)?; + for output in outputs { + delete_path("build step output", &output)?; + } + } + } + } + + { + log_action("Cleaning", "component stubs"); + let _indent = LogIndent::new(); + + for component_name in app.all_wasm_rpc_dependencies() { + log_action( + "Cleaning", + format!("component stub {}", component_name.log_color_highlight()), + ); + let _indent = LogIndent::new(); + + delete_path("stub wit", &app.stub_wit(&component_name))?; + delete_path("stub wasm", &app.stub_wasm(&component_name))?; + } + } + + { + log_action("Cleaning", "application build dir"); + let _indent = LogIndent::new(); + + delete_path("application build dir", &app.build_dir())?; + } + + Ok(()) +} + +fn delete_path(context: &str, path: &Path) -> anyhow::Result<()> { + if path.exists() { + log_warn_action( + "Deleting", + format!("{} {}", context, path.log_color_highlight()), + ); + if path.is_dir() { + std::fs::remove_dir_all(path) + .with_context(|| anyhow!("Failed to delete directory {}", path.display()))?; + } else { + std::fs::remove_file(path) + .with_context(|| anyhow!("Failed to delete file {}", path.display()))?; + } + } + Ok(()) +} + fn create_context(config: Config) -> anyhow::Result { to_anyhow( "Failed to create application context, see problems above", @@ -299,7 +376,10 @@ fn load_app_validated(config: &Config) -> ValidatedResult { } else { format!( "components: {}", - app.wasm_components_by_name.keys().join(", ") + app.wasm_components_by_name + .keys() + .map(|s| s.log_color_highlight()) + .join(", ") ) } }); @@ -373,7 +453,10 @@ fn collect_sources(mode: &ApplicationSourceMode) -> ValidatedResult } else { format!( "sources: {}", - sources.iter().map(|source| source.display()).join(", ") + sources + .iter() + .map(|source| source.log_color_highlight()) + .join(", ") ) } }); @@ -607,9 +690,39 @@ async fn build_stubs(ctx: &ApplicationContext) -> Result<(), Error> { let _indent = LogIndent::new(); for component_name in ctx.application.all_wasm_rpc_dependencies() { + // TODO: let's try to skip this temp dir creation if possible + let target_root = TempDir::new()?; + let canonical_target_root = target_root.path().canonicalize()?; + + let stub_def = StubDefinition::new(StubConfig { + source_wit_root: ctx.application.component_output_wit(&component_name), + target_root: canonical_target_root, + selected_world: ctx.application.stub_world(&component_name), + stub_crate_version: ctx.application.stub_crate_version(&component_name), + wasm_rpc_override: WasmRpcOverride { + wasm_rpc_path_override: ctx.application.stub_wasm_rpc_path(&component_name), + wasm_rpc_version_override: ctx + .application + .stub_wasm_rpc_version(&component_name), + }, + extract_source_interface_package: false, + }) + .context("Failed to gather information for the stub generator")?; + + let stub_dep_package_ids = stub_def.stub_dep_package_ids(); + let stub_inputs: Vec = stub_def + .packages_with_wit_sources() + .flat_map(|(package_id, _, sources)| { + (stub_dep_package_ids.contains(&package_id) + || package_id == stub_def.source_package_id) + .then(|| sources.files.iter().cloned()) + .unwrap_or_default() + }) + .collect(); + if is_up_to_date( ctx.config.skip_up_to_date_checks, - || [ctx.application.component_output_wit(&component_name)], + || stub_inputs, || { [ ctx.application.stub_wasm(&component_name), @@ -630,24 +743,6 @@ async fn build_stubs(ctx: &ApplicationContext) -> Result<(), Error> { ); let _indent = LogIndent::new(); - let target_root = TempDir::new()?; - let canonical_target_root = target_root.path().canonicalize()?; - - let stub_def = StubDefinition::new(StubConfig { - source_wit_root: ctx.application.component_output_wit(&component_name), - target_root: canonical_target_root, - selected_world: ctx.application.stub_world(&component_name), - stub_crate_version: ctx.application.stub_crate_version(&component_name), - wasm_rpc_override: WasmRpcOverride { - wasm_rpc_path_override: ctx.application.stub_wasm_rpc_path(&component_name), - wasm_rpc_version_override: ctx - .application - .stub_wasm_rpc_version(&component_name), - }, - extract_source_interface_package: false, - }) - .context("Failed to gather information for the stub generator")?; - commands::generate::build( &stub_def, &ctx.application.stub_wasm(&component_name), diff --git a/wasm-rpc-stubgen/src/fs.rs b/wasm-rpc-stubgen/src/fs.rs index 95a4badd..639bd018 100644 --- a/wasm-rpc-stubgen/src/fs.rs +++ b/wasm-rpc-stubgen/src/fs.rs @@ -241,6 +241,7 @@ impl OverwriteSafeActions { pub fn run( self, allow_overwrite: bool, + allow_skip_by_content: bool, log_action: F, ) -> anyhow::Result> where @@ -253,23 +254,28 @@ impl OverwriteSafeActions { for action in self.0 { let plan = match &action { - OverwriteSafeAction::CopyFile { source, target } => { - Self::plan_for_action(allow_overwrite, target, || { - has_same_string_content(source, target) - })? - } + OverwriteSafeAction::CopyFile { source, target } => Self::plan_for_action( + allow_overwrite, + allow_skip_by_content, + target, + || has_same_string_content(source, target), + )?, OverwriteSafeAction::CopyFileTransformed { source_content_transformed: source_transformed, target, .. - } => Self::plan_for_action(allow_overwrite, target, || { - has_str_content(target, source_transformed) - })?, - OverwriteSafeAction::WriteFile { content, target } => { - Self::plan_for_action(allow_overwrite, target, || { - has_str_content(target, content) - })? - } + } => Self::plan_for_action( + allow_overwrite, + allow_skip_by_content, + target, + || has_str_content(target, source_transformed), + )?, + OverwriteSafeAction::WriteFile { content, target } => Self::plan_for_action( + allow_overwrite, + allow_skip_by_content, + target, + || has_str_content(target, content), + )?, }; match plan { Some(plan) => actions_with_plan.push((action, plan)), @@ -312,6 +318,7 @@ impl OverwriteSafeActions { fn plan_for_action( allow_overwrite: bool, + allow_skip_by_content: bool, target: P, skip_by_content: F, ) -> anyhow::Result> @@ -321,7 +328,7 @@ impl OverwriteSafeActions { { if !target.as_ref().exists() { Ok(Some(OverwriteSafeActionPlan::Create)) - } else if skip_by_content()? { + } else if allow_skip_by_content && skip_by_content()? { Ok(Some(OverwriteSafeActionPlan::SkipSameContent)) } else if allow_overwrite { Ok(Some(OverwriteSafeActionPlan::Overwrite)) diff --git a/wasm-rpc-stubgen/src/lib.rs b/wasm-rpc-stubgen/src/lib.rs index 4e42e7f0..2f4ca877 100644 --- a/wasm-rpc-stubgen/src/lib.rs +++ b/wasm-rpc-stubgen/src/lib.rs @@ -198,6 +198,8 @@ pub enum App { PostComponentBuild(DeclarativeBuildArgs), /// Runs all build steps (pre-component, component, post-component) Build(DeclarativeBuildArgs), + /// Clean outputs + Clean(DeclarativeBuildArgs), } #[derive(clap::Args, Debug)] @@ -293,6 +295,7 @@ pub async fn run_declarative_command(command: App) -> anyhow::Result<()> { commands::declarative::post_component_build(dec_build_args_to_config(args)).await } App::Build(args) => commands::declarative::build(dec_build_args_to_config(args)).await, + App::Clean(args) => commands::declarative::clean(dec_build_args_to_config(args)), } } diff --git a/wasm-rpc-stubgen/src/wit_generate.rs b/wasm-rpc-stubgen/src/wit_generate.rs index 3f43c69e..64852c68 100644 --- a/wasm-rpc-stubgen/src/wit_generate.rs +++ b/wasm-rpc-stubgen/src/wit_generate.rs @@ -400,7 +400,8 @@ pub fn add_stub_as_dependency_to_wit_dir(config: AddStubAsDepConfig) -> anyhow:: }); // Check overwrites - let forbidden_overwrites = actions.run(true, log_action_plan)?; + // TODO: allow_skip_by_content, decide + let forbidden_overwrites = actions.run(true, false, log_action_plan)?; if !forbidden_overwrites.is_empty() { eprintln!("The following files would have been overwritten with new content:"); for action in forbidden_overwrites { diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index c911d576..481b781b 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -171,6 +171,8 @@ impl ResolvedWitApplication { resolved_app.add_components_from_app(&mut validation, app); + // TODO: validate conflicting package names + if !validation.has_any_errors() { resolved_app.collect_package_deps(); } From 81dfdfc2352f52fe0df9b63c1b7fb44776a98ebf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Mon, 11 Nov 2024 11:21:42 +0100 Subject: [PATCH 044/109] collect input deps by referenced, contained, component --- wasm-rpc-stubgen/src/wit_resolve.rs | 140 +++++++++++++++++++--------- 1 file changed, 97 insertions(+), 43 deletions(-) diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index 481b781b..f83f316f 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -6,7 +6,9 @@ use anyhow::{anyhow, bail, Context}; use indexmap::IndexMap; use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; -use wit_parser::{Package, PackageId, PackageName, PackageSourceMap, Resolve}; +use wit_parser::{ + Package, PackageId, PackageName, PackageSourceMap, Resolve, UnresolvedPackageGroup, +}; pub struct PackageSource { pub dir: PathBuf, @@ -144,11 +146,13 @@ fn collect_package_sources( pub struct ResolvedWitComponent { pub main_package_name: PackageName, - pub resolved_input_wit_dir: ResolvedWitDir, + pub unresolved_input_package_group: UnresolvedPackageGroup, pub resolved_output_wit_dir: Option, - pub app_deps: HashSet, - pub input_deps: HashSet, - pub output_deps: Option>, + pub app_component_deps: HashSet, + pub input_referenced_package_deps: HashSet, + pub input_contained_package_deps: HashSet, + pub input_component_deps: HashSet, + pub output_component_deps: Option>, } pub struct ResolvedWitApplication { @@ -174,7 +178,7 @@ impl ResolvedWitApplication { // TODO: validate conflicting package names if !validation.has_any_errors() { - resolved_app.collect_package_deps(); + resolved_app.collect_component_deps(); } validation.build(resolved_app) @@ -221,34 +225,86 @@ impl ResolvedWitApplication { ); let resolved_component = (|| -> anyhow::Result { - let resolved_input_wit_dir = - ResolvedWitDir::new(&input_wit).with_context(|| { + let unresolved_input_package_group = UnresolvedPackageGroup::parse_dir(&input_wit) + .with_context(|| { anyhow!( - "Failed to resolve component {} input wit dir {}", + "Failed to parse component {} main package in input wit dir {}", component_name, input_wit.display() ) })?; - let main_package = resolved_input_wit_dir.main_package()?; + let input_referenced_package_deps = unresolved_input_package_group + .main + .foreign_deps + .keys() + .cloned() + .collect(); + + let input_contained_package_deps = { + let deps_path = component.input_wit.join("deps"); + if !deps_path.exists() { + HashSet::new() + } else { + let mut entries = deps_path + .read_dir() + .and_then(|read_dir| read_dir.collect::>>()) + .with_context(|| { + anyhow!( + "Failed to read component {} wit dependencies from {}", + component_name, + deps_path.display(), + ) + })?; + entries.sort_by_key(|e| e.file_name()); + entries + .iter() + .filter_map(|entry| { + let path = entry.path(); + // NOTE: unlike wit_resolve - for now - we do not support: + // - symlinks + // - single file deps + // - wasm or wat deps + path.is_dir().then(|| { + UnresolvedPackageGroup::parse_dir(&path) + .with_context(|| { + anyhow!( + "Failed to parse component {} wit dependency {}", + component_name, + path.display() + ) + }) + .map(|package_group| package_group.main.name) + }) + }) + .collect::, _>>()? + } + }; + + let main_package_name = unresolved_input_package_group.main.name.clone(); let resolved_output_wit_dir = ResolvedWitDir::new(&output_wit).ok(); let output_has_same_main_package_name = resolved_output_wit_dir .as_ref() .map(|wit| wit.main_package()) .transpose()? - .map(|output_main_package| main_package.name == output_main_package.name) + .map(|output_main_package| main_package_name == output_main_package.name) .unwrap_or_default(); + let resolved_output_wit_dir = output_has_same_main_package_name + .then_some(resolved_output_wit_dir) + .flatten(); + + let app_component_deps = component.wasm_rpc_dependencies.iter().cloned().collect(); Ok(ResolvedWitComponent { - main_package_name: main_package.name.clone(), - resolved_input_wit_dir, - resolved_output_wit_dir: output_has_same_main_package_name - .then_some(resolved_output_wit_dir) - .flatten(), - app_deps: component.wasm_rpc_dependencies.iter().cloned().collect(), - input_deps: Default::default(), - output_deps: Default::default(), + main_package_name, + unresolved_input_package_group, + resolved_output_wit_dir, + app_component_deps, + input_referenced_package_deps, + input_contained_package_deps, + input_component_deps: Default::default(), + output_component_deps: Default::default(), }) })(); @@ -263,21 +319,14 @@ impl ResolvedWitApplication { } } - pub fn collect_package_deps(&mut self) { - fn component_deps( - resolved_app: &ResolvedWitApplication, - resolved_wit_dir: &ResolvedWitDir, + pub fn collect_component_deps(&mut self) { + fn component_deps<'a, I: IntoIterator>( + known_package_deps: &HashMap, + dep_package_names: I, ) -> HashSet { - resolved_wit_dir - .resolve - .package_names - .keys() - .filter_map(|package_name| { - resolved_app - .stub_package_to_component - .get(package_name) - .cloned() - }) + dep_package_names + .into_iter() + .filter_map(|package_name| known_package_deps.get(package_name).cloned()) .collect() } @@ -286,18 +335,23 @@ impl ResolvedWitApplication { deps.insert( component_name.clone(), ( - component_deps(self, &component.resolved_input_wit_dir), - component - .resolved_output_wit_dir - .as_ref() - .map(|wit_dir| component_deps(self, wit_dir)), + component_deps( + &self.interface_package_to_component, + &component.input_referenced_package_deps, + ), + component.resolved_output_wit_dir.as_ref().map(|wit_dir| { + component_deps( + &self.stub_package_to_component, + wit_dir.resolve.package_names.keys(), + ) + }), ), ); } for (component_name, (input_deps, output_deps)) in deps { let component = self.components.get_mut(&component_name).unwrap(); - component.input_deps = input_deps; - component.output_deps = output_deps; + component.input_component_deps = input_deps; + component.output_component_deps = output_deps; } } @@ -306,8 +360,8 @@ impl ResolvedWitApplication { // application model vs in wit dependencies pub fn is_dep_graph_up_to_date(&self, component_name: &String) -> bool { match self.components.get(component_name) { - Some(component) => match &component.output_deps { - Some(output_deps) => &component.app_deps == output_deps, + Some(component) => match &component.output_component_deps { + Some(output_deps) => &component.app_component_deps == output_deps, None => false, }, None => false, @@ -318,7 +372,7 @@ impl ResolvedWitApplication { // only checks if it is present as wit package dependency pub fn has_as_wit_dep(&self, component_name: &String, dep_component_name: &String) -> bool { match self.components.get(component_name) { - Some(component) => match &component.output_deps { + Some(component) => match &component.output_component_deps { Some(output_deps) => output_deps.contains(dep_component_name), None => false, }, From 4341ca445ec275296c6ce23abced2d28348d8e0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Mon, 11 Nov 2024 11:24:35 +0100 Subject: [PATCH 045/109] fix test --- wasm-rpc-stubgen/src/model/wasm_rpc.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/wasm-rpc-stubgen/src/model/wasm_rpc.rs b/wasm-rpc-stubgen/src/model/wasm_rpc.rs index 9116030e..344ce128 100644 --- a/wasm-rpc-stubgen/src/model/wasm_rpc.rs +++ b/wasm-rpc-stubgen/src/model/wasm_rpc.rs @@ -917,13 +917,15 @@ spec: - name: component-two type: wasm properties: - wit: wit + inputWit: input_wit + outputWit: output_wit inputWasm: out/in.wasm outputWasm: out/out.wasm - name: component-three type: wasm properties: - wit: wit + inputWit: input_wit + outputWit: output_wit inputWasm: out/in.wasm outputWasm: out/out.wasm "# From 405586c0add447ecbe7305d7abdff73924f49fa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 12 Nov 2024 12:01:29 +0100 Subject: [PATCH 046/109] pre-component-build: build by interface dep order and add interface deps --- wasm-rpc-stubgen/src/commands/declarative.rs | 459 +++++++++++-------- wasm-rpc-stubgen/src/commands/generate.rs | 20 +- wasm-rpc-stubgen/src/model/wasm_rpc.rs | 29 ++ wasm-rpc-stubgen/src/stub.rs | 31 +- wasm-rpc-stubgen/src/wit_generate.rs | 6 +- wasm-rpc-stubgen/src/wit_resolve.rs | 167 ++++++- 6 files changed, 493 insertions(+), 219 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index 2d8febe6..f6314d52 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -15,7 +15,7 @@ use crate::wit_generate::{ }; use crate::wit_resolve::ResolvedWitApplication; use crate::{commands, WasmRpcOverride}; -use anyhow::{anyhow, Context, Error}; +use anyhow::{anyhow, bail, Context, Error}; use colored::Colorize; use glob::glob; use itertools::Itertools; @@ -24,7 +24,6 @@ use std::io::Write; use std::path::{Path, PathBuf}; use std::process::Command; use std::time::SystemTime; -use tempfile::TempDir; use walkdir::WalkDir; pub struct Config { @@ -44,6 +43,30 @@ struct ApplicationContext { wit: ResolvedWitApplication, } +impl ApplicationContext { + fn new(config: Config) -> anyhow::Result { + to_anyhow( + "Failed to create application context, see problems above", + load_app_validated(&config).and_then(|application| { + ResolvedWitApplication::new(&application).map(|wit| ApplicationContext { + config, + application, + wit, + }) + }), + ) + } + + fn update_wit_context(&mut self) -> anyhow::Result<()> { + to_anyhow( + "Failed to update application wit context, see problems above", + ResolvedWitApplication::new(&self.application).map(|wit| { + self.wit = wit; + }), + ) + } +} + pub fn init(component_name: String) -> anyhow::Result<()> { let file_name = DEFAULT_CONFIG_FILE_NAME; @@ -58,7 +81,7 @@ pub fn init(component_name: String) -> anyhow::Result<()> { } pub async fn pre_component_build(config: Config) -> anyhow::Result<()> { - let mut ctx = create_context(config)?; + let mut ctx = ApplicationContext::new(config)?; pre_component_build_ctx(&mut ctx).await } @@ -66,19 +89,34 @@ async fn pre_component_build_ctx(ctx: &mut ApplicationContext) -> anyhow::Result log_action("Executing", "pre-component-build steps"); let _indent = LogIndent::new(); - if extract_interface_packages(ctx)? { - update_wit_context(ctx)?; + { + let mut any_changed = false; + for component_name in &ctx.wit.component_order { + any_changed |= create_base_output_wit(ctx, component_name)?; + } + for component_name in &ctx.application.all_wasm_rpc_dependencies() { + any_changed |= build_stub(ctx, component_name).await?; + } + if any_changed { + ctx.update_wit_context()?; + } } - build_stubs(ctx).await?; - if add_stub_deps(ctx)? { - update_wit_context(ctx)?; + + { + let mut any_changed = false; + for component_name in ctx.application.wasm_components_by_name.keys() { + any_changed |= create_output_wit(ctx, component_name)?; + } + if any_changed { + ctx.update_wit_context()?; + } } Ok(()) } pub fn component_build(config: Config) -> anyhow::Result<()> { - let ctx = create_context(config)?; + let ctx = ApplicationContext::new(config)?; component_build_ctx(&ctx) } @@ -167,7 +205,7 @@ fn component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<()> { } pub async fn post_component_build(config: Config) -> anyhow::Result<()> { - let ctx = create_context(config)?; + let ctx = ApplicationContext::new(config)?; post_component_build_ctx(&ctx).await } @@ -245,7 +283,7 @@ async fn post_component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<() } pub async fn build(config: Config) -> anyhow::Result<()> { - let mut ctx = create_context(config)?; + let mut ctx = ApplicationContext::new(config)?; pre_component_build_ctx(&mut ctx).await?; component_build_ctx(&ctx)?; @@ -331,28 +369,6 @@ fn delete_path(context: &str, path: &Path) -> anyhow::Result<()> { Ok(()) } -fn create_context(config: Config) -> anyhow::Result { - to_anyhow( - "Failed to create application context, see problems above", - load_app_validated(&config).and_then(|application| { - ResolvedWitApplication::new(&application).map(|wit| ApplicationContext { - config, - application, - wit, - }) - }), - ) -} - -fn update_wit_context(ctx: &mut ApplicationContext) -> anyhow::Result<()> { - to_anyhow( - "Failed to update application wit context, see problems above", - ResolvedWitApplication::new(&ctx.application).map(|wit| { - ctx.wit = wit; - }), - ) -} - fn load_app_validated(config: &Config) -> ValidatedResult { let sources = collect_sources(&config.app_resolve_mode); let oam_apps = sources.and_then(|sources| { @@ -602,191 +618,217 @@ fn compile_and_collect_globs(root_dir: &Path, globs: &[String]) -> Result Result { - let mut changed_any_component = false; - for component_name in ctx.application.wasm_components_by_name.keys() { - let component_input_wit = ctx.application.component_input_wit(component_name); - let component_output_wit = ctx.application.component_output_wit(component_name); - - if is_up_to_date( - ctx.config.skip_up_to_date_checks || !ctx.wit.is_dep_graph_up_to_date(component_name), - || [component_input_wit.clone()], - || [component_output_wit.clone()], - ) { - log_skipping_up_to_date(format!( - "extracting main interface package from {}", - component_name.log_color_highlight() - )); - } else { - log_action( - "Extracting", - format!( - "main interface package from {} to {}", - component_input_wit.log_color_highlight(), - component_output_wit.log_color_highlight() - ), - ); - let _indent = LogIndent::new(); +fn create_base_output_wit(ctx: &ApplicationContext, component_name: &str) -> Result { + let component_input_wit = ctx.application.component_input_wit(component_name); + let component_base_output_wit = ctx.application.component_base_output_wit(component_name); + + if is_up_to_date( + ctx.config.skip_up_to_date_checks || !ctx.wit.is_dep_graph_up_to_date(component_name)?, + || [component_input_wit.clone()], + || [component_base_output_wit.clone()], + ) { + log_skipping_up_to_date(format!( + "creating base output wit directory for {}", + component_name.log_color_highlight() + )); + Ok(false) + } else { + log_action( + "Creating", + format!( + "base output wit directory for {}", + component_name.log_color_highlight(), + ), + ); + let _indent = LogIndent::new(); - changed_any_component = true; + delete_path("base output wit directory", &component_base_output_wit)?; + copy_wit_sources(&component_input_wit, &component_base_output_wit)?; - if component_output_wit.exists() { - log_warn_action( - "Deleting", - format!( - "output wit directory {}", - component_output_wit.log_color_highlight() - ), + { + let missing_package_deps = + ctx.wit.missing_generic_input_package_deps(component_name)?; + if !missing_package_deps.is_empty() { + bail!( + "TODO: implement adding package deps: {}", + missing_package_deps + .iter() + .map(|s| s.to_string()) + .join(", ") ); - std::fs::remove_dir_all(&component_output_wit)?; } + } - { - log_action( - "Copying", - format!( - "wit sources from {} to {}", - component_input_wit.log_color_highlight(), - component_output_wit.log_color_highlight() - ), - ); + { + let component_interface_package_deps = + ctx.wit.component_interface_package_deps(component_name)?; + if !component_interface_package_deps.is_empty() { + log_action("Adding", "interface package deps"); let _indent = LogIndent::new(); - let dir_content = fs_extra::dir::get_dir_content(&component_input_wit) - .with_context(|| { - anyhow!( - "Failed to read component input directory entries for {}", - component_input_wit.display() - ) - })?; - - for file in dir_content.files { - let from = PathBuf::from(&file); - let to = component_output_wit.join( - from.strip_prefix(&component_input_wit).with_context(|| { - anyhow!("Failed to strip prefix for source {}", &file) - })?, - ); - + for (dep_interface_package_name, dep_component_name) in + &component_interface_package_deps + { + let source = ctx + .application + .component_base_output_wit_interface_package_dir( + dep_component_name, + dep_interface_package_name, + ); + let target = ctx + .application + .component_base_output_wit_interface_package_dir( + component_name, + dep_interface_package_name, + ); log_action( "Copying", - format!("wit source {} to {}", from.display(), to.display()), + format!( + "interface package dependency from {} to {}", + source.log_color_highlight(), + target.log_color_highlight() + ), ); - copy(from, to)?; + copy(source, target)?; } } + } - extract_main_interface_as_wit_dep(&component_output_wit)?; + { + log_action( + "Extracting", + format!( + "main interface package from {} to {}", + component_input_wit.log_color_highlight(), + component_base_output_wit.log_color_highlight() + ), + ); + let _indent = LogIndent::new(); + + extract_main_interface_as_wit_dep(&component_base_output_wit)?; } + + Ok(true) + } +} + +fn create_output_wit(ctx: &ApplicationContext, component_name: &str) -> Result { + let component_base_output_wit = ctx.application.component_base_output_wit(component_name); + let component_output_wit = ctx.application.component_output_wit(component_name); + + if is_up_to_date( + ctx.config.skip_up_to_date_checks || !ctx.wit.is_dep_graph_up_to_date(component_name)?, + || [component_base_output_wit.clone()], + || [component_output_wit.clone()], + ) { + log_skipping_up_to_date(format!( + "creating output wit directory for {}", + component_name.log_color_highlight() + )); + Ok(false) + } else { + log_action( + "Creating", + format!( + "output wit directory for {}", + component_name.log_color_highlight(), + ), + ); + let _indent = LogIndent::new(); + + delete_path("output wit directory", &component_output_wit)?; + copy_wit_sources(&component_base_output_wit, &component_output_wit)?; + add_stub_deps(ctx, component_name)?; + + Ok(true) } - Ok(changed_any_component) } -async fn build_stubs(ctx: &ApplicationContext) -> Result<(), Error> { - if ctx.application.all_wasm_rpc_dependencies().is_empty() { - log_warn_action("Skipping", "building wasm rpc stubs, no dependency found"); +async fn build_stub(ctx: &ApplicationContext, component_name: &str) -> anyhow::Result { + let target_root = ctx.application.stub_temp_build_dir(component_name); + + let stub_def = StubDefinition::new(StubConfig { + source_wit_root: ctx.application.component_base_output_wit(component_name), + target_root: target_root.clone(), + selected_world: ctx.application.stub_world(component_name), + stub_crate_version: ctx.application.stub_crate_version(component_name), + wasm_rpc_override: WasmRpcOverride { + wasm_rpc_path_override: ctx.application.stub_wasm_rpc_path(component_name), + wasm_rpc_version_override: ctx.application.stub_wasm_rpc_version(component_name), + }, + extract_source_interface_package: false, + }) + .context("Failed to gather information for the stub generator")?; + + let stub_dep_package_ids = stub_def.stub_dep_package_ids(); + let stub_inputs: Vec = stub_def + .packages_with_wit_sources() + .flat_map(|(package_id, _, sources)| { + (stub_dep_package_ids.contains(&package_id) || package_id == stub_def.source_package_id) + .then(|| sources.files.iter().cloned()) + .unwrap_or_default() + }) + .collect(); + + if is_up_to_date( + ctx.config.skip_up_to_date_checks, + || stub_inputs, + || { + [ + ctx.application.stub_wasm(component_name), + ctx.application.stub_wit(component_name), + ] + }, + ) { + log_skipping_up_to_date(format!( + "building wasm rpc stub for {}", + component_name.log_color_highlight() + )); + Ok(false) } else { - log_action("Building", "wasm rpc stubs"); + log_action( + "Building", + format!("wasm rpc stub: {}", component_name.log_color_highlight()), + ); let _indent = LogIndent::new(); - for component_name in ctx.application.all_wasm_rpc_dependencies() { - // TODO: let's try to skip this temp dir creation if possible - let target_root = TempDir::new()?; - let canonical_target_root = target_root.path().canonicalize()?; - - let stub_def = StubDefinition::new(StubConfig { - source_wit_root: ctx.application.component_output_wit(&component_name), - target_root: canonical_target_root, - selected_world: ctx.application.stub_world(&component_name), - stub_crate_version: ctx.application.stub_crate_version(&component_name), - wasm_rpc_override: WasmRpcOverride { - wasm_rpc_path_override: ctx.application.stub_wasm_rpc_path(&component_name), - wasm_rpc_version_override: ctx - .application - .stub_wasm_rpc_version(&component_name), - }, - extract_source_interface_package: false, - }) - .context("Failed to gather information for the stub generator")?; - - let stub_dep_package_ids = stub_def.stub_dep_package_ids(); - let stub_inputs: Vec = stub_def - .packages_with_wit_sources() - .flat_map(|(package_id, _, sources)| { - (stub_dep_package_ids.contains(&package_id) - || package_id == stub_def.source_package_id) - .then(|| sources.files.iter().cloned()) - .unwrap_or_default() - }) - .collect(); + delete_path("stub temp build dir", &target_root)?; - if is_up_to_date( - ctx.config.skip_up_to_date_checks, - || stub_inputs, - || { - [ - ctx.application.stub_wasm(&component_name), - ctx.application.stub_wit(&component_name), - ] - }, - ) { - log_skipping_up_to_date(format!( - "building wasm rpc stub for {}", - component_name.log_color_highlight() - )); - continue; - } + log_action( + "Creating", + format!("stub temp build dir {}", target_root.log_color_highlight()), + ); + std::fs::create_dir_all(&target_root)?; - log_action( - "Building", - format!("wasm rpc stub: {}", component_name.log_color_highlight()), - ); - let _indent = LogIndent::new(); + commands::generate::build( + &stub_def, + &ctx.application.stub_wasm(component_name), + &ctx.application.stub_wit(component_name), + ) + .await?; - commands::generate::build( - &stub_def, - &ctx.application.stub_wasm(&component_name), - &ctx.application.stub_wit(&component_name), - ) - .await? - } + delete_path("stub temp build dir", &target_root)?; + + Ok(true) } - Ok(()) } -fn add_stub_deps(ctx: &ApplicationContext) -> Result { - let mut changed_any_components = false; +fn add_stub_deps(ctx: &ApplicationContext, component_name: &str) -> Result { + let component = ctx.application.component(component_name); + if component.wasm_rpc_dependencies.is_empty() { + Ok(false) + } else { + log_action( + "Adding", + format!( + "stub wit dependencies to {}", + component_name.log_color_highlight() + ), + ); - for (component_name, component) in &ctx.application.wasm_components_by_name { - if !component.wasm_rpc_dependencies.is_empty() { - log_action( - "Adding", - format!( - "stub wit dependencies to {}", - component_name.log_color_highlight() - ), - ) - } let _indent = LogIndent::new(); for dep_component_name in &component.wasm_rpc_dependencies { - // TODO: this should check into the wit deps for the specific stubs or do folder diffs - if is_up_to_date( - ctx.config.skip_up_to_date_checks - || !ctx.wit.has_as_wit_dep(component_name, dep_component_name), - || [ctx.application.stub_wit(dep_component_name)], - || [ctx.application.component_output_wit(component_name)], - ) { - log_skipping_up_to_date(format!( - "adding {} stub wit dependency to {}", - dep_component_name.log_color_highlight(), - component_name.log_color_highlight() - )); - continue; - } - - changed_any_components = true; - log_action( "Adding", format!( @@ -803,7 +845,46 @@ fn add_stub_deps(ctx: &ApplicationContext) -> Result { update_cargo_toml: UpdateCargoToml::UpdateIfExists, })? } + + Ok(true) } +} + +fn copy_wit_sources(source: &Path, target: &Path) -> anyhow::Result<()> { + log_action( + "Copying", + format!( + "wit sources from {} to {}", + source.log_color_highlight(), + target.log_color_highlight() + ), + ); + let _indent = LogIndent::new(); + + let dir_content = fs_extra::dir::get_dir_content(&source).with_context(|| { + anyhow!( + "Failed to read component input wit directory entries for {}", + source.display() + ) + })?; + + for file in dir_content.files { + let from = PathBuf::from(&file); + let to = target.join( + from.strip_prefix(&source) + .with_context(|| anyhow!("Failed to strip prefix for source {}", &file))?, + ); - Ok(changed_any_components) + log_action( + "Copying", + format!( + "wit source {} to {}", + from.log_color_highlight(), + to.log_color_highlight() + ), + ); + copy(from, to)?; + } + + Ok(()) } diff --git a/wasm-rpc-stubgen/src/commands/generate.rs b/wasm-rpc-stubgen/src/commands/generate.rs index e7580cf2..7e56d94a 100644 --- a/wasm-rpc-stubgen/src/commands/generate.rs +++ b/wasm-rpc-stubgen/src/commands/generate.rs @@ -21,7 +21,7 @@ use crate::rust::generate_stub_source; use crate::stub::StubDefinition; use crate::wit_generate::{add_dependencies_to_stub_wit_dir, generate_stub_wit_to_target}; use crate::wit_resolve::ResolvedWitDir; -use anyhow::Context; +use anyhow::{anyhow, Context}; use fs_extra::dir::CopyOptions; use heck::ToSnakeCase; use std::fs; @@ -60,9 +60,21 @@ pub async fn generate_and_build_stub(stub_def: &StubDefinition) -> anyhow::Resul generate_cargo_toml(stub_def).context("Failed to generate the Cargo.toml file")?; generate_stub_source(stub_def).context("Failed to generate the stub Rust source")?; - compile(&stub_def.config.target_root) - .await - .context("Failed to compile the generated stub")?; + // TODO: search for canonicalize on call site, as it is not needed anymore + compile( + &stub_def + .config + .target_root + .canonicalize() + .with_context(|| { + anyhow!( + "Failed to canonicalize stub target root {}", + stub_def.config.target_root.display() + ) + })?, + ) + .await + .context("Failed to compile the generated stub")?; let wasm_path = stub_def .config diff --git a/wasm-rpc-stubgen/src/model/wasm_rpc.rs b/wasm-rpc-stubgen/src/model/wasm_rpc.rs index 344ce128..b51a4193 100644 --- a/wasm-rpc-stubgen/src/model/wasm_rpc.rs +++ b/wasm-rpc-stubgen/src/model/wasm_rpc.rs @@ -3,12 +3,14 @@ use crate::model::oam::TypedTraitProperties; use crate::model::unknown_properties::{HasUnknownProperties, UnknownProperties}; use crate::model::validation::{ValidatedResult, ValidationBuilder}; use crate::naming; +use crate::naming::wit::package_dep_dir_name_from_parser; use golem_wasm_rpc::WASM_RPC_VERSION; use itertools::Itertools; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet}; use std::fs; use std::path::{Path, PathBuf}; +use wit_parser::PackageName; pub const DEFAULT_CONFIG_FILE_NAME: &str = "golem.yaml"; @@ -297,6 +299,7 @@ impl Application { source: source.to_path_buf(), name: component.name, build_dir: properties.build_dir.map(|s| s.into()), + wit_deps: properties.wit_deps.into_iter().map(|s| s.into()).collect(), }; Some(wasm_rpc_stub_build) @@ -582,6 +585,23 @@ impl Application { component.source_dir().join(component.input_wit.clone()) } + pub fn component_base_output_wit(&self, component_name: &str) -> PathBuf { + self.build_dir() + .join("base_output_wit") + .join(component_name) + } + + pub fn component_base_output_wit_interface_package_dir( + &self, + component_name: &str, + interface_package_name: &PackageName, + ) -> PathBuf { + self.component_base_output_wit(component_name) + .join(naming::wit::DEPS_DIR) + .join(package_dep_dir_name_from_parser(interface_package_name)) + .join(naming::wit::INTERFACE_WIT_FILE_NAME) + } + pub fn component_output_wit(&self, component_name: &str) -> PathBuf { let component = self.component(component_name); component.source_dir().join(component.output_wit.clone()) @@ -630,6 +650,12 @@ impl Application { .join("stub") } + pub fn stub_temp_build_dir(&self, component_name: &str) -> PathBuf { + self.stub_build_dir(component_name) + .join(component_name) + .join("temp-build") + } + pub fn stub_wasm(&self, component_name: &str) -> PathBuf { self.wasm_rpc_stub_builds_by_name .get(component_name) @@ -753,6 +779,8 @@ impl oam::TypedTraitProperties for WasmRpcTraitProperties { pub struct ComponentBuildProperties { include: Option, build_dir: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + wit_deps: Vec, #[serde(flatten)] unknown_properties: UnknownProperties, } @@ -774,6 +802,7 @@ pub struct WasmBuild { source: PathBuf, name: String, build_dir: Option, + wit_deps: Vec, } #[derive(Clone, Debug, Serialize, Deserialize)] diff --git a/wasm-rpc-stubgen/src/stub.rs b/wasm-rpc-stubgen/src/stub.rs index 86b2c921..4cc31626 100644 --- a/wasm-rpc-stubgen/src/stub.rs +++ b/wasm-rpc-stubgen/src/stub.rs @@ -298,11 +298,40 @@ impl StubDefinition { self.stub_dep_package_ids.get_or_init(|| { self.stub_used_type_defs() .iter() - .filter_map(|type_def| type_def.package_id) + .flat_map(|type_def| { + let mut package_ids = Vec::::new(); + self.type_def_owner_package_ids(&type_def.type_def, &mut package_ids); + package_ids + }) .collect() }) } + fn type_def_owner_package_ids(&self, type_def: &TypeDef, package_ids: &mut Vec) { + let package_id = match type_def.owner { + TypeOwner::World(_) => None, + TypeOwner::Interface(interface) => self + .resolve + .interfaces + .get(interface) + .and_then(|interface| interface.package), + TypeOwner::None => None, + }; + + if let Some(package_id) = package_id { + package_ids.push(package_id); + } + + if let TypeDefKind::Type(Type::Id(type_id)) = type_def.kind { + self.type_def_owner_package_ids( + self.resolve.types.get(type_id).unwrap_or_else(|| { + panic!("Type alias target not found, type id: {:?}", type_id) + }), + package_ids, + ); + } + } + fn partition_world_items<'a>( &'a self, world_items: &'a IndexMap, diff --git a/wasm-rpc-stubgen/src/wit_generate.rs b/wasm-rpc-stubgen/src/wit_generate.rs index 64852c68..c3e681eb 100644 --- a/wasm-rpc-stubgen/src/wit_generate.rs +++ b/wasm-rpc-stubgen/src/wit_generate.rs @@ -212,9 +212,9 @@ pub fn add_dependencies_to_stub_wit_dir(def: &StubDefinition) -> anyhow::Result< log_action( "Adding", format!( - "WIT dependencies to {} from {}", - def.config.target_root.log_color_highlight(), + "WIT dependencies from {} to {}", def.config.source_wit_root.log_color_highlight(), + def.config.target_root.log_color_highlight(), ), ); @@ -665,7 +665,7 @@ pub fn extract_main_interface_as_wit_dep(wit_dir: &Path) -> anyhow::Result<()> { log_action( "Extracting", format!( - "interface package from main component in wit directory {}", + "interface package from main package in wit directory {}", wit_dir.log_color_highlight() ), ); diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index f83f316f..583c4533 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -2,9 +2,9 @@ use crate::commands::log::{log_action, LogColorize, LogIndent}; use crate::model::validation::{ValidatedResult, ValidationBuilder}; use crate::model::wasm_rpc::Application; use crate::naming; -use anyhow::{anyhow, bail, Context}; +use anyhow::{anyhow, bail, Context, Error}; use indexmap::IndexMap; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::path::{Path, PathBuf}; use wit_parser::{ Package, PackageId, PackageName, PackageSourceMap, Resolve, UnresolvedPackageGroup, @@ -151,15 +151,16 @@ pub struct ResolvedWitComponent { pub app_component_deps: HashSet, pub input_referenced_package_deps: HashSet, pub input_contained_package_deps: HashSet, - pub input_component_deps: HashSet, + pub input_component_deps: BTreeSet, // NOTE: BTree for making dep sorting deterministic pub output_component_deps: Option>, } pub struct ResolvedWitApplication { - pub components: HashMap, + pub components: BTreeMap, // NOTE: BTree for making dep sorting deterministic pub package_to_component: HashMap, pub stub_package_to_component: HashMap, pub interface_package_to_component: HashMap, + pub component_order: Vec, } impl ResolvedWitApplication { @@ -169,6 +170,7 @@ impl ResolvedWitApplication { package_to_component: Default::default(), stub_package_to_component: Default::default(), interface_package_to_component: Default::default(), + component_order: Default::default(), }; let mut validation = ValidationBuilder::new(); @@ -179,6 +181,7 @@ impl ResolvedWitApplication { if !validation.has_any_errors() { resolved_app.collect_component_deps(); + resolved_app.sort_components_by_input_deps(&mut validation); } validation.build(resolved_app) @@ -319,18 +322,18 @@ impl ResolvedWitApplication { } } - pub fn collect_component_deps(&mut self) { - fn component_deps<'a, I: IntoIterator>( + fn collect_component_deps(&mut self) { + fn component_deps<'a, I: IntoIterator, O: FromIterator>( known_package_deps: &HashMap, dep_package_names: I, - ) -> HashSet { + ) -> O { dep_package_names .into_iter() .filter_map(|package_name| known_package_deps.get(package_name).cloned()) .collect() } - let mut deps = HashMap::, Option>)>::new(); + let mut deps = HashMap::, Option>)>::new(); for (component_name, component) in &self.components { deps.insert( component_name.clone(), @@ -355,28 +358,148 @@ impl ResolvedWitApplication { } } + fn sort_components_by_input_deps(&mut self, validation: &mut ValidationBuilder) { + let mut component_order = Vec::with_capacity(self.components.len()); + + // TODO: use some id instead of Strings? + let mut visited = HashSet::::new(); + let mut visiting = HashSet::::new(); + + fn visit( + resolved_app: &ResolvedWitApplication, + visited: &mut HashSet, + visiting: &mut HashSet, + component_order: &mut Vec, + component_name: &str, + input_deps: &BTreeSet, + ) -> bool { + if visited.contains(component_name) { + true + } else if visiting.contains(component_name) { + false + } else { + visiting.insert(component_name.to_string()); + + for dep_component_name in input_deps { + if !visit( + resolved_app, + visited, + visiting, + component_order, + dep_component_name, + &resolved_app + .components + .get(dep_component_name) + .unwrap() + .input_component_deps, + ) { + return false; + } + } + + visiting.remove(component_name); + visited.insert(component_name.to_string()); + + component_order.push(component_name.to_string()); + + true + } + } + + for (component_name, component) in &self.components { + if !visited.contains(component_name) + && !visit( + self, + &mut visited, + &mut visiting, + &mut component_order, + component_name, + &component.input_component_deps, + ) + { + // TODO: better error message, collect full path + component_order.push(component_name.to_string()); + validation.add_error(format!( + "Found component cycle: {}", + component_order.join(", ") + )); + break; + } + } + + self.component_order = component_order; + } + + fn component(&self, component_name: &str) -> Result<&ResolvedWitComponent, Error> { + self.components + .get(component_name) + .ok_or_else(|| anyhow!("Component not found: {}", component_name)) + } + + // NOTE: Intended to be used for non-component wit package deps, so it does not include + // component interface packages, as those are added from stubs + pub fn missing_generic_input_package_deps( + &self, + component_name: &str, + ) -> anyhow::Result> { + let component = self.component(component_name)?; + Ok(component + .input_referenced_package_deps + .iter() + .filter(|&package_name| { + !component + .input_contained_package_deps + .contains(&package_name) + && !self + .interface_package_to_component + .contains_key(&package_name) + }) + .cloned() + .collect::>()) + } + + pub fn component_interface_package_deps( + &self, + component_name: &str, + ) -> anyhow::Result> { + let component = self.component(component_name)?; + Ok(component + .input_referenced_package_deps + .iter() + .filter_map(|package_name| { + match self.interface_package_to_component.get(package_name) { + Some(dep_component_name) if dep_component_name != component_name => { + Some((package_name.clone(), dep_component_name.clone())) + } + _ => None, + } + }) + .collect()) + } + // NOTE: this does not mean that the dependencies themselves are up-to-date, rather // only checks if there are difference in set of dependencies specified in the // application model vs in wit dependencies - pub fn is_dep_graph_up_to_date(&self, component_name: &String) -> bool { - match self.components.get(component_name) { - Some(component) => match &component.output_component_deps { - Some(output_deps) => &component.app_component_deps == output_deps, - None => false, - }, + pub fn is_dep_graph_up_to_date(&self, component_name: &str) -> anyhow::Result { + let component = self.component(component_name)?; + Ok(match &component.output_component_deps { + Some(output_deps) => &component.app_component_deps == output_deps, None => false, - } + }) } // NOTE: this does not mean that the dependency itself is up-to-date, rather // only checks if it is present as wit package dependency - pub fn has_as_wit_dep(&self, component_name: &String, dep_component_name: &String) -> bool { - match self.components.get(component_name) { - Some(component) => match &component.output_component_deps { - Some(output_deps) => output_deps.contains(dep_component_name), - None => false, - }, + pub fn has_as_wit_dep( + &self, + component_name: &str, + dep_component_name: &str, + ) -> anyhow::Result { + let component = self.component(component_name)?; + + Ok(match &component.output_component_deps { + Some(output_deps) => output_deps.contains(dep_component_name), None => false, - } + }) } } From 9713ad1c168517e1b6ba9c2e72dfdb93c9373dc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 12 Nov 2024 12:16:12 +0100 Subject: [PATCH 047/109] delete all stub targets --- wasm-rpc-stubgen/src/commands/declarative.rs | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index f6314d52..5cc26391 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -770,15 +770,13 @@ async fn build_stub(ctx: &ApplicationContext, component_name: &str) -> anyhow::R }) .collect(); + let stub_wasm = ctx.application.stub_wasm(component_name); + let stub_wit = ctx.application.stub_wit(component_name); + if is_up_to_date( ctx.config.skip_up_to_date_checks, || stub_inputs, - || { - [ - ctx.application.stub_wasm(component_name), - ctx.application.stub_wit(component_name), - ] - }, + || [stub_wit.clone(), stub_wasm.clone()], ) { log_skipping_up_to_date(format!( "building wasm rpc stub for {}", @@ -793,6 +791,8 @@ async fn build_stub(ctx: &ApplicationContext, component_name: &str) -> anyhow::R let _indent = LogIndent::new(); delete_path("stub temp build dir", &target_root)?; + delete_path("stub wit", &stub_wit)?; + delete_path("stub wasm", &stub_wasm)?; log_action( "Creating", @@ -800,12 +800,7 @@ async fn build_stub(ctx: &ApplicationContext, component_name: &str) -> anyhow::R ); std::fs::create_dir_all(&target_root)?; - commands::generate::build( - &stub_def, - &ctx.application.stub_wasm(component_name), - &ctx.application.stub_wit(component_name), - ) - .await?; + commands::generate::build(&stub_def, &stub_wasm, &stub_wit).await?; delete_path("stub temp build dir", &target_root)?; From 05717f9535e2ef9694fc465b8fc17367a289c257 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 12 Nov 2024 12:57:34 +0100 Subject: [PATCH 048/109] WIP: package deps --- wasm-rpc-stubgen/src/commands/declarative.rs | 63 ++++++++++++++++---- wasm-rpc-stubgen/src/wit_resolve.rs | 59 +++++++++--------- 2 files changed, 80 insertions(+), 42 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index 5cc26391..80fd5428 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -13,9 +13,9 @@ use crate::wit_generate::{ add_stub_as_dependency_to_wit_dir, extract_main_interface_as_wit_dep, AddStubAsDepConfig, UpdateCargoToml, }; -use crate::wit_resolve::ResolvedWitApplication; -use crate::{commands, WasmRpcOverride}; -use anyhow::{anyhow, bail, Context, Error}; +use crate::wit_resolve::{parse_wit_deps_dir, ResolvedWitApplication}; +use crate::{commands, naming, WasmRpcOverride}; +use anyhow::{anyhow, Context, Error}; use colored::Colorize; use glob::glob; use itertools::Itertools; @@ -648,14 +648,55 @@ fn create_base_output_wit(ctx: &ApplicationContext, component_name: &str) -> Res { let missing_package_deps = ctx.wit.missing_generic_input_package_deps(component_name)?; + if !missing_package_deps.is_empty() { - bail!( - "TODO: implement adding package deps: {}", - missing_package_deps + log_action("Adding", "common package deps"); + let _indent = LogIndent::new(); + + // TODO: transitive deps? + // TODO: use wit dep from app manifest + // TODO: extract dep management, with preferring higher versions + + let wit_deps = parse_wit_deps_dir(Path::new("wit-deps"))?; + + for package_name in missing_package_deps { + log_action( + "Adding", + format!( + "package dependency {}", + package_name.to_string().log_color_highlight() + ), + ); + let dep = wit_deps .iter() - .map(|s| s.to_string()) - .join(", ") - ); + .find(|package| package.main.name == package_name) + .ok_or_else(|| anyhow!("Package dependency {} not found", package_name))?; + for source in dep.source_map.source_files() { + let parent = source.parent().ok_or_else(|| { + anyhow!( + "Failed to get dependency source parent for {}", + source.display() + ) + })?; + // TODO: un-unwrap + let target = ctx + .application + .component_base_output_wit(component_name) + .join(naming::wit::DEPS_DIR) + .join(parent.file_name().unwrap()) + .join(source.file_name().unwrap()); + + log_action( + "Copying", + format!( + "package dependency from {} to {}", + source.log_color_highlight(), + target.log_color_highlight() + ), + ); + copy(source, target)?; + } + } } } @@ -663,9 +704,11 @@ fn create_base_output_wit(ctx: &ApplicationContext, component_name: &str) -> Res let component_interface_package_deps = ctx.wit.component_interface_package_deps(component_name)?; if !component_interface_package_deps.is_empty() { - log_action("Adding", "interface package deps"); + log_action("Adding", "component interface package dependencies"); let _indent = LogIndent::new(); + // TODO: transitive deps? + for (dep_interface_package_name, dep_component_name) in &component_interface_package_deps { diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index 583c4533..2bcd17aa 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -249,38 +249,10 @@ impl ResolvedWitApplication { if !deps_path.exists() { HashSet::new() } else { - let mut entries = deps_path - .read_dir() - .and_then(|read_dir| read_dir.collect::>>()) - .with_context(|| { - anyhow!( - "Failed to read component {} wit dependencies from {}", - component_name, - deps_path.display(), - ) - })?; - entries.sort_by_key(|e| e.file_name()); - entries - .iter() - .filter_map(|entry| { - let path = entry.path(); - // NOTE: unlike wit_resolve - for now - we do not support: - // - symlinks - // - single file deps - // - wasm or wat deps - path.is_dir().then(|| { - UnresolvedPackageGroup::parse_dir(&path) - .with_context(|| { - anyhow!( - "Failed to parse component {} wit dependency {}", - component_name, - path.display() - ) - }) - .map(|package_group| package_group.main.name) - }) - }) - .collect::, _>>()? + parse_wit_deps_dir(&deps_path)? + .into_iter() + .map(|package_group| package_group.main.name) + .collect::>() } }; @@ -503,3 +475,26 @@ impl ResolvedWitApplication { }) } } + +pub fn parse_wit_deps_dir(path: &Path) -> Result, Error> { + let mut entries = path + .read_dir() + .and_then(|read_dir| read_dir.collect::>>()) + .with_context(|| anyhow!("Failed to read wit dependencies from {}", path.display(),))?; + entries.sort_by_key(|e| e.file_name()); + entries + .iter() + .filter_map(|entry| { + let path = entry.path(); + // NOTE: unlike wit_resolve - for now - we do not support: + // - symlinks + // - single file deps + // - wasm or wat deps + path.is_dir().then(|| { + UnresolvedPackageGroup::parse_dir(&path).with_context(|| { + anyhow!("Failed to parse wit dependency package {}", path.display()) + }) + }) + }) + .collect::, _>>() +} From 6d6db5c3680d927969d79e8f9d0971a4e74ed424 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 12 Nov 2024 13:04:15 +0100 Subject: [PATCH 049/109] clippy, fmt --- wasm-rpc-stubgen/src/commands/declarative.rs | 4 ++-- wasm-rpc-stubgen/src/model/wasm_rpc.rs | 4 ++-- wasm-rpc-stubgen/src/wit_resolve.rs | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index 80fd5428..d37040c1 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -899,7 +899,7 @@ fn copy_wit_sources(source: &Path, target: &Path) -> anyhow::Result<()> { ); let _indent = LogIndent::new(); - let dir_content = fs_extra::dir::get_dir_content(&source).with_context(|| { + let dir_content = fs_extra::dir::get_dir_content(source).with_context(|| { anyhow!( "Failed to read component input wit directory entries for {}", source.display() @@ -909,7 +909,7 @@ fn copy_wit_sources(source: &Path, target: &Path) -> anyhow::Result<()> { for file in dir_content.files { let from = PathBuf::from(&file); let to = target.join( - from.strip_prefix(&source) + from.strip_prefix(source) .with_context(|| anyhow!("Failed to strip prefix for source {}", &file))?, ); diff --git a/wasm-rpc-stubgen/src/model/wasm_rpc.rs b/wasm-rpc-stubgen/src/model/wasm_rpc.rs index b51a4193..0f7b35ec 100644 --- a/wasm-rpc-stubgen/src/model/wasm_rpc.rs +++ b/wasm-rpc-stubgen/src/model/wasm_rpc.rs @@ -299,7 +299,7 @@ impl Application { source: source.to_path_buf(), name: component.name, build_dir: properties.build_dir.map(|s| s.into()), - wit_deps: properties.wit_deps.into_iter().map(|s| s.into()).collect(), + _wit_deps: properties.wit_deps.into_iter().map(|s| s.into()).collect(), }; Some(wasm_rpc_stub_build) @@ -802,7 +802,7 @@ pub struct WasmBuild { source: PathBuf, name: String, build_dir: Option, - wit_deps: Vec, + _wit_deps: Vec, } #[derive(Clone, Debug, Serialize, Deserialize)] diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index 2bcd17aa..2f014724 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -421,10 +421,10 @@ impl ResolvedWitApplication { .filter(|&package_name| { !component .input_contained_package_deps - .contains(&package_name) + .contains(package_name) && !self .interface_package_to_component - .contains_key(&package_name) + .contains_key(package_name) }) .cloned() .collect::>()) From 7446a4818fb31c38b1ad8bee394e29010234645e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 12 Nov 2024 13:28:28 +0100 Subject: [PATCH 050/109] clean: delete wasm input and output --- wasm-rpc-stubgen/src/commands/declarative.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index d37040c1..48cb6f59 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -310,6 +310,8 @@ pub fn clean(config: Config) -> anyhow::Result<()> { let _indent = LogIndent::new(); delete_path("wit output dir", &app.component_output_wit(component_name))?; + delete_path("wasm input", &app.component_input_wasm(component_name))?; + delete_path("wasm output", &app.component_output_wasm(component_name))?; for build_step in &component.build_steps { let build_dir = build_step From 8e530f96bbe079cc9961692332fb10c0baaadd7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 12 Nov 2024 16:45:34 +0100 Subject: [PATCH 051/109] declarative rust fixes --- wasm-rpc-stubgen/src/cargo.rs | 108 ++++++++++++++++-- wasm-rpc-stubgen/src/commands/declarative.rs | 32 +++++- wasm-rpc-stubgen/src/lib.rs | 2 + wasm-rpc-stubgen/src/stub.rs | 1 + wasm-rpc-stubgen/src/wit_generate.rs | 2 +- wasm-rpc-stubgen/src/wit_resolve.rs | 10 ++ .../tests-integration/tests/compose.rs | 1 + .../tests-integration/tests/stub_wasm.rs | 1 + wasm-rpc-stubgen/tests/add_dep.rs | 2 + wasm-rpc-stubgen/tests/wit.rs | 2 + 10 files changed, 148 insertions(+), 13 deletions(-) diff --git a/wasm-rpc-stubgen/src/cargo.rs b/wasm-rpc-stubgen/src/cargo.rs index 85745993..bf48d14c 100644 --- a/wasm-rpc-stubgen/src/cargo.rs +++ b/wasm-rpc-stubgen/src/cargo.rs @@ -12,20 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::{BTreeMap, BTreeSet}; -use std::fs; -use std::path::{Path, PathBuf}; - use crate::commands::log::{log_action, log_warn_action, LogColorize}; use crate::fs::get_file_name; use crate::naming; use crate::stub::StubDefinition; +use crate::wit_resolve::ResolvedWitDir; +use anyhow::{anyhow, Context}; use cargo_toml::{ Dependency, DependencyDetail, DepsSet, Edition, Inheritable, LtoSetting, Manifest, Profile, - Profiles, StripSetting, + Profiles, StripSetting, Workspace, }; use golem_wasm_rpc::WASM_RPC_VERSION; use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::{Path, PathBuf}; use toml::Value; use wit_parser::PackageName; @@ -71,6 +72,10 @@ struct WitDependency { pub fn generate_cargo_toml(def: &StubDefinition) -> anyhow::Result<()> { let mut manifest = Manifest::default(); + if def.config.seal_cargo_workspace { + manifest.workspace = Some(Workspace::default()); + } + let mut wit_dependencies = BTreeMap::new(); wit_dependencies.insert( @@ -259,11 +264,11 @@ pub fn add_workspace_members(path: &Path, members: &[String]) -> anyhow::Result< Ok(()) } -pub fn add_dependencies_to_cargo_toml( - cargo_path: &Path, +pub fn add_cargo_package_component_deps( + cargo_toml_path: &Path, wit_sources: BTreeMap, ) -> anyhow::Result<()> { - let raw_manifest = fs::read_to_string(cargo_path)?; + let raw_manifest = fs::read_to_string(cargo_toml_path)?; let mut manifest: Manifest = Manifest::from_slice_with_metadata(raw_manifest.as_bytes())?; if let Some(ref mut package) = manifest.package { @@ -292,9 +297,9 @@ pub fn add_dependencies_to_cargo_toml( log_warn_action( "Updating", - format!("Cargo.toml at {:?}", cargo_path.log_color_highlight()), + format!("Cargo.toml at {:?}", cargo_toml_path.log_color_highlight()), ); - fs::write(cargo_path, cargo_toml)?; + fs::write(cargo_toml_path, cargo_toml)?; } } } @@ -302,6 +307,89 @@ pub fn add_dependencies_to_cargo_toml( Ok(()) } +pub fn regenerate_cargo_package_component( + cargo_toml_path: &Path, + wit_path: &Path, + world: Option, +) -> anyhow::Result<()> { + log_warn_action( + "Regenerating", + format!( + "package component in {}", + cargo_toml_path.log_color_highlight() + ), + ); + + let project_root = cargo_toml_path.parent().ok_or_else(|| { + anyhow!( + "Failed to get parent directory for {}", + cargo_toml_path.display() + ) + })?; + let relative_wit_path = wit_path.strip_prefix(project_root).with_context(|| { + anyhow!( + "Failed to create relative path for wit dir: {}, project root: {}", + wit_path.display(), + project_root.display() + ) + })?; + + let raw_manifest = fs::read_to_string(cargo_toml_path).with_context(|| { + anyhow!( + "Failed to read Cargo.toml at {}", + cargo_toml_path.log_color_highlight() + ) + })?; + let mut manifest: Manifest = + Manifest::from_slice_with_metadata(raw_manifest.as_bytes()).with_context(|| { + anyhow!( + "Failed to parse Cargo.toml at {}", + cargo_toml_path.log_color_highlight() + ) + })?; + let package = manifest + .package + .as_mut() + .ok_or_else(|| anyhow!("No package found in {}", cargo_toml_path.display()))?; + + let wit_dir = ResolvedWitDir::new(wit_path)?; + + package.metadata = Some(MetadataRoot { + component: Some(ComponentMetadata { + package: None, // TODO: do we need this? + target: Some(ComponentTarget { + world, + path: relative_wit_path.to_string_lossy().to_string(), + dependencies: wit_dir + .package_sources + .iter() + .filter(|(&package_id, _)| package_id != wit_dir.package_id) + .map(|(package_id, package_sources)| { + ( + format_package_name_without_version( + &wit_dir.package(*package_id).unwrap().name, + ), + WitDependency { + path: package_sources + .dir + .strip_prefix(project_root) + .unwrap() // TODO: unwrap + .to_string_lossy() + .to_string(), + }, + ) + }) + .collect(), + }), + }), + }); + + let cargo_toml = toml::to_string(&manifest)?; + fs::write(cargo_toml_path, cargo_toml)?; + + Ok(()) +} + fn format_package_name_without_version(package_name: &PackageName) -> String { format!("{}:{}", package_name.namespace, package_name.name) } diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index 48cb6f59..6b7fb540 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -1,3 +1,4 @@ +use crate::cargo::regenerate_cargo_package_component; use crate::commands::log::{ log_action, log_skipping_up_to_date, log_validated_action_result, log_warn_action, LogColorize, LogIndent, @@ -105,7 +106,11 @@ async fn pre_component_build_ctx(ctx: &mut ApplicationContext) -> anyhow::Result { let mut any_changed = false; for component_name in ctx.application.wasm_components_by_name.keys() { - any_changed |= create_output_wit(ctx, component_name)?; + let changed = create_output_wit(ctx, component_name)?; + if changed { + update_cargo_toml(ctx, component_name)?; + } + any_changed |= changed; } if any_changed { ctx.update_wit_context()?; @@ -789,6 +794,28 @@ fn create_output_wit(ctx: &ApplicationContext, component_name: &str) -> Result anyhow::Result<()> { + let component_input_wit = ctx.application.component_input_wit(component_name); + let component_input_wit_parent = component_input_wit.parent().ok_or_else(|| { + anyhow!( + "Failed to get parent for component {} input wit dir: {}", + component_name.log_color_highlight(), + component_input_wit.log_color_highlight() + ) + })?; + let cargo_toml = component_input_wit_parent.join("Cargo.toml"); + + if cargo_toml.exists() { + regenerate_cargo_package_component( + &cargo_toml, + &ctx.application.component_output_wit(component_name), + ctx.application.stub_world(component_name), + )? + } + + Ok(()) +} + async fn build_stub(ctx: &ApplicationContext, component_name: &str) -> anyhow::Result { let target_root = ctx.application.stub_temp_build_dir(component_name); @@ -802,6 +829,7 @@ async fn build_stub(ctx: &ApplicationContext, component_name: &str) -> anyhow::R wasm_rpc_version_override: ctx.application.stub_wasm_rpc_version(component_name), }, extract_source_interface_package: false, + seal_cargo_workspace: true, }) .context("Failed to gather information for the stub generator")?; @@ -882,7 +910,7 @@ fn add_stub_deps(ctx: &ApplicationContext, component_name: &str) -> Result anyhow::Result<()> { stub_crate_version: args.stub_crate_version, wasm_rpc_override: args.wasm_rpc_override, extract_source_interface_package: true, + seal_cargo_workspace: false, } ) .context("Failed to gather information for the stub generator. Make sure source_wit_root has a valid WIT file.")?; @@ -246,6 +247,7 @@ pub async fn build(args: BuildArgs) -> anyhow::Result<()> { stub_crate_version: args.stub_crate_version, wasm_rpc_override: args.wasm_rpc_override, extract_source_interface_package: true, + seal_cargo_workspace: false, }) .context("Failed to gather information for the stub generator")?; diff --git a/wasm-rpc-stubgen/src/stub.rs b/wasm-rpc-stubgen/src/stub.rs index 4cc31626..8335f5a6 100644 --- a/wasm-rpc-stubgen/src/stub.rs +++ b/wasm-rpc-stubgen/src/stub.rs @@ -35,6 +35,7 @@ pub struct StubConfig { pub stub_crate_version: String, pub wasm_rpc_override: WasmRpcOverride, pub extract_source_interface_package: bool, + pub seal_cargo_workspace: bool, } pub struct StubDefinition { diff --git a/wasm-rpc-stubgen/src/wit_generate.rs b/wasm-rpc-stubgen/src/wit_generate.rs index c3e681eb..ec0198c4 100644 --- a/wasm-rpc-stubgen/src/wit_generate.rs +++ b/wasm-rpc-stubgen/src/wit_generate.rs @@ -421,7 +421,7 @@ pub fn add_stub_as_dependency_to_wit_dir(config: AddStubAsDepConfig) -> anyhow:: cargo::is_cargo_component_toml(&target_cargo_toml).context(format!( "The file {target_cargo_toml:?} is not a valid cargo-component project" ))?; - cargo::add_dependencies_to_cargo_toml( + cargo::add_cargo_package_component_deps( &target_cargo_toml, package_names_to_package_path, )?; diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index 2f014724..f316b7d8 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -37,6 +37,16 @@ impl ResolvedWitDir { }) } + pub fn package_sources(&self, package_id: PackageId) -> Result<&PackageSource, Error> { + self.package_sources.get(&package_id).with_context(|| { + anyhow!( + "Failed to get package sources by id: {:?}, wit dir: {}", + package_id, + self.path.display() + ) + }) + } + pub fn main_package(&self) -> anyhow::Result<&Package> { self.package(self.package_id) } diff --git a/wasm-rpc-stubgen/tests-integration/tests/compose.rs b/wasm-rpc-stubgen/tests-integration/tests/compose.rs index 69845ea0..a9e40e3a 100644 --- a/wasm-rpc-stubgen/tests-integration/tests/compose.rs +++ b/wasm-rpc-stubgen/tests-integration/tests/compose.rs @@ -82,6 +82,7 @@ async fn init_stub(name: &str) -> (TempDir, TempDir, PathBuf) { stub_crate_version: "1.0.0".to_string(), wasm_rpc_override: wasm_rpc_override(), extract_source_interface_package: true, + seal_cargo_workspace: true, }) .unwrap(); let wasm_path = generate_and_build_stub(&def).await.unwrap(); diff --git a/wasm-rpc-stubgen/tests-integration/tests/stub_wasm.rs b/wasm-rpc-stubgen/tests-integration/tests/stub_wasm.rs index 75fe917e..a5c217e6 100644 --- a/wasm-rpc-stubgen/tests-integration/tests/stub_wasm.rs +++ b/wasm-rpc-stubgen/tests-integration/tests/stub_wasm.rs @@ -45,6 +45,7 @@ async fn all_wit_types() { stub_crate_version: "1.0.0".to_string(), wasm_rpc_override: wasm_rpc_override(), extract_source_interface_package: true, + seal_cargo_workspace: false, }) .unwrap(); diff --git a/wasm-rpc-stubgen/tests/add_dep.rs b/wasm-rpc-stubgen/tests/add_dep.rs index 4be74846..1bb1191d 100644 --- a/wasm-rpc-stubgen/tests/add_dep.rs +++ b/wasm-rpc-stubgen/tests/add_dep.rs @@ -582,6 +582,7 @@ fn init_stub(name: &str) -> (TempDir, TempDir) { stub_crate_version: "1.0.0".to_string(), wasm_rpc_override: WasmRpcOverride::default(), extract_source_interface_package: true, + seal_cargo_workspace: false, }) .unwrap(); let _ = generate_stub_wit_dir(&def).unwrap(); @@ -596,6 +597,7 @@ fn regenerate_stub(stub_dir: &Path, source_wit_root: &Path) { stub_crate_version: "1.0.0".to_string(), wasm_rpc_override: WasmRpcOverride::default(), extract_source_interface_package: true, + seal_cargo_workspace: false, }) .unwrap(); let _ = generate_stub_wit_dir(&def).unwrap(); diff --git a/wasm-rpc-stubgen/tests/wit.rs b/wasm-rpc-stubgen/tests/wit.rs index 87b28e3f..9de4570f 100644 --- a/wasm-rpc-stubgen/tests/wit.rs +++ b/wasm-rpc-stubgen/tests/wit.rs @@ -38,6 +38,7 @@ fn all_wit_types() { stub_crate_version: "1.0.0".to_string(), wasm_rpc_override: WasmRpcOverride::default(), extract_source_interface_package: true, + seal_cargo_workspace: false, }) .unwrap(); let resolve = generate_stub_wit_dir(&def).unwrap().resolve; @@ -128,6 +129,7 @@ fn many_ways_to_export() { stub_crate_version: "1.0.0".to_string(), wasm_rpc_override: WasmRpcOverride::default(), extract_source_interface_package: true, + seal_cargo_workspace: false, }) .unwrap(); let resolve = generate_stub_wit_dir(&def).unwrap().resolve; From da9c62d573e6d9169e01c631f9b6eaff7e18e3ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 12 Nov 2024 16:58:28 +0100 Subject: [PATCH 052/109] plug fix --- wasm-rpc-stubgen/src/commands/composition.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/wasm-rpc-stubgen/src/commands/composition.rs b/wasm-rpc-stubgen/src/commands/composition.rs index 54262c1e..4dcb2a77 100644 --- a/wasm-rpc-stubgen/src/commands/composition.rs +++ b/wasm-rpc-stubgen/src/commands/composition.rs @@ -42,6 +42,7 @@ pub async fn compose( let bytes = graph.encode(EncodeOptions::default())?; + std::fs::create_dir_all(dest_wasm.parent().unwrap())?; // TODO: unwrap / context std::fs::write(dest_wasm, bytes).context(format!( "failed to write output file `{path}`", path = dest_wasm.display() From 5e8e7941efccce0eebea3906155a14302d446b19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Wed, 13 Nov 2024 15:17:47 +0100 Subject: [PATCH 053/109] move wit test data, add placeholder for dec app build integ tests --- Cargo.lock | 1 + Cargo.toml | 1 + wasm-rpc-stubgen/Cargo.toml | 2 +- wasm-rpc-stubgen/src/cargo.rs | 2 +- .../all-wit-types-alternative/main.wit | 0 .../{ => wit}/all-wit-types/main.wit | 0 .../caller-no-dep-importstub/Cargo.toml | 0 .../caller-no-dep-importstub/src/lib.rs | 0 .../caller-no-dep-importstub/wit/caller.wit | 0 .../{ => wit}/caller-no-dep/Cargo.toml | 0 .../{ => wit}/caller-no-dep/src/lib.rs | 0 .../{ => wit}/caller-no-dep/wit/caller.wit | 0 .../direct-circular-a-same-world-name/a.wit | 0 .../{ => wit}/direct-circular-a/a.wit | 0 .../direct-circular-b-same-world-name/b.wit | 0 .../{ => wit}/direct-circular-b/b.wit | 0 .../{ => wit}/indirect-circular-a/a.wit | 0 .../{ => wit}/indirect-circular-b/b.wit | 0 .../{ => wit}/indirect-circular-c/c.wit | 0 .../many-ways-to-export/deps/sub/sub.wit | 0 .../many-ways-to-export/deps/sub2/sub2.wit | 0 .../{ => wit}/many-ways-to-export/main.wit | 0 .../test-data/{ => wit}/resources/main.wit | 0 .../test-data/{ => wit}/self-circular/a.wit | 0 wasm-rpc-stubgen/tests-integration/Cargo.toml | 1 + .../tests-integration/tests/app.rs | 22 +++++++++++++++++++ wasm-rpc-stubgen/tests/add_dep.rs | 6 ++--- wasm-rpc-stubgen/tests/wit.rs | 2 +- 28 files changed, 31 insertions(+), 6 deletions(-) rename wasm-rpc-stubgen/test-data/{ => wit}/all-wit-types-alternative/main.wit (100%) rename wasm-rpc-stubgen/test-data/{ => wit}/all-wit-types/main.wit (100%) rename wasm-rpc-stubgen/test-data/{ => wit}/caller-no-dep-importstub/Cargo.toml (100%) rename wasm-rpc-stubgen/test-data/{ => wit}/caller-no-dep-importstub/src/lib.rs (100%) rename wasm-rpc-stubgen/test-data/{ => wit}/caller-no-dep-importstub/wit/caller.wit (100%) rename wasm-rpc-stubgen/test-data/{ => wit}/caller-no-dep/Cargo.toml (100%) rename wasm-rpc-stubgen/test-data/{ => wit}/caller-no-dep/src/lib.rs (100%) rename wasm-rpc-stubgen/test-data/{ => wit}/caller-no-dep/wit/caller.wit (100%) rename wasm-rpc-stubgen/test-data/{ => wit}/direct-circular-a-same-world-name/a.wit (100%) rename wasm-rpc-stubgen/test-data/{ => wit}/direct-circular-a/a.wit (100%) rename wasm-rpc-stubgen/test-data/{ => wit}/direct-circular-b-same-world-name/b.wit (100%) rename wasm-rpc-stubgen/test-data/{ => wit}/direct-circular-b/b.wit (100%) rename wasm-rpc-stubgen/test-data/{ => wit}/indirect-circular-a/a.wit (100%) rename wasm-rpc-stubgen/test-data/{ => wit}/indirect-circular-b/b.wit (100%) rename wasm-rpc-stubgen/test-data/{ => wit}/indirect-circular-c/c.wit (100%) rename wasm-rpc-stubgen/test-data/{ => wit}/many-ways-to-export/deps/sub/sub.wit (100%) rename wasm-rpc-stubgen/test-data/{ => wit}/many-ways-to-export/deps/sub2/sub2.wit (100%) rename wasm-rpc-stubgen/test-data/{ => wit}/many-ways-to-export/main.wit (100%) rename wasm-rpc-stubgen/test-data/{ => wit}/resources/main.wit (100%) rename wasm-rpc-stubgen/test-data/{ => wit}/self-circular/a.wit (100%) create mode 100644 wasm-rpc-stubgen/tests-integration/tests/app.rs diff --git a/Cargo.lock b/Cargo.lock index cef74b9e..81a9198f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5277,6 +5277,7 @@ dependencies = [ name = "wasm-rpc-stubgen-tests-integration" version = "0.0.0" dependencies = [ + "assert2", "fs_extra", "golem-wasm-ast", "golem-wasm-rpc-stubgen", diff --git a/Cargo.toml b/Cargo.toml index aacf5879..37866c7d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ lto = true opt-level = 's' [workspace.dependencies] +assert2 = "0.3.15" fs_extra = "1.3.0" golem-wasm-ast = "1.0.1" tempfile = "3.12.0" diff --git a/wasm-rpc-stubgen/Cargo.toml b/wasm-rpc-stubgen/Cargo.toml index e7aa2da0..095fefa0 100644 --- a/wasm-rpc-stubgen/Cargo.toml +++ b/wasm-rpc-stubgen/Cargo.toml @@ -30,7 +30,7 @@ harness = false [dependencies] anyhow = "1.0.79" -assert2 = "0.3.15" +assert2 = { workspace = true } cargo_toml = "0.20.2" clap = { version = "4.5.7", features = ["derive"] } colored = "2.1.0" diff --git a/wasm-rpc-stubgen/src/cargo.rs b/wasm-rpc-stubgen/src/cargo.rs index bf48d14c..22bc7de5 100644 --- a/wasm-rpc-stubgen/src/cargo.rs +++ b/wasm-rpc-stubgen/src/cargo.rs @@ -356,7 +356,7 @@ pub fn regenerate_cargo_package_component( package.metadata = Some(MetadataRoot { component: Some(ComponentMetadata { - package: None, // TODO: do we need this? + package: None, target: Some(ComponentTarget { world, path: relative_wit_path.to_string_lossy().to_string(), diff --git a/wasm-rpc-stubgen/test-data/all-wit-types-alternative/main.wit b/wasm-rpc-stubgen/test-data/wit/all-wit-types-alternative/main.wit similarity index 100% rename from wasm-rpc-stubgen/test-data/all-wit-types-alternative/main.wit rename to wasm-rpc-stubgen/test-data/wit/all-wit-types-alternative/main.wit diff --git a/wasm-rpc-stubgen/test-data/all-wit-types/main.wit b/wasm-rpc-stubgen/test-data/wit/all-wit-types/main.wit similarity index 100% rename from wasm-rpc-stubgen/test-data/all-wit-types/main.wit rename to wasm-rpc-stubgen/test-data/wit/all-wit-types/main.wit diff --git a/wasm-rpc-stubgen/test-data/caller-no-dep-importstub/Cargo.toml b/wasm-rpc-stubgen/test-data/wit/caller-no-dep-importstub/Cargo.toml similarity index 100% rename from wasm-rpc-stubgen/test-data/caller-no-dep-importstub/Cargo.toml rename to wasm-rpc-stubgen/test-data/wit/caller-no-dep-importstub/Cargo.toml diff --git a/wasm-rpc-stubgen/test-data/caller-no-dep-importstub/src/lib.rs b/wasm-rpc-stubgen/test-data/wit/caller-no-dep-importstub/src/lib.rs similarity index 100% rename from wasm-rpc-stubgen/test-data/caller-no-dep-importstub/src/lib.rs rename to wasm-rpc-stubgen/test-data/wit/caller-no-dep-importstub/src/lib.rs diff --git a/wasm-rpc-stubgen/test-data/caller-no-dep-importstub/wit/caller.wit b/wasm-rpc-stubgen/test-data/wit/caller-no-dep-importstub/wit/caller.wit similarity index 100% rename from wasm-rpc-stubgen/test-data/caller-no-dep-importstub/wit/caller.wit rename to wasm-rpc-stubgen/test-data/wit/caller-no-dep-importstub/wit/caller.wit diff --git a/wasm-rpc-stubgen/test-data/caller-no-dep/Cargo.toml b/wasm-rpc-stubgen/test-data/wit/caller-no-dep/Cargo.toml similarity index 100% rename from wasm-rpc-stubgen/test-data/caller-no-dep/Cargo.toml rename to wasm-rpc-stubgen/test-data/wit/caller-no-dep/Cargo.toml diff --git a/wasm-rpc-stubgen/test-data/caller-no-dep/src/lib.rs b/wasm-rpc-stubgen/test-data/wit/caller-no-dep/src/lib.rs similarity index 100% rename from wasm-rpc-stubgen/test-data/caller-no-dep/src/lib.rs rename to wasm-rpc-stubgen/test-data/wit/caller-no-dep/src/lib.rs diff --git a/wasm-rpc-stubgen/test-data/caller-no-dep/wit/caller.wit b/wasm-rpc-stubgen/test-data/wit/caller-no-dep/wit/caller.wit similarity index 100% rename from wasm-rpc-stubgen/test-data/caller-no-dep/wit/caller.wit rename to wasm-rpc-stubgen/test-data/wit/caller-no-dep/wit/caller.wit diff --git a/wasm-rpc-stubgen/test-data/direct-circular-a-same-world-name/a.wit b/wasm-rpc-stubgen/test-data/wit/direct-circular-a-same-world-name/a.wit similarity index 100% rename from wasm-rpc-stubgen/test-data/direct-circular-a-same-world-name/a.wit rename to wasm-rpc-stubgen/test-data/wit/direct-circular-a-same-world-name/a.wit diff --git a/wasm-rpc-stubgen/test-data/direct-circular-a/a.wit b/wasm-rpc-stubgen/test-data/wit/direct-circular-a/a.wit similarity index 100% rename from wasm-rpc-stubgen/test-data/direct-circular-a/a.wit rename to wasm-rpc-stubgen/test-data/wit/direct-circular-a/a.wit diff --git a/wasm-rpc-stubgen/test-data/direct-circular-b-same-world-name/b.wit b/wasm-rpc-stubgen/test-data/wit/direct-circular-b-same-world-name/b.wit similarity index 100% rename from wasm-rpc-stubgen/test-data/direct-circular-b-same-world-name/b.wit rename to wasm-rpc-stubgen/test-data/wit/direct-circular-b-same-world-name/b.wit diff --git a/wasm-rpc-stubgen/test-data/direct-circular-b/b.wit b/wasm-rpc-stubgen/test-data/wit/direct-circular-b/b.wit similarity index 100% rename from wasm-rpc-stubgen/test-data/direct-circular-b/b.wit rename to wasm-rpc-stubgen/test-data/wit/direct-circular-b/b.wit diff --git a/wasm-rpc-stubgen/test-data/indirect-circular-a/a.wit b/wasm-rpc-stubgen/test-data/wit/indirect-circular-a/a.wit similarity index 100% rename from wasm-rpc-stubgen/test-data/indirect-circular-a/a.wit rename to wasm-rpc-stubgen/test-data/wit/indirect-circular-a/a.wit diff --git a/wasm-rpc-stubgen/test-data/indirect-circular-b/b.wit b/wasm-rpc-stubgen/test-data/wit/indirect-circular-b/b.wit similarity index 100% rename from wasm-rpc-stubgen/test-data/indirect-circular-b/b.wit rename to wasm-rpc-stubgen/test-data/wit/indirect-circular-b/b.wit diff --git a/wasm-rpc-stubgen/test-data/indirect-circular-c/c.wit b/wasm-rpc-stubgen/test-data/wit/indirect-circular-c/c.wit similarity index 100% rename from wasm-rpc-stubgen/test-data/indirect-circular-c/c.wit rename to wasm-rpc-stubgen/test-data/wit/indirect-circular-c/c.wit diff --git a/wasm-rpc-stubgen/test-data/many-ways-to-export/deps/sub/sub.wit b/wasm-rpc-stubgen/test-data/wit/many-ways-to-export/deps/sub/sub.wit similarity index 100% rename from wasm-rpc-stubgen/test-data/many-ways-to-export/deps/sub/sub.wit rename to wasm-rpc-stubgen/test-data/wit/many-ways-to-export/deps/sub/sub.wit diff --git a/wasm-rpc-stubgen/test-data/many-ways-to-export/deps/sub2/sub2.wit b/wasm-rpc-stubgen/test-data/wit/many-ways-to-export/deps/sub2/sub2.wit similarity index 100% rename from wasm-rpc-stubgen/test-data/many-ways-to-export/deps/sub2/sub2.wit rename to wasm-rpc-stubgen/test-data/wit/many-ways-to-export/deps/sub2/sub2.wit diff --git a/wasm-rpc-stubgen/test-data/many-ways-to-export/main.wit b/wasm-rpc-stubgen/test-data/wit/many-ways-to-export/main.wit similarity index 100% rename from wasm-rpc-stubgen/test-data/many-ways-to-export/main.wit rename to wasm-rpc-stubgen/test-data/wit/many-ways-to-export/main.wit diff --git a/wasm-rpc-stubgen/test-data/resources/main.wit b/wasm-rpc-stubgen/test-data/wit/resources/main.wit similarity index 100% rename from wasm-rpc-stubgen/test-data/resources/main.wit rename to wasm-rpc-stubgen/test-data/wit/resources/main.wit diff --git a/wasm-rpc-stubgen/test-data/self-circular/a.wit b/wasm-rpc-stubgen/test-data/wit/self-circular/a.wit similarity index 100% rename from wasm-rpc-stubgen/test-data/self-circular/a.wit rename to wasm-rpc-stubgen/test-data/wit/self-circular/a.wit diff --git a/wasm-rpc-stubgen/tests-integration/Cargo.toml b/wasm-rpc-stubgen/tests-integration/Cargo.toml index 5d73f2c0..a978a0d5 100644 --- a/wasm-rpc-stubgen/tests-integration/Cargo.toml +++ b/wasm-rpc-stubgen/tests-integration/Cargo.toml @@ -21,6 +21,7 @@ name = "stub_wasm" harness = false [dependencies] +assert2 = { workspace = true} fs_extra = { workspace = true } golem-wasm-ast = { workspace = true } golem-wasm-rpc-stubgen = { path = "../../wasm-rpc-stubgen", version = "0.0.0" } diff --git a/wasm-rpc-stubgen/tests-integration/tests/app.rs b/wasm-rpc-stubgen/tests-integration/tests/app.rs new file mode 100644 index 00000000..83dc39e8 --- /dev/null +++ b/wasm-rpc-stubgen/tests-integration/tests/app.rs @@ -0,0 +1,22 @@ +// Copyright 2024 Golem Cloud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use test_r::test; + +test_r::enable!(); + +#[test] +async fn rust_project() { + // TODO +} diff --git a/wasm-rpc-stubgen/tests/add_dep.rs b/wasm-rpc-stubgen/tests/add_dep.rs index 1bb1191d..7f26d9d0 100644 --- a/wasm-rpc-stubgen/tests/add_dep.rs +++ b/wasm-rpc-stubgen/tests/add_dep.rs @@ -161,7 +161,7 @@ fn many_ways_to_export_no_collision() { assert_has_same_wit_package( &PackageName::new("test", "sub", None), &dest_wit_root, - &Path::new("test-data").join("many-ways-to-export/deps/sub/sub.wit"), + Path::new("test-data/wit/many-ways-to-export/deps/sub/sub.wit"), ); } @@ -566,7 +566,7 @@ fn init_stub(name: &str) -> (TempDir, TempDir) { let canonical_source = source.path().canonicalize().unwrap(); fs_extra::dir::copy( - Path::new("test-data").join(name), + Path::new("test-data/wit").join(name), &canonical_source, &CopyOptions::new().content_only(true), ) @@ -605,7 +605,7 @@ fn regenerate_stub(stub_dir: &Path, source_wit_root: &Path) { fn init_caller(name: &str) -> TempDir { let temp_dir = TempDir::new().unwrap(); - let source = Path::new("test-data").join(name); + let source = Path::new("test-data/wit").join(name); fs_extra::dir::copy( source, diff --git a/wasm-rpc-stubgen/tests/wit.rs b/wasm-rpc-stubgen/tests/wit.rs index 9de4570f..c0f637ed 100644 --- a/wasm-rpc-stubgen/tests/wit.rs +++ b/wasm-rpc-stubgen/tests/wit.rs @@ -317,7 +317,7 @@ fn is_owned_by_interface( fn init_source(name: &str) -> TempDir { let temp_dir = TempDir::new().unwrap(); - let source = Path::new("test-data").join(name); + let source = Path::new("test-data/wit").join(name); fs_extra::dir::copy( source, From 8f6a97b1c21001d217328c5d3a33ce58df79c2c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Wed, 13 Nov 2024 15:32:53 +0100 Subject: [PATCH 054/109] leftover --- wasm-rpc-stubgen/tests-integration/Cargo.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wasm-rpc-stubgen/tests-integration/Cargo.toml b/wasm-rpc-stubgen/tests-integration/Cargo.toml index a978a0d5..6592be9d 100644 --- a/wasm-rpc-stubgen/tests-integration/Cargo.toml +++ b/wasm-rpc-stubgen/tests-integration/Cargo.toml @@ -12,6 +12,10 @@ name = "wasm_rpc_stubgen_tests_integration" path = "src/lib.rs" harness = false +[[test]] +name = "app" +harness = false + [[test]] name = "compose" harness = false From f72eef74e6ab9812e91125c72bd9b5273714ef46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Thu, 14 Nov 2024 09:02:40 +0100 Subject: [PATCH 055/109] move validation to src from model --- wasm-rpc-stubgen/src/commands/declarative.rs | 3 ++- wasm-rpc-stubgen/src/commands/log.rs | 2 +- wasm-rpc-stubgen/src/lib.rs | 1 + wasm-rpc-stubgen/src/model/mod.rs | 1 - wasm-rpc-stubgen/src/model/oam.rs | 2 +- wasm-rpc-stubgen/src/model/unknown_properties.rs | 2 +- wasm-rpc-stubgen/src/model/wasm_rpc.rs | 2 +- wasm-rpc-stubgen/src/{model => }/validation.rs | 2 +- wasm-rpc-stubgen/src/wit_resolve.rs | 2 +- 9 files changed, 9 insertions(+), 8 deletions(-) rename wasm-rpc-stubgen/src/{model => }/validation.rs (99%) diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index 6b7fb540..60c522b0 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -5,7 +5,7 @@ use crate::commands::log::{ }; use crate::fs::copy; use crate::model::oam; -use crate::model::validation::ValidatedResult; +use crate::validation::ValidatedResult; use crate::model::wasm_rpc::{ include_glob_patter_from_yaml_file, init_oam_app, Application, DEFAULT_CONFIG_FILE_NAME, }; @@ -264,6 +264,7 @@ async fn post_component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<() component_name.log_color_highlight(), ), ); + let _indent = LogIndent::new(); let stub_wasms = component .wasm_rpc_dependencies diff --git a/wasm-rpc-stubgen/src/commands/log.rs b/wasm-rpc-stubgen/src/commands/log.rs index 8b403350..31073129 100644 --- a/wasm-rpc-stubgen/src/commands/log.rs +++ b/wasm-rpc-stubgen/src/commands/log.rs @@ -1,5 +1,5 @@ use crate::fs::{OverwriteSafeAction, OverwriteSafeActionPlan}; -use crate::model::validation::ValidatedResult; +use crate::validation::ValidatedResult; use colored::{ColoredString, Colorize}; use std::path::{Path, PathBuf}; use std::sync::{LazyLock, RwLock}; diff --git a/wasm-rpc-stubgen/src/lib.rs b/wasm-rpc-stubgen/src/lib.rs index 86f93b0c..c4c50319 100644 --- a/wasm-rpc-stubgen/src/lib.rs +++ b/wasm-rpc-stubgen/src/lib.rs @@ -24,6 +24,7 @@ pub mod stub; pub mod wit_encode; pub mod wit_generate; pub mod wit_resolve; +pub mod validation; use crate::stub::{StubConfig, StubDefinition}; use crate::wit_generate::UpdateCargoToml; diff --git a/wasm-rpc-stubgen/src/model/mod.rs b/wasm-rpc-stubgen/src/model/mod.rs index fdaddf47..6e669204 100644 --- a/wasm-rpc-stubgen/src/model/mod.rs +++ b/wasm-rpc-stubgen/src/model/mod.rs @@ -1,4 +1,3 @@ pub mod oam; pub mod unknown_properties; -pub mod validation; pub mod wasm_rpc; diff --git a/wasm-rpc-stubgen/src/model/oam.rs b/wasm-rpc-stubgen/src/model/oam.rs index 600e7d89..b3e706ce 100644 --- a/wasm-rpc-stubgen/src/model/oam.rs +++ b/wasm-rpc-stubgen/src/model/oam.rs @@ -1,4 +1,4 @@ -use crate::model::validation::{ValidatedResult, ValidationBuilder}; +use crate::validation::{ValidatedResult, ValidationBuilder}; use anyhow::Context; use itertools::Itertools; use serde::de::DeserializeOwned; diff --git a/wasm-rpc-stubgen/src/model/unknown_properties.rs b/wasm-rpc-stubgen/src/model/unknown_properties.rs index 30519dea..3b09ca59 100644 --- a/wasm-rpc-stubgen/src/model/unknown_properties.rs +++ b/wasm-rpc-stubgen/src/model/unknown_properties.rs @@ -1,4 +1,4 @@ -use crate::model::validation::ValidationBuilder; +use crate::validation::ValidationBuilder; use std::collections::BTreeMap; pub type UnknownProperties = BTreeMap; diff --git a/wasm-rpc-stubgen/src/model/wasm_rpc.rs b/wasm-rpc-stubgen/src/model/wasm_rpc.rs index 0f7b35ec..f1de84c1 100644 --- a/wasm-rpc-stubgen/src/model/wasm_rpc.rs +++ b/wasm-rpc-stubgen/src/model/wasm_rpc.rs @@ -1,7 +1,7 @@ use crate::model::oam; use crate::model::oam::TypedTraitProperties; use crate::model::unknown_properties::{HasUnknownProperties, UnknownProperties}; -use crate::model::validation::{ValidatedResult, ValidationBuilder}; +use crate::validation::{ValidatedResult, ValidationBuilder}; use crate::naming; use crate::naming::wit::package_dep_dir_name_from_parser; use golem_wasm_rpc::WASM_RPC_VERSION; diff --git a/wasm-rpc-stubgen/src/model/validation.rs b/wasm-rpc-stubgen/src/validation.rs similarity index 99% rename from wasm-rpc-stubgen/src/model/validation.rs rename to wasm-rpc-stubgen/src/validation.rs index 5b039426..afa1be66 100644 --- a/wasm-rpc-stubgen/src/model/validation.rs +++ b/wasm-rpc-stubgen/src/validation.rs @@ -1,7 +1,7 @@ use itertools::Itertools; use std::fmt::Display; -// TODO: move to common + pub struct ValidationContext { pub name: &'static str, diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index f316b7d8..46714b0a 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -1,5 +1,5 @@ use crate::commands::log::{log_action, LogColorize, LogIndent}; -use crate::model::validation::{ValidatedResult, ValidationBuilder}; +use crate::validation::{ValidatedResult, ValidationBuilder}; use crate::model::wasm_rpc::Application; use crate::naming; use anyhow::{anyhow, bail, Context, Error}; From de67dd0955b0989c73b8183ba8c3abca703772a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Thu, 14 Nov 2024 09:54:06 +0100 Subject: [PATCH 056/109] add new type for ComponentName and cleanup LogColorize --- wasm-rpc-stubgen/src/commands/declarative.rs | 34 +++-- wasm-rpc-stubgen/src/commands/log.rs | 50 +++----- wasm-rpc-stubgen/src/lib.rs | 4 +- wasm-rpc-stubgen/src/model/wasm_rpc.rs | 124 ++++++++++++------- wasm-rpc-stubgen/src/wit_resolve.rs | 76 +++++++----- 5 files changed, 166 insertions(+), 122 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index 60c522b0..adc8b4fe 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -5,11 +5,12 @@ use crate::commands::log::{ }; use crate::fs::copy; use crate::model::oam; -use crate::validation::ValidatedResult; use crate::model::wasm_rpc::{ - include_glob_patter_from_yaml_file, init_oam_app, Application, DEFAULT_CONFIG_FILE_NAME, + include_glob_patter_from_yaml_file, init_oam_app, Application, ComponentName, + DEFAULT_CONFIG_FILE_NAME, }; use crate::stub::{StubConfig, StubDefinition}; +use crate::validation::ValidatedResult; use crate::wit_generate::{ add_stub_as_dependency_to_wit_dir, extract_main_interface_as_wit_dep, AddStubAsDepConfig, UpdateCargoToml, @@ -68,7 +69,7 @@ impl ApplicationContext { } } -pub fn init(component_name: String) -> anyhow::Result<()> { +pub fn init(component_name: ComponentName) -> anyhow::Result<()> { let file_name = DEFAULT_CONFIG_FILE_NAME; let mut file = std::fs::File::create_new(file_name) @@ -341,7 +342,10 @@ pub fn clean(config: Config) -> anyhow::Result<()> { for component_name in app.all_wasm_rpc_dependencies() { log_action( "Cleaning", - format!("component stub {}", component_name.log_color_highlight()), + format!( + "component stub {}", + component_name.as_str().log_color_highlight() + ), ); let _indent = LogIndent::new(); @@ -626,7 +630,10 @@ fn compile_and_collect_globs(root_dir: &Path, globs: &[String]) -> Result Result { +fn create_base_output_wit( + ctx: &ApplicationContext, + component_name: &ComponentName, +) -> Result { let component_input_wit = ctx.application.component_input_wit(component_name); let component_base_output_wit = ctx.application.component_base_output_wit(component_name); @@ -763,7 +770,10 @@ fn create_base_output_wit(ctx: &ApplicationContext, component_name: &str) -> Res } } -fn create_output_wit(ctx: &ApplicationContext, component_name: &str) -> Result { +fn create_output_wit( + ctx: &ApplicationContext, + component_name: &ComponentName, +) -> Result { let component_base_output_wit = ctx.application.component_base_output_wit(component_name); let component_output_wit = ctx.application.component_output_wit(component_name); @@ -795,7 +805,10 @@ fn create_output_wit(ctx: &ApplicationContext, component_name: &str) -> Result anyhow::Result<()> { +fn update_cargo_toml( + ctx: &ApplicationContext, + component_name: &ComponentName, +) -> anyhow::Result<()> { let component_input_wit = ctx.application.component_input_wit(component_name); let component_input_wit_parent = component_input_wit.parent().ok_or_else(|| { anyhow!( @@ -817,7 +830,10 @@ fn update_cargo_toml(ctx: &ApplicationContext, component_name: &str) -> anyhow:: Ok(()) } -async fn build_stub(ctx: &ApplicationContext, component_name: &str) -> anyhow::Result { +async fn build_stub( + ctx: &ApplicationContext, + component_name: &ComponentName, +) -> anyhow::Result { let target_root = ctx.application.stub_temp_build_dir(component_name); let stub_def = StubDefinition::new(StubConfig { @@ -882,7 +898,7 @@ async fn build_stub(ctx: &ApplicationContext, component_name: &str) -> anyhow::R } } -fn add_stub_deps(ctx: &ApplicationContext, component_name: &str) -> Result { +fn add_stub_deps(ctx: &ApplicationContext, component_name: &ComponentName) -> Result { let component = ctx.application.component(component_name); if component.wasm_rpc_dependencies.is_empty() { Ok(false) diff --git a/wasm-rpc-stubgen/src/commands/log.rs b/wasm-rpc-stubgen/src/commands/log.rs index 31073129..e1cafbbd 100644 --- a/wasm-rpc-stubgen/src/commands/log.rs +++ b/wasm-rpc-stubgen/src/commands/log.rs @@ -1,10 +1,9 @@ use crate::fs::{OverwriteSafeAction, OverwriteSafeActionPlan}; +use crate::model::wasm_rpc::ComponentName; use crate::validation::ValidatedResult; use colored::{ColoredString, Colorize}; use std::path::{Path, PathBuf}; use std::sync::{LazyLock, RwLock}; -use std::thread::JoinHandle; - // TODO: move package under lib static LOG_STATE: LazyLock> = LazyLock::new(RwLock::default); @@ -185,60 +184,51 @@ pub fn log_action_plan(action: &OverwriteSafeAction, plan: OverwriteSafeActionPl } pub trait LogColorize { - type Colorize: Colorize; - - fn to_colorize(&self) -> Self::Colorize; + fn as_str(&self) -> impl Colorize; fn log_color_action(&self) -> ColoredString { - self.to_colorize().green() + self.as_str().green() } fn log_color_warn(&self) -> ColoredString { - self.to_colorize().yellow() + self.as_str().yellow() } fn log_color_error(&self) -> ColoredString { - self.to_colorize().red() + self.as_str().red() } fn log_color_highlight(&self) -> ColoredString { - self.to_colorize().bold() + self.as_str().bold() } } impl<'a> LogColorize for &'a str { - type Colorize = &'a str; - - fn to_colorize(&self) -> Self::Colorize { - self + fn as_str(&self) -> impl Colorize { + self.as_ref() } } impl LogColorize for String { - type Colorize = ColoredString; - - fn to_colorize(&self) -> Self::Colorize { - self.clone().into() + fn as_str(&self) -> impl Colorize { + self.as_str() } } -impl LogColorize for PathBuf { - type Colorize = ColoredString; - - fn to_colorize(&self) -> Self::Colorize { - self.display().to_string().into() +impl<'a> LogColorize for &'a Path { + fn as_str(&self) -> impl Colorize { + ColoredString::from(self.display().to_string()) } } -impl<'a> LogColorize for &'a Path { - type Colorize = ColoredString; - - fn to_colorize(&self) -> Self::Colorize { - self.display().to_string().into() +impl LogColorize for PathBuf { + fn as_str(&self) -> impl Colorize { + ColoredString::from(self.display().to_string()) } } -pub struct ChildProcessLogger { - _out_handle: JoinHandle<()>, - _err_handle: JoinHandle<()>, +impl LogColorize for ComponentName { + fn as_str(&self) -> impl Colorize { + self.as_str() + } } diff --git a/wasm-rpc-stubgen/src/lib.rs b/wasm-rpc-stubgen/src/lib.rs index c4c50319..e9df1178 100644 --- a/wasm-rpc-stubgen/src/lib.rs +++ b/wasm-rpc-stubgen/src/lib.rs @@ -21,10 +21,10 @@ pub mod model; pub mod naming; pub mod rust; pub mod stub; +pub mod validation; pub mod wit_encode; pub mod wit_generate; pub mod wit_resolve; -pub mod validation; use crate::stub::{StubConfig, StubDefinition}; use crate::wit_generate::UpdateCargoToml; @@ -287,7 +287,7 @@ pub fn initialize_workspace( pub async fn run_declarative_command(command: App) -> anyhow::Result<()> { match command { - App::Init(args) => commands::declarative::init(args.component_name), + App::Init(args) => commands::declarative::init(args.component_name.into()), App::PreComponentBuild(args) => { commands::declarative::pre_component_build(dec_build_args_to_config(args)).await } diff --git a/wasm-rpc-stubgen/src/model/wasm_rpc.rs b/wasm-rpc-stubgen/src/model/wasm_rpc.rs index f1de84c1..08e9cf3e 100644 --- a/wasm-rpc-stubgen/src/model/wasm_rpc.rs +++ b/wasm-rpc-stubgen/src/model/wasm_rpc.rs @@ -1,13 +1,15 @@ use crate::model::oam; use crate::model::oam::TypedTraitProperties; use crate::model::unknown_properties::{HasUnknownProperties, UnknownProperties}; -use crate::validation::{ValidatedResult, ValidationBuilder}; use crate::naming; use crate::naming::wit::package_dep_dir_name_from_parser; +use crate::validation::{ValidatedResult, ValidationBuilder}; use golem_wasm_rpc::WASM_RPC_VERSION; use itertools::Itertools; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::Display; +use std::fmt::Formatter; use std::fs; use std::path::{Path, PathBuf}; use wit_parser::PackageName; @@ -20,9 +22,34 @@ pub const OAM_COMPONENT_TYPE_WASM: &str = "wasm"; pub const OAM_COMPONENT_TYPE_WASM_BUILD: &str = "wasm-build"; pub const OAM_COMPONENT_TYPE_WASM_RPC_STUB_BUILD: &str = "wasm-rpc-stub-build"; -// TODO: ComponentName new type +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ComponentName(String); + +impl ComponentName { + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl Display for ComponentName { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl From for ComponentName { + fn from(value: String) -> Self { + ComponentName(value) + } +} + +impl From<&str> for ComponentName { + fn from(value: &str) -> Self { + ComponentName(value.to_string()) + } +} -pub fn init_oam_app(_component_name: String) -> oam::Application { +pub fn init_oam_app(_component_name: ComponentName) -> oam::Application { // TODO: let's do it as part of https://github.com/golemcloud/wasm-rpc/issues/89 todo!() } @@ -61,8 +88,8 @@ pub fn include_glob_patter_from_yaml_file(source: &Path) -> Option { pub struct Application { pub common_wasm_build: Option, pub common_wasm_rpc_stub_build: Option, - pub wasm_rpc_stub_builds_by_name: BTreeMap, - pub wasm_components_by_name: BTreeMap, + pub wasm_rpc_stub_builds_by_name: BTreeMap, + pub wasm_components_by_name: BTreeMap, } impl Application { @@ -200,7 +227,7 @@ impl Application { .remove(OAM_TRAIT_TYPE_WASM_RPC) .unwrap_or_default(); - let mut wasm_rpc_dependencies = Vec::::new(); + let mut wasm_rpc_dependencies = Vec::::new(); for wasm_rpc in wasm_rpc_traits { validation.push_context("trait type", wasm_rpc.trait_type.clone()); @@ -210,7 +237,7 @@ impl Application { || vec![("dep component name", wasm_rpc.component_name.clone())], validation, ); - wasm_rpc_dependencies.push(wasm_rpc.component_name) + wasm_rpc_dependencies.push(wasm_rpc.component_name.into()) } Err(err) => validation .add_error(format!("Failed to get wasm-rpc trait properties: {}", err)), @@ -272,7 +299,7 @@ impl Application { } Some(WasmComponent { - name: component.name, + name: component.name.into(), source: source.to_path_buf(), build_steps: properties.build, input_wit: properties.input_wit.into(), @@ -335,7 +362,7 @@ impl Application { let wasm_rpc_stub_build = WasmRpcStubBuild { source: source.to_path_buf(), name: component.name, - component_name: properties.component_name, + component_name: properties.component_name.map(Into::into), build_dir: properties.build_dir.map(|s| s.into()), wasm: properties.wasm.map(|s| s.into()), wit: properties.wit.map(|s| s.into()), @@ -401,10 +428,10 @@ impl Application { pub fn validate_components( validation: &mut ValidationBuilder, components: Vec, - ) -> BTreeMap { + ) -> BTreeMap { let (wasm_components_by_name, sources) = { - let mut wasm_components_by_name = BTreeMap::::new(); - let mut sources = BTreeMap::>::new(); + let mut wasm_components_by_name = BTreeMap::::new(); + let mut sources = BTreeMap::>::new(); for component in components { sources .entry(component.name.clone()) @@ -418,7 +445,7 @@ impl Application { let non_unique_components = sources.into_iter().filter(|(_, sources)| sources.len() > 1); validation.add_errors(non_unique_components, |(component_name, sources)| { Some(( - vec![("component name", component_name)], + vec![("component name", component_name.0)], format!( "Component is specified multiple times in sources: {}", sources.join(", ") @@ -466,9 +493,12 @@ impl Application { fn validate_wasm_rpc_stub_builds( validation: &mut ValidationBuilder, - wasm_components_by_name: &BTreeMap, + wasm_components_by_name: &BTreeMap, wasm_rpc_stub_builds: Vec, - ) -> (Option, BTreeMap) { + ) -> ( + Option, + BTreeMap, + ) { let ( common_wasm_rpc_stub_builds, wasm_rpc_stub_builds_by_component_name, @@ -477,10 +507,10 @@ impl Application { ) = { let mut common_wasm_rpc_stub_builds = Vec::::new(); let mut wasm_rpc_stub_builds_by_component_name = - BTreeMap::::new(); + BTreeMap::::new(); let mut common_sources = Vec::::new(); - let mut by_name_sources = BTreeMap::>::new(); + let mut by_name_sources = BTreeMap::>::new(); for wasm_rpc_stub_build in wasm_rpc_stub_builds { match &wasm_rpc_stub_build.component_name { @@ -516,7 +546,7 @@ impl Application { non_unique_wasm_rpc_stub_builds, |(component_name, sources)| { Some(( - vec![("component name", component_name)], + vec![("component name", component_name.0)], format!( "Wasm rpc stub build is specified multiple times in sources: {}", sources.join(", ") @@ -555,14 +585,14 @@ impl Application { ) } - pub fn all_wasm_rpc_dependencies(&self) -> BTreeSet { + pub fn all_wasm_rpc_dependencies(&self) -> BTreeSet { self.wasm_components_by_name .iter() .flat_map(|(_, component)| { component .wasm_rpc_dependencies .iter() - .map(|component_name| component_name.to_string()) + .map(|component_name| component_name.clone()) }) .collect() } @@ -574,26 +604,26 @@ impl Application { .unwrap_or_else(|| PathBuf::from("build")) } - pub fn component(&self, component_name: &str) -> &WasmComponent { + pub fn component(&self, component_name: &ComponentName) -> &WasmComponent { self.wasm_components_by_name .get(component_name) .unwrap_or_else(|| panic!("Component not found: {}", component_name)) } - pub fn component_input_wit(&self, component_name: &str) -> PathBuf { + pub fn component_input_wit(&self, component_name: &ComponentName) -> PathBuf { let component = self.component(component_name); component.source_dir().join(component.input_wit.clone()) } - pub fn component_base_output_wit(&self, component_name: &str) -> PathBuf { + pub fn component_base_output_wit(&self, component_name: &ComponentName) -> PathBuf { self.build_dir() .join("base_output_wit") - .join(component_name) + .join(component_name.as_str()) } pub fn component_base_output_wit_interface_package_dir( &self, - component_name: &str, + component_name: &ComponentName, interface_package_name: &PackageName, ) -> PathBuf { self.component_base_output_wit(component_name) @@ -602,43 +632,43 @@ impl Application { .join(naming::wit::INTERFACE_WIT_FILE_NAME) } - pub fn component_output_wit(&self, component_name: &str) -> PathBuf { + pub fn component_output_wit(&self, component_name: &ComponentName) -> PathBuf { let component = self.component(component_name); component.source_dir().join(component.output_wit.clone()) } - pub fn component_input_wasm(&self, component_name: &str) -> PathBuf { + pub fn component_input_wasm(&self, component_name: &ComponentName) -> PathBuf { let component = self.component(component_name); component.source_dir().join(component.input_wasm.clone()) } - pub fn component_output_wasm(&self, component_name: &str) -> PathBuf { + pub fn component_output_wasm(&self, component_name: &ComponentName) -> PathBuf { let component = self.component(component_name); component.source_dir().join(component.output_wasm.clone()) } - pub fn stub_world(&self, component_name: &str) -> Option { + pub fn stub_world(&self, component_name: &ComponentName) -> Option { self.stub_gen_property(component_name, |build| build.world.clone()) .flatten() } - pub fn stub_crate_version(&self, component_name: &str) -> String { + pub fn stub_crate_version(&self, component_name: &ComponentName) -> String { self.stub_gen_property(component_name, |build| build.crate_version.clone()) .flatten() .unwrap_or_else(|| WASM_RPC_VERSION.to_string()) } - pub fn stub_wasm_rpc_path(&self, component_name: &str) -> Option { + pub fn stub_wasm_rpc_path(&self, component_name: &ComponentName) -> Option { self.stub_gen_property(component_name, |build| build.wasm_rpc_path.clone()) .flatten() } - pub fn stub_wasm_rpc_version(&self, component_name: &str) -> Option { + pub fn stub_wasm_rpc_version(&self, component_name: &ComponentName) -> Option { self.stub_gen_property(component_name, |build| build.wasm_rpc_version.clone()) .flatten() } - pub fn stub_build_dir(&self, component_name: &str) -> PathBuf { + pub fn stub_build_dir(&self, component_name: &ComponentName) -> PathBuf { self.stub_gen_property(component_name, |build| { build .build_dir @@ -650,13 +680,13 @@ impl Application { .join("stub") } - pub fn stub_temp_build_dir(&self, component_name: &str) -> PathBuf { + pub fn stub_temp_build_dir(&self, component_name: &ComponentName) -> PathBuf { self.stub_build_dir(component_name) - .join(component_name) + .join(component_name.as_str()) .join("temp-build") } - pub fn stub_wasm(&self, component_name: &str) -> PathBuf { + pub fn stub_wasm(&self, component_name: &ComponentName) -> PathBuf { self.wasm_rpc_stub_builds_by_name .get(component_name) .and_then(|build| { @@ -667,23 +697,23 @@ impl Application { }) .unwrap_or_else(|| { self.stub_build_dir(component_name) - .join(component_name) + .join(component_name.as_str()) .join("stub.wasm") }) } - pub fn stub_wit(&self, component_name: &str) -> PathBuf { + pub fn stub_wit(&self, component_name: &ComponentName) -> PathBuf { self.wasm_rpc_stub_builds_by_name .get(component_name) .and_then(|build| build.wit.as_ref().map(|wit| build.source_dir().join(wit))) .unwrap_or_else(|| { self.stub_build_dir(component_name) - .join(component_name) + .join(component_name.as_str()) .join(naming::wit::WIT_DIR) }) } - fn stub_gen_property(&self, component_name: &str, get_property: F) -> Option + fn stub_gen_property(&self, component_name: &ComponentName, get_property: F) -> Option where F: Fn(&WasmRpcStubBuild) -> T, { @@ -696,14 +726,14 @@ impl Application { #[derive(Clone, Debug)] pub struct WasmComponent { - pub name: String, + pub name: ComponentName, pub source: PathBuf, pub build_steps: Vec, pub input_wit: PathBuf, pub output_wit: PathBuf, pub input_wasm: PathBuf, pub output_wasm: PathBuf, - pub wasm_rpc_dependencies: Vec, + pub wasm_rpc_dependencies: Vec, } impl WasmComponent { @@ -836,7 +866,7 @@ impl HasUnknownProperties for ComponentStubBuildProperties { pub struct WasmRpcStubBuild { source: PathBuf, name: String, - component_name: Option, + component_name: Option, build_dir: Option, wasm: Option, wit: Option, @@ -883,16 +913,16 @@ mod tests { let (component_name, component) = app.wasm_components_by_name.iter().next().unwrap(); - assert!(component_name == "component-one"); - assert!(component.name == "component-one"); + assert!(component_name.as_str() == "component-one"); + assert!(component.name.as_str() == "component-one"); assert!(component.input_wit.to_string_lossy() == "input_wit"); assert!(component.output_wit.to_string_lossy() == "output_wit"); assert!(component.input_wasm.to_string_lossy() == "out/in.wasm"); assert!(component.output_wasm.to_string_lossy() == "out/out.wasm"); assert!(component.wasm_rpc_dependencies.len() == 2); - assert!(component.wasm_rpc_dependencies[0] == "component-three"); - assert!(component.wasm_rpc_dependencies[1] == "component-two"); + assert!(component.wasm_rpc_dependencies[0].as_str() == "component-three"); + assert!(component.wasm_rpc_dependencies[1].as_str() == "component-two"); } #[test] diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index 46714b0a..1ca935b4 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -1,9 +1,10 @@ use crate::commands::log::{log_action, LogColorize, LogIndent}; -use crate::validation::{ValidatedResult, ValidationBuilder}; -use crate::model::wasm_rpc::Application; +use crate::model::wasm_rpc::{Application, ComponentName}; use crate::naming; +use crate::validation::{ValidatedResult, ValidationBuilder}; use anyhow::{anyhow, bail, Context, Error}; use indexmap::IndexMap; +use itertools::Itertools; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::path::{Path, PathBuf}; use wit_parser::{ @@ -158,19 +159,19 @@ pub struct ResolvedWitComponent { pub main_package_name: PackageName, pub unresolved_input_package_group: UnresolvedPackageGroup, pub resolved_output_wit_dir: Option, - pub app_component_deps: HashSet, + pub app_component_deps: HashSet, pub input_referenced_package_deps: HashSet, pub input_contained_package_deps: HashSet, - pub input_component_deps: BTreeSet, // NOTE: BTree for making dep sorting deterministic - pub output_component_deps: Option>, + pub input_component_deps: BTreeSet, // NOTE: BTree for making dep sorting deterministic + pub output_component_deps: Option>, } pub struct ResolvedWitApplication { - pub components: BTreeMap, // NOTE: BTree for making dep sorting deterministic - pub package_to_component: HashMap, - pub stub_package_to_component: HashMap, - pub interface_package_to_component: HashMap, - pub component_order: Vec, + pub components: BTreeMap, // NOTE: BTree for making dep sorting deterministic + pub package_to_component: HashMap, + pub stub_package_to_component: HashMap, + pub interface_package_to_component: HashMap, + pub component_order: Vec, } impl ResolvedWitApplication { @@ -199,7 +200,7 @@ impl ResolvedWitApplication { fn add_resolved_component( &mut self, - component_name: String, + component_name: ComponentName, resolved_component: ResolvedWitComponent, ) { self.package_to_component.insert( @@ -222,7 +223,7 @@ impl ResolvedWitApplication { let _indent = LogIndent::new(); for (component_name, component) in &app.wasm_components_by_name { - validation.push_context("component name", component_name.clone()); + validation.push_context("component name", component_name.to_string()); let input_wit = app.component_input_wit(component_name); let output_wit = app.component_output_wit(component_name); @@ -305,8 +306,12 @@ impl ResolvedWitApplication { } fn collect_component_deps(&mut self) { - fn component_deps<'a, I: IntoIterator, O: FromIterator>( - known_package_deps: &HashMap, + fn component_deps< + 'a, + I: IntoIterator, + O: FromIterator, + >( + known_package_deps: &HashMap, dep_package_names: I, ) -> O { dep_package_names @@ -315,7 +320,10 @@ impl ResolvedWitApplication { .collect() } - let mut deps = HashMap::, Option>)>::new(); + let mut deps = HashMap::< + ComponentName, + (BTreeSet, Option>), + >::new(); for (component_name, component) in &self.components { deps.insert( component_name.clone(), @@ -344,23 +352,23 @@ impl ResolvedWitApplication { let mut component_order = Vec::with_capacity(self.components.len()); // TODO: use some id instead of Strings? - let mut visited = HashSet::::new(); - let mut visiting = HashSet::::new(); + let mut visited = HashSet::::new(); + let mut visiting = HashSet::::new(); fn visit( resolved_app: &ResolvedWitApplication, - visited: &mut HashSet, - visiting: &mut HashSet, - component_order: &mut Vec, - component_name: &str, - input_deps: &BTreeSet, + visited: &mut HashSet, + visiting: &mut HashSet, + component_order: &mut Vec, + component_name: &ComponentName, + input_deps: &BTreeSet, ) -> bool { if visited.contains(component_name) { true } else if visiting.contains(component_name) { false } else { - visiting.insert(component_name.to_string()); + visiting.insert(component_name.clone()); for dep_component_name in input_deps { if !visit( @@ -380,9 +388,9 @@ impl ResolvedWitApplication { } visiting.remove(component_name); - visited.insert(component_name.to_string()); + visited.insert(component_name.clone()); - component_order.push(component_name.to_string()); + component_order.push(component_name.clone()); true } @@ -400,10 +408,10 @@ impl ResolvedWitApplication { ) { // TODO: better error message, collect full path - component_order.push(component_name.to_string()); + component_order.push(component_name.clone()); validation.add_error(format!( "Found component cycle: {}", - component_order.join(", ") + component_order.iter().map(|s| s.as_str()).join(", ") )); break; } @@ -412,7 +420,7 @@ impl ResolvedWitApplication { self.component_order = component_order; } - fn component(&self, component_name: &str) -> Result<&ResolvedWitComponent, Error> { + fn component(&self, component_name: &ComponentName) -> Result<&ResolvedWitComponent, Error> { self.components .get(component_name) .ok_or_else(|| anyhow!("Component not found: {}", component_name)) @@ -422,7 +430,7 @@ impl ResolvedWitApplication { // component interface packages, as those are added from stubs pub fn missing_generic_input_package_deps( &self, - component_name: &str, + component_name: &ComponentName, ) -> anyhow::Result> { let component = self.component(component_name)?; Ok(component @@ -442,8 +450,8 @@ impl ResolvedWitApplication { pub fn component_interface_package_deps( &self, - component_name: &str, - ) -> anyhow::Result> { + component_name: &ComponentName, + ) -> anyhow::Result> { let component = self.component(component_name)?; Ok(component .input_referenced_package_deps @@ -462,7 +470,7 @@ impl ResolvedWitApplication { // NOTE: this does not mean that the dependencies themselves are up-to-date, rather // only checks if there are difference in set of dependencies specified in the // application model vs in wit dependencies - pub fn is_dep_graph_up_to_date(&self, component_name: &str) -> anyhow::Result { + pub fn is_dep_graph_up_to_date(&self, component_name: &ComponentName) -> anyhow::Result { let component = self.component(component_name)?; Ok(match &component.output_component_deps { Some(output_deps) => &component.app_component_deps == output_deps, @@ -474,8 +482,8 @@ impl ResolvedWitApplication { // only checks if it is present as wit package dependency pub fn has_as_wit_dep( &self, - component_name: &str, - dep_component_name: &str, + component_name: &ComponentName, + dep_component_name: &ComponentName, ) -> anyhow::Result { let component = self.component(component_name)?; From 2057f13896312e8077809419cef4e8b920a994bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Thu, 14 Nov 2024 11:50:54 +0100 Subject: [PATCH 057/109] fs cleanups --- wasm-rpc-stubgen/src/cargo.rs | 22 +-- wasm-rpc-stubgen/src/commands/composition.rs | 17 +- wasm-rpc-stubgen/src/commands/declarative.rs | 72 +++----- wasm-rpc-stubgen/src/commands/generate.rs | 6 +- wasm-rpc-stubgen/src/commands/log.rs | 8 +- wasm-rpc-stubgen/src/fs.rs | 180 ++++++++++++------- wasm-rpc-stubgen/src/make.rs | 3 +- wasm-rpc-stubgen/src/model/oam.rs | 10 +- wasm-rpc-stubgen/src/model/wasm_rpc.rs | 13 +- wasm-rpc-stubgen/src/rust.rs | 12 +- wasm-rpc-stubgen/src/validation.rs | 2 - wasm-rpc-stubgen/src/wit_generate.rs | 17 +- 12 files changed, 193 insertions(+), 169 deletions(-) diff --git a/wasm-rpc-stubgen/src/cargo.rs b/wasm-rpc-stubgen/src/cargo.rs index 22bc7de5..656b9411 100644 --- a/wasm-rpc-stubgen/src/cargo.rs +++ b/wasm-rpc-stubgen/src/cargo.rs @@ -13,10 +13,10 @@ // limitations under the License. use crate::commands::log::{log_action, log_warn_action, LogColorize}; -use crate::fs::get_file_name; -use crate::naming; +use crate::fs::PathExtra; use crate::stub::StubDefinition; use crate::wit_resolve::ResolvedWitDir; +use crate::{fs, naming}; use anyhow::{anyhow, Context}; use cargo_toml::{ Dependency, DependencyDetail, DepsSet, Edition, Inheritable, LtoSetting, Manifest, Profile, @@ -25,7 +25,6 @@ use cargo_toml::{ use golem_wasm_rpc::WASM_RPC_VERSION; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet}; -use std::fs; use std::path::{Path, PathBuf}; use toml::Value; use wit_parser::PackageName; @@ -111,9 +110,9 @@ pub fn generate_cargo_toml(def: &StubDefinition) -> anyhow::Result<()> { wit_dependencies.insert( format_package_name_without_version(&dep_package.name), WitDependency { - path: naming::wit::package_wit_dep_dir_from_package_dir_name(&get_file_name( - &dep_package_sources.dir, - )?) + path: naming::wit::package_wit_dep_dir_from_package_dir_name( + &PathExtra::new(&dep_package_sources.dir).file_name_to_string()?, + ) .to_string_lossy() .to_string(), }, @@ -312,6 +311,8 @@ pub fn regenerate_cargo_package_component( wit_path: &Path, world: Option, ) -> anyhow::Result<()> { + let cargo_toml_path = PathExtra::new(cargo_toml_path); + log_warn_action( "Regenerating", format!( @@ -320,12 +321,7 @@ pub fn regenerate_cargo_package_component( ), ); - let project_root = cargo_toml_path.parent().ok_or_else(|| { - anyhow!( - "Failed to get parent directory for {}", - cargo_toml_path.display() - ) - })?; + let project_root = cargo_toml_path.parent()?; let relative_wit_path = wit_path.strip_prefix(project_root).with_context(|| { anyhow!( "Failed to create relative path for wit dir: {}, project root: {}", @@ -334,7 +330,7 @@ pub fn regenerate_cargo_package_component( ) })?; - let raw_manifest = fs::read_to_string(cargo_toml_path).with_context(|| { + let raw_manifest = fs::read_to_string(&cargo_toml_path).with_context(|| { anyhow!( "Failed to read Cargo.toml at {}", cargo_toml_path.log_color_highlight() diff --git a/wasm-rpc-stubgen/src/commands/composition.rs b/wasm-rpc-stubgen/src/commands/composition.rs index 4dcb2a77..641a4c17 100644 --- a/wasm-rpc-stubgen/src/commands/composition.rs +++ b/wasm-rpc-stubgen/src/commands/composition.rs @@ -1,4 +1,5 @@ use crate::commands::log::{log_warn_action, LogColorize}; +use crate::fs::{create_dir_all, read_to_string, write, PathExtra}; use anyhow::Context; use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; @@ -14,14 +15,11 @@ pub async fn compose( // with allowing missing plugs (through the also customized plug function below) // and using local packages only (for now) + let dest_wasm = PathExtra::new(dest_wasm); + let mut graph = CompositionGraph::new(); - let socket = std::fs::read(source_wasm).with_context(|| { - format!( - "failed to read socket component `{socket}`", - socket = source_wasm.display() - ) - })?; + let socket = read_to_string(source_wasm).context("Failed to read socket component")?; let socket = Package::from_bytes("socket", None, socket, graph.types_mut())?; let socket = graph.register_package(socket)?; @@ -42,11 +40,8 @@ pub async fn compose( let bytes = graph.encode(EncodeOptions::default())?; - std::fs::create_dir_all(dest_wasm.parent().unwrap())?; // TODO: unwrap / context - std::fs::write(dest_wasm, bytes).context(format!( - "failed to write output file `{path}`", - path = dest_wasm.display() - ))?; + create_dir_all(dest_wasm.parent()?)?; + write(dest_wasm, bytes)?; Ok(()) } diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index adc8b4fe..51b8e03d 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -3,7 +3,8 @@ use crate::commands::log::{ log_action, log_skipping_up_to_date, log_validated_action_result, log_warn_action, LogColorize, LogIndent, }; -use crate::fs::copy; +use crate::fs; +use crate::fs::PathExtra; use crate::model::oam; use crate::model::wasm_rpc::{ include_glob_patter_from_yaml_file, init_oam_app, Application, ComponentName, @@ -22,7 +23,6 @@ use colored::Colorize; use glob::glob; use itertools::Itertools; use std::cmp::Ordering; -use std::io::Write; use std::path::{Path, PathBuf}; use std::process::Command; use std::time::SystemTime; @@ -70,16 +70,11 @@ impl ApplicationContext { } pub fn init(component_name: ComponentName) -> anyhow::Result<()> { - let file_name = DEFAULT_CONFIG_FILE_NAME; - - let mut file = std::fs::File::create_new(file_name) - .with_context(|| format!("Failed to create {}", file_name))?; - - let app = init_oam_app(component_name); - file.write_all(app.to_yaml_string().as_bytes()) - .with_context(|| format!("Failed to write {}", file_name))?; - - Ok(()) + fs::write_str( + DEFAULT_CONFIG_FILE_NAME, + init_oam_app(component_name).to_yaml_string(), + ) + .context("Failed to init component application manifest") } pub async fn pre_component_build(config: Config) -> anyhow::Result<()> { @@ -251,7 +246,7 @@ async fn post_component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<() output_wasm.log_color_highlight(), ), ); - copy(&input_wasm, &output_wasm)?; + fs::copy(&input_wasm, &output_wasm)?; } else { log_action( "Composing", @@ -370,13 +365,8 @@ fn delete_path(context: &str, path: &Path) -> anyhow::Result<()> { "Deleting", format!("{} {}", context, path.log_color_highlight()), ); - if path.is_dir() { - std::fs::remove_dir_all(path) - .with_context(|| anyhow!("Failed to delete directory {}", path.display()))?; - } else { - std::fs::remove_file(path) - .with_context(|| anyhow!("Failed to delete file {}", path.display()))?; - } + fs::remove(path) + .with_context(|| anyhow!("Failed to delete {}, path: {}", context, path.display()))?; } Ok(()) } @@ -422,7 +412,7 @@ fn collect_sources(mode: &ApplicationSourceMode) -> ValidatedResult let sources = match mode { ApplicationSourceMode::Automatic => match find_main_source() { Some(source) => { - std::env::set_current_dir(source.parent().expect("Failed ot get config parent")) + std::env::set_current_dir(PathExtra::new(&source).parent().unwrap()) .expect("Failed to set current dir for config parent"); match include_glob_patter_from_yaml_file(source.as_path()) { @@ -563,7 +553,7 @@ where } }; - if let Ok(metadata) = std::fs::metadata(path) { + if let Ok(metadata) = fs::metadata(path) { if metadata.is_dir() { WalkDir::new(path) .into_iter() @@ -687,19 +677,19 @@ fn create_base_output_wit( .find(|package| package.main.name == package_name) .ok_or_else(|| anyhow!("Package dependency {} not found", package_name))?; for source in dep.source_map.source_files() { - let parent = source.parent().ok_or_else(|| { - anyhow!( - "Failed to get dependency source parent for {}", - source.display() - ) - })?; - // TODO: un-unwrap + let source = PathExtra::new(source); + let parent = PathExtra::new( + source + .parent() + .context("Failed to get dependency source parent")?, + ); + let target = ctx .application .component_base_output_wit(component_name) .join(naming::wit::DEPS_DIR) - .join(parent.file_name().unwrap()) - .join(source.file_name().unwrap()); + .join(parent.file_name_to_string()?) + .join(source.file_name_to_string()?); log_action( "Copying", @@ -709,7 +699,7 @@ fn create_base_output_wit( target.log_color_highlight() ), ); - copy(source, target)?; + fs::copy(source, target)?; } } } @@ -747,7 +737,7 @@ fn create_base_output_wit( target.log_color_highlight() ), ); - copy(source, target)?; + fs::copy(source, target)?; } } } @@ -809,14 +799,10 @@ fn update_cargo_toml( ctx: &ApplicationContext, component_name: &ComponentName, ) -> anyhow::Result<()> { - let component_input_wit = ctx.application.component_input_wit(component_name); - let component_input_wit_parent = component_input_wit.parent().ok_or_else(|| { - anyhow!( - "Failed to get parent for component {} input wit dir: {}", - component_name.log_color_highlight(), - component_input_wit.log_color_highlight() - ) - })?; + let component_input_wit = PathExtra::new(ctx.application.component_input_wit(component_name)); + let component_input_wit_parent = component_input_wit + .parent() + .with_context(|| anyhow!("Failed to get parent for component {}", component_name))?; let cargo_toml = component_input_wit_parent.join("Cargo.toml"); if cargo_toml.exists() { @@ -888,7 +874,7 @@ async fn build_stub( "Creating", format!("stub temp build dir {}", target_root.log_color_highlight()), ); - std::fs::create_dir_all(&target_root)?; + fs::create_dir_all(&target_root)?; commands::generate::build(&stub_def, &stub_wasm, &stub_wit).await?; @@ -968,7 +954,7 @@ fn copy_wit_sources(source: &Path, target: &Path) -> anyhow::Result<()> { to.log_color_highlight() ), ); - copy(from, to)?; + fs::copy(from, to)?; } Ok(()) diff --git a/wasm-rpc-stubgen/src/commands/generate.rs b/wasm-rpc-stubgen/src/commands/generate.rs index 7e56d94a..a077a87c 100644 --- a/wasm-rpc-stubgen/src/commands/generate.rs +++ b/wasm-rpc-stubgen/src/commands/generate.rs @@ -15,7 +15,7 @@ use crate::cargo::generate_cargo_toml; use crate::commands::log::{log_action, LogColorize, LogIndent}; use crate::compilation::compile; -use crate::fs::copy; +use crate::fs; use crate::naming; use crate::rust::generate_stub_source; use crate::stub::StubDefinition; @@ -24,7 +24,6 @@ use crate::wit_resolve::ResolvedWitDir; use anyhow::{anyhow, Context}; use fs_extra::dir::CopyOptions; use heck::ToSnakeCase; -use std::fs; use std::path::{Path, PathBuf}; pub fn generate(stub_def: &StubDefinition) -> anyhow::Result<()> { @@ -41,8 +40,7 @@ pub async fn build( ) -> anyhow::Result<()> { let wasm_path = generate_and_build_stub(stub_def).await?; - copy(wasm_path, dest_wasm).context("Failed to copy the WASM file to the destination")?; - + fs::copy(wasm_path, dest_wasm).context("Failed to copy the WASM file to the destination")?; fs::create_dir_all(dest_wit_root).context("Failed to create the target WIT root directory")?; fs_extra::dir::copy( diff --git a/wasm-rpc-stubgen/src/commands/log.rs b/wasm-rpc-stubgen/src/commands/log.rs index e1cafbbd..ff2e8e89 100644 --- a/wasm-rpc-stubgen/src/commands/log.rs +++ b/wasm-rpc-stubgen/src/commands/log.rs @@ -1,4 +1,4 @@ -use crate::fs::{OverwriteSafeAction, OverwriteSafeActionPlan}; +use crate::fs::{OverwriteSafeAction, OverwriteSafeActionPlan, PathExtra}; use crate::model::wasm_rpc::ComponentName; use crate::validation::ValidatedResult; use colored::{ColoredString, Colorize}; @@ -227,6 +227,12 @@ impl LogColorize for PathBuf { } } +impl> LogColorize for PathExtra

{ + fn as_str(&self) -> impl Colorize { + ColoredString::from(self.display().to_string()) + } +} + impl LogColorize for ComponentName { fn as_str(&self) -> impl Colorize { self.as_str() diff --git a/wasm-rpc-stubgen/src/fs.rs b/wasm-rpc-stubgen/src/fs.rs index 639bd018..c2222894 100644 --- a/wasm-rpc-stubgen/src/fs.rs +++ b/wasm-rpc-stubgen/src/fs.rs @@ -1,33 +1,37 @@ use anyhow::{anyhow, Context}; use std::cmp::PartialEq; +use std::fs::Metadata; use std::path::{Path, PathBuf}; use std::time::SystemTime; +pub fn create_dir_all>(path: P) -> anyhow::Result<()> { + let path = path.as_ref(); + if path.exists() { + Ok(()) + } else { + std::fs::create_dir_all(path) + .with_context(|| anyhow!("Failed to create directory {}", path.display())) + } +} + // Differences compared to std::fs::copy // - ensures that the target dir exists // - updated the modtime after copy, which is not guaranteed to happen, making it not usable for // modtime based up-to-date checks (see https://github.com/rust-lang/rust/issues/115982 for more info) // - uses anyhow error with added context pub fn copy, Q: AsRef>(from: P, to: Q) -> anyhow::Result { - let from = from.as_ref(); - let to = to.as_ref(); + let from = PathExtra(from); + let to = PathExtra(to); let context = || format!("Failed to copy from {} to {}", from.display(), to.display()); - let target_parent = to - .parent() - .ok_or_else(|| anyhow!("Failed to get target parent dir")) - .with_context(context)?; + create_dir_all(from.parent()?).with_context(context)?; - std::fs::create_dir_all(target_parent) - .with_context(|| anyhow!("Failed to create target dir")) - .with_context(context)?; + let bytes = std::fs::copy(&from, &to).with_context(context)?; - let bytes = std::fs::copy(from, to).with_context(context)?; - - std::fs::File::open(to) + std::fs::File::open(&to) .and_then(|to| to.set_modified(SystemTime::now())) - .with_context(|| anyhow!("Failed to update target modification time")) + .context("Failed to update target modification time") .with_context(context)?; Ok(bytes) @@ -39,8 +43,8 @@ pub fn copy_transformed, Q: AsRef, T: Fn(String) -> anyhow: to: Q, transform: T, ) -> anyhow::Result { - let from = from.as_ref(); - let to = to.as_ref(); + let from = PathExtra(from); + let to = PathExtra(to); let context = || { format!( @@ -50,50 +54,66 @@ pub fn copy_transformed, Q: AsRef, T: Fn(String) -> anyhow: ) }; - let target_parent = to - .parent() - .ok_or_else(|| anyhow!("Failed to get target parent dir")) - .with_context(context)?; + let target_parent = from.parent().with_context(context)?; - std::fs::create_dir_all(target_parent) - .with_context(|| anyhow!("Failed to create target dir")) + create_dir_all(target_parent) + .context("Failed to create target dir") .with_context(context)?; - let content = std::fs::read_to_string(from) - .with_context(|| anyhow!("Failed to read source content")) - .with_context(context)?; + let content = read_to_string(&from).with_context(context)?; let transformed_content = transform(content) - .with_context(|| anyhow!("Failed to transform source content")) + .context("Failed to transform source content") .with_context(context)?; let bytes_count = transformed_content.as_bytes().len(); - std::fs::write(to, transformed_content.as_bytes()) - .with_context(|| anyhow!("Failed to write transformed content")) + write(&to, transformed_content.as_bytes()) + .context("Failed to write transformed content") .with_context(context)?; Ok(bytes_count as u64) } +pub fn read_to_string>(path: P) -> anyhow::Result { + let path = path.as_ref(); + fs_extra::file::read_to_string(path) + .with_context(|| anyhow!("Failed to read to string, file: {}", path.display())) +} + // Creates all missing parent directories if necessary and writes str to path. pub fn write_str, S: AsRef>(path: P, str: S) -> anyhow::Result<()> { - let path = path.as_ref(); + let path = PathExtra(path); let str = str.as_ref(); let context = || format!("Failed to write string to {}", path.display()); - let target_parent = path - .parent() - .ok_or_else(|| anyhow!("Failed to get parent dir")) - .with_context(context)?; + let target_parent = path.parent().with_context(context)?; + create_dir_all(target_parent).with_context(context)?; + std::fs::write(&path, str.as_bytes()).with_context(context) +} - std::fs::create_dir_all(target_parent) - .with_context(|| anyhow!("Failed to create parent dir")) - .with_context(context)?; +pub fn write, C: AsRef<[u8]>>(path: P, contents: C) -> anyhow::Result<()> { + let path = PathExtra(path); - std::fs::write(path, str.as_bytes()).with_context(context)?; + let context = || format!("Failed to write to {}", path.display()); + let target_parent = path.parent().with_context(context)?; + create_dir_all(target_parent).with_context(context)?; + std::fs::write(&path, contents).with_context(context) +} + +pub fn remove>(path: P) -> anyhow::Result<()> { + let path = path.as_ref(); + if path.exists() { + if path.is_dir() { + std::fs::remove_dir_all(path) + .with_context(|| anyhow!("Failed to delete directory {}", path.display()))?; + } else { + std::fs::remove_file(path) + .with_context(|| anyhow!("Failed to delete file {}", path.display()))?; + } + } Ok(()) } @@ -103,7 +123,7 @@ pub fn has_str_content, S: AsRef>(path: P, str: S) -> anyhow let context = || format!("Failed to compare content to string for {}", path.display()); - let content = std::fs::read_to_string(path) + let content = read_to_string(path) .with_context(|| anyhow!("Failed to read as string: {}", path.display())) .with_context(context)?; @@ -122,43 +142,69 @@ pub fn has_same_string_content, Q: AsRef>(a: P, b: Q) -> an ) }; - let content_a = std::fs::read_to_string(a) - .with_context(|| anyhow!("Failed to read as string: {}", a.display())) - .with_context(context)?; - - let content_b = std::fs::read_to_string(b) - .with_context(|| anyhow!("Failed to read as string: {}", b.display())) - .with_context(context)?; + let content_a = read_to_string(a).with_context(context)?; + let content_b = read_to_string(b).with_context(context)?; Ok(content_a == content_b) } -pub fn get_file_name>(path: P) -> anyhow::Result { +pub fn metadata>(path: P) -> anyhow::Result { let path = path.as_ref(); - path.file_name() - .ok_or_else(|| anyhow!("Failed to get file name for path: {}", path.display(),))? - .to_os_string() - .into_string() - .map_err(|_| anyhow!("Failed to convert filename for path: {}", path.display())) + std::fs::metadata(path) + .with_context(|| anyhow!("Failed to get metadata for {}", path.display())) } -pub fn strip_path_prefix, Q: AsRef>( - prefix: Q, - path: P, -) -> anyhow::Result { - let path = path.as_ref(); - let prefix = prefix.as_ref(); - - Ok(path - .strip_prefix(prefix) - .with_context(|| { - anyhow!( - "Failed to strip prefix from path, prefix: {}, path: {}", - prefix.display(), - path.display() - ) - })? - .to_path_buf()) +pub struct PathExtra>(P); + +impl> PathExtra

{ + pub fn new(path: P) -> Self { + Self(path) + } + + pub fn parent(&self) -> anyhow::Result<&Path> { + let path = self.0.as_ref(); + path.parent() + .ok_or_else(|| anyhow!("Failed to get parent dir for path: {}", path.display())) + } + + pub fn file_name_to_string(&self) -> anyhow::Result { + let path = self.0.as_ref(); + path.file_name() + .ok_or_else(|| anyhow!("Failed to get file name for path: {}", path.display(),))? + .to_os_string() + .into_string() + .map_err(|_| anyhow!("Failed to convert filename for path: {}", path.display())) + } + + pub fn strip_prefix>(&self, prefix: Q) -> anyhow::Result { + let path = self.0.as_ref(); + let prefix = prefix.as_ref(); + + Ok(path + .strip_prefix(prefix) + .with_context(|| { + anyhow!( + "Failed to strip prefix from path, prefix: {}, path: {}", + prefix.display(), + path.display() + ) + })? + .to_path_buf()) + } + + pub fn as_path(&self) -> &Path { + self.0.as_ref() + } + + pub fn display(&self) -> std::path::Display { + self.as_path().display() + } +} + +impl> AsRef for PathExtra

{ + fn as_ref(&self) -> &Path { + self.as_path() + } } pub enum OverwriteSafeAction { diff --git a/wasm-rpc-stubgen/src/make.rs b/wasm-rpc-stubgen/src/make.rs index 107320bd..9eb41f46 100644 --- a/wasm-rpc-stubgen/src/make.rs +++ b/wasm-rpc-stubgen/src/make.rs @@ -13,9 +13,8 @@ // limitations under the License. use crate::commands::log::{log_action, log_warn_action, LogColorize}; -use crate::{cargo, GenerateArgs, WasmRpcOverride}; +use crate::{cargo, fs, GenerateArgs, WasmRpcOverride}; use heck::ToSnakeCase; -use std::fs; use std::path::Path; use std::process::Command; use toml::map::Map; diff --git a/wasm-rpc-stubgen/src/model/oam.rs b/wasm-rpc-stubgen/src/model/oam.rs index b3e706ce..0006a70d 100644 --- a/wasm-rpc-stubgen/src/model/oam.rs +++ b/wasm-rpc-stubgen/src/model/oam.rs @@ -1,5 +1,5 @@ +use crate::fs; use crate::validation::{ValidatedResult, ValidationBuilder}; -use anyhow::Context; use itertools::Itertools; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; @@ -17,10 +17,10 @@ pub struct ApplicationWithSource { impl ApplicationWithSource { pub fn from_yaml_file(file: PathBuf) -> anyhow::Result { - let content = std::fs::read_to_string(file.as_path()) - .with_context(|| format!("Failed to load file: {}", file.display()))?; - - Ok(Self::from_yaml_string(file, content)?) + Ok(Self::from_yaml_string( + file.clone(), + fs::read_to_string(file)?, + )?) } pub fn from_yaml_string(source: PathBuf, string: String) -> serde_yaml::Result { diff --git a/wasm-rpc-stubgen/src/model/wasm_rpc.rs b/wasm-rpc-stubgen/src/model/wasm_rpc.rs index 08e9cf3e..5969660d 100644 --- a/wasm-rpc-stubgen/src/model/wasm_rpc.rs +++ b/wasm-rpc-stubgen/src/model/wasm_rpc.rs @@ -1,16 +1,15 @@ use crate::model::oam; use crate::model::oam::TypedTraitProperties; use crate::model::unknown_properties::{HasUnknownProperties, UnknownProperties}; -use crate::naming; use crate::naming::wit::package_dep_dir_name_from_parser; use crate::validation::{ValidatedResult, ValidationBuilder}; +use crate::{fs, naming}; use golem_wasm_rpc::WASM_RPC_VERSION; use itertools::Itertools; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet}; use std::fmt::Display; use std::fmt::Formatter; -use std::fs; use std::path::{Path, PathBuf}; use wit_parser::PackageName; @@ -742,9 +741,13 @@ impl WasmComponent { } pub fn source_dir(&self) -> &Path { - self.source - .parent() - .expect("Failed to get parent for source") + self.source.parent().unwrap_or_else(|| { + panic!( + "Failed to get parent for component {}, source: {}", + self.name, + self.source.display() + ) + }) } } diff --git a/wasm-rpc-stubgen/src/rust.rs b/wasm-rpc-stubgen/src/rust.rs index 3a1c0924..0259cd48 100644 --- a/wasm-rpc-stubgen/src/rust.rs +++ b/wasm-rpc-stubgen/src/rust.rs @@ -13,12 +13,13 @@ // limitations under the License. use crate::commands::log::{log_action, LogColorize}; +use crate::fs; +use crate::fs::PathExtra; use crate::stub::{FunctionResultStub, FunctionStub, InterfaceStub, StubDefinition}; use anyhow::anyhow; use heck::{ToShoutySnakeCase, ToSnakeCase, ToUpperCamelCase}; use proc_macro2::{Ident, Span, TokenStream}; use quote::quote; -use std::fs; use wit_bindgen_rust::to_rust_ident; use wit_parser::{ Enum, Flags, Handle, Record, Result_, Tuple, Type, TypeDef, TypeDefKind, TypeOwner, Variant, @@ -284,14 +285,13 @@ pub fn generate_stub_source(def: &StubDefinition) -> anyhow::Result<()> { let syntax_tree = syn::parse2(lib)?; let src = prettyplease::unparse(&syntax_tree); + let target_rust_path = PathExtra::new(def.target_rust_path()); + log_action( "Generating", - format!( - "stub source to {}", - def.target_rust_path().log_color_highlight() - ), + format!("stub source to {}", target_rust_path.log_color_highlight()), ); - fs::create_dir_all(def.target_rust_path().parent().unwrap())?; + fs::create_dir_all(target_rust_path.parent()?)?; fs::write(def.target_rust_path(), src)?; Ok(()) } diff --git a/wasm-rpc-stubgen/src/validation.rs b/wasm-rpc-stubgen/src/validation.rs index afa1be66..caa0a38d 100644 --- a/wasm-rpc-stubgen/src/validation.rs +++ b/wasm-rpc-stubgen/src/validation.rs @@ -1,8 +1,6 @@ use itertools::Itertools; use std::fmt::Display; - - pub struct ValidationContext { pub name: &'static str, pub value: String, diff --git a/wasm-rpc-stubgen/src/wit_generate.rs b/wasm-rpc-stubgen/src/wit_generate.rs index ec0198c4..18e979aa 100644 --- a/wasm-rpc-stubgen/src/wit_generate.rs +++ b/wasm-rpc-stubgen/src/wit_generate.rs @@ -13,9 +13,7 @@ // limitations under the License. use crate::commands::log::{log_action, log_action_plan, log_warn_action, LogColorize, LogIndent}; -use crate::fs::{ - copy, get_file_name, strip_path_prefix, write_str, OverwriteSafeAction, OverwriteSafeActions, -}; +use crate::fs::{copy, write_str, OverwriteSafeAction, OverwriteSafeActions, PathExtra}; use crate::naming::wit::package_dep_dir_name_from_encoder; use crate::stub::{ FunctionParamStub, FunctionResultStub, FunctionStub, InterfaceStub, InterfaceStubTypeDef, @@ -23,10 +21,9 @@ use crate::stub::{ }; use crate::wit_encode::EncodedWitDir; use crate::wit_resolve::ResolvedWitDir; -use crate::{cargo, naming}; +use crate::{cargo, fs, naming}; use anyhow::{anyhow, bail, Context}; use std::collections::{BTreeMap, HashSet}; -use std::fs; use std::path::{Path, PathBuf}; use wit_encoder::{ Ident, Interface, Package, PackageItem, PackageName, Params, ResourceFunc, Results, @@ -346,15 +343,15 @@ pub fn add_stub_as_dependency_to_wit_dir(config: AddStubAsDepConfig) -> anyhow:: .dest_wit_root .join(naming::wit::DEPS_DIR) .join(naming::wit::package_dep_dir_name_from_parser(package_name)) - .join(get_file_name(source)?), + .join(PathExtra::new(&source).file_name_to_string()?), }); } } else { package_names_to_package_path.insert( package_name.clone(), - naming::wit::package_wit_dep_dir_from_package_dir_name(&get_file_name( - &package_sources.dir, - )?), + naming::wit::package_wit_dep_dir_from_package_dir_name( + &PathExtra::new(&package_sources.dir).file_name_to_string()?, + ), ); for source in &package_sources.files { @@ -362,7 +359,7 @@ pub fn add_stub_as_dependency_to_wit_dir(config: AddStubAsDepConfig) -> anyhow:: source: source.clone(), target: config .dest_wit_root - .join(strip_path_prefix(&config.stub_wit_root, source)?), + .join(PathExtra::new(&source).strip_prefix(&config.stub_wit_root)?), }); } } From 938aec1c082d4cedb861616e5d9e98178331432c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Thu, 14 Nov 2024 11:53:58 +0100 Subject: [PATCH 058/109] clippy --- wasm-rpc-stubgen/src/commands/log.rs | 2 +- wasm-rpc-stubgen/src/model/wasm_rpc.rs | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/log.rs b/wasm-rpc-stubgen/src/commands/log.rs index ff2e8e89..d30a26ad 100644 --- a/wasm-rpc-stubgen/src/commands/log.rs +++ b/wasm-rpc-stubgen/src/commands/log.rs @@ -205,7 +205,7 @@ pub trait LogColorize { impl<'a> LogColorize for &'a str { fn as_str(&self) -> impl Colorize { - self.as_ref() + *self } } diff --git a/wasm-rpc-stubgen/src/model/wasm_rpc.rs b/wasm-rpc-stubgen/src/model/wasm_rpc.rs index 5969660d..05453a6c 100644 --- a/wasm-rpc-stubgen/src/model/wasm_rpc.rs +++ b/wasm-rpc-stubgen/src/model/wasm_rpc.rs @@ -587,12 +587,7 @@ impl Application { pub fn all_wasm_rpc_dependencies(&self) -> BTreeSet { self.wasm_components_by_name .iter() - .flat_map(|(_, component)| { - component - .wasm_rpc_dependencies - .iter() - .map(|component_name| component_name.clone()) - }) + .flat_map(|(_, component)| component.wasm_rpc_dependencies.iter().cloned()) .collect() } From 0ea09c7dadf949522d6ceb04a0e14b4fb70da29d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Thu, 14 Nov 2024 12:06:19 +0100 Subject: [PATCH 059/109] cleanup --- wasm-rpc-stubgen/src/commands/generate.rs | 1 - wasm-rpc-stubgen/src/lib.rs | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/generate.rs b/wasm-rpc-stubgen/src/commands/generate.rs index a077a87c..c40f8d20 100644 --- a/wasm-rpc-stubgen/src/commands/generate.rs +++ b/wasm-rpc-stubgen/src/commands/generate.rs @@ -58,7 +58,6 @@ pub async fn generate_and_build_stub(stub_def: &StubDefinition) -> anyhow::Resul generate_cargo_toml(stub_def).context("Failed to generate the Cargo.toml file")?; generate_stub_source(stub_def).context("Failed to generate the stub Rust source")?; - // TODO: search for canonicalize on call site, as it is not needed anymore compile( &stub_def .config diff --git a/wasm-rpc-stubgen/src/lib.rs b/wasm-rpc-stubgen/src/lib.rs index e9df1178..af65d3af 100644 --- a/wasm-rpc-stubgen/src/lib.rs +++ b/wasm-rpc-stubgen/src/lib.rs @@ -239,11 +239,10 @@ pub fn generate(args: GenerateArgs) -> anyhow::Result<()> { pub async fn build(args: BuildArgs) -> anyhow::Result<()> { let target_root = TempDir::new()?; - let canonical_target_root = target_root.path().canonicalize()?; let stub_def = StubDefinition::new(StubConfig { source_wit_root: args.source_wit_root, - target_root: canonical_target_root, + target_root: target_root.path().to_path_buf(), selected_world: args.world, stub_crate_version: args.stub_crate_version, wasm_rpc_override: args.wasm_rpc_override, From 96db313302fc1cb1eb89b2296d90db4c9a753804 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Thu, 14 Nov 2024 12:32:16 +0100 Subject: [PATCH 060/109] fs fixes and cleanups --- wasm-rpc-stubgen/src/commands/composition.rs | 9 +++++---- wasm-rpc-stubgen/src/fs.rs | 8 ++++---- wasm-rpc-stubgen/src/wit_generate.rs | 8 ++++---- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/composition.rs b/wasm-rpc-stubgen/src/commands/composition.rs index 641a4c17..8d034c62 100644 --- a/wasm-rpc-stubgen/src/commands/composition.rs +++ b/wasm-rpc-stubgen/src/commands/composition.rs @@ -1,5 +1,6 @@ use crate::commands::log::{log_warn_action, LogColorize}; -use crate::fs::{create_dir_all, read_to_string, write, PathExtra}; +use crate::fs; +use crate::fs::PathExtra; use anyhow::Context; use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; @@ -19,7 +20,7 @@ pub async fn compose( let mut graph = CompositionGraph::new(); - let socket = read_to_string(source_wasm).context("Failed to read socket component")?; + let socket = fs::read_to_string(source_wasm).context("Failed to read socket component")?; let socket = Package::from_bytes("socket", None, socket, graph.types_mut())?; let socket = graph.register_package(socket)?; @@ -40,8 +41,8 @@ pub async fn compose( let bytes = graph.encode(EncodeOptions::default())?; - create_dir_all(dest_wasm.parent()?)?; - write(dest_wasm, bytes)?; + fs::create_dir_all(dest_wasm.parent()?)?; + fs::write(dest_wasm, bytes)?; Ok(()) } diff --git a/wasm-rpc-stubgen/src/fs.rs b/wasm-rpc-stubgen/src/fs.rs index c2222894..73b502f8 100644 --- a/wasm-rpc-stubgen/src/fs.rs +++ b/wasm-rpc-stubgen/src/fs.rs @@ -25,7 +25,9 @@ pub fn copy, Q: AsRef>(from: P, to: Q) -> anyhow::Result, Q: AsRef, T: Fn(String) -> anyhow: ) }; - let target_parent = from.parent().with_context(context)?; - - create_dir_all(target_parent) + create_dir_all(from.parent()?) .context("Failed to create target dir") .with_context(context)?; diff --git a/wasm-rpc-stubgen/src/wit_generate.rs b/wasm-rpc-stubgen/src/wit_generate.rs index 18e979aa..c0224002 100644 --- a/wasm-rpc-stubgen/src/wit_generate.rs +++ b/wasm-rpc-stubgen/src/wit_generate.rs @@ -13,7 +13,7 @@ // limitations under the License. use crate::commands::log::{log_action, log_action_plan, log_warn_action, LogColorize, LogIndent}; -use crate::fs::{copy, write_str, OverwriteSafeAction, OverwriteSafeActions, PathExtra}; +use crate::fs::{OverwriteSafeAction, OverwriteSafeActions, PathExtra}; use crate::naming::wit::package_dep_dir_name_from_encoder; use crate::stub::{ FunctionParamStub, FunctionResultStub, FunctionStub, InterfaceStub, InterfaceStubTypeDef, @@ -254,7 +254,7 @@ pub fn add_dependencies_to_stub_wit_dir(def: &StubDefinition) -> anyhow::Result< dest.log_color_highlight() ), ); - copy(source, &dest)?; + fs::copy(source, &dest)?; } } @@ -705,7 +705,7 @@ pub fn extract_main_interface_as_wit_dep(wit_dir: &Path) -> anyhow::Result<()> { interface_package_path.log_color_highlight() ), ); - write_str(&interface_package_path, interface_package.to_string())?; + fs::write_str(&interface_package_path, interface_package.to_string())?; let main_package_path = &sources.files[0]; log_action( @@ -715,7 +715,7 @@ pub fn extract_main_interface_as_wit_dep(wit_dir: &Path) -> anyhow::Result<()> { main_package_path.log_color_highlight() ), ); - write_str(main_package_path, main_package.to_string())?; + fs::write_str(main_package_path, main_package.to_string())?; Ok(()) } From 951fa54e12978468abaa8c246ffda645a5c76cd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Thu, 14 Nov 2024 12:35:23 +0100 Subject: [PATCH 061/109] move to src from commands --- wasm-rpc-stubgen/src/cargo.rs | 2 +- wasm-rpc-stubgen/src/commands/composition.rs | 2 +- wasm-rpc-stubgen/src/commands/declarative.rs | 6 +++--- wasm-rpc-stubgen/src/commands/generate.rs | 2 +- wasm-rpc-stubgen/src/commands/mod.rs | 3 --- wasm-rpc-stubgen/src/lib.rs | 1 + wasm-rpc-stubgen/src/{commands => }/log.rs | 1 - wasm-rpc-stubgen/src/make.rs | 2 +- wasm-rpc-stubgen/src/rust.rs | 2 +- wasm-rpc-stubgen/src/wit_generate.rs | 2 +- wasm-rpc-stubgen/src/wit_resolve.rs | 2 +- 11 files changed, 11 insertions(+), 14 deletions(-) rename wasm-rpc-stubgen/src/{commands => }/log.rs (99%) diff --git a/wasm-rpc-stubgen/src/cargo.rs b/wasm-rpc-stubgen/src/cargo.rs index 656b9411..110d5d18 100644 --- a/wasm-rpc-stubgen/src/cargo.rs +++ b/wasm-rpc-stubgen/src/cargo.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::commands::log::{log_action, log_warn_action, LogColorize}; use crate::fs::PathExtra; +use crate::log::{log_action, log_warn_action, LogColorize}; use crate::stub::StubDefinition; use crate::wit_resolve::ResolvedWitDir; use crate::{fs, naming}; diff --git a/wasm-rpc-stubgen/src/commands/composition.rs b/wasm-rpc-stubgen/src/commands/composition.rs index 8d034c62..256b4763 100644 --- a/wasm-rpc-stubgen/src/commands/composition.rs +++ b/wasm-rpc-stubgen/src/commands/composition.rs @@ -1,6 +1,6 @@ -use crate::commands::log::{log_warn_action, LogColorize}; use crate::fs; use crate::fs::PathExtra; +use crate::log::{log_warn_action, LogColorize}; use anyhow::Context; use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index 51b8e03d..4f9426d7 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -1,10 +1,10 @@ use crate::cargo::regenerate_cargo_package_component; -use crate::commands::log::{ +use crate::fs; +use crate::fs::PathExtra; +use crate::log::{ log_action, log_skipping_up_to_date, log_validated_action_result, log_warn_action, LogColorize, LogIndent, }; -use crate::fs; -use crate::fs::PathExtra; use crate::model::oam; use crate::model::wasm_rpc::{ include_glob_patter_from_yaml_file, init_oam_app, Application, ComponentName, diff --git a/wasm-rpc-stubgen/src/commands/generate.rs b/wasm-rpc-stubgen/src/commands/generate.rs index c40f8d20..e4a5bf3a 100644 --- a/wasm-rpc-stubgen/src/commands/generate.rs +++ b/wasm-rpc-stubgen/src/commands/generate.rs @@ -13,9 +13,9 @@ // limitations under the License. use crate::cargo::generate_cargo_toml; -use crate::commands::log::{log_action, LogColorize, LogIndent}; use crate::compilation::compile; use crate::fs; +use crate::log::{log_action, LogColorize, LogIndent}; use crate::naming; use crate::rust::generate_stub_source; use crate::stub::StubDefinition; diff --git a/wasm-rpc-stubgen/src/commands/mod.rs b/wasm-rpc-stubgen/src/commands/mod.rs index aa03f6aa..b91cb5c3 100644 --- a/wasm-rpc-stubgen/src/commands/mod.rs +++ b/wasm-rpc-stubgen/src/commands/mod.rs @@ -23,6 +23,3 @@ pub mod composition; /// Declarative subcommands pub mod declarative; - -/// Logging -pub mod log; diff --git a/wasm-rpc-stubgen/src/lib.rs b/wasm-rpc-stubgen/src/lib.rs index af65d3af..88cc2994 100644 --- a/wasm-rpc-stubgen/src/lib.rs +++ b/wasm-rpc-stubgen/src/lib.rs @@ -16,6 +16,7 @@ pub mod cargo; pub mod commands; pub mod compilation; pub mod fs; +pub mod log; pub mod make; pub mod model; pub mod naming; diff --git a/wasm-rpc-stubgen/src/commands/log.rs b/wasm-rpc-stubgen/src/log.rs similarity index 99% rename from wasm-rpc-stubgen/src/commands/log.rs rename to wasm-rpc-stubgen/src/log.rs index d30a26ad..8e91d7ae 100644 --- a/wasm-rpc-stubgen/src/commands/log.rs +++ b/wasm-rpc-stubgen/src/log.rs @@ -4,7 +4,6 @@ use crate::validation::ValidatedResult; use colored::{ColoredString, Colorize}; use std::path::{Path, PathBuf}; use std::sync::{LazyLock, RwLock}; -// TODO: move package under lib static LOG_STATE: LazyLock> = LazyLock::new(RwLock::default); diff --git a/wasm-rpc-stubgen/src/make.rs b/wasm-rpc-stubgen/src/make.rs index 9eb41f46..18b68304 100644 --- a/wasm-rpc-stubgen/src/make.rs +++ b/wasm-rpc-stubgen/src/make.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::commands::log::{log_action, log_warn_action, LogColorize}; +use crate::log::{log_action, log_warn_action, LogColorize}; use crate::{cargo, fs, GenerateArgs, WasmRpcOverride}; use heck::ToSnakeCase; use std::path::Path; diff --git a/wasm-rpc-stubgen/src/rust.rs b/wasm-rpc-stubgen/src/rust.rs index 0259cd48..de2fb2c9 100644 --- a/wasm-rpc-stubgen/src/rust.rs +++ b/wasm-rpc-stubgen/src/rust.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::commands::log::{log_action, LogColorize}; use crate::fs; use crate::fs::PathExtra; +use crate::log::{log_action, LogColorize}; use crate::stub::{FunctionResultStub, FunctionStub, InterfaceStub, StubDefinition}; use anyhow::anyhow; use heck::{ToShoutySnakeCase, ToSnakeCase, ToUpperCamelCase}; diff --git a/wasm-rpc-stubgen/src/wit_generate.rs b/wasm-rpc-stubgen/src/wit_generate.rs index c0224002..64f4d146 100644 --- a/wasm-rpc-stubgen/src/wit_generate.rs +++ b/wasm-rpc-stubgen/src/wit_generate.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::commands::log::{log_action, log_action_plan, log_warn_action, LogColorize, LogIndent}; use crate::fs::{OverwriteSafeAction, OverwriteSafeActions, PathExtra}; +use crate::log::{log_action, log_action_plan, log_warn_action, LogColorize, LogIndent}; use crate::naming::wit::package_dep_dir_name_from_encoder; use crate::stub::{ FunctionParamStub, FunctionResultStub, FunctionStub, InterfaceStub, InterfaceStubTypeDef, diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index 1ca935b4..60f59fd0 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -1,4 +1,4 @@ -use crate::commands::log::{log_action, LogColorize, LogIndent}; +use crate::log::{log_action, LogColorize, LogIndent}; use crate::model::wasm_rpc::{Application, ComponentName}; use crate::naming; use crate::validation::{ValidatedResult, ValidationBuilder}; From 438d81e965c97413ee005d0e6493fe89b8040ddb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Thu, 14 Nov 2024 13:54:34 +0100 Subject: [PATCH 062/109] transitive resolve for package deps --- wasm-rpc-stubgen/src/commands/declarative.rs | 33 +++++---- wasm-rpc-stubgen/src/model/wasm_rpc.rs | 7 ++ wasm-rpc-stubgen/src/wit_resolve.rs | 74 ++++++++++++++++++++ 3 files changed, 102 insertions(+), 12 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index 4f9426d7..a3ad5de8 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -16,12 +16,13 @@ use crate::wit_generate::{ add_stub_as_dependency_to_wit_dir, extract_main_interface_as_wit_dep, AddStubAsDepConfig, UpdateCargoToml, }; -use crate::wit_resolve::{parse_wit_deps_dir, ResolvedWitApplication}; +use crate::wit_resolve::{ResolvedWitApplication, WitDepsResolver}; use crate::{commands, naming, WasmRpcOverride}; use anyhow::{anyhow, Context, Error}; use colored::Colorize; use glob::glob; use itertools::Itertools; +use std::cell::OnceCell; use std::cmp::Ordering; use std::path::{Path, PathBuf}; use std::process::Command; @@ -43,6 +44,7 @@ struct ApplicationContext { config: Config, application: Application, wit: ResolvedWitApplication, + wit_deps: OnceCell>, } impl ApplicationContext { @@ -54,6 +56,7 @@ impl ApplicationContext { config, application, wit, + wit_deps: OnceCell::new(), }) }), ) @@ -67,6 +70,17 @@ impl ApplicationContext { }), ) } + + fn wit_deps(&self) -> anyhow::Result<&WitDepsResolver> { + match self + .wit_deps + .get_or_init(|| WitDepsResolver::new(self.application.wit_deps())) + .as_ref() + { + Ok(wit_deps) => Ok(wit_deps), + Err(err) => Err(anyhow!("Failed to init wit dependency resolver? {}", err)), + } + } } pub fn init(component_name: ComponentName) -> anyhow::Result<()> { @@ -655,16 +669,14 @@ fn create_base_output_wit( ctx.wit.missing_generic_input_package_deps(component_name)?; if !missing_package_deps.is_empty() { - log_action("Adding", "common package deps"); + log_action("Adding", "package deps"); let _indent = LogIndent::new(); - // TODO: transitive deps? - // TODO: use wit dep from app manifest - // TODO: extract dep management, with preferring higher versions + let wit_deps = ctx.wit_deps()?; - let wit_deps = parse_wit_deps_dir(Path::new("wit-deps"))?; + let all_package_deps = wit_deps.package_names_with_deps(&missing_package_deps)?; - for package_name in missing_package_deps { + for package_name in all_package_deps { log_action( "Adding", format!( @@ -672,11 +684,8 @@ fn create_base_output_wit( package_name.to_string().log_color_highlight() ), ); - let dep = wit_deps - .iter() - .find(|package| package.main.name == package_name) - .ok_or_else(|| anyhow!("Package dependency {} not found", package_name))?; - for source in dep.source_map.source_files() { + + for source in wit_deps.package_sources(&package_name)? { let source = PathExtra::new(source); let parent = PathExtra::new( source diff --git a/wasm-rpc-stubgen/src/model/wasm_rpc.rs b/wasm-rpc-stubgen/src/model/wasm_rpc.rs index 05453a6c..2e7a5b52 100644 --- a/wasm-rpc-stubgen/src/model/wasm_rpc.rs +++ b/wasm-rpc-stubgen/src/model/wasm_rpc.rs @@ -584,6 +584,13 @@ impl Application { ) } + pub fn wit_deps(&self) -> Vec { + self.common_wasm_build + .as_ref() + .map(|wasm_build| wasm_build._wit_deps.clone()) + .unwrap_or_default() + } + pub fn all_wasm_rpc_dependencies(&self) -> BTreeSet { self.wasm_components_by_name .iter() diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index 60f59fd0..1f78cd7d 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -516,3 +516,77 @@ pub fn parse_wit_deps_dir(path: &Path) -> Result, Er }) .collect::, _>>() } + +pub struct WitDepsResolver { + sources: Vec, + packages: HashMap>, +} + +impl WitDepsResolver { + pub fn new(sources: Vec) -> anyhow::Result { + let mut packages = HashMap::>::new(); + + for source in &sources { + packages.insert( + source.clone(), + parse_wit_deps_dir(source)? + .into_iter() + .map(|package| (package.main.name.clone(), package)) + .collect(), + ); + } + + Ok(Self { sources, packages }) + } + + pub fn package(&self, package_name: &PackageName) -> anyhow::Result<&UnresolvedPackageGroup> { + for source in &self.sources { + if let Some(package) = self.packages.get(source).unwrap().get(package_name) { + return Ok(package); + } + } + bail!( + "Package {} not found, sources searched: {}", + package_name, + self.sources + .iter() + .map(|s| s.display().to_string()) + .join(", ") + ) + } + + pub fn package_sources( + &self, + package_name: &PackageName, + ) -> anyhow::Result> { + self.package(package_name) + .map(|package| package.source_map.source_files()) + } + + pub fn package_names_with_deps( + &self, + package_names: &[PackageName], + ) -> anyhow::Result> { + fn visit( + resolver: &WitDepsResolver, + all_package_names: &mut HashSet, + package_name: &PackageName, + ) -> anyhow::Result<()> { + if !all_package_names.contains(package_name) { + let package = resolver.package(package_name)?; + all_package_names.insert(package_name.clone()); + for package_name in package.main.foreign_deps.keys() { + visit(resolver, all_package_names, package_name)?; + } + } + + Ok(()) + } + + let mut all_package_names = HashSet::::new(); + for package_name in package_names { + visit(self, &mut all_package_names, package_name)?; + } + Ok(all_package_names) + } +} From bc57c3412808756009d732cdfbff8eb97b83b068 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Thu, 14 Nov 2024 14:00:24 +0100 Subject: [PATCH 063/109] fix test data path --- wasm-rpc-stubgen/tests-integration/tests/compose.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wasm-rpc-stubgen/tests-integration/tests/compose.rs b/wasm-rpc-stubgen/tests-integration/tests/compose.rs index a9e40e3a..808e7b8f 100644 --- a/wasm-rpc-stubgen/tests-integration/tests/compose.rs +++ b/wasm-rpc-stubgen/tests-integration/tests/compose.rs @@ -66,7 +66,7 @@ async fn init_stub(name: &str) -> (TempDir, TempDir, PathBuf) { let source_wit_root = source_dir.path().canonicalize().unwrap(); fs_extra::dir::copy( - test_data_path().join(name), + test_data_path().join("wit").join(name), &source_wit_root, &CopyOptions::new().content_only(true), ) @@ -91,7 +91,7 @@ async fn init_stub(name: &str) -> (TempDir, TempDir, PathBuf) { fn init_caller(name: &str) -> TempDir { let temp_dir = TempDir::new().unwrap(); - let source = test_data_path().join(name); + let source = test_data_path().join("wit").join(name); fs_extra::dir::copy( source, From 7c5d0d4e1c1fbc2f05aa84dfc4b57d6cabd76c5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Thu, 14 Nov 2024 15:48:33 +0100 Subject: [PATCH 064/109] transitive resolve for package deps for building base output wit dir --- wasm-rpc-stubgen/src/commands/declarative.rs | 118 +++++++------------ wasm-rpc-stubgen/src/wit_resolve.rs | 104 +++++++++++----- 2 files changed, 119 insertions(+), 103 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index a3ad5de8..291c63f2 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -24,6 +24,7 @@ use glob::glob; use itertools::Itertools; use std::cell::OnceCell; use std::cmp::Ordering; +use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::process::Command; use std::time::SystemTime; @@ -44,7 +45,8 @@ struct ApplicationContext { config: Config, application: Application, wit: ResolvedWitApplication, - wit_deps: OnceCell>, + common_wit_deps: OnceCell>, + component_base_output_wit_deps: HashMap, } impl ApplicationContext { @@ -56,7 +58,8 @@ impl ApplicationContext { config, application, wit, - wit_deps: OnceCell::new(), + common_wit_deps: OnceCell::new(), + component_base_output_wit_deps: HashMap::new(), }) }), ) @@ -71,9 +74,9 @@ impl ApplicationContext { ) } - fn wit_deps(&self) -> anyhow::Result<&WitDepsResolver> { + fn common_wit_deps(&self) -> anyhow::Result<&WitDepsResolver> { match self - .wit_deps + .common_wit_deps .get_or_init(|| WitDepsResolver::new(self.application.wit_deps())) .as_ref() { @@ -81,6 +84,29 @@ impl ApplicationContext { Err(err) => Err(anyhow!("Failed to init wit dependency resolver? {}", err)), } } + + fn component_base_output_wit_deps( + &mut self, + component_name: &ComponentName, + ) -> anyhow::Result<&WitDepsResolver> { + // Not using the entry API, so we can skip copying the component name + if !self + .component_base_output_wit_deps + .contains_key(component_name) + { + self.component_base_output_wit_deps.insert( + component_name.clone(), + WitDepsResolver::new(vec![self + .application + .component_base_output_wit(component_name) + .join(naming::wit::DEPS_DIR)])?, + ); + } + Ok(self + .component_base_output_wit_deps + .get(component_name) + .unwrap()) + } } pub fn init(component_name: ComponentName) -> anyhow::Result<()> { @@ -101,15 +127,11 @@ async fn pre_component_build_ctx(ctx: &mut ApplicationContext) -> anyhow::Result let _indent = LogIndent::new(); { - let mut any_changed = false; - for component_name in &ctx.wit.component_order { - any_changed |= create_base_output_wit(ctx, component_name)?; + for component_name in ctx.wit.component_order_cloned() { + create_base_output_wit(ctx, &component_name)?; } for component_name in &ctx.application.all_wasm_rpc_dependencies() { - any_changed |= build_stub(ctx, component_name).await?; - } - if any_changed { - ctx.update_wit_context()?; + build_stub(ctx, component_name).await?; } } @@ -635,7 +657,7 @@ fn compile_and_collect_globs(root_dir: &Path, globs: &[String]) -> Result Result { let component_input_wit = ctx.application.component_input_wit(component_name); @@ -672,45 +694,11 @@ fn create_base_output_wit( log_action("Adding", "package deps"); let _indent = LogIndent::new(); - let wit_deps = ctx.wit_deps()?; - - let all_package_deps = wit_deps.package_names_with_deps(&missing_package_deps)?; - - for package_name in all_package_deps { - log_action( - "Adding", - format!( - "package dependency {}", - package_name.to_string().log_color_highlight() - ), - ); - - for source in wit_deps.package_sources(&package_name)? { - let source = PathExtra::new(source); - let parent = PathExtra::new( - source - .parent() - .context("Failed to get dependency source parent")?, - ); - - let target = ctx - .application - .component_base_output_wit(component_name) - .join(naming::wit::DEPS_DIR) - .join(parent.file_name_to_string()?) - .join(source.file_name_to_string()?); - - log_action( - "Copying", - format!( - "package dependency from {} to {}", - source.log_color_highlight(), - target.log_color_highlight() - ), - ); - fs::copy(source, target)?; - } - } + ctx.common_wit_deps()? + .add_packages_with_transitive_deps_to_wit_dir( + &missing_package_deps, + &component_base_output_wit, + )?; } } @@ -721,32 +709,14 @@ fn create_base_output_wit( log_action("Adding", "component interface package dependencies"); let _indent = LogIndent::new(); - // TODO: transitive deps? - for (dep_interface_package_name, dep_component_name) in &component_interface_package_deps { - let source = ctx - .application - .component_base_output_wit_interface_package_dir( - dep_component_name, - dep_interface_package_name, - ); - let target = ctx - .application - .component_base_output_wit_interface_package_dir( - component_name, - dep_interface_package_name, - ); - log_action( - "Copying", - format!( - "interface package dependency from {} to {}", - source.log_color_highlight(), - target.log_color_highlight() - ), - ); - fs::copy(source, target)?; + ctx.component_base_output_wit_deps(dep_component_name)? + .add_packages_with_transitive_deps_to_wit_dir( + &[dep_interface_package_name.clone()], + &component_base_output_wit, + )?; } } } diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index 1f78cd7d..a267b97f 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -1,7 +1,8 @@ +use crate::fs::PathExtra; use crate::log::{log_action, LogColorize, LogIndent}; use crate::model::wasm_rpc::{Application, ComponentName}; -use crate::naming; use crate::validation::{ValidatedResult, ValidationBuilder}; +use crate::{fs, naming}; use anyhow::{anyhow, bail, Context, Error}; use indexmap::IndexMap; use itertools::Itertools; @@ -156,26 +157,28 @@ fn collect_package_sources( } pub struct ResolvedWitComponent { - pub main_package_name: PackageName, - pub unresolved_input_package_group: UnresolvedPackageGroup, - pub resolved_output_wit_dir: Option, - pub app_component_deps: HashSet, - pub input_referenced_package_deps: HashSet, - pub input_contained_package_deps: HashSet, - pub input_component_deps: BTreeSet, // NOTE: BTree for making dep sorting deterministic - pub output_component_deps: Option>, + main_package_name: PackageName, + resolved_output_wit_dir: Option, + app_component_deps: HashSet, + input_referenced_package_deps: HashSet, + input_contained_package_deps: HashSet, + input_component_deps: BTreeSet, // NOTE: BTree for making dep sorting deterministic + output_component_deps: Option>, } pub struct ResolvedWitApplication { - pub components: BTreeMap, // NOTE: BTree for making dep sorting deterministic - pub package_to_component: HashMap, - pub stub_package_to_component: HashMap, - pub interface_package_to_component: HashMap, - pub component_order: Vec, + components: BTreeMap, // NOTE: BTree for making dep sorting deterministic + package_to_component: HashMap, + stub_package_to_component: HashMap, + interface_package_to_component: HashMap, + component_order: Vec, } impl ResolvedWitApplication { pub fn new(app: &Application) -> ValidatedResult { + log_action("Resolving", "application wit directories"); + let _indent = LogIndent::new(); + let mut resolved_app = Self { components: Default::default(), package_to_component: Default::default(), @@ -219,32 +222,29 @@ impl ResolvedWitApplication { } fn add_components_from_app(&mut self, validation: &mut ValidationBuilder, app: &Application) { - log_action("Resolving", "application wit directories"); - let _indent = LogIndent::new(); - for (component_name, component) in &app.wasm_components_by_name { validation.push_context("component name", component_name.to_string()); - let input_wit = app.component_input_wit(component_name); - let output_wit = app.component_output_wit(component_name); + let input_wit_dir = app.component_input_wit(component_name); + let output_wit_dir = app.component_output_wit(component_name); log_action( "Resolving", format!( "component wit dirs for {} ({}, {})", component_name.log_color_highlight(), - input_wit.log_color_highlight(), - output_wit.log_color_highlight(), + input_wit_dir.log_color_highlight(), + output_wit_dir.log_color_highlight(), ), ); let resolved_component = (|| -> anyhow::Result { - let unresolved_input_package_group = UnresolvedPackageGroup::parse_dir(&input_wit) - .with_context(|| { + let unresolved_input_package_group = + UnresolvedPackageGroup::parse_dir(&input_wit_dir).with_context(|| { anyhow!( "Failed to parse component {} main package in input wit dir {}", component_name, - input_wit.display() + input_wit_dir.display() ) })?; @@ -269,7 +269,7 @@ impl ResolvedWitApplication { let main_package_name = unresolved_input_package_group.main.name.clone(); - let resolved_output_wit_dir = ResolvedWitDir::new(&output_wit).ok(); + let resolved_output_wit_dir = ResolvedWitDir::new(&output_wit_dir).ok(); let output_has_same_main_package_name = resolved_output_wit_dir .as_ref() .map(|wit| wit.main_package()) @@ -284,7 +284,6 @@ impl ResolvedWitApplication { Ok(ResolvedWitComponent { main_package_name, - unresolved_input_package_group, resolved_output_wit_dir, app_component_deps, input_referenced_package_deps, @@ -420,6 +419,14 @@ impl ResolvedWitApplication { self.component_order = component_order; } + pub fn component_order(&self) -> &[ComponentName] { + &self.component_order + } + + pub fn component_order_cloned(&self) -> Vec { + self.component_order.clone() + } + fn component(&self, component_name: &ComponentName) -> Result<&ResolvedWitComponent, Error> { self.components .get(component_name) @@ -563,9 +570,9 @@ impl WitDepsResolver { .map(|package| package.source_map.source_files()) } - pub fn package_names_with_deps( + pub fn package_names_with_transitive_deps( &self, - package_names: &[PackageName], + packages: &[PackageName], ) -> anyhow::Result> { fn visit( resolver: &WitDepsResolver, @@ -584,9 +591,48 @@ impl WitDepsResolver { } let mut all_package_names = HashSet::::new(); - for package_name in package_names { + for package_name in packages { visit(self, &mut all_package_names, package_name)?; } Ok(all_package_names) } + + pub fn add_packages_with_transitive_deps_to_wit_dir( + &self, + packages: &[PackageName], + target_deps_dir: &Path, + ) -> anyhow::Result<()> { + for package_name in self.package_names_with_transitive_deps(packages)? { + log_action( + "Adding", + format!( + "package dependency {} to {}", + package_name.to_string().log_color_highlight(), + target_deps_dir.log_color_highlight(), + ), + ); + + for source in self.package_sources(&package_name)? { + let source = PathExtra::new(source); + let source_parent = PathExtra::new(source.parent()?); + + let target = target_deps_dir + .join(naming::wit::DEPS_DIR) + .join(source_parent.file_name_to_string()?) + .join(source.file_name_to_string()?); + + log_action( + "Copying", + format!( + "package dependency source from {} to {}", + source.log_color_highlight(), + target.log_color_highlight() + ), + ); + fs::copy(source, target)?; + } + } + + Ok(()) + } } From 5d961b585003b886bf031bede9d2efbe23fec435 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Thu, 14 Nov 2024 17:05:28 +0100 Subject: [PATCH 065/109] fix compose wasm read --- wasm-rpc-stubgen/src/commands/composition.rs | 2 +- wasm-rpc-stubgen/src/fs.rs | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/wasm-rpc-stubgen/src/commands/composition.rs b/wasm-rpc-stubgen/src/commands/composition.rs index 256b4763..4746b64a 100644 --- a/wasm-rpc-stubgen/src/commands/composition.rs +++ b/wasm-rpc-stubgen/src/commands/composition.rs @@ -20,7 +20,7 @@ pub async fn compose( let mut graph = CompositionGraph::new(); - let socket = fs::read_to_string(source_wasm).context("Failed to read socket component")?; + let socket = fs::read(source_wasm).context("Failed to read socket component")?; let socket = Package::from_bytes("socket", None, socket, graph.types_mut())?; let socket = graph.register_package(socket)?; diff --git a/wasm-rpc-stubgen/src/fs.rs b/wasm-rpc-stubgen/src/fs.rs index 73b502f8..42048f4b 100644 --- a/wasm-rpc-stubgen/src/fs.rs +++ b/wasm-rpc-stubgen/src/fs.rs @@ -81,6 +81,11 @@ pub fn read_to_string>(path: P) -> anyhow::Result { .with_context(|| anyhow!("Failed to read to string, file: {}", path.display())) } +pub fn read>(path: P) -> anyhow::Result> { + let path = path.as_ref(); + std::fs::read(path).with_context(|| anyhow!("Failed to read file: {}", path.display())) +} + // Creates all missing parent directories if necessary and writes str to path. pub fn write_str, S: AsRef>(path: P, str: S) -> anyhow::Result<()> { let path = PathExtra(path); From 16080231d3155439c4bb8664e1292370b9519239 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Thu, 14 Nov 2024 17:19:44 +0100 Subject: [PATCH 066/109] error and logging fixes --- wasm-rpc-stubgen/src/commands/declarative.rs | 12 +++++++++--- wasm-rpc-stubgen/src/wit_resolve.rs | 13 +++++++++---- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index 291c63f2..f50152ae 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -18,7 +18,7 @@ use crate::wit_generate::{ }; use crate::wit_resolve::{ResolvedWitApplication, WitDepsResolver}; use crate::{commands, naming, WasmRpcOverride}; -use anyhow::{anyhow, Context, Error}; +use anyhow::{anyhow, bail, Context, Error}; use colored::Colorize; use glob::glob; use itertools::Itertools; @@ -77,11 +77,17 @@ impl ApplicationContext { fn common_wit_deps(&self) -> anyhow::Result<&WitDepsResolver> { match self .common_wit_deps - .get_or_init(|| WitDepsResolver::new(self.application.wit_deps())) + .get_or_init(|| { + let sources = self.application.wit_deps(); + if sources.is_empty() { + bail!("No common witDeps were defined in the application manifest") + } + WitDepsResolver::new(sources) + }) .as_ref() { Ok(wit_deps) => Ok(wit_deps), - Err(err) => Err(anyhow!("Failed to init wit dependency resolver? {}", err)), + Err(err) => Err(anyhow!("Failed to init wit dependency resolver: {}", err)), } } diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index a267b97f..0cda2fb5 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -555,10 +555,14 @@ impl WitDepsResolver { bail!( "Package {} not found, sources searched: {}", package_name, - self.sources - .iter() - .map(|s| s.display().to_string()) - .join(", ") + if self.sources.is_empty() { + "no sources were provided".to_string() + } else { + self.sources + .iter() + .map(|s| s.display().to_string()) + .join(", ") + } ) } @@ -611,6 +615,7 @@ impl WitDepsResolver { target_deps_dir.log_color_highlight(), ), ); + let _indent = LogIndent::new(); for source in self.package_sources(&package_name)? { let source = PathExtra::new(source); From e2b8e76dc643d5b561e1801c0e4fd4cce29c2b78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Thu, 14 Nov 2024 17:24:30 +0100 Subject: [PATCH 067/109] fix stub-wasm test --- .../tests-integration/tests/stub_wasm.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/wasm-rpc-stubgen/tests-integration/tests/stub_wasm.rs b/wasm-rpc-stubgen/tests-integration/tests/stub_wasm.rs index a5c217e6..739f1a4b 100644 --- a/wasm-rpc-stubgen/tests-integration/tests/stub_wasm.rs +++ b/wasm-rpc-stubgen/tests-integration/tests/stub_wasm.rs @@ -15,6 +15,7 @@ //! Tests in this module are verifying the STUB WASM created by the stub generator //! regardless of how the actual wasm generator is implemented. (Currently generates Rust code and compiles it) +use fs_extra::dir::CopyOptions; use test_r::test; use golem_wasm_ast::analysis::analysed_type::*; @@ -34,12 +35,21 @@ test_r::enable!(); #[test] async fn all_wit_types() { - let source_wit_root = test_data_path().join("all-wit-types"); + let source = test_data_path().join("wit/all-wit-types"); + let source_wit_root = tempdir().unwrap(); + + fs_extra::dir::copy( + source, + source_wit_root.path(), + &CopyOptions::new().content_only(true), + ) + .unwrap(); + let target_root = tempdir().unwrap(); let canonical_target_root = target_root.path().canonicalize().unwrap(); let def = StubDefinition::new(StubConfig { - source_wit_root, + source_wit_root: source_wit_root.path().to_path_buf(), target_root: canonical_target_root, selected_world: None, stub_crate_version: "1.0.0".to_string(), From a2789df7a839b33b0e6e06c4c12e0a478e13c0c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Thu, 14 Nov 2024 17:43:36 +0100 Subject: [PATCH 068/109] log message --- wasm-rpc-stubgen/src/commands/declarative.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index f50152ae..5773bd54 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -847,7 +847,7 @@ async fn build_stub( } else { log_action( "Building", - format!("wasm rpc stub: {}", component_name.log_color_highlight()), + format!("wasm rpc stub for {}", component_name.log_color_highlight()), ); let _indent = LogIndent::new(); From aa2ae4c93f53b1d033cb17ef84fb2f104ea91370 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Thu, 14 Nov 2024 18:54:47 +0100 Subject: [PATCH 069/109] validate conflicting package names --- wasm-rpc-stubgen/src/wit_resolve.rs | 33 ++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index 0cda2fb5..0d217a12 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -191,7 +191,7 @@ impl ResolvedWitApplication { resolved_app.add_components_from_app(&mut validation, app); - // TODO: validate conflicting package names + resolved_app.validate_package_names(&mut validation); if !validation.has_any_errors() { resolved_app.collect_component_deps(); @@ -201,6 +201,37 @@ impl ResolvedWitApplication { validation.build(resolved_app) } + fn validate_package_names(&self, validation: &mut ValidationBuilder) { + if self.package_to_component.len() != self.components.len() { + let mut package_names_to_component_names = + BTreeMap::<&PackageName, Vec<&ComponentName>>::new(); + for (component_name, component) in &self.components { + package_names_to_component_names + .entry(&component.main_package_name) + .and_modify(|component_names| component_names.push(&component_name)) + .or_insert_with(|| vec![&component_name]); + } + + validation.add_errors( + package_names_to_component_names, + |(package_name, component_names)| { + (component_names.len() > 1).then(|| { + ( + vec![ + ("package name", package_name.to_string()), + ( + "component names", + component_names.iter().map(|s| s.as_str()).join(", "), + ), + ], + "Same package name is used for multiple components".to_string(), + ) + }) + }, + ); + } + } + fn add_resolved_component( &mut self, component_name: ComponentName, From d34b40cf3056173429b8523a2b2daff2f5019ba0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Fri, 15 Nov 2024 12:48:36 +0100 Subject: [PATCH 070/109] cleanups --- wasm-rpc-stubgen/src/cargo.rs | 13 +- wasm-rpc-stubgen/src/fs.rs | 10 + wasm-rpc-stubgen/src/rust.rs | 6 +- wasm-rpc-stubgen/src/stub.rs | 27 +-- wasm-rpc-stubgen/src/wit_generate.rs | 270 ++++++++++++--------------- wasm-rpc-stubgen/src/wit_resolve.rs | 2 +- 6 files changed, 150 insertions(+), 178 deletions(-) diff --git a/wasm-rpc-stubgen/src/cargo.rs b/wasm-rpc-stubgen/src/cargo.rs index 110d5d18..079509fa 100644 --- a/wasm-rpc-stubgen/src/cargo.rs +++ b/wasm-rpc-stubgen/src/cargo.rs @@ -366,12 +366,13 @@ pub fn regenerate_cargo_package_component( &wit_dir.package(*package_id).unwrap().name, ), WitDependency { - path: package_sources - .dir - .strip_prefix(project_root) - .unwrap() // TODO: unwrap - .to_string_lossy() - .to_string(), + path: PathExtra::new( + PathExtra::new(&package_sources.dir) + .strip_prefix(project_root) + .unwrap(), + ) + .to_string() + .unwrap(), }, ) }) diff --git a/wasm-rpc-stubgen/src/fs.rs b/wasm-rpc-stubgen/src/fs.rs index 42048f4b..e3b0c4d0 100644 --- a/wasm-rpc-stubgen/src/fs.rs +++ b/wasm-rpc-stubgen/src/fs.rs @@ -181,6 +181,16 @@ impl> PathExtra

{ .map_err(|_| anyhow!("Failed to convert filename for path: {}", path.display())) } + pub fn to_str(&self) -> anyhow::Result<&str> { + let path = self.0.as_ref(); + path.to_str() + .ok_or_else(|| anyhow!("Failed to convert path to string: {}", path.display())) + } + + pub fn to_string(&self) -> anyhow::Result { + Ok(self.to_str()?.to_string()) + } + pub fn strip_prefix>(&self, prefix: Q) -> anyhow::Result { let path = self.0.as_ref(); let prefix = prefix.as_ref(); diff --git a/wasm-rpc-stubgen/src/rust.rs b/wasm-rpc-stubgen/src/rust.rs index de2fb2c9..38885fc7 100644 --- a/wasm-rpc-stubgen/src/rust.rs +++ b/wasm-rpc-stubgen/src/rust.rs @@ -1287,10 +1287,10 @@ fn extract_from_wit_value( extract_from_result_value(result, def, base_expr) } TypeDefKind::List(elem) => extract_from_list_value(elem, def, base_expr), - TypeDefKind::Future(_) => Ok(quote!(todo!("future"))), - TypeDefKind::Stream(_) => Ok(quote!(todo!("stream"))), + TypeDefKind::Future(_) => Ok(quote!(panic!("Future is not supported yet"))), + TypeDefKind::Stream(_) => Ok(quote!(panic!("Stream is not supported yet"))), TypeDefKind::Type(typ) => extract_from_wit_value(typ, def, base_expr), - TypeDefKind::Unknown => Ok(quote!(todo!("unknown"))), + TypeDefKind::Unknown => Ok(quote!(panic!("Unexpected unknown type!"))), } } } diff --git a/wasm-rpc-stubgen/src/stub.rs b/wasm-rpc-stubgen/src/stub.rs index 8335f5a6..22f4c2f0 100644 --- a/wasm-rpc-stubgen/src/stub.rs +++ b/wasm-rpc-stubgen/src/stub.rs @@ -664,33 +664,20 @@ impl InterfaceStubTypeDef { pub struct FunctionStub { pub name: String, pub params: Vec, - pub results: FunctionResultStub, // TODO: option? + pub results: FunctionResultStub, } impl FunctionStub { pub fn as_method(&self) -> Option { - // TODO: duplicated code - self.name.strip_prefix("[method]").and_then(|method_name| { - let parts = method_name.split('.').collect::>(); - if parts.len() != 2 { - None - } else { - Some(FunctionStub { - name: parts[1].to_string(), - params: self - .params - .iter() - .filter(|p| p.name != "self") - .cloned() - .collect(), - results: self.results.clone(), - }) - } - }) + self.as_function_stub_without_prefix("[method]") } pub fn as_static(&self) -> Option { - self.name.strip_prefix("[static]").and_then(|method_name| { + self.as_function_stub_without_prefix("[static]") + } + + fn as_function_stub_without_prefix(&self, prefix: &str) -> Option { + self.name.strip_prefix(prefix).and_then(|method_name| { let parts = method_name.split('.').collect::>(); if parts.len() != 2 { None diff --git a/wasm-rpc-stubgen/src/wit_generate.rs b/wasm-rpc-stubgen/src/wit_generate.rs index 64f4d146..263fb5bc 100644 --- a/wasm-rpc-stubgen/src/wit_generate.rs +++ b/wasm-rpc-stubgen/src/wit_generate.rs @@ -16,8 +16,7 @@ use crate::fs::{OverwriteSafeAction, OverwriteSafeActions, PathExtra}; use crate::log::{log_action, log_action_plan, log_warn_action, LogColorize, LogIndent}; use crate::naming::wit::package_dep_dir_name_from_encoder; use crate::stub::{ - FunctionParamStub, FunctionResultStub, FunctionStub, InterfaceStub, InterfaceStubTypeDef, - StubDefinition, + FunctionParamStub, FunctionResultStub, FunctionStub, InterfaceStub, StubDefinition, }; use crate::wit_encode::EncodedWitDir; use crate::wit_resolve::ResolvedWitDir; @@ -27,7 +26,7 @@ use std::collections::{BTreeMap, HashSet}; use std::path::{Path, PathBuf}; use wit_encoder::{ Ident, Interface, Package, PackageItem, PackageName, Params, ResourceFunc, Results, - StandaloneFunc, Type, TypeDef, TypeDefKind, VariantCase, World, WorldItem, + StandaloneFunc, Type, TypeDef, World, WorldItem, }; use wit_parser::PackageId; @@ -47,10 +46,10 @@ pub fn generate_stub_wit_to_target(def: &StubDefinition) -> anyhow::Result<()> { } pub fn generate_stub_wit_from_stub_def(def: &StubDefinition) -> anyhow::Result { - Ok(generate_stub_package_from_stub_def(def).to_string()) + Ok(generate_stub_package_from_stub_def(def)?.to_string()) } -pub fn generate_stub_package_from_stub_def(def: &StubDefinition) -> Package { +pub fn generate_stub_package_from_stub_def(def: &StubDefinition) -> anyhow::Result { let mut package = Package::new(PackageName::new( def.source_package_name.namespace.clone(), format!("{}-stub", def.source_package_name.name), @@ -88,7 +87,7 @@ pub fn generate_stub_package_from_stub_def(def: &StubDefinition) -> Package { for interface in def.stub_imported_interfaces() { for (function, _) in interface.all_functions() { if !function.results.is_empty() { - add_async_return_type(def, &mut stub_interface, function, interface); + add_async_return_type(def, &mut stub_interface, function, interface)?; } } } @@ -101,7 +100,7 @@ pub fn generate_stub_package_from_stub_def(def: &StubDefinition) -> Package { { let mut constructor = ResourceFunc::constructor(); let mut params = match &interface.constructor_params { - Some(constructor_params) => constructor_params.to_encoder(def), + Some(constructor_params) => constructor_params.to_encoder(def)?, None => Params::empty(), }; params.items_mut().insert( @@ -127,9 +126,9 @@ pub fn generate_stub_package_from_stub_def(def: &StubDefinition) -> Package { ResourceFunc::method(function_name) } }; - blocking_function.set_params(function.params.to_encoder(def)); + blocking_function.set_params(function.params.to_encoder(def)?); if !function.results.is_empty() { - blocking_function.set_results(function.results.to_encoder(def)); + blocking_function.set_results(function.results.to_encoder(def)?); } stub_functions.push(blocking_function); } @@ -143,7 +142,7 @@ pub fn generate_stub_package_from_stub_def(def: &StubDefinition) -> Package { ResourceFunc::method(function.name.clone()) } }; - async_function.set_params(function.params.to_encoder(def)); + async_function.set_params(function.params.to_encoder(def)?); if !function.results.is_empty() { async_function.set_results(Results::Anon(Type::Named(Ident::new( function.async_result_type(interface), @@ -166,7 +165,7 @@ pub fn generate_stub_package_from_stub_def(def: &StubDefinition) -> Package { package.world(stub_world); } - package + Ok(package) } fn add_async_return_type( @@ -174,7 +173,12 @@ fn add_async_return_type( stub_interface: &mut Interface, function: &FunctionStub, owner_interface: &InterfaceStub, -) { +) -> anyhow::Result<()> { + let context = || { + anyhow!( + "Failed to add async return type to stub, stub interface: {}, owner interface: {}, function: {:?}", stub_interface.name(), owner_interface.name, function.name) + }; + stub_interface.type_def(TypeDef::resource( function.async_result_type(owner_interface), [ @@ -187,22 +191,27 @@ fn add_async_return_type( let mut get = ResourceFunc::method("get"); match &function.results { FunctionResultStub::Anon(typ) => { - get.set_results(Results::Anon(Type::option(typ.to_encoder(def)))); + get.set_results(Results::Anon(Type::option( + typ.to_encoder(def).with_context(context)?, + ))); } - FunctionResultStub::Named(_params) => { - // TODO: this case was generating in the previous version, check it - // TODO: this should be a proper error message with context - panic!("Named parameters are not supported"); + FunctionResultStub::Named(params) => { + Err(anyhow!( + "Named parameters are not supported as async stub result, params: {:?}", + params + )) + .with_context(context)?; } FunctionResultStub::SelfType => { - // TODO: this should be a proper error message with context - panic!("Unexpected self return type in wit generator"); + Err(anyhow!("Unexpected self return type")).with_context(context)?; } } get }, ], - )) + )); + + Ok(()) } pub fn add_dependencies_to_stub_wit_dir(def: &StubDefinition) -> anyhow::Result<()> { @@ -438,14 +447,14 @@ pub fn add_stub_as_dependency_to_wit_dir(config: AddStubAsDepConfig) -> anyhow:: trait ToEncoder { type EncoderType; - fn to_encoder(&self, stub_definition: &StubDefinition) -> Self::EncoderType; + fn to_encoder(&self, stub_definition: &StubDefinition) -> anyhow::Result; } impl ToEncoder for wit_parser::Type { type EncoderType = Type; - fn to_encoder(&self, def: &StubDefinition) -> Self::EncoderType { - match self { + fn to_encoder(&self, def: &StubDefinition) -> anyhow::Result { + Ok(match self { wit_parser::Type::Bool => Type::Bool, wit_parser::Type::U8 => Type::U8, wit_parser::Type::U16 => Type::U16, @@ -463,167 +472,132 @@ impl ToEncoder for wit_parser::Type { if let Some(type_alias) = def.get_stub_used_type_alias(*type_id) { Type::Named(Ident::new(type_alias.to_string())) } else { - let typ = def.get_type_def(*type_id).expect("type not found"); + let typ = def.get_type_def(*type_id)?; + + let context = || { + anyhow!( + "Failed to convert wit parser type to encoder type, type: {:?}", + typ + ) + }; + match &typ.kind { - wit_parser::TypeDefKind::Option(inner) => { - Type::option(inner.to_encoder(def)) - } - wit_parser::TypeDefKind::List(inner) => Type::list(inner.to_encoder(def)), - wit_parser::TypeDefKind::Tuple(tuple) => { - Type::tuple(tuple.types.iter().map(|t| t.to_encoder(def))) - } + wit_parser::TypeDefKind::Option(inner) => Type::option( + inner + .to_encoder(def) + .context("Failed to convert option inner type") + .with_context(context)?, + ), + wit_parser::TypeDefKind::List(inner) => Type::list( + inner + .to_encoder(def) + .context("Failed to convert list inner type") + .with_context(context)?, + ), + wit_parser::TypeDefKind::Tuple(tuple) => Type::tuple( + tuple + .types + .iter() + .map(|t| { + t.to_encoder(def).with_context(|| { + anyhow!("Failed to convert tuple elem type: {:?}", t) + }) + }) + .collect::, _>>() + .with_context(context)?, + ), wit_parser::TypeDefKind::Result(result) => { match (&result.ok, &result.err) { - (Some(ok), Some(err)) => { - Type::result_both(ok.to_encoder(def), err.to_encoder(def)) - } - (Some(ok), None) => Type::result_ok(ok.to_encoder(def)), - (None, Some(err)) => Type::result_err(err.to_encoder(def)), - (None, None) => { - panic!("result type has no ok or err types") - } + (Some(ok), Some(err)) => Type::result_both( + ok.to_encoder(def) + .context("Failed to convert result ok type") + .with_context(context)?, + err.to_encoder(def) + .context("Failed to convert result error type") + .with_context(context)?, + ), + (Some(ok), None) => Type::result_ok( + ok.to_encoder(def) + .context("Failed to convert result ok type (no error case)") + .with_context(context)?, + ), + (None, Some(err)) => Type::result_err( + err.to_encoder(def) + .context("Failed to convert result error type (no ok case)") + .with_context(context)?, + ), + (None, None) => Err(anyhow!("Result type has no ok or err types")) + .with_context(context)?, } } wit_parser::TypeDefKind::Handle(handle) => match handle { wit_parser::Handle::Own(type_id) => { - wit_parser::Type::Id(*type_id).to_encoder(def) + wit_parser::Type::Id(*type_id).to_encoder(def)? } wit_parser::Handle::Borrow(type_id) => Type::borrow( - wit_parser::Type::Id(*type_id).to_encoder(def).to_string(), + wit_parser::Type::Id(*type_id).to_encoder(def)?.to_string(), ), }, _ => { let name = typ .name .clone() - .unwrap_or_else(|| panic!("missing name for type: {:?}", typ)); + .ok_or_else(|| anyhow!("Expected name for type: {:?}", typ)) + .with_context(context)?; Type::Named(Ident::new(name)) } } } } - } + }) } } impl ToEncoder for Vec { type EncoderType = Params; - fn to_encoder(&self, def: &StubDefinition) -> Self::EncoderType { - Params::from_iter( + fn to_encoder(&self, def: &StubDefinition) -> anyhow::Result { + Ok(Params::from_iter( self.iter() - .map(|param| (param.name.clone(), param.typ.to_encoder(def))), - ) + .map(|param| { + param + .typ + .to_encoder(def) + .with_context(|| { + anyhow!( + "Failed to convert parameter to encoder type, parameter: {:?}", + param + ) + }) + .map(|typ| (param.name.clone(), typ)) + }) + .collect::, _>>() + .with_context(|| anyhow!("Failed to convert params to encoder type: {:?}", self))?, + )) } } impl ToEncoder for FunctionResultStub { type EncoderType = Results; - fn to_encoder(&self, def: &StubDefinition) -> Self::EncoderType { - match self { - FunctionResultStub::Anon(typ) => Results::Anon(typ.to_encoder(def)), - FunctionResultStub::Named(types) => Results::Named(types.to_encoder(def)), - FunctionResultStub::SelfType => { - // TODO: This should be a proper error message - panic!("Unexpected self type function result") - } - } - } -} - -impl ToEncoder for InterfaceStubTypeDef { - type EncoderType = TypeDef; - - fn to_encoder(&self, def: &StubDefinition) -> Self::EncoderType { - (self.stub_type_name(), &self.type_def).to_encoder(def) - } -} - -impl ToEncoder for (&str, &wit_parser::TypeDef) { - type EncoderType = TypeDef; - - fn to_encoder(&self, def: &StubDefinition) -> Self::EncoderType { - let (name, type_def) = self; - match &type_def.kind { - wit_parser::TypeDefKind::Record(record) => TypeDef::record( - name.to_string(), - record - .fields - .iter() - .map(|field| (field.name.clone(), field.ty.to_encoder(def))), - ), - wit_parser::TypeDefKind::Flags(flags) => TypeDef::flags( - name.to_string(), - flags.flags.iter().map(|flag| (flag.name.clone(),)), - ), - wit_parser::TypeDefKind::Tuple(tuple) => TypeDef::new( - name.to_string(), - TypeDefKind::Type(Type::tuple( - tuple.types.iter().map(|typ| typ.to_encoder(def)), - )), - ), - wit_parser::TypeDefKind::Variant(variant) => TypeDef::variant( - name.to_string(), - variant.cases.iter().map(|case| match case.ty { - Some(ty) => VariantCase::value(case.name.clone(), ty.to_encoder(def)), - None => VariantCase::empty(case.name.clone()), - }), - ), - wit_parser::TypeDefKind::Enum(enum_ty) => TypeDef::enum_( - name.to_string(), - enum_ty.cases.iter().map(|case| case.name.clone()), - ), - wit_parser::TypeDefKind::Option(typ) => TypeDef::new( - name.to_string(), - TypeDefKind::Type(Type::option(typ.to_encoder(def))), - ), - wit_parser::TypeDefKind::Result(result) => TypeDef::new( - name.to_string(), - // TODO: match is duplicated - TypeDefKind::Type(match (&result.ok, &result.err) { - (Some(ok), Some(err)) => { - Type::result_both(ok.to_encoder(def), err.to_encoder(def)) - } - (Some(ok), None) => Type::result_ok(ok.to_encoder(def)), - (None, Some(err)) => Type::result_err(err.to_encoder(def)), - (None, None) => { - panic!("result type has no ok or err types") - } - }), - ), - wit_parser::TypeDefKind::List(typ) => TypeDef::new( - name.to_string(), - TypeDefKind::Type(Type::list(typ.to_encoder(def))), - ), - wit_parser::TypeDefKind::Future(_) => { - panic!("Future type is not supported yet") - } - wit_parser::TypeDefKind::Stream(_) => { - panic!("Stream type is not supported yet") - } - wit_parser::TypeDefKind::Type(typ) => { - TypeDef::new(name.to_string(), TypeDefKind::Type(typ.to_encoder(def))) + fn to_encoder(&self, def: &StubDefinition) -> anyhow::Result { + let context = || { + anyhow!( + "Failed to convert function result stub to encoder type: {:?}", + self + ) + }; + Ok(match self { + FunctionResultStub::Anon(typ) => { + Results::Anon(typ.to_encoder(def).with_context(context)?) } - wit_parser::TypeDefKind::Unknown => { - panic!("Unexpected unknown type") + FunctionResultStub::Named(types) => { + Results::Named(types.to_encoder(def).with_context(context)?) } - wit_parser::TypeDefKind::Resource => { - panic!("Unexpected resource type") + FunctionResultStub::SelfType => { + Err(anyhow!("Unexpected self type")).with_context(context)? } - wit_parser::TypeDefKind::Handle(handle) => TypeDef::new( - name.to_string(), - // TODO: match is duplicated - TypeDefKind::Type(match handle { - wit_parser::Handle::Own(type_id) => { - wit_parser::Type::Id(*type_id).to_encoder(def) - } - wit_parser::Handle::Borrow(type_id) => { - Type::borrow(wit_parser::Type::Id(*type_id).to_encoder(def).to_string()) - } - }), - ), - } + }) } } diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index 0d217a12..14c17539 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -208,7 +208,7 @@ impl ResolvedWitApplication { for (component_name, component) in &self.components { package_names_to_component_names .entry(&component.main_package_name) - .and_modify(|component_names| component_names.push(&component_name)) + .and_modify(|component_names| component_names.push(component_name)) .or_insert_with(|| vec![&component_name]); } From 84e02800c85c20a421d6d89d36683b76bea9a394 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Fri, 15 Nov 2024 14:32:42 +0100 Subject: [PATCH 071/109] mark generated dirs with a marker file --- wasm-rpc-stubgen/src/commands/declarative.rs | 39 ++++++++++++++++++-- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index 5773bd54..48990ca6 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -668,9 +668,12 @@ fn create_base_output_wit( ) -> Result { let component_input_wit = ctx.application.component_input_wit(component_name); let component_base_output_wit = ctx.application.component_base_output_wit(component_name); + let gen_dir_done_marker = GeneratedDirDoneMarker::new(&component_base_output_wit); if is_up_to_date( - ctx.config.skip_up_to_date_checks || !ctx.wit.is_dep_graph_up_to_date(component_name)?, + ctx.config.skip_up_to_date_checks + || !gen_dir_done_marker.is_done() + || !ctx.wit.is_dep_graph_up_to_date(component_name)?, || [component_input_wit.clone()], || [component_base_output_wit.clone()], ) { @@ -741,6 +744,8 @@ fn create_base_output_wit( extract_main_interface_as_wit_dep(&component_base_output_wit)?; } + gen_dir_done_marker.mark_as_done()?; + Ok(true) } } @@ -751,9 +756,12 @@ fn create_output_wit( ) -> Result { let component_base_output_wit = ctx.application.component_base_output_wit(component_name); let component_output_wit = ctx.application.component_output_wit(component_name); + let gen_dir_done_marker = GeneratedDirDoneMarker::new(&component_output_wit); if is_up_to_date( - ctx.config.skip_up_to_date_checks || !ctx.wit.is_dep_graph_up_to_date(component_name)?, + ctx.config.skip_up_to_date_checks + || !gen_dir_done_marker.is_done() + || !ctx.wit.is_dep_graph_up_to_date(component_name)?, || [component_base_output_wit.clone()], || [component_output_wit.clone()], ) { @@ -776,6 +784,8 @@ fn create_output_wit( copy_wit_sources(&component_base_output_wit, &component_output_wit)?; add_stub_deps(ctx, component_name)?; + gen_dir_done_marker.mark_as_done()?; + Ok(true) } } @@ -833,9 +843,10 @@ async fn build_stub( let stub_wasm = ctx.application.stub_wasm(component_name); let stub_wit = ctx.application.stub_wit(component_name); + let gen_dir_done_marker = GeneratedDirDoneMarker::new(&stub_wit); if is_up_to_date( - ctx.config.skip_up_to_date_checks, + ctx.config.skip_up_to_date_checks || !gen_dir_done_marker.is_done(), || stub_inputs, || [stub_wit.clone(), stub_wasm.clone()], ) { @@ -863,6 +874,8 @@ async fn build_stub( commands::generate::build(&stub_def, &stub_wasm, &stub_wit).await?; + gen_dir_done_marker.mark_as_done()?; + delete_path("stub temp build dir", &target_root)?; Ok(true) @@ -944,3 +957,23 @@ fn copy_wit_sources(source: &Path, target: &Path) -> anyhow::Result<()> { Ok(()) } + +static GENERATED_DIR_DONE_MARKER_FILE_NAME: &str = &".done"; + +struct GeneratedDirDoneMarker<'a> { + dir: &'a Path, +} + +impl<'a> GeneratedDirDoneMarker<'a> { + fn new(dir: &'a Path) -> Self { + Self { dir } + } + + fn is_done(&self) -> bool { + self.dir.join(GENERATED_DIR_DONE_MARKER_FILE_NAME).exists() + } + + fn mark_as_done(&self) -> anyhow::Result<()> { + fs::write_str(self.dir.join(GENERATED_DIR_DONE_MARKER_FILE_NAME), "") + } +} From 494af7c66253f3c6710ff1a72d0a83078ca240e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Fri, 15 Nov 2024 14:56:50 +0100 Subject: [PATCH 072/109] error format --- Cargo.lock | 7 + wasm-rpc-stubgen/Cargo.toml | 7 +- wasm-rpc-stubgen/src/cargo.rs | 14 +- wasm-rpc-stubgen/src/commands/declarative.rs | 34 +++-- wasm-rpc-stubgen/src/commands/generate.rs | 2 +- wasm-rpc-stubgen/src/fs.rs | 89 +++++++++---- wasm-rpc-stubgen/src/log.rs | 4 + wasm-rpc-stubgen/src/model/oam.rs | 13 +- wasm-rpc-stubgen/src/model/wasm_rpc.rs | 19 ++- wasm-rpc-stubgen/src/naming.rs | 3 +- wasm-rpc-stubgen/src/validation.rs | 77 ++++++++--- wasm-rpc-stubgen/src/wit_generate.rs | 15 ++- wasm-rpc-stubgen/src/wit_resolve.rs | 128 +++++++++++++++---- 13 files changed, 308 insertions(+), 104 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 81a9198f..acadca8b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1921,6 +1921,7 @@ dependencies = [ "heck 0.5.0", "id-arena", "indexmap 2.6.0", + "indoc", "itertools 0.12.1", "pretty_env_logger", "prettyplease", @@ -2300,6 +2301,12 @@ dependencies = [ "serde 1.0.210", ] +[[package]] +name = "indoc" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" + [[package]] name = "inout" version = "0.1.3" diff --git a/wasm-rpc-stubgen/Cargo.toml b/wasm-rpc-stubgen/Cargo.toml index 095fefa0..0949144a 100644 --- a/wasm-rpc-stubgen/Cargo.toml +++ b/wasm-rpc-stubgen/Cargo.toml @@ -31,11 +31,11 @@ harness = false [dependencies] anyhow = "1.0.79" assert2 = { workspace = true } +cargo-component = "=0.13.2" +cargo-component-core = "=0.13.2" cargo_toml = "0.20.2" clap = { version = "4.5.7", features = ["derive"] } colored = "2.1.0" -cargo-component-core = "=0.13.2" -cargo-component = "=0.13.2" dir-diff = "0.3.3" fs_extra = { workspace = true } glob = "0.3.1" @@ -44,6 +44,7 @@ golem-wasm-rpc = { path = "../wasm-rpc", version = "0.0.0" } heck = "0.5.0" id-arena = "2.2.1" indexmap = "2.2.6" +indoc = "2.0.5" itertools = "0.12.1" pretty_env_logger = "0.5.0" prettyplease = "0.2.20" @@ -58,8 +59,8 @@ syn = "2.0.66" tempfile = { workspace = true } tokio = { workspace = true } toml = "0.8.14" -walkdir = "2.5.0" wac-graph = "=0.6.0" +walkdir = "2.5.0" wit-bindgen-rust = "=0.26.0" wit-encoder = { git = "https://github.com/golemcloud/wasm-tools.git" } wit-parser = { git = "https://github.com/golemcloud/wasm-tools.git" } diff --git a/wasm-rpc-stubgen/src/cargo.rs b/wasm-rpc-stubgen/src/cargo.rs index 079509fa..e84beef9 100644 --- a/wasm-rpc-stubgen/src/cargo.rs +++ b/wasm-rpc-stubgen/src/cargo.rs @@ -325,8 +325,8 @@ pub fn regenerate_cargo_package_component( let relative_wit_path = wit_path.strip_prefix(project_root).with_context(|| { anyhow!( "Failed to create relative path for wit dir: {}, project root: {}", - wit_path.display(), - project_root.display() + wit_path.log_color_highlight(), + project_root.log_color_highlight() ) })?; @@ -343,10 +343,12 @@ pub fn regenerate_cargo_package_component( cargo_toml_path.log_color_highlight() ) })?; - let package = manifest - .package - .as_mut() - .ok_or_else(|| anyhow!("No package found in {}", cargo_toml_path.display()))?; + let package = manifest.package.as_mut().ok_or_else(|| { + anyhow!( + "No package found in {}", + cargo_toml_path.log_color_highlight() + ) + })?; let wit_dir = ResolvedWitDir::new(wit_path)?; diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index 48990ca6..6ee09976 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -237,7 +237,7 @@ fn component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<()> { "Command failed with exit code: {}", result .code() - .map(|code| code.to_string()) + .map(|code| code.to_string().log_color_error_highlight().to_string()) .unwrap_or_else(|| "?".to_string()) ))); } @@ -407,8 +407,13 @@ fn delete_path(context: &str, path: &Path) -> anyhow::Result<()> { "Deleting", format!("{} {}", context, path.log_color_highlight()), ); - fs::remove(path) - .with_context(|| anyhow!("Failed to delete {}, path: {}", context, path.display()))?; + fs::remove(path).with_context(|| { + anyhow!( + "Failed to delete {}, path: {}", + context.log_color_highlight(), + path.log_color_highlight() + ) + })?; } Ok(()) } @@ -562,11 +567,13 @@ fn to_anyhow(message: &str, result: ValidatedResult) -> anyhow::Result match result { ValidatedResult::Ok(value) => Ok(value), ValidatedResult::OkWithWarns(components, warns) => { + println!(); print_warns(warns); println!(); Ok(components) } ValidatedResult::WarnsAndErrors(warns, errors) => { + println!(); print_warns(warns); print_errors(errors); println!(); @@ -795,9 +802,12 @@ fn update_cargo_toml( component_name: &ComponentName, ) -> anyhow::Result<()> { let component_input_wit = PathExtra::new(ctx.application.component_input_wit(component_name)); - let component_input_wit_parent = component_input_wit - .parent() - .with_context(|| anyhow!("Failed to get parent for component {}", component_name))?; + let component_input_wit_parent = component_input_wit.parent().with_context(|| { + anyhow!( + "Failed to get parent for component {}", + component_name.log_color_highlight() + ) + })?; let cargo_toml = component_input_wit_parent.join("Cargo.toml"); if cargo_toml.exists() { @@ -933,16 +943,18 @@ fn copy_wit_sources(source: &Path, target: &Path) -> anyhow::Result<()> { let dir_content = fs_extra::dir::get_dir_content(source).with_context(|| { anyhow!( "Failed to read component input wit directory entries for {}", - source.display() + source.log_color_highlight() ) })?; for file in dir_content.files { let from = PathBuf::from(&file); - let to = target.join( - from.strip_prefix(source) - .with_context(|| anyhow!("Failed to strip prefix for source {}", &file))?, - ); + let to = target.join(from.strip_prefix(source).with_context(|| { + anyhow!( + "Failed to strip prefix for source {}", + &file.log_color_highlight() + ) + })?); log_action( "Copying", diff --git a/wasm-rpc-stubgen/src/commands/generate.rs b/wasm-rpc-stubgen/src/commands/generate.rs index e4a5bf3a..c13bc803 100644 --- a/wasm-rpc-stubgen/src/commands/generate.rs +++ b/wasm-rpc-stubgen/src/commands/generate.rs @@ -66,7 +66,7 @@ pub async fn generate_and_build_stub(stub_def: &StubDefinition) -> anyhow::Resul .with_context(|| { anyhow!( "Failed to canonicalize stub target root {}", - stub_def.config.target_root.display() + stub_def.config.target_root.log_color_error_highlight() ) })?, ) diff --git a/wasm-rpc-stubgen/src/fs.rs b/wasm-rpc-stubgen/src/fs.rs index e3b0c4d0..dfc0a28a 100644 --- a/wasm-rpc-stubgen/src/fs.rs +++ b/wasm-rpc-stubgen/src/fs.rs @@ -1,3 +1,4 @@ +use crate::log::LogColorize; use anyhow::{anyhow, Context}; use std::cmp::PartialEq; use std::fs::Metadata; @@ -10,7 +11,7 @@ pub fn create_dir_all>(path: P) -> anyhow::Result<()> { Ok(()) } else { std::fs::create_dir_all(path) - .with_context(|| anyhow!("Failed to create directory {}", path.display())) + .with_context(|| anyhow!("Failed to create directory {}", path.log_color_highlight())) } } @@ -77,13 +78,18 @@ pub fn copy_transformed, Q: AsRef, T: Fn(String) -> anyhow: pub fn read_to_string>(path: P) -> anyhow::Result { let path = path.as_ref(); - fs_extra::file::read_to_string(path) - .with_context(|| anyhow!("Failed to read to string, file: {}", path.display())) + fs_extra::file::read_to_string(path).with_context(|| { + anyhow!( + "Failed to read to string, file: {}", + path.log_color_highlight() + ) + }) } pub fn read>(path: P) -> anyhow::Result> { let path = path.as_ref(); - std::fs::read(path).with_context(|| anyhow!("Failed to read file: {}", path.display())) + std::fs::read(path) + .with_context(|| anyhow!("Failed to read file: {}", path.log_color_highlight())) } // Creates all missing parent directories if necessary and writes str to path. @@ -91,7 +97,7 @@ pub fn write_str, S: AsRef>(path: P, str: S) -> anyhow::Resu let path = PathExtra(path); let str = str.as_ref(); - let context = || format!("Failed to write string to {}", path.display()); + let context = || anyhow!("Failed to write string to {}", path.log_color_highlight()); let target_parent = path.parent().with_context(context)?; create_dir_all(target_parent).with_context(context)?; @@ -101,7 +107,7 @@ pub fn write_str, S: AsRef>(path: P, str: S) -> anyhow::Resu pub fn write, C: AsRef<[u8]>>(path: P, contents: C) -> anyhow::Result<()> { let path = PathExtra(path); - let context = || format!("Failed to write to {}", path.display()); + let context = || anyhow!("Failed to write to {}", path.log_color_highlight()); let target_parent = path.parent().with_context(context)?; create_dir_all(target_parent).with_context(context)?; @@ -112,11 +118,12 @@ pub fn remove>(path: P) -> anyhow::Result<()> { let path = path.as_ref(); if path.exists() { if path.is_dir() { - std::fs::remove_dir_all(path) - .with_context(|| anyhow!("Failed to delete directory {}", path.display()))?; + std::fs::remove_dir_all(path).with_context(|| { + anyhow!("Failed to delete directory {}", path.log_color_highlight()) + })?; } else { std::fs::remove_file(path) - .with_context(|| anyhow!("Failed to delete file {}", path.display()))?; + .with_context(|| anyhow!("Failed to delete file {}", path.log_color_highlight()))?; } } Ok(()) @@ -126,10 +133,15 @@ pub fn has_str_content, S: AsRef>(path: P, str: S) -> anyhow let path = path.as_ref(); let str = str.as_ref(); - let context = || format!("Failed to compare content to string for {}", path.display()); + let context = || { + anyhow!( + "Failed to compare content to string for {}", + path.log_color_highlight() + ) + }; let content = read_to_string(path) - .with_context(|| anyhow!("Failed to read as string: {}", path.display())) + .with_context(|| anyhow!("Failed to read as string: {}", path.log_color_highlight())) .with_context(context)?; Ok(content == str) @@ -140,10 +152,10 @@ pub fn has_same_string_content, Q: AsRef>(a: P, b: Q) -> an let b = b.as_ref(); let context = || { - format!( + anyhow!( "Failed to compare string contents of {} and {}", - a.display(), - b.display() + a.log_color_highlight(), + b.log_color_highlight() ) }; @@ -156,7 +168,7 @@ pub fn has_same_string_content, Q: AsRef>(a: P, b: Q) -> an pub fn metadata>(path: P) -> anyhow::Result { let path = path.as_ref(); std::fs::metadata(path) - .with_context(|| anyhow!("Failed to get metadata for {}", path.display())) + .with_context(|| anyhow!("Failed to get metadata for {}", path.log_color_highlight())) } pub struct PathExtra>(P); @@ -168,23 +180,41 @@ impl> PathExtra

{ pub fn parent(&self) -> anyhow::Result<&Path> { let path = self.0.as_ref(); - path.parent() - .ok_or_else(|| anyhow!("Failed to get parent dir for path: {}", path.display())) + path.parent().ok_or_else(|| { + anyhow!( + "Failed to get parent dir for path: {}", + path.log_color_highlight() + ) + }) } pub fn file_name_to_string(&self) -> anyhow::Result { let path = self.0.as_ref(); path.file_name() - .ok_or_else(|| anyhow!("Failed to get file name for path: {}", path.display(),))? + .ok_or_else(|| { + anyhow!( + "Failed to get file name for path: {}", + path.log_color_highlight(), + ) + })? .to_os_string() .into_string() - .map_err(|_| anyhow!("Failed to convert filename for path: {}", path.display())) + .map_err(|_| { + anyhow!( + "Failed to convert filename for path: {}", + path.log_color_highlight() + ) + }) } pub fn to_str(&self) -> anyhow::Result<&str> { let path = self.0.as_ref(); - path.to_str() - .ok_or_else(|| anyhow!("Failed to convert path to string: {}", path.display())) + path.to_str().ok_or_else(|| { + anyhow!( + "Failed to convert path to string: {}", + path.log_color_highlight() + ) + }) } pub fn to_string(&self) -> anyhow::Result { @@ -200,8 +230,8 @@ impl> PathExtra

{ .with_context(|| { anyhow!( "Failed to strip prefix from path, prefix: {}, path: {}", - prefix.display(), - path.display() + prefix.log_color_highlight(), + path.log_color_highlight() ) })? .to_path_buf()) @@ -248,11 +278,18 @@ impl OverwriteSafeAction { F: FnOnce(String) -> anyhow::Result, { let content = std::fs::read_to_string(&source).with_context(|| { - anyhow!("Failed to read file as string, path: {}", source.display()) + anyhow!( + "Failed to read file as string, path: {}", + source.log_color_highlight() + ) })?; - let source_transformed = transform(content) - .with_context(|| anyhow!("Failed to transform file, path: {}", source.display()))?; + let source_transformed = transform(content).with_context(|| { + anyhow!( + "Failed to transform file, path: {}", + source.log_color_highlight() + ) + })?; Ok(OverwriteSafeAction::CopyFileTransformed { source, diff --git a/wasm-rpc-stubgen/src/log.rs b/wasm-rpc-stubgen/src/log.rs index 8e91d7ae..711b5187 100644 --- a/wasm-rpc-stubgen/src/log.rs +++ b/wasm-rpc-stubgen/src/log.rs @@ -200,6 +200,10 @@ pub trait LogColorize { fn log_color_highlight(&self) -> ColoredString { self.as_str().bold() } + + fn log_color_error_highlight(&self) -> ColoredString { + self.as_str().bold().red().underline() + } } impl<'a> LogColorize for &'a str { diff --git a/wasm-rpc-stubgen/src/model/oam.rs b/wasm-rpc-stubgen/src/model/oam.rs index 0006a70d..7c0e913c 100644 --- a/wasm-rpc-stubgen/src/model/oam.rs +++ b/wasm-rpc-stubgen/src/model/oam.rs @@ -1,4 +1,5 @@ use crate::fs; +use crate::log::LogColorize; use crate::validation::{ValidatedResult, ValidationBuilder}; use itertools::Itertools; use serde::de::DeserializeOwned; @@ -40,11 +41,19 @@ impl ApplicationWithSource { validation.push_context("source", self.source_as_string()); if self.application.api_version != API_VERSION_V1BETA1 { - validation.add_warn(format!("Expected apiVersion: {}", API_VERSION_V1BETA1)) + validation.add_warn(format!( + "Expected apiVersion: {}, got: {}", + API_VERSION_V1BETA1.log_color_highlight(), + self.application.api_version.log_color_error_highlight() + )) } if self.application.kind != KIND_APPLICATION { - validation.add_error(format!("Expected kind: {}", KIND_APPLICATION)) + validation.add_error(format!( + "Expected kind: {}, got: {}", + KIND_APPLICATION.log_color_highlight(), + self.application.kind.log_color_error_highlight() + )) } self.application diff --git a/wasm-rpc-stubgen/src/model/wasm_rpc.rs b/wasm-rpc-stubgen/src/model/wasm_rpc.rs index 2e7a5b52..a0af5b74 100644 --- a/wasm-rpc-stubgen/src/model/wasm_rpc.rs +++ b/wasm-rpc-stubgen/src/model/wasm_rpc.rs @@ -1,3 +1,4 @@ +use crate::log::LogColorize; use crate::model::oam; use crate::model::oam::TypedTraitProperties; use crate::model::unknown_properties::{HasUnknownProperties, UnknownProperties}; @@ -447,7 +448,7 @@ impl Application { vec![("component name", component_name.0)], format!( "Component is specified multiple times in sources: {}", - sources.join(", ") + sources.iter().map(|s| s.log_color_highlight()).join(", ") ), )) }); @@ -461,7 +462,8 @@ impl Application { vec![], format!( "Component {} references unknown component {} as dependency", - component_name, dep_component_name, + component_name.log_color_highlight(), + dep_component_name.log_color_error_highlight(), ), ) }) @@ -482,7 +484,11 @@ impl Application { "Component Build is specified multiple times in sources: {}", wasm_builds .iter() - .map(|c| format!("{} in {}", c.name, c.source.display())) + .map(|c| format!( + "{} in {}", + c.name.log_color_highlight(), + c.source.log_color_highlight() + )) .join(", ") )); } @@ -548,7 +554,7 @@ impl Application { vec![("component name", component_name.0)], format!( "Wasm rpc stub build is specified multiple times in sources: {}", - sources.join(", ") + sources.iter().map(|s| s.log_color_highlight()).join(", ") ), )) }, @@ -558,7 +564,7 @@ impl Application { validation.add_error( format!( "Common (without component name) wasm rpc build is specified multiple times in sources: {}", - common_sources.join(", "), + common_sources.iter().map(|s| s.log_color_highlight()).join(", "), ) ) } @@ -571,7 +577,8 @@ impl Application { vec![("source", wasm_rpc_stub_build.source_as_string())], format!( "Wasm rpc stub build {} references unknown component {}", - wasm_rpc_stub_build.name, component_name + wasm_rpc_stub_build.name.log_color_highlight(), + component_name.log_color_error_highlight(), ), ) }) diff --git a/wasm-rpc-stubgen/src/naming.rs b/wasm-rpc-stubgen/src/naming.rs index daccf02a..8860dc90 100644 --- a/wasm-rpc-stubgen/src/naming.rs +++ b/wasm-rpc-stubgen/src/naming.rs @@ -1,4 +1,5 @@ pub mod wit { + use crate::log::LogColorize; use anyhow::{anyhow, bail}; use std::path::{Path, PathBuf}; @@ -69,7 +70,7 @@ pub mod wit { if stub_package.interfaces.len() != 1 { bail!( "Expected exactly one interface in stub package, package name: {}", - package_name + package_name.to_string().log_color_highlight() ); } diff --git a/wasm-rpc-stubgen/src/validation.rs b/wasm-rpc-stubgen/src/validation.rs index caa0a38d..178fad71 100644 --- a/wasm-rpc-stubgen/src/validation.rs +++ b/wasm-rpc-stubgen/src/validation.rs @@ -1,3 +1,4 @@ +use crate::log::LogColorize; use itertools::Itertools; use std::fmt::Display; @@ -188,12 +189,64 @@ impl ValidationBuilder { _ = self.context.pop(); } + fn format(&mut self, message: String) -> String { + let multiline = message.contains("\n"); + + let message = { + if multiline { + message.lines().map(|l| format!(" {}", l)).join("\n") + } else { + message + } + }; + + let context = { + if self.context.is_empty() { + "".to_string() + } else if multiline { + format!( + "{}{}", + if message.ends_with("\n") { + "\n " + } else { + "\n\n " + }, + self.context + .iter() + .map(|c| format!("{}: {}", c.name, c.value.log_color_highlight())) + .join(", ") + ) + } else { + format!( + ", {}", + self.context + .iter() + .map(|c| format!("{}: {}", c.name, c.value.log_color_highlight())) + .join("\n") + ) + } + }; + + format!( + "{}{}{}", + if multiline && !message.starts_with("\n") { + "\n" + } else { + "" + }, + message, + context + ) + } + pub fn add_error(&mut self, error: String) { - self.errors.push(format!("{}{}", error, self.context(),)); + let error = self.format(error); + self.errors.push(error); } pub fn add_warn(&mut self, warn: String) { - self.warns.push(format!("{}{}", warn, self.context(),)); + let warn = self.format(warn); + self.warns.push(warn); } pub fn add_errors(&mut self, elems: C, context_and_error: F) @@ -201,7 +254,7 @@ impl ValidationBuilder { C: IntoIterator, F: Fn(T) -> Option<(Vec<(&'static str, String)>, String)>, { - self.add(elems, context_and_error, Self::add_error); + self.add_all(elems, context_and_error, Self::add_error); } pub fn add_warns(&mut self, elems: C, context_and_error: F) @@ -209,10 +262,10 @@ impl ValidationBuilder { C: IntoIterator, F: Fn(T) -> Option<(Vec<(&'static str, String)>, String)>, { - self.add(elems, context_and_error, Self::add_warn); + self.add_all(elems, context_and_error, Self::add_warn); } - pub fn add(&mut self, elems: C, context_and_error: CE, add: A) + fn add_all(&mut self, elems: C, context_and_error: CE, add: A) where C: IntoIterator, CE: Fn(T) -> Option<(Vec<(&'static str, String)>, String)>, @@ -247,20 +300,6 @@ impl ValidationBuilder { ValidatedResult::WarnsAndErrors(self.warns, self.errors) } } - - fn context(&self) -> String { - if self.context.is_empty() { - "".to_string() - } else { - format!( - ", {}", - self.context - .iter() - .map(|c| format!("{}: {}", c.name, c.value)) - .join(", ") - ) - } - } } impl Default for ValidationBuilder { diff --git a/wasm-rpc-stubgen/src/wit_generate.rs b/wasm-rpc-stubgen/src/wit_generate.rs index 263fb5bc..d7f94241 100644 --- a/wasm-rpc-stubgen/src/wit_generate.rs +++ b/wasm-rpc-stubgen/src/wit_generate.rs @@ -22,6 +22,7 @@ use crate::wit_encode::EncodedWitDir; use crate::wit_resolve::ResolvedWitDir; use crate::{cargo, fs, naming}; use anyhow::{anyhow, bail, Context}; +use itertools::Itertools; use std::collections::{BTreeMap, HashSet}; use std::path::{Path, PathBuf}; use wit_encoder::{ @@ -384,8 +385,12 @@ pub fn add_stub_as_dependency_to_wit_dir(config: AddStubAsDepConfig) -> anyhow:: if dest_main_package_sources.files.len() != 1 { bail!( - "Expected exactly one dest main package source, got sources: {:?}", - dest_main_package_sources.files + "Expected exactly one dest main package source, got sources: {}", + dest_main_package_sources + .files + .iter() + .map(|s| s.log_color_highlight()) + .join(", ") ); } @@ -435,7 +440,7 @@ pub fn add_stub_as_dependency_to_wit_dir(config: AddStubAsDepConfig) -> anyhow:: } else if config.update_cargo_toml == UpdateCargoToml::Update { return Err(anyhow!( "Cannot update {:?} file because it does not exist or is not a file", - target_cargo_toml + target_cargo_toml.log_color_highlight() )); } } else if config.update_cargo_toml == UpdateCargoToml::Update { @@ -655,14 +660,14 @@ pub fn extract_main_interface_as_wit_dep(wit_dir: &Path) -> anyhow::Result<()> { .ok_or_else(|| { anyhow!( "Failed to get sources for main package, wit dir: {}", - wit_dir.display() + wit_dir.log_color_highlight() ) })?; if sources.files.len() != 1 { bail!( "Expected exactly one source for main package, wit dir: {}", - wit_dir.display() + wit_dir.log_color_highlight() ); } diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index 14c17539..8b8919da 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -5,6 +5,7 @@ use crate::validation::{ValidatedResult, ValidationBuilder}; use crate::{fs, naming}; use anyhow::{anyhow, bail, Context, Error}; use indexmap::IndexMap; +use indoc::formatdoc; use itertools::Itertools; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::path::{Path, PathBuf}; @@ -34,7 +35,7 @@ impl ResolvedWitDir { anyhow!( "Failed to get package by id: {:?}, wit dir: {}", package_id, - self.path.display() + self.path.log_color_highlight() ) }) } @@ -44,7 +45,7 @@ impl ResolvedWitDir { anyhow!( "Failed to get package sources by id: {:?}, wit dir: {}", package_id, - self.path.display() + self.path.log_color_highlight() ) }) } @@ -62,7 +63,7 @@ fn resolve_wit_dir(path: &Path) -> anyhow::Result { let (package_id, package_source_map) = resolve .push_dir(path) - .with_context(|| anyhow!("Failed to resolve wit dir: {}", path.display()))?; + .with_context(|| anyhow!("Failed to resolve wit dir: {}", path.log_color_highlight()))?; let package_sources = collect_package_sources(path, &resolve, package_id, &package_source_map)?; @@ -98,7 +99,7 @@ fn collect_package_sources( .ok_or_else(|| { anyhow!( "Failed to get package source map for package {}", - package.name + package.name.to_string().log_color_highlight() ) })? .map(|path| path.to_path_buf()) @@ -114,7 +115,10 @@ fn collect_package_sources( ); } else { if sources.is_empty() { - bail!("Expected at least one source for package: {}", package.name); + bail!( + "Expected at least one source for package: {}", + package.name.to_string().log_color_error_highlight() + ); }; let source = &sources[0]; @@ -122,25 +126,28 @@ fn collect_package_sources( let extension = source.extension().ok_or_else(|| { anyhow!( "Failed to get extension for wit source: {}", - source.display() + source.log_color_highlight() ) })?; if extension != "wit" { bail!( "Only wit sources are supported, source: {}", - source.display() + source.log_color_highlight() ); } let parent = source.parent().ok_or_else(|| { - anyhow!("Failed to get parent for wit source: {}", source.display()) + anyhow!( + "Failed to get parent for wit source: {}", + source.log_color_highlight() + ) })?; if parent == deps_dir { bail!( "Single-file wit packages without folder are not supported, source: {}", - source.display() + source.log_color_highlight() ); } @@ -192,11 +199,8 @@ impl ResolvedWitApplication { resolved_app.add_components_from_app(&mut validation, app); resolved_app.validate_package_names(&mut validation); - - if !validation.has_any_errors() { - resolved_app.collect_component_deps(); - resolved_app.sort_components_by_input_deps(&mut validation); - } + resolved_app.collect_component_deps(app, &mut validation); + resolved_app.sort_components_by_input_deps(&mut validation); validation.build(resolved_app) } @@ -274,8 +278,8 @@ impl ResolvedWitApplication { UnresolvedPackageGroup::parse_dir(&input_wit_dir).with_context(|| { anyhow!( "Failed to parse component {} main package in input wit dir {}", - component_name, - input_wit_dir.display() + component_name.log_color_error_highlight(), + input_wit_dir.log_color_highlight(), ) })?; @@ -335,7 +339,7 @@ impl ResolvedWitApplication { } } - fn collect_component_deps(&mut self) { + fn collect_component_deps(&mut self, app: &Application, validation: &mut ValidationBuilder) { fn component_deps< 'a, I: IntoIterator, @@ -376,6 +380,71 @@ impl ResolvedWitApplication { component.input_component_deps = input_deps; component.output_component_deps = output_deps; } + + for (component_name, component) in &self.components { + let main_deps: HashSet = component_deps( + &self.package_to_component, + &component.input_referenced_package_deps, + ); + + let stub_deps: HashSet = component_deps( + &self.stub_package_to_component, + &component.input_referenced_package_deps, + ); + + if !main_deps.is_empty() || !stub_deps.is_empty() { + validation.push_context("component name", component_name.to_string()); + validation.push_context("package name", component.main_package_name.to_string()); + { + let component = app.component(component_name); + validation + .push_context("source", component.source.to_string_lossy().to_string()); + } + + for dep_component_name in main_deps { + let dep_package_name = &self + .component(&dep_component_name) + .unwrap() + .main_package_name; + + validation + .push_context("referenced package name", dep_package_name.to_string()); + + validation.add_error(formatdoc!(" + Direct WIT package reference to component ({}) main package ({}) is not supported. + For using component stubs, declare them in the app manifest. + For using exported types from another component, use the component interface package (e.g.: ns:package-name-interface).", + dep_component_name.to_string().log_color_error_highlight(), + dep_package_name.to_string().log_color_highlight() + )); + + validation.pop_context(); + } + + for dep_component_name in stub_deps { + let dep_package_name = &self + .component(&dep_component_name) + .unwrap() + .main_package_name; + + validation + .push_context("referenced package name", dep_package_name.to_string()); + + validation.add_error(formatdoc!(" + Direct WIT package reference to component ({}) stub packages ({}) is not supported. + For using component stubs, declare them in the app manifest. + For using exported types from another component, use the component interface package (e.g.: ns:package-name-interface).", + dep_component_name.to_string().log_color_error_highlight(), + dep_package_name.to_string().log_color_highlight() + )); + + validation.pop_context(); + } + + validation.pop_context(); + validation.pop_context(); + } + } } fn sort_components_by_input_deps(&mut self, validation: &mut ValidationBuilder) { @@ -459,9 +528,12 @@ impl ResolvedWitApplication { } fn component(&self, component_name: &ComponentName) -> Result<&ResolvedWitComponent, Error> { - self.components - .get(component_name) - .ok_or_else(|| anyhow!("Component not found: {}", component_name)) + self.components.get(component_name).ok_or_else(|| { + anyhow!( + "Component not found: {}", + component_name.log_color_error_highlight() + ) + }) } // NOTE: Intended to be used for non-component wit package deps, so it does not include @@ -536,7 +608,12 @@ pub fn parse_wit_deps_dir(path: &Path) -> Result, Er let mut entries = path .read_dir() .and_then(|read_dir| read_dir.collect::>>()) - .with_context(|| anyhow!("Failed to read wit dependencies from {}", path.display(),))?; + .with_context(|| { + anyhow!( + "Failed to read wit dependencies from {}", + path.log_color_error_highlight() + ) + })?; entries.sort_by_key(|e| e.file_name()); entries .iter() @@ -548,7 +625,10 @@ pub fn parse_wit_deps_dir(path: &Path) -> Result, Er // - wasm or wat deps path.is_dir().then(|| { UnresolvedPackageGroup::parse_dir(&path).with_context(|| { - anyhow!("Failed to parse wit dependency package {}", path.display()) + anyhow!( + "Failed to parse wit dependency package {}", + path.log_color_error_highlight() + ) }) }) }) @@ -585,13 +665,13 @@ impl WitDepsResolver { } bail!( "Package {} not found, sources searched: {}", - package_name, + package_name.to_string().log_color_error_highlight(), if self.sources.is_empty() { "no sources were provided".to_string() } else { self.sources .iter() - .map(|s| s.display().to_string()) + .map(|s| s.log_color_highlight()) .join(", ") } ) From 3d066947da9e248b7be82e1b386e89a2b8482fe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Fri, 15 Nov 2024 15:17:45 +0100 Subject: [PATCH 073/109] clippy --- wasm-rpc-stubgen/src/commands/declarative.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index 6ee09976..6a654916 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -970,7 +970,7 @@ fn copy_wit_sources(source: &Path, target: &Path) -> anyhow::Result<()> { Ok(()) } -static GENERATED_DIR_DONE_MARKER_FILE_NAME: &str = &".done"; +static GENERATED_DIR_DONE_MARKER_FILE_NAME: &str = ".done"; struct GeneratedDirDoneMarker<'a> { dir: &'a Path, From 7256838e7a4a39c9bd554b2689c5f0e62187eb50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Sun, 17 Nov 2024 08:38:08 +0100 Subject: [PATCH 074/109] log format --- wasm-rpc-stubgen/src/wit_resolve.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index 8b8919da..b3372ee0 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -411,11 +411,11 @@ impl ResolvedWitApplication { .push_context("referenced package name", dep_package_name.to_string()); validation.add_error(formatdoc!(" - Direct WIT package reference to component ({}) main package ({}) is not supported. + Direct WIT package reference to component {} main package {} is not supported. For using component stubs, declare them in the app manifest. For using exported types from another component, use the component interface package (e.g.: ns:package-name-interface).", - dep_component_name.to_string().log_color_error_highlight(), - dep_package_name.to_string().log_color_highlight() + dep_component_name.to_string().log_color_highlight(), + dep_package_name.to_string().log_color_error_highlight() )); validation.pop_context(); @@ -431,11 +431,11 @@ impl ResolvedWitApplication { .push_context("referenced package name", dep_package_name.to_string()); validation.add_error(formatdoc!(" - Direct WIT package reference to component ({}) stub packages ({}) is not supported. + Direct WIT package reference to component {} stub packages {} is not supported. For using component stubs, declare them in the app manifest. For using exported types from another component, use the component interface package (e.g.: ns:package-name-interface).", - dep_component_name.to_string().log_color_error_highlight(), - dep_package_name.to_string().log_color_highlight() + dep_component_name.to_string().log_color_highlight(), + dep_package_name.to_string().log_color_error_highlight() )); validation.pop_context(); From 6b3f266c9b54a515423db251e4dd559dd77130e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Mon, 18 Nov 2024 16:12:30 -0500 Subject: [PATCH 075/109] interface extraction: fix exporting / importing the same interface --- wasm-rpc-stubgen/src/wit_generate.rs | 26 ++++++++++++------- .../wit/many-ways-to-export/main.wit | 1 + 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/wasm-rpc-stubgen/src/wit_generate.rs b/wasm-rpc-stubgen/src/wit_generate.rs index d7f94241..01d5243d 100644 --- a/wasm-rpc-stubgen/src/wit_generate.rs +++ b/wasm-rpc-stubgen/src/wit_generate.rs @@ -711,18 +711,17 @@ fn extract_main_interface_package( let mut interface_package = package.clone(); interface_package.set_name(naming::wit::interface_encoder_package_name(package.name())); - let interface_export_prefix = format!( + let interface_prefix = format!( "{}:{}/", package.name().namespace(), interface_package.name().name() ); - let interface_export_suffix = package + let interface_suffix = package .name() .version() .map(|version| format!("@{}", version)) .unwrap_or_default(); - // TODO: is import and exporting the same interface allowed? let mut exported_interface_identifiers = HashSet::::new(); let mut inline_interface_exports = BTreeMap::>::new(); let mut inline_function_exports = BTreeMap::>::new(); @@ -772,11 +771,11 @@ fn extract_main_interface_package( if inline_function_exports.contains_key(&world_name) { world.named_interface_export(format!( "{}{}{}", - interface_export_prefix, + interface_prefix, naming::wit::interface_package_world_inline_functions_interface_name( &world_name ), - interface_export_suffix + interface_suffix )); } } @@ -799,17 +798,26 @@ fn extract_main_interface_package( PackageItem::World(_) => false, }); - // Rename named self imports to use the extracted interface names + // Rename named self export and imports to use the extracted interface names for package_item in package.items_mut() { if let PackageItem::World(world) = package_item { for world_item in world.items_mut() { - if let WorldItem::NamedInterfaceExport(export) = world_item { + if let WorldItem::NamedInterfaceImport(import) = world_item { + if !import.name().raw_name().contains("/") { + import.set_name(format!( + "{}{}{}", + interface_prefix, + import.name(), + interface_suffix + )); + } + } else if let WorldItem::NamedInterfaceExport(export) = world_item { if !export.name().raw_name().contains("/") { export.set_name(format!( "{}{}{}", - interface_export_prefix, + interface_prefix, export.name(), - interface_export_suffix + interface_suffix )); } } diff --git a/wasm-rpc-stubgen/test-data/wit/many-ways-to-export/main.wit b/wasm-rpc-stubgen/test-data/wit/many-ways-to-export/main.wit index b4951dd5..b33e3087 100644 --- a/wasm-rpc-stubgen/test-data/wit/many-ways-to-export/main.wit +++ b/wasm-rpc-stubgen/test-data/wit/many-ways-to-export/main.wit @@ -56,6 +56,7 @@ world api { export func1: func(); export iface1; + import iface1; // TODO: // export iface2; From d2a7114d33853940e0a569bc05871b1c508cc85b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 19 Nov 2024 11:36:14 -0500 Subject: [PATCH 076/109] use id for dep order building, collect loops properly --- wasm-rpc-stubgen/src/wit_resolve.rs | 130 ++++++++++++++++------------ 1 file changed, 75 insertions(+), 55 deletions(-) diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index b3372ee0..de9446a2 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -448,75 +448,95 @@ impl ResolvedWitApplication { } fn sort_components_by_input_deps(&mut self, validation: &mut ValidationBuilder) { - let mut component_order = Vec::with_capacity(self.components.len()); + struct Visit<'a> { + resolved_app: &'a ResolvedWitApplication, + component_names_by_id: Vec<&'a ComponentName>, + component_names_to_id: HashMap<&'a ComponentName, usize>, + visited: HashSet, + visiting: HashSet, + path: Vec, + component_order: Vec, + } - // TODO: use some id instead of Strings? - let mut visited = HashSet::::new(); - let mut visiting = HashSet::::new(); + impl<'a> Visit<'a> { + fn new(resolved_app: &'a ResolvedWitApplication) -> Self { + let component_names_by_id = resolved_app.components.keys().collect::>(); + let component_names_to_id = component_names_by_id + .iter() + .enumerate() + .map(|(id, name)| (*name, id)) + .collect(); - fn visit( - resolved_app: &ResolvedWitApplication, - visited: &mut HashSet, - visiting: &mut HashSet, - component_order: &mut Vec, - component_name: &ComponentName, - input_deps: &BTreeSet, - ) -> bool { - if visited.contains(component_name) { - true - } else if visiting.contains(component_name) { - false - } else { - visiting.insert(component_name.clone()); - - for dep_component_name in input_deps { - if !visit( - resolved_app, - visited, - visiting, - component_order, - dep_component_name, - &resolved_app - .components - .get(dep_component_name) - .unwrap() - .input_component_deps, - ) { - return false; + Self { + resolved_app, + component_names_by_id, + component_names_to_id, + visited: Default::default(), + visiting: Default::default(), + path: vec![], + component_order: vec![], + } + } + + fn visit_all(mut self) -> Result, Vec<&'a ComponentName>> { + for (component_id, &component_name) in + self.component_names_by_id.clone().iter().enumerate() + { + if !self.visited.contains(&component_id) + && !self.visit(component_id, component_name) + { + return Err(self + .path + .into_iter() + .map(|component_id| self.component_names_by_id[component_id]) + .collect::>()); } } + Ok(self.component_order) + } - visiting.remove(component_name); - visited.insert(component_name.clone()); + fn visit(&mut self, component_id: usize, component_name: &ComponentName) -> bool { + if self.visited.contains(&component_id) { + true + } else if self.visiting.contains(&component_id) { + self.path.push(component_id); + false + } else { + let component = self.resolved_app.components.get(component_name).unwrap(); + + self.path.push(component_id); + self.visiting.insert(component_id); + + for dep_component_name in &component.input_component_deps { + if !self.visit( + self.component_names_to_id[dep_component_name], + dep_component_name, + ) { + return false; + } + } - component_order.push(component_name.clone()); + self.visiting.remove(&component_id); + self.visited.insert(component_id); + self.component_order.push(component_name.clone()); + self.path.pop(); - true + true + } } } - for (component_name, component) in &self.components { - if !visited.contains(component_name) - && !visit( - self, - &mut visited, - &mut visiting, - &mut component_order, - component_name, - &component.input_component_deps, - ) - { - // TODO: better error message, collect full path - component_order.push(component_name.clone()); + match Visit::new(&self).visit_all() { + Ok(component_order) => { + self.component_order = component_order; + } + Err(recursive_path) => { validation.add_error(format!( - "Found component cycle: {}", - component_order.iter().map(|s| s.as_str()).join(", ") + "Found component interface package use cycle: {}", + recursive_path.iter().map(|s| s.as_str()).join(", ") )); - break; } } - - self.component_order = component_order; } pub fn component_order(&self) -> &[ComponentName] { From 5696a32078b6506e12ea5d58ed2023c0763ffe86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 19 Nov 2024 11:37:36 -0500 Subject: [PATCH 077/109] highlight --- wasm-rpc-stubgen/src/wit_resolve.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index de9446a2..3d1037c1 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -533,7 +533,10 @@ impl ResolvedWitApplication { Err(recursive_path) => { validation.add_error(format!( "Found component interface package use cycle: {}", - recursive_path.iter().map(|s| s.as_str()).join(", ") + recursive_path + .iter() + .map(|s| s.as_str().log_color_error_highlight()) + .join(", ") )); } } From f2d5313d4eddabd9cce31d10548b780de427b214 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 19 Nov 2024 11:39:39 -0500 Subject: [PATCH 078/109] clippy --- wasm-rpc-stubgen/src/wit_resolve.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index 3d1037c1..83acafd9 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -526,7 +526,7 @@ impl ResolvedWitApplication { } } - match Visit::new(&self).visit_all() { + match Visit::new(self).visit_all() { Ok(component_order) => { self.component_order = component_order; } From 0847d4cbce78f71bfc9a109e7261cc385a029ff8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Thu, 21 Nov 2024 11:15:12 -0500 Subject: [PATCH 079/109] build profiles and component templates --- Cargo.lock | 10 + wasm-rpc-stubgen/Cargo.toml | 1 + wasm-rpc-stubgen/src/commands/declarative.rs | 174 +++-- wasm-rpc-stubgen/src/lib.rs | 4 + wasm-rpc-stubgen/src/model/wasm_rpc.rs | 722 +++++++++++++++---- wasm-rpc-stubgen/src/wit_resolve.rs | 4 +- 6 files changed, 712 insertions(+), 203 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index acadca8b..9815011e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1923,6 +1923,7 @@ dependencies = [ "indexmap 2.6.0", "indoc", "itertools 0.12.1", + "minijinja", "pretty_env_logger", "prettyplease", "proc-macro2", @@ -2680,6 +2681,15 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minijinja" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c37e1b517d1dcd0e51dc36c4567b9d5a29262b3ec8da6cb5d35e27a8fb529b5" +dependencies = [ + "serde 1.0.210", +] + [[package]] name = "miniz_oxide" version = "0.8.0" diff --git a/wasm-rpc-stubgen/Cargo.toml b/wasm-rpc-stubgen/Cargo.toml index 0949144a..d7980652 100644 --- a/wasm-rpc-stubgen/Cargo.toml +++ b/wasm-rpc-stubgen/Cargo.toml @@ -46,6 +46,7 @@ id-arena = "2.2.1" indexmap = "2.2.6" indoc = "2.0.5" itertools = "0.12.1" +minijinja = "2.5.0" pretty_env_logger = "0.5.0" prettyplease = "0.2.20" proc-macro2 = "1.0.85" diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index 6a654916..5c02471d 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -7,8 +7,8 @@ use crate::log::{ }; use crate::model::oam; use crate::model::wasm_rpc::{ - include_glob_patter_from_yaml_file, init_oam_app, Application, ComponentName, - DEFAULT_CONFIG_FILE_NAME, + include_glob_patter_from_yaml_file, init_oam_app, Application, BuildStep, + BuildStepsLookupResult, ComponentName, WasmComponent, DEFAULT_CONFIG_FILE_NAME, }; use crate::stub::{StubConfig, StubDefinition}; use crate::validation::ValidatedResult; @@ -33,6 +33,7 @@ use walkdir::WalkDir; pub struct Config { pub app_resolve_mode: ApplicationSourceMode, pub skip_up_to_date_checks: bool, + pub profile: Option, } #[derive(Debug, Clone)] @@ -51,7 +52,7 @@ struct ApplicationContext { impl ApplicationContext { fn new(config: Config) -> anyhow::Result { - to_anyhow( + let ctx = to_anyhow( "Failed to create application context, see problems above", load_app_validated(&config).and_then(|application| { ResolvedWitApplication::new(&application).map(|wit| ApplicationContext { @@ -62,7 +63,23 @@ impl ApplicationContext { component_base_output_wit_deps: HashMap::new(), }) }), - ) + )?; + + if let Some(profile) = &ctx.config.profile { + let all_profiles = ctx.application.all_profiles(); + if !all_profiles.contains(profile) { + bail!( + "Profile {} not found. Available profiles: {}", + profile.log_color_error_highlight(), + all_profiles + .into_iter() + .map(|s| s.log_color_highlight()) + .join(", ") + ); + } + } + + Ok(ctx) } fn update_wit_context(&mut self) -> anyhow::Result<()> { @@ -143,7 +160,7 @@ async fn pre_component_build_ctx(ctx: &mut ApplicationContext) -> anyhow::Result { let mut any_changed = false; - for component_name in ctx.application.wasm_components_by_name.keys() { + for component_name in ctx.application.component_names() { let changed = create_output_wit(ctx, component_name)?; if changed { update_cargo_toml(ctx, component_name)?; @@ -167,36 +184,79 @@ fn component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<()> { log_action("Executing", "component-build steps"); let _indent = LogIndent::new(); - let components_with_build_steps = ctx - .application - .wasm_components_by_name - .values() - .filter(|component| !component.build_steps.is_empty()) - .collect::>(); - - if components_with_build_steps.is_empty() { - log_warn_action( - "Skipping", - "building components, no components with build steps found", - ); - return Ok(()); - } - log_action("Building", "components"); let _indent = LogIndent::new(); - for component in components_with_build_steps { - log_action( - "Building", - format!( - "component {} in directory {}", - component.name.log_color_highlight(), - component.source_dir().log_color_highlight(), - ), - ); + for (component_name, component) in ctx.application.components() { + let build_steps = match ctx + .application + .component_build_steps(component_name, ctx.config.profile.as_deref()) + { + BuildStepsLookupResult::NoBuildSteps => { + log_warn_action( + "Skipping", + format!( + "building {}, no build steps are defined", + component_name.log_color_highlight() + ), + ); + None + } + BuildStepsLookupResult::NoBuildStepsForRequestedProfile => { + log_warn_action( + "Skipping", + format!( + "building {}, no build steps for profile {}", + component_name.log_color_highlight(), + ctx.config.profile.as_ref().unwrap().log_color_highlight() + ), + ); + None + } + BuildStepsLookupResult::BuildSteps { build_steps } => { + log_action( + "Building", + format!("{}", component_name.log_color_highlight()), + ); + Some(build_steps) + } + BuildStepsLookupResult::BuildStepsForDefaultProfile { + profile, + build_steps, + } => { + log_action( + "Building", + format!( + "{} using requested profile {}", + component_name.log_color_highlight(), + profile.log_color_highlight() + ), + ); + Some(build_steps) + } + BuildStepsLookupResult::BuildStepsForRequestedProfile { + profile, + build_steps, + } => { + log_action( + "Building", + format!( + "{} using default profile {}", + component_name.log_color_highlight(), + profile.log_color_highlight() + ), + ); + Some(build_steps) + } + }; + + let Some(build_steps) = build_steps else { + continue; + }; + let _indent = LogIndent::new(); - for build_step in &component.build_steps { + for build_step in build_steps { let build_dir = build_step .dir .as_ref() @@ -209,8 +269,9 @@ fn component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<()> { if is_up_to_date(ctx.config.skip_up_to_date_checks, || inputs, || outputs) { log_skipping_up_to_date(format!( - "executing command: {}", - build_step.command.log_color_highlight() + "executing command: {} in directory {}", + build_step.command.log_color_highlight(), + build_dir.log_color_highlight() )); continue; } @@ -256,7 +317,7 @@ async fn post_component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<() log_action("Executing", "post-component-build steps"); let _indent = LogIndent::new(); - for (component_name, component) in &ctx.application.wasm_components_by_name { + for (component_name, component) in ctx.application.components() { let input_wasm = ctx.application.component_input_wasm(component_name); let output_wasm = ctx.application.component_output_wasm(component_name); @@ -346,18 +407,11 @@ pub fn clean(config: Config) -> anyhow::Result<()> { log_action("Cleaning", "components"); let _indent = LogIndent::new(); - for (component_name, component) in &app.wasm_components_by_name { - log_action( - "Cleaning", - format!("component {}", component_name.log_color_highlight()), - ); - let _indent = LogIndent::new(); - - delete_path("wit output dir", &app.component_output_wit(component_name))?; - delete_path("wasm input", &app.component_input_wasm(component_name))?; - delete_path("wasm output", &app.component_output_wasm(component_name))?; - - for build_step in &component.build_steps { + fn clean_build_steps( + component: &WasmComponent, + build_steps: &[BuildStep], + ) -> anyhow::Result<()> { + for build_step in build_steps { let build_dir = build_step .dir .as_ref() @@ -369,6 +423,31 @@ pub fn clean(config: Config) -> anyhow::Result<()> { delete_path("build step output", &output)?; } } + + Ok(()) + } + + for (component_name, component) in app.components() { + log_action( + "Cleaning", + format!("component {}", component_name.log_color_highlight()), + ); + let _indent = LogIndent::new(); + + delete_path("wit output dir", &app.component_output_wit(component_name))?; + delete_path("wasm input", &app.component_input_wasm(component_name))?; + delete_path("wasm output", &app.component_output_wasm(component_name))?; + + let component_build = app.component_properties(component_name); + clean_build_steps(component, &component_build.build)?; + for (profile, build_steps) in &component_build.build_profiles { + log_action( + "Cleaning", + format!("profile {}", profile.log_color_highlight()), + ); + let _indent = LogIndent::new(); + clean_build_steps(component, build_steps)?; + } } } @@ -436,13 +515,12 @@ fn load_app_validated(config: &Config) -> ValidatedResult { let app = oam_apps.and_then(Application::from_oam_apps); log_validated_action_result("Found", &app, |app| { - if app.wasm_components_by_name.is_empty() { + if app.component_names().next().is_none() { "no components".to_string() } else { format!( "components: {}", - app.wasm_components_by_name - .keys() + app.component_names() .map(|s| s.log_color_highlight()) .join(", ") ) diff --git a/wasm-rpc-stubgen/src/lib.rs b/wasm-rpc-stubgen/src/lib.rs index 88cc2994..e995a539 100644 --- a/wasm-rpc-stubgen/src/lib.rs +++ b/wasm-rpc-stubgen/src/lib.rs @@ -220,6 +220,9 @@ pub struct DeclarativeBuildArgs { /// When set to true will skip modification time based up-to-date checks, defaults to false #[clap(long, short, default_value = "false")] pub force_build: bool, + /// Selects a build profile + #[clap(long, short)] + pub profile: Option, } pub fn generate(args: GenerateArgs) -> anyhow::Result<()> { @@ -312,5 +315,6 @@ fn dec_build_args_to_config(args: DeclarativeBuildArgs) -> commands::declarative } }, skip_up_to_date_checks: args.force_build, + profile: args.profile, } } diff --git a/wasm-rpc-stubgen/src/model/wasm_rpc.rs b/wasm-rpc-stubgen/src/model/wasm_rpc.rs index a0af5b74..7c4f4a6c 100644 --- a/wasm-rpc-stubgen/src/model/wasm_rpc.rs +++ b/wasm-rpc-stubgen/src/model/wasm_rpc.rs @@ -1,14 +1,14 @@ use crate::log::LogColorize; use crate::model::oam; use crate::model::oam::TypedTraitProperties; -use crate::model::unknown_properties::{HasUnknownProperties, UnknownProperties}; +use crate::model::unknown_properties::HasUnknownProperties; +use crate::model::wasm_rpc::template::Template; use crate::naming::wit::package_dep_dir_name_from_parser; use crate::validation::{ValidatedResult, ValidationBuilder}; use crate::{fs, naming}; use golem_wasm_rpc::WASM_RPC_VERSION; use itertools::Itertools; -use serde::{Deserialize, Serialize}; -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::fmt::Display; use std::fmt::Formatter; use std::path::{Path, PathBuf}; @@ -16,12 +16,6 @@ use wit_parser::PackageName; pub const DEFAULT_CONFIG_FILE_NAME: &str = "golem.yaml"; -pub const OAM_TRAIT_TYPE_WASM_RPC: &str = "wasm-rpc"; - -pub const OAM_COMPONENT_TYPE_WASM: &str = "wasm"; -pub const OAM_COMPONENT_TYPE_WASM_BUILD: &str = "wasm-build"; -pub const OAM_COMPONENT_TYPE_WASM_RPC_STUB_BUILD: &str = "wasm-rpc-stub-build"; - #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct ComponentName(String); @@ -49,6 +43,260 @@ impl From<&str> for ComponentName { } } +mod raw { + use crate::model::oam; + use crate::model::unknown_properties::{HasUnknownProperties, UnknownProperties}; + use serde::{Deserialize, Serialize}; + use std::collections::HashMap; + + pub const OAM_TRAIT_TYPE_WASM_RPC: &str = "wasm-rpc"; + + pub const OAM_COMPONENT_TYPE_WASM: &str = "wasm"; + pub const OAM_COMPONENT_TYPE_WASM_BUILD: &str = "wasm-build"; + pub const OAM_COMPONENT_TYPE_WASM_RPC_STUB_BUILD: &str = "wasm-rpc-stub-build"; + + #[derive(Clone, Debug, Serialize, Deserialize)] + pub struct BuildStep { + pub command: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub dir: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub inputs: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub outputs: Vec, + } + + #[derive(Clone, Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct WasmComponentProperties { + pub component_template: Option, + pub input_wit: Option, + pub output_wit: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub build: Vec, + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub build_profiles: HashMap>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub default_build_profile: Option, + pub input_wasm: Option, + pub output_wasm: Option, + #[serde(flatten)] + pub unknown_properties: UnknownProperties, + } + + #[derive(Clone, Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct WasmComponentTemplateProperties { + pub input_wit: String, + pub output_wit: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub build: Vec, + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub build_profiles: HashMap>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub default_build_profile: Option, + pub input_wasm: String, + pub output_wasm: String, + #[serde(flatten)] + pub unknown_properties: UnknownProperties, + } + + impl HasUnknownProperties for WasmComponentProperties { + fn unknown_properties(&self) -> &UnknownProperties { + &self.unknown_properties + } + } + + impl oam::TypedComponentProperties for WasmComponentProperties { + fn component_type() -> &'static str { + OAM_COMPONENT_TYPE_WASM + } + } + + #[derive(Clone, Debug, Serialize, Deserialize)] + pub struct WasmRpcTraitProperties { + #[serde(rename = "componentName")] + pub component_name: String, + #[serde(flatten)] + pub unknown_properties: UnknownProperties, + } + + impl HasUnknownProperties for WasmRpcTraitProperties { + fn unknown_properties(&self) -> &UnknownProperties { + &self.unknown_properties + } + } + + impl oam::TypedTraitProperties for WasmRpcTraitProperties { + fn trait_type() -> &'static str { + OAM_TRAIT_TYPE_WASM_RPC + } + } + + #[derive(Clone, Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct ComponentBuildProperties { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub include: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub build_dir: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub wit_deps: Vec, + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub component_templates: HashMap, + #[serde(flatten)] + pub unknown_properties: UnknownProperties, + } + + impl oam::TypedComponentProperties for ComponentBuildProperties { + fn component_type() -> &'static str { + OAM_COMPONENT_TYPE_WASM_BUILD + } + } + + impl HasUnknownProperties for ComponentBuildProperties { + fn unknown_properties(&self) -> &UnknownProperties { + &self.unknown_properties + } + } + + #[derive(Clone, Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct ComponentStubBuildProperties { + pub component_name: Option, + pub build_dir: Option, + pub wasm: Option, + pub wit: Option, + pub world: Option, + pub crate_version: Option, + pub wasm_rpc_path: Option, + pub wasm_rpc_version: Option, + #[serde(flatten)] + pub unknown_properties: UnknownProperties, + } + + impl oam::TypedComponentProperties for ComponentStubBuildProperties { + fn component_type() -> &'static str { + OAM_COMPONENT_TYPE_WASM_RPC_STUB_BUILD + } + } + + impl HasUnknownProperties for ComponentStubBuildProperties { + fn unknown_properties(&self) -> &UnknownProperties { + &self.unknown_properties + } + } +} + +mod template { + use crate::model::wasm_rpc::raw::{BuildStep, WasmComponentTemplateProperties}; + use crate::model::wasm_rpc::WasmComponentProperties; + use serde::Serialize; + use std::collections::HashMap; + + pub trait Template { + type Rendered; + + fn render( + &self, + env: &minijinja::Environment, + ctx: &C, + ) -> Result; + } + + impl Template for String { + type Rendered = String; + + fn render( + &self, + env: &minijinja::Environment, + ctx: &C, + ) -> Result { + env.render_str(self, ctx) + } + } + + impl> Template for Option { + type Rendered = Option; + + fn render( + &self, + env: &minijinja::Environment, + ctx: &C, + ) -> Result { + match self { + Some(template) => Ok(Some(template.render(env, ctx)?)), + None => Ok(None), + } + } + } + + impl> Template for Vec { + type Rendered = Vec; + + fn render( + &self, + env: &minijinja::Environment, + ctx: &C, + ) -> Result { + self.iter().map(|elem| elem.render(env, ctx)).collect() + } + } + + impl> Template for HashMap { + type Rendered = HashMap; + + fn render( + &self, + env: &minijinja::Environment, + ctx: &C, + ) -> Result { + let mut rendered = HashMap::::new(); + for (key, template) in self { + rendered.insert(key.clone(), template.render(env, ctx)?); + } + Ok(rendered) + } + } + + impl Template for BuildStep { + type Rendered = BuildStep; + + fn render( + &self, + env: &minijinja::Environment, + ctx: &C, + ) -> Result { + Ok(BuildStep { + command: self.command.render(env, ctx)?, + dir: self.dir.render(env, ctx)?, + inputs: self.inputs.render(env, ctx)?, + outputs: self.outputs.render(env, ctx)?, + }) + } + } + + impl Template for WasmComponentTemplateProperties { + type Rendered = WasmComponentProperties; + + fn render( + &self, + env: &minijinja::Environment, + ctx: &C, + ) -> Result { + Ok(WasmComponentProperties { + build: self.build.render(env, ctx)?, + build_profiles: self.build_profiles.render(env, ctx)?, + default_build_profile: self.default_build_profile.render(env, ctx)?, + input_wit: self.input_wit.render(env, ctx)?.into(), + output_wit: self.output_wit.render(env, ctx)?.into(), + input_wasm: self.input_wasm.render(env, ctx)?.into(), + output_wasm: self.output_wasm.render(env, ctx)?.into(), + }) + } + } +} + pub fn init_oam_app(_component_name: ComponentName) -> oam::Application { // TODO: let's do it as part of https://github.com/golemcloud/wasm-rpc/issues/89 todo!() @@ -63,13 +311,13 @@ pub fn include_glob_patter_from_yaml_file(source: &Path) -> Option { .and_then(|mut oam_app| { let mut includes = oam_app .spec - .extract_components_by_type(&BTreeSet::from([OAM_COMPONENT_TYPE_WASM_BUILD])) - .remove(OAM_COMPONENT_TYPE_WASM_BUILD) + .extract_components_by_type(&BTreeSet::from([raw::OAM_COMPONENT_TYPE_WASM_BUILD])) + .remove(raw::OAM_COMPONENT_TYPE_WASM_BUILD) .unwrap_or_default() .into_iter() .filter_map(|component| { component - .typed_properties::() + .typed_properties::() .ok() .and_then(|properties| properties.include) }); @@ -86,10 +334,11 @@ pub fn include_glob_patter_from_yaml_file(source: &Path) -> Option { #[derive(Clone, Debug)] pub struct Application { - pub common_wasm_build: Option, - pub common_wasm_rpc_stub_build: Option, - pub wasm_rpc_stub_builds_by_name: BTreeMap, - pub wasm_components_by_name: BTreeMap, + common_wasm_build: Option, + common_wasm_rpc_stub_build: Option, + wasm_rpc_stub_builds_by_name: BTreeMap, + wasm_components_by_name: BTreeMap, + wasm_component_rendered_templates_by_name: BTreeMap, } impl Application { @@ -123,11 +372,82 @@ impl Application { let common_wasm_build = Self::validate_wasm_builds(&mut validation, all_wasm_builds); + let wasm_component_rendered_templates_by_name = { + let env = minijinja::Environment::new(); + + wasm_components_by_name + .iter() + .filter_map(|(component_name, component)| { + if let WasmComponentPropertySource::Template { template_name } = + &component.properties + { + Some((component_name, template_name)) + } else { + None + } + }) + .filter_map(|(component_name, template_name)| { + let templates = common_wasm_build + .as_ref() + .map(|build| &build.component_templates); + + match templates.and_then(|templates| templates.get(template_name)) { + Some(template) => { + match template.render( + &env, + &minijinja::context! { componentName => component_name.as_str() }, + ) { + Ok(properties) => Some((component_name.clone(), properties)), + Err(err) => { + validation + .push_context("component_name", component_name.to_string()); + validation + .push_context("template_name", template_name.to_string()); + + validation.add_error(format!( + "Failed to render component template: {}", + err + )); + validation.pop_context(); + validation.pop_context(); + + None + } + } + } + None => { + validation.add_error(format!( + "Component template {} not found, {}", + template_name.log_color_error_highlight(), + match templates { + Some(templates) if !templates.is_empty() => { + format!( + "available templates: {}", + templates + .keys() + .map(|s| s.log_color_highlight()) + .join(", ") + ) + } + _ => { + "no templates are defined".to_string() + } + } + )); + validation.pop_context(); + None + } + } + }) + .collect() + }; + validation.build(Self { common_wasm_build, common_wasm_rpc_stub_build, wasm_rpc_stub_builds_by_name, wasm_components_by_name, + wasm_component_rendered_templates_by_name, }) } @@ -142,16 +462,16 @@ impl Application { .application .spec .extract_components_by_type(&BTreeSet::from([ - OAM_COMPONENT_TYPE_WASM, - OAM_COMPONENT_TYPE_WASM_BUILD, - OAM_COMPONENT_TYPE_WASM_RPC_STUB_BUILD, + raw::OAM_COMPONENT_TYPE_WASM, + raw::OAM_COMPONENT_TYPE_WASM_BUILD, + raw::OAM_COMPONENT_TYPE_WASM_RPC_STUB_BUILD, ])); let wasm_components = Self::convert_components( &oam_app.source, validation, &mut components_by_type, - OAM_COMPONENT_TYPE_WASM, + raw::OAM_COMPONENT_TYPE_WASM, Self::convert_wasm_component, ); @@ -159,7 +479,7 @@ impl Application { &oam_app.source, validation, &mut components_by_type, - OAM_COMPONENT_TYPE_WASM_BUILD, + raw::OAM_COMPONENT_TYPE_WASM_BUILD, Self::convert_wasm_build, ); @@ -167,7 +487,7 @@ impl Application { &oam_app.source, validation, &mut components_by_type, - OAM_COMPONENT_TYPE_WASM_RPC_STUB_BUILD, + raw::OAM_COMPONENT_TYPE_WASM_RPC_STUB_BUILD, Self::convert_wasm_rpc_stub_build, ); @@ -216,22 +536,22 @@ impl Application { validation: &mut ValidationBuilder, mut component: oam::Component, ) -> Option { - let properties = component.typed_properties::(); + let properties = component.typed_properties::(); if let Some(err) = properties.as_ref().err() { validation.add_error(format!("Failed to get component properties: {}", err)) } let wasm_rpc_traits = component - .extract_traits_by_type(&BTreeSet::from([OAM_TRAIT_TYPE_WASM_RPC])) - .remove(OAM_TRAIT_TYPE_WASM_RPC) + .extract_traits_by_type(&BTreeSet::from([raw::OAM_TRAIT_TYPE_WASM_RPC])) + .remove(raw::OAM_TRAIT_TYPE_WASM_RPC) .unwrap_or_default(); let mut wasm_rpc_dependencies = Vec::::new(); for wasm_rpc in wasm_rpc_traits { validation.push_context("trait type", wasm_rpc.trait_type.clone()); - match WasmRpcTraitProperties::from_generic_trait(wasm_rpc) { + match raw::WasmRpcTraitProperties::from_generic_trait(wasm_rpc) { Ok(wasm_rpc) => { wasm_rpc.add_unknown_property_warns( || vec![("dep component name", wasm_rpc.component_name.clone())], @@ -298,16 +618,97 @@ impl Application { } } - Some(WasmComponent { - name: component.name.into(), - source: source.to_path_buf(), - build_steps: properties.build, - input_wit: properties.input_wit.into(), - output_wit: properties.output_wit.into(), - input_wasm: properties.input_wasm.into(), - output_wasm: properties.output_wasm.into(), - wasm_rpc_dependencies, - }) + if !properties.build_profiles.is_empty() && !properties.build.is_empty() { + validation.add_warn( + "If buildProfiles is defined then build will be ignored".to_string(), + ); + } + + if !properties.build_profiles.is_empty() + && properties.default_build_profile.is_none() + { + validation.add_error("If buildProfiles is defined then defaultBuildProfile also have to be defined".to_string()); + } + + if properties.build_profiles.is_empty() + && properties.default_build_profile.is_some() + { + validation.add_error("If defaultBuildProfile is defined then buildProfiles also have to be defined".to_string()); + } + + if let Some(default_build_profile) = &properties.default_build_profile { + if !properties + .build_profiles + .contains_key(default_build_profile) + { + validation.add_error(format!( + "The defined defaultBuildProfile ({}) if not found in buildProfiles", + default_build_profile.log_color_error_highlight() + )) + } + } + + match properties.component_template { + Some(build_template) => { + for (property_defined, property_name) in [ + (properties.input_wit.is_some(), "inputWit"), + (properties.output_wit.is_some(), "outputWit"), + (!properties.build.is_empty(), "build"), + (!properties.build_profiles.is_empty(), "buildProfiles"), + ( + properties.default_build_profile.is_some(), + "defaultBuildProfile", + ), + (properties.input_wasm.is_some(), "inputWasm"), + (properties.output_wasm.is_some(), "outputWasm"), + ] { + if property_defined { + validation.add_warn(format!( + "Component property {} is ignored when componentTemplate is defined", + property_name.log_color_error_highlight() + )) + } + } + + Some(WasmComponent { + name: component.name.into(), + source: source.to_path_buf(), + properties: WasmComponentPropertySource::Template { + template_name: build_template, + }, + wasm_rpc_dependencies, + }) + } + None => { + for (property, property_name) in [ + (&properties.input_wit, "inputWit"), + (&properties.output_wit, "outputWit"), + (&properties.input_wasm, "inputWasm"), + (&properties.output_wasm, "outputWasm"), + ] { + if property.is_none() { + validation.add_error(format!("Component property {} must be defined, unless componentTemplate is defined", property_name.log_color_error_highlight())) + } + } + + Some(WasmComponent { + name: component.name.into(), + source: source.to_path_buf(), + properties: WasmComponentPropertySource::Concrete { + build: WasmComponentProperties { + build: properties.build, + build_profiles: properties.build_profiles, + default_build_profile: properties.default_build_profile, + input_wit: properties.input_wit.unwrap_or_default().into(), + output_wit: properties.output_wit.unwrap_or_default().into(), + input_wasm: properties.input_wasm.unwrap_or_default().into(), + output_wasm: properties.output_wasm.unwrap_or_default().into(), + }, + }, + wasm_rpc_dependencies, + }) + } + } } _ => None, } @@ -318,15 +719,17 @@ impl Application { validation: &mut ValidationBuilder, component: oam::Component, ) -> Option { - let result = match component.typed_properties::() { + let result = match component.typed_properties::() { Ok(properties) => { + // TODO: validate component templates properties.add_unknown_property_warns(Vec::new, validation); let wasm_rpc_stub_build = WasmBuild { source: source.to_path_buf(), name: component.name, build_dir: properties.build_dir.map(|s| s.into()), - _wit_deps: properties.wit_deps.into_iter().map(|s| s.into()).collect(), + component_templates: properties.component_templates, + wit_deps: properties.wit_deps.into_iter().map(|s| s.into()).collect(), }; Some(wasm_rpc_stub_build) @@ -355,7 +758,7 @@ impl Application { validation: &mut ValidationBuilder, component: oam::Component, ) -> Option { - let result = match component.typed_properties::() { + let result = match component.typed_properties::() { Ok(properties) => { properties.add_unknown_property_warns(Vec::new, validation); @@ -493,6 +896,12 @@ impl Application { )); } + if wasm_builds.len() == 1 { + for (_template_name, _build) in &wasm_builds[0].component_templates { + // TODO: validate templates + } + } + wasm_builds.into_iter().next() } @@ -591,10 +1000,18 @@ impl Application { ) } + pub fn components(&self) -> impl Iterator { + self.wasm_components_by_name.iter() + } + + pub fn component_names(&self) -> impl Iterator { + self.wasm_components_by_name.keys() + } + pub fn wit_deps(&self) -> Vec { self.common_wasm_build .as_ref() - .map(|wasm_build| wasm_build._wit_deps.clone()) + .map(|wasm_build| wasm_build.wit_deps.clone()) .unwrap_or_default() } @@ -605,6 +1022,17 @@ impl Application { .collect() } + pub fn all_profiles(&self) -> BTreeSet { + self.component_names() + .flat_map(|component_name| { + self.component_properties(component_name) + .build_profiles + .keys() + .cloned() + }) + .collect() + } + pub fn build_dir(&self) -> PathBuf { self.common_wasm_build .as_ref() @@ -618,9 +1046,21 @@ impl Application { .unwrap_or_else(|| panic!("Component not found: {}", component_name)) } + pub fn component_properties(&self, component_name: &ComponentName) -> &WasmComponentProperties { + match &self.component(component_name).properties { + WasmComponentPropertySource::Concrete { build } => build, + WasmComponentPropertySource::Template { .. } => self + .wasm_component_rendered_templates_by_name + .get(component_name) + .expect("Missing rendered template"), + } + } + pub fn component_input_wit(&self, component_name: &ComponentName) -> PathBuf { let component = self.component(component_name); - component.source_dir().join(component.input_wit.clone()) + component + .source_dir() + .join(self.component_properties(component_name).input_wit.clone()) } pub fn component_base_output_wit(&self, component_name: &ComponentName) -> PathBuf { @@ -642,17 +1082,64 @@ impl Application { pub fn component_output_wit(&self, component_name: &ComponentName) -> PathBuf { let component = self.component(component_name); - component.source_dir().join(component.output_wit.clone()) + component + .source_dir() + .join(self.component_properties(component_name).output_wit.clone()) } pub fn component_input_wasm(&self, component_name: &ComponentName) -> PathBuf { let component = self.component(component_name); - component.source_dir().join(component.input_wasm.clone()) + component + .source_dir() + .join(self.component_properties(component_name).input_wasm.clone()) } pub fn component_output_wasm(&self, component_name: &ComponentName) -> PathBuf { let component = self.component(component_name); - component.source_dir().join(component.output_wasm.clone()) + component.source_dir().join( + self.component_properties(component_name) + .output_wasm + .clone(), + ) + } + + pub fn component_build_steps<'a>( + &'a self, + component_name: &ComponentName, + profile: Option<&'a str>, + ) -> BuildStepsLookupResult<'a> { + let component_build = self.component_properties(component_name); + match profile { + Some(profile) => match component_build.build_profiles.get(profile) { + Some(build_steps) => BuildStepsLookupResult::BuildStepsForRequestedProfile { + profile, + build_steps, + }, + None => BuildStepsLookupResult::NoBuildStepsForRequestedProfile, + }, + None => { + if !component_build.build_profiles.is_empty() { + let default_profile = component_build + .default_build_profile + .as_ref() + .expect("Missing build profile"); + + BuildStepsLookupResult::BuildStepsForDefaultProfile { + profile: default_profile, + build_steps: component_build + .build_profiles + .get(default_profile) + .expect("Missing build steps for profile"), + } + } else if !component_build.build.is_empty() { + BuildStepsLookupResult::BuildSteps { + build_steps: &component_build.build, + } + } else { + BuildStepsLookupResult::NoBuildSteps + } + } + } } pub fn stub_world(&self, component_name: &ComponentName) -> Option { @@ -732,15 +1219,30 @@ impl Application { } } +pub type BuildStep = raw::BuildStep; + #[derive(Clone, Debug)] -pub struct WasmComponent { - pub name: ComponentName, - pub source: PathBuf, - pub build_steps: Vec, +pub struct WasmComponentProperties { + pub build: Vec, + pub build_profiles: HashMap>, + pub default_build_profile: Option, pub input_wit: PathBuf, pub output_wit: PathBuf, pub input_wasm: PathBuf, pub output_wasm: PathBuf, +} + +#[derive(Clone, Debug)] +pub enum WasmComponentPropertySource { + Concrete { build: WasmComponentProperties }, + Template { template_name: String }, +} + +#[derive(Clone, Debug)] +pub struct WasmComponent { + pub name: ComponentName, + pub source: PathBuf, + pub properties: WasmComponentPropertySource, pub wasm_rpc_dependencies: Vec, } @@ -760,83 +1262,20 @@ impl WasmComponent { } } -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct BuildStep { - pub command: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub dir: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub inputs: Vec, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub outputs: Vec, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct WasmComponentProperties { - pub input_wit: String, - pub output_wit: String, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub build: Vec, - pub input_wasm: String, - pub output_wasm: String, - #[serde(flatten)] - pub unknown_properties: UnknownProperties, -} - -impl HasUnknownProperties for WasmComponentProperties { - fn unknown_properties(&self) -> &UnknownProperties { - &self.unknown_properties - } -} - -impl oam::TypedComponentProperties for WasmComponentProperties { - fn component_type() -> &'static str { - OAM_COMPONENT_TYPE_WASM - } -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct WasmRpcTraitProperties { - #[serde(rename = "componentName")] - pub component_name: String, - #[serde(flatten)] - pub unknown_properties: UnknownProperties, -} - -impl HasUnknownProperties for WasmRpcTraitProperties { - fn unknown_properties(&self) -> &UnknownProperties { - &self.unknown_properties - } -} - -impl oam::TypedTraitProperties for WasmRpcTraitProperties { - fn trait_type() -> &'static str { - OAM_TRAIT_TYPE_WASM_RPC - } -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ComponentBuildProperties { - include: Option, - build_dir: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - wit_deps: Vec, - #[serde(flatten)] - unknown_properties: UnknownProperties, -} - -impl oam::TypedComponentProperties for ComponentBuildProperties { - fn component_type() -> &'static str { - OAM_COMPONENT_TYPE_WASM_BUILD - } -} - -impl HasUnknownProperties for ComponentBuildProperties { - fn unknown_properties(&self) -> &UnknownProperties { - &self.unknown_properties - } +pub enum BuildStepsLookupResult<'a> { + NoBuildSteps, + NoBuildStepsForRequestedProfile, + BuildSteps { + build_steps: &'a [BuildStep], + }, + BuildStepsForDefaultProfile { + profile: &'a str, + build_steps: &'a [BuildStep], + }, + BuildStepsForRequestedProfile { + profile: &'a str, + build_steps: &'a [BuildStep], + }, } #[derive(Clone, Debug)] @@ -844,34 +1283,8 @@ pub struct WasmBuild { source: PathBuf, name: String, build_dir: Option, - _wit_deps: Vec, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ComponentStubBuildProperties { - component_name: Option, - build_dir: Option, - wasm: Option, - wit: Option, - world: Option, - crate_version: Option, - wasm_rpc_path: Option, - wasm_rpc_version: Option, - #[serde(flatten)] - unknown_properties: UnknownProperties, -} - -impl oam::TypedComponentProperties for ComponentStubBuildProperties { - fn component_type() -> &'static str { - OAM_COMPONENT_TYPE_WASM_RPC_STUB_BUILD - } -} - -impl HasUnknownProperties for ComponentStubBuildProperties { - fn unknown_properties(&self) -> &UnknownProperties { - &self.unknown_properties - } + component_templates: HashMap, + wit_deps: Vec, } #[derive(Clone, Debug)] @@ -927,10 +1340,13 @@ mod tests { assert!(component_name.as_str() == "component-one"); assert!(component.name.as_str() == "component-one"); + // TODO: + /* assert!(component.input_wit.to_string_lossy() == "input_wit"); assert!(component.output_wit.to_string_lossy() == "output_wit"); assert!(component.input_wasm.to_string_lossy() == "out/in.wasm"); assert!(component.output_wasm.to_string_lossy() == "out/out.wasm"); + */ assert!(component.wasm_rpc_dependencies.len() == 2); assert!(component.wasm_rpc_dependencies[0].as_str() == "component-three"); diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index 83acafd9..17f9e25f 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -257,7 +257,7 @@ impl ResolvedWitApplication { } fn add_components_from_app(&mut self, validation: &mut ValidationBuilder, app: &Application) { - for (component_name, component) in &app.wasm_components_by_name { + for (component_name, component) in app.components() { validation.push_context("component name", component_name.to_string()); let input_wit_dir = app.component_input_wit(component_name); @@ -291,7 +291,7 @@ impl ResolvedWitApplication { .collect(); let input_contained_package_deps = { - let deps_path = component.input_wit.join("deps"); + let deps_path = app.component_properties(component_name).input_wit.join("deps"); if !deps_path.exists() { HashSet::new() } else { From 6181c51365c9ff59c2b647eef836955f87c37600 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Thu, 21 Nov 2024 20:51:21 -0500 Subject: [PATCH 080/109] WIP: reworking profiles --- wasm-rpc-stubgen/src/commands/declarative.rs | 185 +++--- wasm-rpc-stubgen/src/lib.rs | 2 +- wasm-rpc-stubgen/src/model/wasm_rpc.rs | 647 +++++++++++++------ wasm-rpc-stubgen/src/wit_resolve.rs | 27 +- 4 files changed, 564 insertions(+), 297 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index 5c02471d..e9126730 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -7,8 +7,8 @@ use crate::log::{ }; use crate::model::oam; use crate::model::wasm_rpc::{ - include_glob_patter_from_yaml_file, init_oam_app, Application, BuildStep, - BuildStepsLookupResult, ComponentName, WasmComponent, DEFAULT_CONFIG_FILE_NAME, + include_glob_patter_from_yaml_file, init_oam_app, Application, BuildStep, ComponentName, + ProfileName, WasmComponent, DEFAULT_CONFIG_FILE_NAME, }; use crate::stub::{StubConfig, StubDefinition}; use crate::validation::ValidatedResult; @@ -22,6 +22,7 @@ use anyhow::{anyhow, bail, Context, Error}; use colored::Colorize; use glob::glob; use itertools::Itertools; +use log::log; use std::cell::OnceCell; use std::cmp::Ordering; use std::collections::HashMap; @@ -33,7 +34,7 @@ use walkdir::WalkDir; pub struct Config { pub app_resolve_mode: ApplicationSourceMode, pub skip_up_to_date_checks: bool, - pub profile: Option, + pub profile: Option, } #[derive(Debug, Clone)] @@ -55,37 +56,58 @@ impl ApplicationContext { let ctx = to_anyhow( "Failed to create application context, see problems above", load_app_validated(&config).and_then(|application| { - ResolvedWitApplication::new(&application).map(|wit| ApplicationContext { - config, - application, - wit, - common_wit_deps: OnceCell::new(), - component_base_output_wit_deps: HashMap::new(), + ResolvedWitApplication::new(&application, config.profile.as_ref()).map(|wit| { + ApplicationContext { + config, + application, + wit, + common_wit_deps: OnceCell::new(), + component_base_output_wit_deps: HashMap::new(), + } }) }), )?; if let Some(profile) = &ctx.config.profile { let all_profiles = ctx.application.all_profiles(); - if !all_profiles.contains(profile) { + if all_profiles.is_empty() { bail!( - "Profile {} not found. Available profiles: {}", - profile.log_color_error_highlight(), + "Profile {} not found, no available profiles", + profile.as_str().log_color_error_highlight(), + ); + } else if !all_profiles.contains(profile) { + bail!( + "Profile {} not found, available profiles: {}", + profile.as_str().log_color_error_highlight(), all_profiles .into_iter() - .map(|s| s.log_color_highlight()) + .map(|s| s.as_str().log_color_highlight()) .join(", ") ); + } else { + log_action( + "Selected", + format!("profile: {}", profile.as_str().log_color_highlight()), + ); } } Ok(ctx) } + fn profile(&self) -> Option<&ProfileName> { + self.config.profile.as_ref() + } + + fn component_matches_profile(&self, component_name: &ComponentName) -> bool { + self.application + .component_matches_profile(component_name, self.profile()) + } + fn update_wit_context(&mut self) -> anyhow::Result<()> { to_anyhow( "Failed to update application wit context, see problems above", - ResolvedWitApplication::new(&self.application).map(|wit| { + ResolvedWitApplication::new(&self.application, self.profile()).map(|wit| { self.wit = wit; }), ) @@ -188,75 +210,40 @@ fn component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<()> { let _indent = LogIndent::new(); for (component_name, component) in ctx.application.components() { - let build_steps = match ctx + if !ctx.component_matches_profile(component_name) { + log_warn_action( + "Skipping", + format!( + "building {}, no build steps for profile {}", + component_name.log_color_highlight(), + ctx.profile().unwrap().as_str().log_color_highlight() + ), + ); + continue; + } + + let component_properties = ctx .application - .component_build_steps(component_name, ctx.config.profile.as_deref()) - { - BuildStepsLookupResult::NoBuildSteps => { - log_warn_action( - "Skipping", - format!( - "building {}, no build steps are defined", - component_name.log_color_highlight() - ), - ); - None - } - BuildStepsLookupResult::NoBuildStepsForRequestedProfile => { - log_warn_action( - "Skipping", - format!( - "building {}, no build steps for profile {}", - component_name.log_color_highlight(), - ctx.config.profile.as_ref().unwrap().log_color_highlight() - ), - ); - None - } - BuildStepsLookupResult::BuildSteps { build_steps } => { - log_action( - "Building", - format!("{}", component_name.log_color_highlight()), - ); - Some(build_steps) - } - BuildStepsLookupResult::BuildStepsForDefaultProfile { - profile, - build_steps, - } => { - log_action( - "Building", - format!( - "{} using requested profile {}", - component_name.log_color_highlight(), - profile.log_color_highlight() - ), - ); - Some(build_steps) - } - BuildStepsLookupResult::BuildStepsForRequestedProfile { - profile, - build_steps, - } => { - log_action( - "Building", - format!( - "{} using default profile {}", - component_name.log_color_highlight(), - profile.log_color_highlight() - ), - ); - Some(build_steps) - } - }; + .component_properties(component_name, ctx.profile()); - let Some(build_steps) = build_steps else { + if component_properties.build.is_empty() { + log_warn_action( + "Skipping", + format!( + "building {}, no build steps", + component_name.log_color_highlight(), + ), + ); continue; - }; + } + log_action( + "Building", + format!("{}", component_name.log_color_highlight()), + ); let _indent = LogIndent::new(); - for build_step in build_steps { + for build_step in &component_properties.build { let build_dir = build_step .dir .as_ref() @@ -318,8 +305,17 @@ async fn post_component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<() let _indent = LogIndent::new(); for (component_name, component) in ctx.application.components() { - let input_wasm = ctx.application.component_input_wasm(component_name); - let output_wasm = ctx.application.component_output_wasm(component_name); + if !ctx.component_matches_profile(component_name) { + // TODO: log + continue; + } + + let input_wasm = ctx + .application + .component_input_wasm(component_name, ctx.profile()); + let output_wasm = ctx + .application + .component_output_wasm(component_name, ctx.profile()); if is_up_to_date( ctx.config.skip_up_to_date_checks, @@ -373,11 +369,11 @@ async fn post_component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<() commands::composition::compose( ctx.application - .component_input_wasm(component_name) + .component_input_wasm(component_name, ctx.profile()) .as_path(), &stub_wasms, ctx.application - .component_output_wasm(component_name) + .component_output_wasm(component_name, ctx.profile()) .as_path(), ) .await?; @@ -427,19 +423,22 @@ pub fn clean(config: Config) -> anyhow::Result<()> { Ok(()) } - for (component_name, component) in app.components() { + for (component_name, _component) in app.components() { log_action( "Cleaning", format!("component {}", component_name.log_color_highlight()), ); let _indent = LogIndent::new(); + // TODO + /* delete_path("wit output dir", &app.component_output_wit(component_name))?; delete_path("wasm input", &app.component_input_wasm(component_name))?; delete_path("wasm output", &app.component_output_wasm(component_name))?; let component_build = app.component_properties(component_name); clean_build_steps(component, &component_build.build)?; + for (profile, build_steps) in &component_build.build_profiles { log_action( "Cleaning", @@ -447,7 +446,7 @@ pub fn clean(config: Config) -> anyhow::Result<()> { ); let _indent = LogIndent::new(); clean_build_steps(component, build_steps)?; - } + }*/ } } @@ -751,7 +750,9 @@ fn create_base_output_wit( ctx: &mut ApplicationContext, component_name: &ComponentName, ) -> Result { - let component_input_wit = ctx.application.component_input_wit(component_name); + let component_input_wit = ctx + .application + .component_input_wit(component_name, ctx.profile()); let component_base_output_wit = ctx.application.component_base_output_wit(component_name); let gen_dir_done_marker = GeneratedDirDoneMarker::new(&component_base_output_wit); @@ -840,7 +841,9 @@ fn create_output_wit( component_name: &ComponentName, ) -> Result { let component_base_output_wit = ctx.application.component_base_output_wit(component_name); - let component_output_wit = ctx.application.component_output_wit(component_name); + let component_output_wit = ctx + .application + .component_output_wit(component_name, ctx.profile()); let gen_dir_done_marker = GeneratedDirDoneMarker::new(&component_output_wit); if is_up_to_date( @@ -879,7 +882,10 @@ fn update_cargo_toml( ctx: &ApplicationContext, component_name: &ComponentName, ) -> anyhow::Result<()> { - let component_input_wit = PathExtra::new(ctx.application.component_input_wit(component_name)); + let component_input_wit = PathExtra::new( + ctx.application + .component_input_wit(component_name, ctx.profile()), + ); let component_input_wit_parent = component_input_wit.parent().with_context(|| { anyhow!( "Failed to get parent for component {}", @@ -891,7 +897,8 @@ fn update_cargo_toml( if cargo_toml.exists() { regenerate_cargo_package_component( &cargo_toml, - &ctx.application.component_output_wit(component_name), + &ctx.application + .component_output_wit(component_name, ctx.profile()), ctx.application.stub_world(component_name), )? } @@ -998,7 +1005,9 @@ fn add_stub_deps(ctx: &ApplicationContext, component_name: &ComponentName) -> Re add_stub_as_dependency_to_wit_dir(AddStubAsDepConfig { stub_wit_root: ctx.application.stub_wit(dep_component_name), - dest_wit_root: ctx.application.component_output_wit(component_name), + dest_wit_root: ctx + .application + .component_output_wit(component_name, ctx.profile()), update_cargo_toml: UpdateCargoToml::NoUpdate, })? } diff --git a/wasm-rpc-stubgen/src/lib.rs b/wasm-rpc-stubgen/src/lib.rs index e995a539..0c948e1f 100644 --- a/wasm-rpc-stubgen/src/lib.rs +++ b/wasm-rpc-stubgen/src/lib.rs @@ -315,6 +315,6 @@ fn dec_build_args_to_config(args: DeclarativeBuildArgs) -> commands::declarative } }, skip_up_to_date_checks: args.force_build, - profile: args.profile, + profile: args.profile.map(|profile| profile.into()), } } diff --git a/wasm-rpc-stubgen/src/model/wasm_rpc.rs b/wasm-rpc-stubgen/src/model/wasm_rpc.rs index 7c4f4a6c..a7036963 100644 --- a/wasm-rpc-stubgen/src/model/wasm_rpc.rs +++ b/wasm-rpc-stubgen/src/model/wasm_rpc.rs @@ -7,8 +7,13 @@ use crate::naming::wit::package_dep_dir_name_from_parser; use crate::validation::{ValidatedResult, ValidationBuilder}; use crate::{fs, naming}; use golem_wasm_rpc::WASM_RPC_VERSION; +use heck::{ + ToKebabCase, ToLowerCamelCase, ToPascalCase, ToShoutyKebabCase, ToShoutySnakeCase, ToSnakeCase, + ToTitleCase, ToTrainCase, ToUpperCamelCase, +}; use itertools::Itertools; -use std::collections::{BTreeMap, BTreeSet, HashMap}; +use minijinja::Environment; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::fmt::Display; use std::fmt::Formatter; use std::path::{Path, PathBuf}; @@ -43,6 +48,33 @@ impl From<&str> for ComponentName { } } +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ProfileName(String); + +impl ProfileName { + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl Display for ProfileName { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl From for ProfileName { + fn from(value: String) -> Self { + ProfileName(value) + } +} + +impl From<&str> for ProfileName { + fn from(value: &str) -> Self { + ProfileName(value.to_string()) + } +} + mod raw { use crate::model::oam; use crate::model::unknown_properties::{HasUnknownProperties, UnknownProperties}; @@ -69,16 +101,15 @@ mod raw { #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct WasmComponentProperties { - pub component_template: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub input_wit: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub output_wit: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub build: Vec, - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub build_profiles: HashMap>, #[serde(default, skip_serializing_if = "Option::is_none")] - pub default_build_profile: Option, pub input_wasm: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub output_wasm: Option, #[serde(flatten)] pub unknown_properties: UnknownProperties, @@ -86,33 +117,51 @@ mod raw { #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] - pub struct WasmComponentTemplateProperties { - pub input_wit: String, - pub output_wit: String, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub build: Vec, + pub struct WasmComponentPropertiesComponentSpecific { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub component_template: Option, + #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub build_profiles: HashMap>, + pub profiles: HashMap, + #[serde(default, skip_serializing_if = "Option::is_none")] - pub default_build_profile: Option, - pub input_wasm: String, - pub output_wasm: String, + pub default_profile: Option, + #[serde(flatten)] - pub unknown_properties: UnknownProperties, + pub properties: WasmComponentProperties, } - impl HasUnknownProperties for WasmComponentProperties { + impl HasUnknownProperties for WasmComponentPropertiesComponentSpecific { fn unknown_properties(&self) -> &UnknownProperties { - &self.unknown_properties + &self.properties.unknown_properties } } - impl oam::TypedComponentProperties for WasmComponentProperties { + impl oam::TypedComponentProperties for WasmComponentPropertiesComponentSpecific { fn component_type() -> &'static str { OAM_COMPONENT_TYPE_WASM } } + #[derive(Clone, Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct WasmComponentPropertiesTemplate { + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub profiles: HashMap, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub default_profile: Option, + + #[serde(flatten)] + pub properties: WasmComponentProperties, + } + + impl HasUnknownProperties for WasmComponentPropertiesTemplate { + fn unknown_properties(&self) -> &UnknownProperties { + &self.properties.unknown_properties + } + } + #[derive(Clone, Debug, Serialize, Deserialize)] pub struct WasmRpcTraitProperties { #[serde(rename = "componentName")] @@ -135,7 +184,7 @@ mod raw { #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] - pub struct ComponentBuildProperties { + pub struct WasmComponentBuildProperties { #[serde(default, skip_serializing_if = "Option::is_none")] pub include: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -143,18 +192,18 @@ mod raw { #[serde(default, skip_serializing_if = "Vec::is_empty")] pub wit_deps: Vec, #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub component_templates: HashMap, + pub component_templates: HashMap, #[serde(flatten)] pub unknown_properties: UnknownProperties, } - impl oam::TypedComponentProperties for ComponentBuildProperties { + impl oam::TypedComponentProperties for WasmComponentBuildProperties { fn component_type() -> &'static str { OAM_COMPONENT_TYPE_WASM_BUILD } } - impl HasUnknownProperties for ComponentBuildProperties { + impl HasUnknownProperties for WasmComponentBuildProperties { fn unknown_properties(&self) -> &UnknownProperties { &self.unknown_properties } @@ -189,8 +238,7 @@ mod raw { } mod template { - use crate::model::wasm_rpc::raw::{BuildStep, WasmComponentTemplateProperties}; - use crate::model::wasm_rpc::WasmComponentProperties; + use crate::model::wasm_rpc::{raw, BuildStep, WasmComponentProperties}; use serde::Serialize; use std::collections::HashMap; @@ -259,7 +307,7 @@ mod template { } } - impl Template for BuildStep { + impl Template for raw::BuildStep { type Rendered = BuildStep; fn render( @@ -276,7 +324,7 @@ mod template { } } - impl Template for WasmComponentTemplateProperties { + impl Template for raw::WasmComponentProperties { type Rendered = WasmComponentProperties; fn render( @@ -286,12 +334,14 @@ mod template { ) -> Result { Ok(WasmComponentProperties { build: self.build.render(env, ctx)?, - build_profiles: self.build_profiles.render(env, ctx)?, - default_build_profile: self.default_build_profile.render(env, ctx)?, - input_wit: self.input_wit.render(env, ctx)?.into(), - output_wit: self.output_wit.render(env, ctx)?.into(), - input_wasm: self.input_wasm.render(env, ctx)?.into(), - output_wasm: self.output_wasm.render(env, ctx)?.into(), + input_wit: self.input_wit.render(env, ctx)?.unwrap_or_default().into(), + output_wit: self.output_wit.render(env, ctx)?.unwrap_or_default().into(), + input_wasm: self.input_wasm.render(env, ctx)?.unwrap_or_default().into(), + output_wasm: self + .output_wasm + .render(env, ctx)? + .unwrap_or_default() + .into(), }) } } @@ -317,7 +367,7 @@ pub fn include_glob_patter_from_yaml_file(source: &Path) -> Option { .into_iter() .filter_map(|component| { component - .typed_properties::() + .typed_properties::() .ok() .and_then(|properties| properties.include) }); @@ -332,13 +382,24 @@ pub fn include_glob_patter_from_yaml_file(source: &Path) -> Option { }) } +#[derive(Clone, Debug)] +enum RenderedWasmComponentProperties { + Single { + properties: WasmComponentProperties, + }, + Profiles { + default_profile: ProfileName, + profiles: HashMap, + }, +} + #[derive(Clone, Debug)] pub struct Application { common_wasm_build: Option, common_wasm_rpc_stub_build: Option, wasm_rpc_stub_builds_by_name: BTreeMap, wasm_components_by_name: BTreeMap, - wasm_component_rendered_templates_by_name: BTreeMap, + rendered_wasm_component_properties: HashMap, } impl Application { @@ -372,82 +433,18 @@ impl Application { let common_wasm_build = Self::validate_wasm_builds(&mut validation, all_wasm_builds); - let wasm_component_rendered_templates_by_name = { - let env = minijinja::Environment::new(); - - wasm_components_by_name - .iter() - .filter_map(|(component_name, component)| { - if let WasmComponentPropertySource::Template { template_name } = - &component.properties - { - Some((component_name, template_name)) - } else { - None - } - }) - .filter_map(|(component_name, template_name)| { - let templates = common_wasm_build - .as_ref() - .map(|build| &build.component_templates); - - match templates.and_then(|templates| templates.get(template_name)) { - Some(template) => { - match template.render( - &env, - &minijinja::context! { componentName => component_name.as_str() }, - ) { - Ok(properties) => Some((component_name.clone(), properties)), - Err(err) => { - validation - .push_context("component_name", component_name.to_string()); - validation - .push_context("template_name", template_name.to_string()); - - validation.add_error(format!( - "Failed to render component template: {}", - err - )); - validation.pop_context(); - validation.pop_context(); - - None - } - } - } - None => { - validation.add_error(format!( - "Component template {} not found, {}", - template_name.log_color_error_highlight(), - match templates { - Some(templates) if !templates.is_empty() => { - format!( - "available templates: {}", - templates - .keys() - .map(|s| s.log_color_highlight()) - .join(", ") - ) - } - _ => { - "no templates are defined".to_string() - } - } - )); - validation.pop_context(); - None - } - } - }) - .collect() - }; + let rendered_wasm_component_properties = Self::render_templates( + &mut validation, + common_wasm_build.as_ref(), + &wasm_components_by_name, + ); validation.build(Self { common_wasm_build, common_wasm_rpc_stub_build, wasm_rpc_stub_builds_by_name, wasm_components_by_name, - wasm_component_rendered_templates_by_name, + rendered_wasm_component_properties, }) } @@ -536,7 +533,8 @@ impl Application { validation: &mut ValidationBuilder, mut component: oam::Component, ) -> Option { - let properties = component.typed_properties::(); + let properties = + component.typed_properties::(); if let Some(err) = properties.as_ref().err() { validation.add_error(format!("Failed to get component properties: {}", err)) @@ -604,7 +602,8 @@ impl Application { (Ok(properties), false) => { properties.add_unknown_property_warns(Vec::new, validation); - for build_step in &properties.build { + // TODO: validate + /*for build_step in &properties.properties.build { let has_inputs = !build_step.inputs.is_empty(); let has_outputs = !build_step.outputs.is_empty(); @@ -646,11 +645,12 @@ impl Application { default_build_profile.log_color_error_highlight() )) } - } + }*/ - match properties.component_template { - Some(build_template) => { - for (property_defined, property_name) in [ + let properties = match properties.component_template { + Some(template_name) => { + // TODO: validate + /*for (property_defined, property_name) in [ (properties.input_wit.is_some(), "inputWit"), (properties.output_wit.is_some(), "outputWit"), (!properties.build.is_empty(), "build"), @@ -668,18 +668,13 @@ impl Application { property_name.log_color_error_highlight() )) } - } + }*/ - Some(WasmComponent { - name: component.name.into(), - source: source.to_path_buf(), - properties: WasmComponentPropertySource::Template { - template_name: build_template, - }, - wasm_rpc_dependencies, - }) + Some(WasmComponentPropertySource::Template { template_name }) } None => { + // TODO: validate + /* for (property, property_name) in [ (&properties.input_wit, "inputWit"), (&properties.output_wit, "outputWit"), @@ -689,26 +684,72 @@ impl Application { if property.is_none() { validation.add_error(format!("Component property {} must be defined, unless componentTemplate is defined", property_name.log_color_error_highlight())) } - } + }*/ - Some(WasmComponent { - name: component.name.into(), - source: source.to_path_buf(), - properties: WasmComponentPropertySource::Concrete { - build: WasmComponentProperties { + if properties.profiles.is_empty() { + let properties = properties.properties; + Some(WasmComponentPropertySource::Explicit { + properties: WasmComponentProperties { build: properties.build, - build_profiles: properties.build_profiles, - default_build_profile: properties.default_build_profile, input_wit: properties.input_wit.unwrap_or_default().into(), output_wit: properties.output_wit.unwrap_or_default().into(), input_wasm: properties.input_wasm.unwrap_or_default().into(), output_wasm: properties.output_wasm.unwrap_or_default().into(), }, - }, - wasm_rpc_dependencies, - }) + }) + } else { + match properties.default_profile { + Some(default_profile) => { + Some(WasmComponentPropertySource::ExplicitWithProfiles { + default_profile: default_profile.into(), + profiles: properties + .profiles + .into_iter() + .map(|(profile_name, properties)| { + ( + profile_name.into(), + // TODO: dup + WasmComponentProperties { + build: properties.build, + input_wit: properties + .input_wit + .unwrap_or_default() + .into(), + output_wit: properties + .output_wit + .unwrap_or_default() + .into(), + input_wasm: properties + .input_wasm + .unwrap_or_default() + .into(), + output_wasm: properties + .output_wasm + .unwrap_or_default() + .into(), + }, + ) + }) + .collect(), + }) + } + None => { + // TODO: + validation + .add_error("TODO: missing default profile".to_string()); + None + } + } + } } - } + }; + + properties.map(|properties| WasmComponent { + name: component.name.into(), + source: source.to_path_buf(), + properties, + wasm_rpc_dependencies, + }) } _ => None, } @@ -719,7 +760,7 @@ impl Application { validation: &mut ValidationBuilder, component: oam::Component, ) -> Option { - let result = match component.typed_properties::() { + let result = match component.typed_properties::() { Ok(properties) => { // TODO: validate component templates properties.add_unknown_property_warns(Vec::new, validation); @@ -1000,6 +1041,131 @@ impl Application { ) } + fn render_templates( + validation: &mut ValidationBuilder, + common_wasm_build: Option<&WasmBuild>, + wasm_components_by_name: &BTreeMap, + ) -> HashMap { + let env = Self::template_env(); + let templates = common_wasm_build.map(|build| &build.component_templates); + let mut rendered = HashMap::::new(); + + for (component_name, component) in wasm_components_by_name { + let WasmComponentPropertySource::Template { template_name } = &component.properties + else { + continue; + }; + + let template_context = minijinja::context! { componentName => component_name.as_str() }; + + match templates.and_then(|templates| templates.get(template_name)) { + Some(template) => match &template.default_profile { + Some(default_profile) => { + let profiles = template + .profiles + .iter() + .filter_map(|(profile_name, template)| { + match template.render(&env, &template_context) { + Ok(properties) => { + Some((profile_name.as_str().into(), properties)) + } + Err(err) => { + validation.push_context( + "component name", + component_name.to_string(), + ); + validation.push_context( + "template name", + template_name.to_string(), + ); + validation.push_context( + "template profile name", + profile_name.to_string(), + ); + validation.add_error(format!( + "Failed to render component template: {}", + err + )); + validation.pop_context(); + validation.pop_context(); + validation.pop_context(); + None + } + } + }) + .collect(); + rendered.insert( + component_name.clone(), + RenderedWasmComponentProperties::Profiles { + default_profile: default_profile.as_str().into(), + profiles, + }, + ); + } + None => match template.properties.render(&env, &template_context) { + Ok(properties) => { + rendered.insert( + component_name.clone(), + RenderedWasmComponentProperties::Single { properties }, + ); + } + Err(err) => { + validation.push_context("component name", component_name.to_string()); + validation.push_context("template name", template_name.to_string()); + + validation + .add_error(format!("Failed to render component template: {}", err)); + validation.pop_context(); + validation.pop_context(); + } + }, + }, + None => { + validation.add_error(format!( + "Component template {} not found, {}", + template_name.log_color_error_highlight(), + match templates { + Some(templates) if !templates.is_empty() => { + format!( + "available templates: {}", + templates.keys().map(|s| s.log_color_highlight()).join(", ") + ) + } + _ => { + "no templates are defined".to_string() + } + } + )); + validation.pop_context(); + } + } + } + + rendered + } + + fn template_env<'a>() -> Environment<'a> { + let mut env = minijinja::Environment::new(); + + env.add_filter("to_snake_case", |str: &str| str.to_snake_case()); + + env.add_filter("to_kebab_case", |str: &str| str.to_kebab_case()); + env.add_filter("to_lower_camel_case", |str: &str| str.to_lower_camel_case()); + env.add_filter("to_pascal_case", |str: &str| str.to_pascal_case()); + env.add_filter("to_shouty_kebab_case", |str: &str| { + str.to_shouty_kebab_case() + }); + env.add_filter("to_shouty_snake_case", |str: &str| { + str.to_shouty_snake_case() + }); + env.add_filter("to_snake_case", |str: &str| str.to_snake_case()); + env.add_filter("to_title_case", |str: &str| str.to_title_case()); + env.add_filter("to_train_case", |str: &str| str.to_train_case()); + env.add_filter("to_upper_camel_case", |str: &str| str.to_upper_camel_case()); + + env + } + pub fn components(&self) -> impl Iterator { self.wasm_components_by_name.iter() } @@ -1022,14 +1188,9 @@ impl Application { .collect() } - pub fn all_profiles(&self) -> BTreeSet { + pub fn all_profiles(&self) -> BTreeSet { self.component_names() - .flat_map(|component_name| { - self.component_properties(component_name) - .build_profiles - .keys() - .cloned() - }) + .flat_map(|component_name| self.component_profiles(component_name)) .collect() } @@ -1046,21 +1207,122 @@ impl Application { .unwrap_or_else(|| panic!("Component not found: {}", component_name)) } - pub fn component_properties(&self, component_name: &ComponentName) -> &WasmComponentProperties { + fn component_profiles(&self, component_name: &ComponentName) -> HashSet { match &self.component(component_name).properties { - WasmComponentPropertySource::Concrete { build } => build, + WasmComponentPropertySource::Explicit { .. } => HashSet::new(), + WasmComponentPropertySource::ExplicitWithProfiles { profiles, .. } => { + profiles.keys().cloned().collect() + } WasmComponentPropertySource::Template { .. } => self - .wasm_component_rendered_templates_by_name + .rendered_wasm_component_properties .get(component_name) - .expect("Missing rendered template"), + .and_then(|properties| match properties { + RenderedWasmComponentProperties::Single { .. } => None, + RenderedWasmComponentProperties::Profiles { profiles, .. } => { + Some(profiles.keys().cloned().collect()) + } + }) + .unwrap_or_default(), + } + } + + pub fn component_matches_profile( + &self, + component_name: &ComponentName, + profile: Option<&ProfileName>, + ) -> bool { + match &self.component(component_name).properties { + WasmComponentPropertySource::Explicit { .. } => profile.is_none(), + WasmComponentPropertySource::ExplicitWithProfiles { profiles, .. } => match profile { + Some(profile) => profiles.contains_key(profile), + None => true, + }, + WasmComponentPropertySource::Template { .. } => self + .rendered_wasm_component_properties + .get(component_name) + .map(|properties| match properties { + RenderedWasmComponentProperties::Single { .. } => profile.is_none(), + RenderedWasmComponentProperties::Profiles { profiles, .. } => match profile { + Some(profile) => profiles.contains_key(profile), + None => true, + }, + }) + .unwrap_or_default(), + } + } + + pub fn component_properties( + &self, + component_name: &ComponentName, + profile: Option<&ProfileName>, + ) -> &WasmComponentProperties { + match &self.component(component_name).properties { + WasmComponentPropertySource::Explicit { properties } => match profile { + Some(profile) => { + panic!( + "Component {} has not profiles, requested profile: {}", + component_name, profile, + ) + } + None => properties, + }, + WasmComponentPropertySource::ExplicitWithProfiles { + default_profile, + profiles, + } => { + let profile = profile.unwrap_or(&default_profile); + profiles.get(profile).unwrap_or_else(|| { + panic!( + "Requested profile {} for component {} not found", + profile, component_name + ) + }) + } + WasmComponentPropertySource::Template { .. } => { + let properties = self + .rendered_wasm_component_properties + .get(component_name) + .unwrap_or_else(|| { + panic!( + "Rendered component properties not found: {}", + component_name + ) + }); + match properties { + RenderedWasmComponentProperties::Single { properties } => match profile { + Some(profile) => { + panic!("Requested profile {} for templated component {}, which does not have profiles", profile, component_name); + } + None => properties, + }, + RenderedWasmComponentProperties::Profiles { + default_profile, + profiles, + } => { + let profile = profile.unwrap_or(&default_profile); + profiles.get(profile).unwrap_or_else(|| { + panic!( + "Requested profile {} for templated component {} not found", + profile, component_name + ) + }) + } + } + } } } - pub fn component_input_wit(&self, component_name: &ComponentName) -> PathBuf { + pub fn component_input_wit( + &self, + component_name: &ComponentName, + profile: Option<&ProfileName>, + ) -> PathBuf { let component = self.component(component_name); - component - .source_dir() - .join(self.component_properties(component_name).input_wit.clone()) + component.source_dir().join( + self.component_properties(component_name, profile) + .input_wit + .clone(), + ) } pub fn component_base_output_wit(&self, component_name: &ComponentName) -> PathBuf { @@ -1080,68 +1342,45 @@ impl Application { .join(naming::wit::INTERFACE_WIT_FILE_NAME) } - pub fn component_output_wit(&self, component_name: &ComponentName) -> PathBuf { + pub fn component_output_wit( + &self, + component_name: &ComponentName, + profile: Option<&ProfileName>, + ) -> PathBuf { let component = self.component(component_name); - component - .source_dir() - .join(self.component_properties(component_name).output_wit.clone()) + component.source_dir().join( + self.component_properties(component_name, profile) + .output_wit + .clone(), + ) } - pub fn component_input_wasm(&self, component_name: &ComponentName) -> PathBuf { + pub fn component_input_wasm( + &self, + component_name: &ComponentName, + profile: Option<&ProfileName>, + ) -> PathBuf { let component = self.component(component_name); - component - .source_dir() - .join(self.component_properties(component_name).input_wasm.clone()) + component.source_dir().join( + self.component_properties(component_name, profile) + .input_wasm + .clone(), + ) } - pub fn component_output_wasm(&self, component_name: &ComponentName) -> PathBuf { + pub fn component_output_wasm( + &self, + component_name: &ComponentName, + profile: Option<&ProfileName>, + ) -> PathBuf { let component = self.component(component_name); component.source_dir().join( - self.component_properties(component_name) + self.component_properties(component_name, profile) .output_wasm .clone(), ) } - pub fn component_build_steps<'a>( - &'a self, - component_name: &ComponentName, - profile: Option<&'a str>, - ) -> BuildStepsLookupResult<'a> { - let component_build = self.component_properties(component_name); - match profile { - Some(profile) => match component_build.build_profiles.get(profile) { - Some(build_steps) => BuildStepsLookupResult::BuildStepsForRequestedProfile { - profile, - build_steps, - }, - None => BuildStepsLookupResult::NoBuildStepsForRequestedProfile, - }, - None => { - if !component_build.build_profiles.is_empty() { - let default_profile = component_build - .default_build_profile - .as_ref() - .expect("Missing build profile"); - - BuildStepsLookupResult::BuildStepsForDefaultProfile { - profile: default_profile, - build_steps: component_build - .build_profiles - .get(default_profile) - .expect("Missing build steps for profile"), - } - } else if !component_build.build.is_empty() { - BuildStepsLookupResult::BuildSteps { - build_steps: &component_build.build, - } - } else { - BuildStepsLookupResult::NoBuildSteps - } - } - } - } - pub fn stub_world(&self, component_name: &ComponentName) -> Option { self.stub_gen_property(component_name, |build| build.world.clone()) .flatten() @@ -1224,8 +1463,6 @@ pub type BuildStep = raw::BuildStep; #[derive(Clone, Debug)] pub struct WasmComponentProperties { pub build: Vec, - pub build_profiles: HashMap>, - pub default_build_profile: Option, pub input_wit: PathBuf, pub output_wit: PathBuf, pub input_wasm: PathBuf, @@ -1234,8 +1471,16 @@ pub struct WasmComponentProperties { #[derive(Clone, Debug)] pub enum WasmComponentPropertySource { - Concrete { build: WasmComponentProperties }, - Template { template_name: String }, + Explicit { + properties: WasmComponentProperties, + }, + ExplicitWithProfiles { + default_profile: ProfileName, + profiles: HashMap, + }, + Template { + template_name: String, + }, } #[derive(Clone, Debug)] @@ -1283,7 +1528,7 @@ pub struct WasmBuild { source: PathBuf, name: String, build_dir: Option, - component_templates: HashMap, + component_templates: HashMap, wit_deps: Vec, } diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index 17f9e25f..dcce9978 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -1,6 +1,6 @@ use crate::fs::PathExtra; use crate::log::{log_action, LogColorize, LogIndent}; -use crate::model::wasm_rpc::{Application, ComponentName}; +use crate::model::wasm_rpc::{Application, ComponentName, ProfileName}; use crate::validation::{ValidatedResult, ValidationBuilder}; use crate::{fs, naming}; use anyhow::{anyhow, bail, Context, Error}; @@ -182,7 +182,7 @@ pub struct ResolvedWitApplication { } impl ResolvedWitApplication { - pub fn new(app: &Application) -> ValidatedResult { + pub fn new(app: &Application, profile: Option<&ProfileName>) -> ValidatedResult { log_action("Resolving", "application wit directories"); let _indent = LogIndent::new(); @@ -196,7 +196,7 @@ impl ResolvedWitApplication { let mut validation = ValidationBuilder::new(); - resolved_app.add_components_from_app(&mut validation, app); + resolved_app.add_components_from_app(&mut validation, app, profile); resolved_app.validate_package_names(&mut validation); resolved_app.collect_component_deps(app, &mut validation); @@ -256,12 +256,22 @@ impl ResolvedWitApplication { self.components.insert(component_name, resolved_component); } - fn add_components_from_app(&mut self, validation: &mut ValidationBuilder, app: &Application) { + fn add_components_from_app( + &mut self, + validation: &mut ValidationBuilder, + app: &Application, + profile: Option<&ProfileName>, + ) { for (component_name, component) in app.components() { + if !app.component_matches_profile(component_name, profile) { + // TODO: check if it is okay to skip here + continue; + } + validation.push_context("component name", component_name.to_string()); - let input_wit_dir = app.component_input_wit(component_name); - let output_wit_dir = app.component_output_wit(component_name); + let input_wit_dir = app.component_input_wit(component_name, profile); + let output_wit_dir = app.component_output_wit(component_name, profile); log_action( "Resolving", @@ -291,7 +301,10 @@ impl ResolvedWitApplication { .collect(); let input_contained_package_deps = { - let deps_path = app.component_properties(component_name).input_wit.join("deps"); + let deps_path = app + .component_properties(component_name, profile) + .input_wit + .join("deps"); if !deps_path.exists() { HashSet::new() } else { From d63e6f0fdfb7fd184a40b96f959cc5955aa6e689 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Thu, 21 Nov 2024 21:48:05 -0500 Subject: [PATCH 081/109] cleanup --- wasm-rpc-stubgen/src/commands/declarative.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index e9126730..7db33d7a 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -22,7 +22,6 @@ use anyhow::{anyhow, bail, Context, Error}; use colored::Colorize; use glob::glob; use itertools::Itertools; -use log::log; use std::cell::OnceCell; use std::cmp::Ordering; use std::collections::HashMap; From 465e8c9b6a78c28d81282cdf46f02f9cb8c678fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Fri, 22 Nov 2024 08:44:20 -0500 Subject: [PATCH 082/109] always choose profile, even if it is not matching the requested one --- wasm-rpc-stubgen/src/commands/declarative.rs | 252 ++++++++++++++----- wasm-rpc-stubgen/src/log.rs | 7 - wasm-rpc-stubgen/src/model/wasm_rpc.rs | 132 ++++++---- wasm-rpc-stubgen/src/wit_resolve.rs | 11 +- 4 files changed, 283 insertions(+), 119 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index 7db33d7a..4aeb9742 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -67,27 +67,174 @@ impl ApplicationContext { }), )?; - if let Some(profile) = &ctx.config.profile { - let all_profiles = ctx.application.all_profiles(); - if all_profiles.is_empty() { - bail!( - "Profile {} not found, no available profiles", - profile.as_str().log_color_error_highlight(), - ); - } else if !all_profiles.contains(profile) { - bail!( - "Profile {} not found, available profiles: {}", - profile.as_str().log_color_error_highlight(), - all_profiles - .into_iter() - .map(|s| s.as_str().log_color_highlight()) - .join(", ") - ); - } else { - log_action( - "Selected", - format!("profile: {}", profile.as_str().log_color_highlight()), - ); + // Selecting and validating profiles + { + match &ctx.config.profile { + Some(profile) => { + let all_profiles = ctx.application.all_profiles(); + if all_profiles.is_empty() { + bail!( + "Profile {} not found, no available profiles", + profile.as_str().log_color_error_highlight(), + ); + } else if !all_profiles.contains(profile) { + bail!( + "Profile {} not found, available profiles: {}", + profile.as_str().log_color_error_highlight(), + all_profiles + .into_iter() + .map(|s| s.as_str().log_color_highlight()) + .join(", ") + ); + } + log_action( + "Selecting", + format!( + "profiles, requested profile: {}", + profile.as_str().log_color_highlight() + ), + ); + } + None => { + log_action("Selecting", "profiles, no profile was requested"); + } + } + + let _indent = LogIndent::new(); + for component_name in ctx.application.component_names() { + let selection = ctx + .application + .component_effective_template_and_profile(component_name, ctx.profile()); + + match ( + selection.profile, + selection.template_name, + ctx.profile().is_some(), + selection.is_requested_profile, + ) { + (None, None, false, _) => { + log_action( + "Selected", + format!( + "default build for {}", + component_name.as_str().log_color_highlight() + ), + ); + } + (None, None, true, _) => { + log_action( + "Selected", + format!( + "default build for {}, component has no profiles", + component_name.as_str().log_color_highlight() + ), + ); + } + (None, Some(template), false, _) => { + log_action( + "Selected", + format!( + "default build for {} using template {}", + component_name.as_str().log_color_highlight(), + template.log_color_highlight() + ), + ); + } + (None, Some(template), true, _) => { + log_action( + "Selected", + format!( + "default build for {} using template {}, component has no profiles", + component_name.as_str().log_color_highlight(), + template.log_color_highlight() + ), + ); + } + (Some(profile), None, false, false) => { + log_action( + "Selected", + format!( + "default profile {} for {}", + profile.as_str().log_color_highlight(), + component_name.as_str().log_color_highlight() + ), + ); + } + (Some(profile), None, true, false) => { + log_action( + "Selected", + format!( + "default profile {} for {}, component has no matching requested profile", + profile.as_str().log_color_highlight(), + component_name.as_str().log_color_highlight() + ), + ); + } + (Some(profile), Some(template), false, false) => { + log_action( + "Selected", + format!( + "default profile {} for {} using template {}", + profile.as_str().log_color_highlight(), + component_name.as_str().log_color_highlight(), + template.log_color_highlight() + ), + ); + } + (Some(profile), Some(template), true, false) => { + log_action( + "Selected", + format!( + "default profile {} for {} using template {}, component has no matching requested profile", + profile.as_str().log_color_highlight(), + component_name.as_str().log_color_highlight(), + template.log_color_highlight() + ), + ); + } + (Some(profile), None, false, true) => { + log_action( + "Selected", + format!( + "profile {} for {}", + profile.as_str().log_color_highlight(), + component_name.as_str().log_color_highlight() + ), + ); + } + (Some(profile), None, true, true) => { + log_action( + "Selected", + format!( + "requested profile {} for {}", + profile.as_str().log_color_highlight(), + component_name.as_str().log_color_highlight() + ), + ); + } + (Some(profile), Some(template), false, true) => { + log_action( + "Selected", + format!( + "profile {} for {} using template {}", + profile.as_str().log_color_highlight(), + component_name.as_str().log_color_highlight(), + template.log_color_highlight() + ), + ); + } + (Some(profile), Some(template), true, true) => { + log_action( + "Selected", + format!( + "requested profile {} for {} using template {}", + profile.as_str().log_color_highlight(), + component_name.as_str().log_color_highlight(), + template.log_color_highlight() + ), + ); + } + } } } @@ -98,10 +245,10 @@ impl ApplicationContext { self.config.profile.as_ref() } - fn component_matches_profile(&self, component_name: &ComponentName) -> bool { + /*fn component_matches_profile(&self, component_name: &ComponentName) -> bool { self.application .component_matches_profile(component_name, self.profile()) - } + }*/ fn update_wit_context(&mut self) -> anyhow::Result<()> { to_anyhow( @@ -209,18 +356,6 @@ fn component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<()> { let _indent = LogIndent::new(); for (component_name, component) in ctx.application.components() { - if !ctx.component_matches_profile(component_name) { - log_warn_action( - "Skipping", - format!( - "building {}, no build steps for profile {}", - component_name.log_color_highlight(), - ctx.profile().unwrap().as_str().log_color_highlight() - ), - ); - continue; - } - let component_properties = ctx .application .component_properties(component_name, ctx.profile()); @@ -230,7 +365,7 @@ fn component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<()> { "Skipping", format!( "building {}, no build steps", - component_name.log_color_highlight(), + component_name.as_str().log_color_highlight(), ), ); continue; @@ -238,7 +373,7 @@ fn component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<()> { log_action( "Building", - format!("{}", component_name.log_color_highlight()), + format!("{}", component_name.as_str().log_color_highlight()), ); let _indent = LogIndent::new(); @@ -304,11 +439,6 @@ async fn post_component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<() let _indent = LogIndent::new(); for (component_name, component) in ctx.application.components() { - if !ctx.component_matches_profile(component_name) { - // TODO: log - continue; - } - let input_wasm = ctx .application .component_input_wasm(component_name, ctx.profile()); @@ -328,9 +458,9 @@ async fn post_component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<() component .wasm_rpc_dependencies .iter() - .map(|s| s.log_color_highlight()) + .map(|s| s.as_str().log_color_highlight()) .join(", "), - component_name.log_color_highlight(), + component_name.as_str().log_color_highlight(), )); continue; } @@ -353,9 +483,9 @@ async fn post_component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<() component .wasm_rpc_dependencies .iter() - .map(|s| s.log_color_highlight()) + .map(|s| s.as_str().log_color_highlight()) .join(", "), - component_name.log_color_highlight(), + component_name.as_str().log_color_highlight(), ), ); let _indent = LogIndent::new(); @@ -425,7 +555,10 @@ pub fn clean(config: Config) -> anyhow::Result<()> { for (component_name, _component) in app.components() { log_action( "Cleaning", - format!("component {}", component_name.log_color_highlight()), + format!( + "component {}", + component_name.as_str().log_color_highlight() + ), ); let _indent = LogIndent::new(); @@ -519,7 +652,7 @@ fn load_app_validated(config: &Config) -> ValidatedResult { format!( "components: {}", app.component_names() - .map(|s| s.log_color_highlight()) + .map(|s| s.as_str().log_color_highlight()) .join(", ") ) } @@ -764,7 +897,7 @@ fn create_base_output_wit( ) { log_skipping_up_to_date(format!( "creating base output wit directory for {}", - component_name.log_color_highlight() + component_name.as_str().log_color_highlight() )); Ok(false) } else { @@ -772,7 +905,7 @@ fn create_base_output_wit( "Creating", format!( "base output wit directory for {}", - component_name.log_color_highlight(), + component_name.as_str().log_color_highlight(), ), ); let _indent = LogIndent::new(); @@ -854,7 +987,7 @@ fn create_output_wit( ) { log_skipping_up_to_date(format!( "creating output wit directory for {}", - component_name.log_color_highlight() + component_name.as_str().log_color_highlight() )); Ok(false) } else { @@ -862,7 +995,7 @@ fn create_output_wit( "Creating", format!( "output wit directory for {}", - component_name.log_color_highlight(), + component_name.as_str().log_color_highlight(), ), ); let _indent = LogIndent::new(); @@ -888,7 +1021,7 @@ fn update_cargo_toml( let component_input_wit_parent = component_input_wit.parent().with_context(|| { anyhow!( "Failed to get parent for component {}", - component_name.log_color_highlight() + component_name.as_str().log_color_highlight() ) })?; let cargo_toml = component_input_wit_parent.join("Cargo.toml"); @@ -946,13 +1079,16 @@ async fn build_stub( ) { log_skipping_up_to_date(format!( "building wasm rpc stub for {}", - component_name.log_color_highlight() + component_name.as_str().log_color_highlight() )); Ok(false) } else { log_action( "Building", - format!("wasm rpc stub for {}", component_name.log_color_highlight()), + format!( + "wasm rpc stub for {}", + component_name.as_str().log_color_highlight() + ), ); let _indent = LogIndent::new(); @@ -985,7 +1121,7 @@ fn add_stub_deps(ctx: &ApplicationContext, component_name: &ComponentName) -> Re "Adding", format!( "stub wit dependencies to {}", - component_name.log_color_highlight() + component_name.as_str().log_color_highlight() ), ); @@ -996,8 +1132,8 @@ fn add_stub_deps(ctx: &ApplicationContext, component_name: &ComponentName) -> Re "Adding", format!( "{} stub wit dependency to {}", - dep_component_name.log_color_highlight(), - component_name.log_color_highlight() + dep_component_name.as_str().log_color_highlight(), + component_name.as_str().log_color_highlight() ), ); let _indent = LogIndent::new(); diff --git a/wasm-rpc-stubgen/src/log.rs b/wasm-rpc-stubgen/src/log.rs index 711b5187..85e2d136 100644 --- a/wasm-rpc-stubgen/src/log.rs +++ b/wasm-rpc-stubgen/src/log.rs @@ -1,5 +1,4 @@ use crate::fs::{OverwriteSafeAction, OverwriteSafeActionPlan, PathExtra}; -use crate::model::wasm_rpc::ComponentName; use crate::validation::ValidatedResult; use colored::{ColoredString, Colorize}; use std::path::{Path, PathBuf}; @@ -235,9 +234,3 @@ impl> LogColorize for PathExtra

{ ColoredString::from(self.display().to_string()) } } - -impl LogColorize for ComponentName { - fn as_str(&self) -> impl Colorize { - self.as_str() - } -} diff --git a/wasm-rpc-stubgen/src/model/wasm_rpc.rs b/wasm-rpc-stubgen/src/model/wasm_rpc.rs index a7036963..c3525dce 100644 --- a/wasm-rpc-stubgen/src/model/wasm_rpc.rs +++ b/wasm-rpc-stubgen/src/model/wasm_rpc.rs @@ -393,6 +393,12 @@ enum RenderedWasmComponentProperties { }, } +pub struct ComponentEffectiveTemplateAndProfile<'a> { + pub template_name: Option<&'a str>, + pub profile: Option<&'a ProfileName>, + pub is_requested_profile: bool, +} + #[derive(Clone, Debug)] pub struct Application { common_wasm_build: Option, @@ -906,8 +912,8 @@ impl Application { vec![], format!( "Component {} references unknown component {} as dependency", - component_name.log_color_highlight(), - dep_component_name.log_color_error_highlight(), + component_name.as_str().log_color_highlight(), + dep_component_name.as_str().log_color_error_highlight(), ), ) }) @@ -1028,7 +1034,7 @@ impl Application { format!( "Wasm rpc stub build {} references unknown component {}", wasm_rpc_stub_build.name.log_color_highlight(), - component_name.log_color_error_highlight(), + component_name.as_str().log_color_error_highlight(), ), ) }) @@ -1226,28 +1232,71 @@ impl Application { } } - pub fn component_matches_profile( - &self, + pub fn component_effective_template_and_profile<'a>( + &'a self, component_name: &ComponentName, - profile: Option<&ProfileName>, - ) -> bool { + profile: Option<&'a ProfileName>, + ) -> ComponentEffectiveTemplateAndProfile<'a> { match &self.component(component_name).properties { - WasmComponentPropertySource::Explicit { .. } => profile.is_none(), - WasmComponentPropertySource::ExplicitWithProfiles { profiles, .. } => match profile { - Some(profile) => profiles.contains_key(profile), - None => true, + WasmComponentPropertySource::Explicit { .. } => ComponentEffectiveTemplateAndProfile { + template_name: None, + profile: None, + is_requested_profile: false, }, - WasmComponentPropertySource::Template { .. } => self - .rendered_wasm_component_properties - .get(component_name) - .map(|properties| match properties { - RenderedWasmComponentProperties::Single { .. } => profile.is_none(), - RenderedWasmComponentProperties::Profiles { profiles, .. } => match profile { - Some(profile) => profiles.contains_key(profile), - None => true, + WasmComponentPropertySource::ExplicitWithProfiles { + default_profile, + profiles, + } => match profile { + Some(profile) if profiles.contains_key(profile) => { + ComponentEffectiveTemplateAndProfile { + template_name: None, + profile: Some(profile), + is_requested_profile: true, + } + } + _ => ComponentEffectiveTemplateAndProfile { + template_name: None, + profile: Some(default_profile), + is_requested_profile: false, + }, + }, + WasmComponentPropertySource::Template { template_name } => { + let properties = self + .rendered_wasm_component_properties + .get(component_name) + .unwrap_or_else(|| { + panic!( + "Rendered component properties not found for {}", + component_name + ) + }); + match properties { + RenderedWasmComponentProperties::Single { .. } => { + ComponentEffectiveTemplateAndProfile { + template_name: Some(template_name), + profile: None, + is_requested_profile: false, + } + } + RenderedWasmComponentProperties::Profiles { + default_profile, + profiles, + } => match profile { + Some(profile) if profiles.contains_key(profile) => { + ComponentEffectiveTemplateAndProfile { + template_name: Some(template_name), + profile: Some(profile), + is_requested_profile: true, + } + } + _ => ComponentEffectiveTemplateAndProfile { + template_name: Some(template_name), + profile: Some(default_profile), + is_requested_profile: false, + }, }, - }) - .unwrap_or_default(), + } + } } } @@ -1257,25 +1306,19 @@ impl Application { profile: Option<&ProfileName>, ) -> &WasmComponentProperties { match &self.component(component_name).properties { - WasmComponentPropertySource::Explicit { properties } => match profile { - Some(profile) => { - panic!( - "Component {} has not profiles, requested profile: {}", - component_name, profile, - ) - } - None => properties, - }, + WasmComponentPropertySource::Explicit { properties } => properties, WasmComponentPropertySource::ExplicitWithProfiles { default_profile, profiles, } => { let profile = profile.unwrap_or(&default_profile); profiles.get(profile).unwrap_or_else(|| { - panic!( - "Requested profile {} for component {} not found", - profile, component_name - ) + profiles.get(default_profile).unwrap_or_else(|| { + panic!( + "Default profile {} for {} not found", + default_profile, component_name + ) + }) }) } WasmComponentPropertySource::Template { .. } => { @@ -1284,27 +1327,24 @@ impl Application { .get(component_name) .unwrap_or_else(|| { panic!( - "Rendered component properties not found: {}", + "Rendered component properties not found for {}", component_name ) }); match properties { - RenderedWasmComponentProperties::Single { properties } => match profile { - Some(profile) => { - panic!("Requested profile {} for templated component {}, which does not have profiles", profile, component_name); - } - None => properties, - }, + RenderedWasmComponentProperties::Single { properties } => properties, RenderedWasmComponentProperties::Profiles { default_profile, profiles, } => { - let profile = profile.unwrap_or(&default_profile); + let profile = profile.unwrap_or(default_profile); profiles.get(profile).unwrap_or_else(|| { - panic!( - "Requested profile {} for templated component {} not found", - profile, component_name - ) + profiles.get(default_profile).unwrap_or_else(|| { + panic!( + "Default (rendered) profile {} for {} not found", + default_profile, component_name + ) + }) }) } } diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index dcce9978..aa9e668e 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -263,11 +263,6 @@ impl ResolvedWitApplication { profile: Option<&ProfileName>, ) { for (component_name, component) in app.components() { - if !app.component_matches_profile(component_name, profile) { - // TODO: check if it is okay to skip here - continue; - } - validation.push_context("component name", component_name.to_string()); let input_wit_dir = app.component_input_wit(component_name, profile); @@ -277,7 +272,7 @@ impl ResolvedWitApplication { "Resolving", format!( "component wit dirs for {} ({}, {})", - component_name.log_color_highlight(), + component_name.as_str().log_color_highlight(), input_wit_dir.log_color_highlight(), output_wit_dir.log_color_highlight(), ), @@ -288,7 +283,7 @@ impl ResolvedWitApplication { UnresolvedPackageGroup::parse_dir(&input_wit_dir).with_context(|| { anyhow!( "Failed to parse component {} main package in input wit dir {}", - component_name.log_color_error_highlight(), + component_name.as_str().log_color_error_highlight(), input_wit_dir.log_color_highlight(), ) })?; @@ -567,7 +562,7 @@ impl ResolvedWitApplication { self.components.get(component_name).ok_or_else(|| { anyhow!( "Component not found: {}", - component_name.log_color_error_highlight() + component_name.as_str().log_color_error_highlight() ) }) } From c4fe6ebbe5fdda0693d3482b0081ee81dae2b3d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Fri, 22 Nov 2024 09:34:08 -0500 Subject: [PATCH 083/109] fix clean to support profiles --- wasm-rpc-stubgen/src/commands/declarative.rs | 99 ++++++++++---------- 1 file changed, 48 insertions(+), 51 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index 4aeb9742..c24c0f6b 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -7,8 +7,8 @@ use crate::log::{ }; use crate::model::oam; use crate::model::wasm_rpc::{ - include_glob_patter_from_yaml_file, init_oam_app, Application, BuildStep, ComponentName, - ProfileName, WasmComponent, DEFAULT_CONFIG_FILE_NAME, + include_glob_patter_from_yaml_file, init_oam_app, Application, ComponentName, ProfileName, + DEFAULT_CONFIG_FILE_NAME, }; use crate::stub::{StubConfig, StubDefinition}; use crate::validation::ValidatedResult; @@ -24,7 +24,7 @@ use glob::glob; use itertools::Itertools; use std::cell::OnceCell; use std::cmp::Ordering; -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap, HashSet}; use std::path::{Path, PathBuf}; use std::process::Command; use std::time::SystemTime; @@ -245,11 +245,6 @@ impl ApplicationContext { self.config.profile.as_ref() } - /*fn component_matches_profile(&self, component_name: &ComponentName) -> bool { - self.application - .component_matches_profile(component_name, self.profile()) - }*/ - fn update_wit_context(&mut self) -> anyhow::Result<()> { to_anyhow( "Failed to update application wit context, see problems above", @@ -532,53 +527,55 @@ pub fn clean(config: Config) -> anyhow::Result<()> { log_action("Cleaning", "components"); let _indent = LogIndent::new(); - fn clean_build_steps( - component: &WasmComponent, - build_steps: &[BuildStep], - ) -> anyhow::Result<()> { - for build_step in build_steps { - let build_dir = build_step - .dir - .as_ref() - .map(|dir| component.source_dir().join(dir)) - .unwrap_or_else(|| component.source_dir().to_path_buf()); + let all_profiles = { + let mut profiles = app + .all_profiles() + .into_iter() + .map(Some) + .collect::>(); + profiles.insert(None); + profiles + }; + let paths = { + let mut paths = BTreeSet::<(&'static str, PathBuf)>::new(); + for (component_name, component) in app.components() { + for profile in &all_profiles { + paths.insert(( + "output wit", + app.component_output_wit(component_name, profile.as_ref()), + )); + paths.insert(( + "input wasm", + app.component_input_wasm(component_name, profile.as_ref()), + )); + paths.insert(( + "output wasm", + app.component_output_wasm(component_name, profile.as_ref()), + )); - let outputs = compile_and_collect_globs(&build_dir, &build_step.outputs)?; - for output in outputs { - delete_path("build step output", &output)?; + let build = &app + .component_properties(component_name, profile.as_ref()) + .build; + for build_step in build { + let build_dir = build_step + .dir + .as_ref() + .map(|dir| component.source_dir().join(dir)) + .unwrap_or_else(|| component.source_dir().to_path_buf()); + + paths.extend( + compile_and_collect_globs(&build_dir, &build_step.outputs)? + .into_iter() + .map(|path| ("build output", path)), + ); + } } } + paths + }; - Ok(()) - } - - for (component_name, _component) in app.components() { - log_action( - "Cleaning", - format!( - "component {}", - component_name.as_str().log_color_highlight() - ), - ); - let _indent = LogIndent::new(); - - // TODO - /* - delete_path("wit output dir", &app.component_output_wit(component_name))?; - delete_path("wasm input", &app.component_input_wasm(component_name))?; - delete_path("wasm output", &app.component_output_wasm(component_name))?; - - let component_build = app.component_properties(component_name); - clean_build_steps(component, &component_build.build)?; - - for (profile, build_steps) in &component_build.build_profiles { - log_action( - "Cleaning", - format!("profile {}", profile.log_color_highlight()), - ); - let _indent = LogIndent::new(); - clean_build_steps(component, build_steps)?; - }*/ + for (context, path) in paths { + delete_path(context, &path)?; } } From ab3a8b686199add4396fd3f74a9c2ac9fcfeae04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Fri, 22 Nov 2024 09:59:49 -0500 Subject: [PATCH 084/109] up-to-date coloring --- wasm-rpc-stubgen/src/log.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/wasm-rpc-stubgen/src/log.rs b/wasm-rpc-stubgen/src/log.rs index 85e2d136..eff0d723 100644 --- a/wasm-rpc-stubgen/src/log.rs +++ b/wasm-rpc-stubgen/src/log.rs @@ -82,7 +82,11 @@ pub fn log_warn_action>(action: &str, subject: T) { pub fn log_skipping_up_to_date>(subject: T) { log_warn_action( "Skipping", - format!("{}, already up-to-date", subject.as_ref()), + format!( + "{}, {}", + subject.as_ref(), + "UP-TO-DATE".log_color_ok_highlight() + ), ); } @@ -203,6 +207,10 @@ pub trait LogColorize { fn log_color_error_highlight(&self) -> ColoredString { self.as_str().bold().red().underline() } + + fn log_color_ok_highlight(&self) -> ColoredString { + self.as_str().bold().green() + } } impl<'a> LogColorize for &'a str { From 44cadee78509278f470834593e433e8c6a286393 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Fri, 22 Nov 2024 13:00:54 -0500 Subject: [PATCH 085/109] extract wasm component prop validation --- .../src/model/unknown_properties.rs | 6 +- wasm-rpc-stubgen/src/model/wasm_rpc.rs | 427 ++++++++++-------- wasm-rpc-stubgen/src/validation.rs | 4 +- wasm-rpc-stubgen/src/wit_resolve.rs | 7 +- 4 files changed, 259 insertions(+), 185 deletions(-) diff --git a/wasm-rpc-stubgen/src/model/unknown_properties.rs b/wasm-rpc-stubgen/src/model/unknown_properties.rs index 3b09ca59..02be77ba 100644 --- a/wasm-rpc-stubgen/src/model/unknown_properties.rs +++ b/wasm-rpc-stubgen/src/model/unknown_properties.rs @@ -1,3 +1,4 @@ +use crate::log::LogColorize; use crate::validation::ValidationBuilder; use std::collections::BTreeMap; @@ -25,7 +26,10 @@ pub trait HasUnknownProperties { validation_builder.add_warns(self.unknown_properties().keys(), |unknown_property_name| { Some(( vec![], - format!("Unknown property: {}", unknown_property_name), + format!( + "Unknown property: {}", + unknown_property_name.log_color_highlight() + ), )) }); diff --git a/wasm-rpc-stubgen/src/model/wasm_rpc.rs b/wasm-rpc-stubgen/src/model/wasm_rpc.rs index c3525dce..4b76e5ea 100644 --- a/wasm-rpc-stubgen/src/model/wasm_rpc.rs +++ b/wasm-rpc-stubgen/src/model/wasm_rpc.rs @@ -115,6 +115,12 @@ mod raw { pub unknown_properties: UnknownProperties, } + impl HasUnknownProperties for WasmComponentProperties { + fn unknown_properties(&self) -> &UnknownProperties { + &self.unknown_properties + } + } + #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct WasmComponentPropertiesComponentSpecific { @@ -324,7 +330,7 @@ mod template { } } - impl Template for raw::WasmComponentProperties { + impl Template for WasmComponentProperties { type Rendered = WasmComponentProperties; fn render( @@ -334,14 +340,10 @@ mod template { ) -> Result { Ok(WasmComponentProperties { build: self.build.render(env, ctx)?, - input_wit: self.input_wit.render(env, ctx)?.unwrap_or_default().into(), - output_wit: self.output_wit.render(env, ctx)?.unwrap_or_default().into(), - input_wasm: self.input_wasm.render(env, ctx)?.unwrap_or_default().into(), - output_wasm: self - .output_wasm - .render(env, ctx)? - .unwrap_or_default() - .into(), + input_wit: self.input_wit.render(env, ctx)?, + output_wit: self.output_wit.render(env, ctx)?, + input_wasm: self.input_wasm.render(env, ctx)?, + output_wasm: self.output_wasm.render(env, ctx)?, }) } } @@ -608,147 +610,13 @@ impl Application { (Ok(properties), false) => { properties.add_unknown_property_warns(Vec::new, validation); - // TODO: validate - /*for build_step in &properties.properties.build { - let has_inputs = !build_step.inputs.is_empty(); - let has_outputs = !build_step.outputs.is_empty(); - - if (has_inputs && !has_outputs) || (!has_inputs && has_outputs) { - validation.push_context("command", build_step.command.clone()); - validation.add_warn( - "Using inputs and outputs only has effect when both defined" - .to_string(), - ); - validation.pop_context(); - } - } - - if !properties.build_profiles.is_empty() && !properties.build.is_empty() { - validation.add_warn( - "If buildProfiles is defined then build will be ignored".to_string(), - ); - } - - if !properties.build_profiles.is_empty() - && properties.default_build_profile.is_none() - { - validation.add_error("If buildProfiles is defined then defaultBuildProfile also have to be defined".to_string()); - } - - if properties.build_profiles.is_empty() - && properties.default_build_profile.is_some() - { - validation.add_error("If defaultBuildProfile is defined then buildProfiles also have to be defined".to_string()); - } - - if let Some(default_build_profile) = &properties.default_build_profile { - if !properties - .build_profiles - .contains_key(default_build_profile) - { - validation.add_error(format!( - "The defined defaultBuildProfile ({}) if not found in buildProfiles", - default_build_profile.log_color_error_highlight() - )) - } - }*/ - - let properties = match properties.component_template { - Some(template_name) => { - // TODO: validate - /*for (property_defined, property_name) in [ - (properties.input_wit.is_some(), "inputWit"), - (properties.output_wit.is_some(), "outputWit"), - (!properties.build.is_empty(), "build"), - (!properties.build_profiles.is_empty(), "buildProfiles"), - ( - properties.default_build_profile.is_some(), - "defaultBuildProfile", - ), - (properties.input_wasm.is_some(), "inputWasm"), - (properties.output_wasm.is_some(), "outputWasm"), - ] { - if property_defined { - validation.add_warn(format!( - "Component property {} is ignored when componentTemplate is defined", - property_name.log_color_error_highlight() - )) - } - }*/ - - Some(WasmComponentPropertySource::Template { template_name }) - } - None => { - // TODO: validate - /* - for (property, property_name) in [ - (&properties.input_wit, "inputWit"), - (&properties.output_wit, "outputWit"), - (&properties.input_wasm, "inputWasm"), - (&properties.output_wasm, "outputWasm"), - ] { - if property.is_none() { - validation.add_error(format!("Component property {} must be defined, unless componentTemplate is defined", property_name.log_color_error_highlight())) - } - }*/ - - if properties.profiles.is_empty() { - let properties = properties.properties; - Some(WasmComponentPropertySource::Explicit { - properties: WasmComponentProperties { - build: properties.build, - input_wit: properties.input_wit.unwrap_or_default().into(), - output_wit: properties.output_wit.unwrap_or_default().into(), - input_wasm: properties.input_wasm.unwrap_or_default().into(), - output_wasm: properties.output_wasm.unwrap_or_default().into(), - }, - }) - } else { - match properties.default_profile { - Some(default_profile) => { - Some(WasmComponentPropertySource::ExplicitWithProfiles { - default_profile: default_profile.into(), - profiles: properties - .profiles - .into_iter() - .map(|(profile_name, properties)| { - ( - profile_name.into(), - // TODO: dup - WasmComponentProperties { - build: properties.build, - input_wit: properties - .input_wit - .unwrap_or_default() - .into(), - output_wit: properties - .output_wit - .unwrap_or_default() - .into(), - input_wasm: properties - .input_wasm - .unwrap_or_default() - .into(), - output_wasm: properties - .output_wasm - .unwrap_or_default() - .into(), - }, - ) - }) - .collect(), - }) - } - None => { - // TODO: - validation - .add_error("TODO: missing default profile".to_string()); - None - } - } - } - } - }; + let properties = Self::convert_wasm_component_properties( + validation, + Some(properties.component_template), + properties.profiles, + properties.default_profile, + properties.properties, + ); properties.map(|properties| WasmComponent { name: component.name.into(), @@ -768,14 +636,35 @@ impl Application { ) -> Option { let result = match component.typed_properties::() { Ok(properties) => { - // TODO: validate component templates properties.add_unknown_property_warns(Vec::new, validation); + let component_templates = properties + .component_templates + .into_iter() + .filter_map(|(template_name, template)| { + validation.push_context("template", template_name.clone()); + + template.add_unknown_property_warns(Vec::new, validation); + + let component_source = Self::convert_wasm_component_properties( + validation, + None, + template.profiles, + template.default_profile, + template.properties, + ); + + validation.pop_context(); + + component_source.map(|component_source| (template_name, component_source)) + }) + .collect::>(); + let wasm_rpc_stub_build = WasmBuild { source: source.to_path_buf(), name: component.name, build_dir: properties.build_dir.map(|s| s.into()), - component_templates: properties.component_templates, + component_templates, wit_deps: properties.wit_deps.into_iter().map(|s| s.into()).collect(), }; @@ -800,6 +689,180 @@ impl Application { result } + fn convert_wasm_component_properties( + validation: &mut ValidationBuilder, + component_template: Option>, + profiles: HashMap, + default_profile: Option, + properties: raw::WasmComponentProperties, + ) -> Option { + fn unwrap_validated_properties( + properties: raw::WasmComponentProperties, + ) -> WasmComponentProperties { + WasmComponentProperties { + build: properties.build, + input_wit: properties.input_wit.unwrap(), + output_wit: properties.output_wit.unwrap(), + input_wasm: properties.input_wasm.unwrap(), + output_wasm: properties.output_wasm.unwrap(), + } + } + + let property_ignored_unless_defined_message = { + if component_template.is_some() { + format!( + "{} or {}", + "componentTemplate".log_color_highlight(), + "profile".log_color_highlight() + ) + } else { + "profile".log_color_highlight().to_string() + } + }; + + for build_step in &properties.build { + let has_inputs = !build_step.inputs.is_empty(); + let has_outputs = !build_step.outputs.is_empty(); + + if (has_inputs && !has_outputs) || (!has_inputs && has_outputs) { + validation.push_context("command", build_step.command.clone()); + validation.add_warn(format!( + "Using {} and {} only has effect when both defined", + "inputs".log_color_highlight(), + "outputs".log_color_highlight() + )); + validation.pop_context(); + } + } + + if !profiles.is_empty() && !properties.build.is_empty() { + validation.add_warn(format!( + "If {} is defined then {} will be ignored", + "profiles".log_color_highlight(), + "build".log_color_highlight() + )); + } + + if !profiles.is_empty() && default_profile.is_none() { + validation.add_error(format!( + "If {} is defined then {} also have to be defined", + "profiles".log_color_highlight(), + "defaultProfile".log_color_highlight() + )); + } + + if profiles.is_empty() && default_profile.is_some() { + validation.add_error(format!( + "If {} is defined then {} also have to be defined", + "defaultProfile".log_color_highlight(), + "profiles".log_color_highlight(), + )); + } + + if let Some(default_profile) = &default_profile { + if !profiles.contains_key(default_profile) { + validation.add_error(format!( + "The defined {} ({}) if not found in {}", + "defaultProfile".log_color_highlight(), + default_profile.log_color_error_highlight(), + "profiles".log_color_highlight(), + )); + } + } + + match component_template { + Some(Some(template_name)) => { + for (property_defined, property_name) in [ + (properties.input_wit.is_some(), "inputWit"), + (properties.output_wit.is_some(), "outputWit"), + (!properties.build.is_empty(), "build"), + (properties.input_wasm.is_some(), "inputWasm"), + (properties.output_wasm.is_some(), "outputWasm"), + (!profiles.is_empty(), "profiles"), + (default_profile.is_some(), "defaultProfile"), + ] { + if property_defined { + validation.add_warn(format!( + "Component property {} is ignored when {} is defined", + property_name.log_color_error_highlight(), + "componentTemplate".log_color_highlight(), + )) + } + } + + Some(WasmComponentPropertySource::Template { template_name }) + } + _ => { + if profiles.is_empty() { + let mut any_mandatory_property_missing = false; + for (property, property_name) in [ + (&properties.input_wit, "inputWit"), + (&properties.output_wit, "outputWit"), + (&properties.input_wasm, "inputWasm"), + (&properties.output_wasm, "outputWasm"), + ] { + if property.is_none() { + any_mandatory_property_missing = true; + validation.add_error(format!( + "Component property {} must be defined, unless {} is defined", + property_name.log_color_error_highlight(), + property_ignored_unless_defined_message + )) + } + } + + (!any_mandatory_property_missing).then_some( + WasmComponentPropertySource::Explicit { + properties: unwrap_validated_properties(properties), + }, + ) + } else { + default_profile.map(|default_profile| + WasmComponentPropertySource::ExplicitWithProfiles { + default_profile: default_profile.into(), + profiles: profiles + .into_iter() + .filter_map(|(profile_name, properties)| { + validation.push_context("profile", profile_name.clone()); + + properties.add_unknown_property_warns(Vec::new, validation); + + let mut any_mandatory_property_missing = false; + for (property, property_name) in [ + (&properties.input_wit, "inputWit"), + (&properties.output_wit, "outputWit"), + (&properties.input_wasm, "inputWasm"), + (&properties.output_wasm, "outputWasm"), + ] { + if property.is_none() { + any_mandatory_property_missing = true; + validation.add_error( + format!( + "Component property {} for profile {} must be defined, unless {} is defined", + property_name.log_color_error_highlight(), + profile_name.log_color_highlight(), + property_ignored_unless_defined_message, + ) + ); + } + } + + validation.pop_context(); + + (!any_mandatory_property_missing).then_some( + ( + profile_name.into(), + unwrap_validated_properties(properties), + ) + ) + }) + .collect(), + }) + } + } + } + } + fn convert_wasm_rpc_stub_build( source: &Path, validation: &mut ValidationBuilder, @@ -1065,10 +1128,31 @@ impl Application { let template_context = minijinja::context! { componentName => component_name.as_str() }; match templates.and_then(|templates| templates.get(template_name)) { - Some(template) => match &template.default_profile { - Some(default_profile) => { - let profiles = template - .profiles + Some(template) => match template { + WasmComponentPropertySource::Explicit { properties } => match properties + .render(&env, &template_context) + { + Ok(properties) => { + rendered.insert( + component_name.clone(), + RenderedWasmComponentProperties::Single { properties }, + ); + } + Err(err) => { + validation.push_context("component name", component_name.to_string()); + validation.push_context("template name", template_name.to_string()); + + validation + .add_error(format!("Failed to render component template: {}", err)); + validation.pop_context(); + validation.pop_context(); + } + }, + WasmComponentPropertySource::ExplicitWithProfiles { + default_profile, + profiles, + } => { + let profiles = profiles .iter() .filter_map(|(profile_name, template)| { match template.render(&env, &template_context) { @@ -1108,23 +1192,9 @@ impl Application { }, ); } - None => match template.properties.render(&env, &template_context) { - Ok(properties) => { - rendered.insert( - component_name.clone(), - RenderedWasmComponentProperties::Single { properties }, - ); - } - Err(err) => { - validation.push_context("component name", component_name.to_string()); - validation.push_context("template name", template_name.to_string()); - - validation - .add_error(format!("Failed to render component template: {}", err)); - validation.pop_context(); - validation.pop_context(); - } - }, + WasmComponentPropertySource::Template { .. } => { + panic!("Unexpected template property source for template"); + } }, None => { validation.add_error(format!( @@ -1500,13 +1570,14 @@ impl Application { pub type BuildStep = raw::BuildStep; +// NOTE: we do not use PathBuf here for fields, rather Strings, as values might be templated #[derive(Clone, Debug)] pub struct WasmComponentProperties { pub build: Vec, - pub input_wit: PathBuf, - pub output_wit: PathBuf, - pub input_wasm: PathBuf, - pub output_wasm: PathBuf, + pub input_wit: String, + pub output_wit: String, + pub input_wasm: String, + pub output_wasm: String, } #[derive(Clone, Debug)] @@ -1568,7 +1639,7 @@ pub struct WasmBuild { source: PathBuf, name: String, build_dir: Option, - component_templates: HashMap, + component_templates: HashMap, wit_deps: Vec, } diff --git a/wasm-rpc-stubgen/src/validation.rs b/wasm-rpc-stubgen/src/validation.rs index 178fad71..a5ccec92 100644 --- a/wasm-rpc-stubgen/src/validation.rs +++ b/wasm-rpc-stubgen/src/validation.rs @@ -214,7 +214,7 @@ impl ValidationBuilder { self.context .iter() .map(|c| format!("{}: {}", c.name, c.value.log_color_highlight())) - .join(", ") + .join("\n") ) } else { format!( @@ -222,7 +222,7 @@ impl ValidationBuilder { self.context .iter() .map(|c| format!("{}: {}", c.name, c.value.log_color_highlight())) - .join("\n") + .join(", ") ) } }; diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index aa9e668e..f193799d 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -296,10 +296,9 @@ impl ResolvedWitApplication { .collect(); let input_contained_package_deps = { - let deps_path = app - .component_properties(component_name, profile) - .input_wit - .join("deps"); + let deps_path = + Path::new(&app.component_properties(component_name, profile).input_wit) + .join("deps"); if !deps_path.exists() { HashSet::new() } else { From 51a225969122d40c05acfe926618533d3efd7b80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Fri, 22 Nov 2024 14:00:51 -0500 Subject: [PATCH 086/109] custom commands and clean targets --- wasm-rpc-stubgen/src/commands/declarative.rs | 170 +++++++++++++------ wasm-rpc-stubgen/src/lib.rs | 14 ++ wasm-rpc-stubgen/src/model/wasm_rpc.rs | 43 +++-- 3 files changed, 165 insertions(+), 62 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index c24c0f6b..30135176 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -7,8 +7,8 @@ use crate::log::{ }; use crate::model::oam; use crate::model::wasm_rpc::{ - include_glob_patter_from_yaml_file, init_oam_app, Application, ComponentName, ProfileName, - DEFAULT_CONFIG_FILE_NAME, + include_glob_patter_from_yaml_file, init_oam_app, Application, ComponentName, CustomCommand, + ProfileName, WasmComponent, DEFAULT_CONFIG_FILE_NAME, }; use crate::stub::{StubConfig, StubDefinition}; use crate::validation::ValidatedResult; @@ -373,51 +373,7 @@ fn component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<()> { let _indent = LogIndent::new(); for build_step in &component_properties.build { - let build_dir = build_step - .dir - .as_ref() - .map(|dir| component.source_dir().join(dir)) - .unwrap_or_else(|| component.source_dir().to_path_buf()); - - if !build_step.inputs.is_empty() && !build_step.outputs.is_empty() { - let inputs = compile_and_collect_globs(&build_dir, &build_step.inputs)?; - let outputs = compile_and_collect_globs(&build_dir, &build_step.outputs)?; - - if is_up_to_date(ctx.config.skip_up_to_date_checks, || inputs, || outputs) { - log_skipping_up_to_date(format!( - "executing command: {} in directory {}", - build_step.command.log_color_highlight(), - build_dir.log_color_highlight() - )); - continue; - } - } - - log_action( - "Executing", - format!("command: {}", build_step.command.log_color_highlight()), - ); - - let command_tokens = build_step.command.split(' ').collect::>(); - if command_tokens.is_empty() { - return Err(anyhow!("Empty command!")); - } - - let result = Command::new(command_tokens[0]) - .args(command_tokens.iter().skip(1)) - .current_dir(build_dir) - .status() - .with_context(|| "Failed to execute command".to_string())?; - - if !result.success() { - return Err(anyhow!(format!( - "Command failed with exit code: {}", - result - .code() - .map(|code| code.to_string().log_color_error_highlight().to_string()) - .unwrap_or_else(|| "?".to_string()) - ))); - } + execute_custom_command(&ctx, component, &build_step)?; } } @@ -553,10 +509,9 @@ pub fn clean(config: Config) -> anyhow::Result<()> { app.component_output_wasm(component_name, profile.as_ref()), )); - let build = &app - .component_properties(component_name, profile.as_ref()) - .build; - for build_step in build { + let properties = &app.component_properties(component_name, profile.as_ref()); + + for build_step in &properties.build { let build_dir = build_step .dir .as_ref() @@ -569,6 +524,13 @@ pub fn clean(config: Config) -> anyhow::Result<()> { .map(|path| ("build output", path)), ); } + + paths.extend( + properties + .clean + .iter() + .map(|path| ("clean target", component.source_dir().join(path))), + ); } } paths @@ -608,6 +570,58 @@ pub fn clean(config: Config) -> anyhow::Result<()> { Ok(()) } +pub fn custom_command(config: Config, command: String) -> anyhow::Result<()> { + let ctx = ApplicationContext::new(config)?; + + let all_custom_commands = ctx.application.all_custom_commands(ctx.profile()); + if !all_custom_commands.contains(&command) { + if all_custom_commands.is_empty() { + bail!( + "Custom command {} not found, available custom commands: {}", + command.log_color_error_highlight(), + all_custom_commands + .iter() + .map(|s| s.log_color_highlight()) + .join(", ") + ); + } else { + bail!( + "Custom command {} not found, no custom command is available", + command.log_color_error_highlight(), + ); + } + } + + log_action( + "Executing", + format!("custom command {}", command.log_color_highlight()), + ); + let _indent = LogIndent::new(); + + for (component_name, component) in ctx.application.components() { + let properties = &ctx + .application + .component_properties(component_name, ctx.profile()); + if let Some(custom_command) = properties.custom_commands.get(&command) { + log_action( + "Executing", + format!( + "custom command {} for component {}", + command.log_color_highlight(), + component_name.as_str().log_color_highlight() + ), + ); + let _indent = LogIndent::new(); + + for step in custom_command { + execute_custom_command(&ctx, component, step)?; + } + } + } + + Ok(()) +} + fn delete_path(context: &str, path: &Path) -> anyhow::Result<()> { if path.exists() { log_warn_action( @@ -1189,6 +1203,60 @@ fn copy_wit_sources(source: &Path, target: &Path) -> anyhow::Result<()> { Ok(()) } +fn execute_custom_command( + ctx: &ApplicationContext, + component: &WasmComponent, + command: &CustomCommand, +) -> anyhow::Result<()> { + let build_dir = command + .dir + .as_ref() + .map(|dir| component.source_dir().join(dir)) + .unwrap_or_else(|| component.source_dir().to_path_buf()); + + if !command.inputs.is_empty() && !command.outputs.is_empty() { + let inputs = compile_and_collect_globs(&build_dir, &command.inputs)?; + let outputs = compile_and_collect_globs(&build_dir, &command.outputs)?; + + if is_up_to_date(ctx.config.skip_up_to_date_checks, || inputs, || outputs) { + log_skipping_up_to_date(format!( + "executing command: {} in directory {}", + command.command.log_color_highlight(), + build_dir.log_color_highlight() + )); + return Ok(()); + } + } + + log_action( + "Executing", + format!("command: {}", command.command.log_color_highlight()), + ); + + let command_tokens = command.command.split(' ').collect::>(); + if command_tokens.is_empty() { + return Err(anyhow!("Empty command!")); + } + + let result = Command::new(command_tokens[0]) + .args(command_tokens.iter().skip(1)) + .current_dir(build_dir) + .status() + .with_context(|| "Failed to execute command".to_string())?; + + if !result.success() { + return Err(anyhow!(format!( + "Command failed with exit code: {}", + result + .code() + .map(|code| code.to_string().log_color_error_highlight().to_string()) + .unwrap_or_else(|| "?".to_string()) + ))); + } + + Ok(()) +} + static GENERATED_DIR_DONE_MARKER_FILE_NAME: &str = ".done"; struct GeneratedDirDoneMarker<'a> { diff --git a/wasm-rpc-stubgen/src/lib.rs b/wasm-rpc-stubgen/src/lib.rs index 0c948e1f..1419037a 100644 --- a/wasm-rpc-stubgen/src/lib.rs +++ b/wasm-rpc-stubgen/src/lib.rs @@ -202,6 +202,8 @@ pub enum App { Build(DeclarativeBuildArgs), /// Clean outputs Clean(DeclarativeBuildArgs), + ///x Run custom command + CustomCommand(DeclarativeCustomCommand), } #[derive(clap::Args, Debug)] @@ -225,6 +227,15 @@ pub struct DeclarativeBuildArgs { pub profile: Option, } +#[derive(clap::Args, Debug)] +#[command(version, about, long_about = None)] +pub struct DeclarativeCustomCommand { + #[clap(flatten)] + args: DeclarativeBuildArgs, + #[arg(value_name = "custom command")] + command: String, +} + pub fn generate(args: GenerateArgs) -> anyhow::Result<()> { let stub_def = StubDefinition::new( StubConfig { @@ -302,6 +313,9 @@ pub async fn run_declarative_command(command: App) -> anyhow::Result<()> { } App::Build(args) => commands::declarative::build(dec_build_args_to_config(args)).await, App::Clean(args) => commands::declarative::clean(dec_build_args_to_config(args)), + App::CustomCommand(args) => { + commands::declarative::custom_command(dec_build_args_to_config(args.args), args.command) + } } } diff --git a/wasm-rpc-stubgen/src/model/wasm_rpc.rs b/wasm-rpc-stubgen/src/model/wasm_rpc.rs index 4b76e5ea..e12102f5 100644 --- a/wasm-rpc-stubgen/src/model/wasm_rpc.rs +++ b/wasm-rpc-stubgen/src/model/wasm_rpc.rs @@ -88,7 +88,7 @@ mod raw { pub const OAM_COMPONENT_TYPE_WASM_RPC_STUB_BUILD: &str = "wasm-rpc-stub-build"; #[derive(Clone, Debug, Serialize, Deserialize)] - pub struct BuildStep { + pub struct CustomCommand { pub command: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub dir: Option, @@ -106,11 +106,15 @@ mod raw { #[serde(default, skip_serializing_if = "Option::is_none")] pub output_wit: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub build: Vec, + pub build: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] pub input_wasm: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub output_wasm: Option, + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub custom_commands: HashMap>, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub clean: Vec, #[serde(flatten)] pub unknown_properties: UnknownProperties, } @@ -244,7 +248,7 @@ mod raw { } mod template { - use crate::model::wasm_rpc::{raw, BuildStep, WasmComponentProperties}; + use crate::model::wasm_rpc::{raw, CustomCommand, WasmComponentProperties}; use serde::Serialize; use std::collections::HashMap; @@ -313,15 +317,15 @@ mod template { } } - impl Template for raw::BuildStep { - type Rendered = BuildStep; + impl Template for raw::CustomCommand { + type Rendered = CustomCommand; fn render( &self, env: &minijinja::Environment, ctx: &C, ) -> Result { - Ok(BuildStep { + Ok(CustomCommand { command: self.command.render(env, ctx)?, dir: self.dir.render(env, ctx)?, inputs: self.inputs.render(env, ctx)?, @@ -344,6 +348,8 @@ mod template { output_wit: self.output_wit.render(env, ctx)?, input_wasm: self.input_wasm.render(env, ctx)?, output_wasm: self.output_wasm.render(env, ctx)?, + custom_commands: self.custom_commands.render(env, ctx)?, + clean: self.clean.render(env, ctx)?, }) } } @@ -705,6 +711,8 @@ impl Application { output_wit: properties.output_wit.unwrap(), input_wasm: properties.input_wasm.unwrap(), output_wasm: properties.output_wasm.unwrap(), + custom_commands: properties.custom_commands, + clean: properties.clean, } } @@ -1270,6 +1278,17 @@ impl Application { .collect() } + pub fn all_custom_commands(&self, profile: Option<&ProfileName>) -> BTreeSet { + self.component_names() + .flat_map(|component_name| { + self.component_properties(component_name, profile) + .custom_commands + .keys() + .cloned() + }) + .collect() + } + pub fn build_dir(&self) -> PathBuf { self.common_wasm_build .as_ref() @@ -1568,16 +1587,18 @@ impl Application { } } -pub type BuildStep = raw::BuildStep; +pub type CustomCommand = raw::CustomCommand; // NOTE: we do not use PathBuf here for fields, rather Strings, as values might be templated #[derive(Clone, Debug)] pub struct WasmComponentProperties { - pub build: Vec, + pub build: Vec, pub input_wit: String, pub output_wit: String, pub input_wasm: String, pub output_wasm: String, + pub custom_commands: HashMap>, + pub clean: Vec, } #[derive(Clone, Debug)] @@ -1622,15 +1643,15 @@ pub enum BuildStepsLookupResult<'a> { NoBuildSteps, NoBuildStepsForRequestedProfile, BuildSteps { - build_steps: &'a [BuildStep], + build_steps: &'a [CustomCommand], }, BuildStepsForDefaultProfile { profile: &'a str, - build_steps: &'a [BuildStep], + build_steps: &'a [CustomCommand], }, BuildStepsForRequestedProfile { profile: &'a str, - build_steps: &'a [BuildStep], + build_steps: &'a [CustomCommand], }, } From 2e7f0591a9e7b78bf91d2bea2f34c7ab1e66e268 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Fri, 22 Nov 2024 14:02:25 -0500 Subject: [PATCH 087/109] custom commands and clean targets --- wasm-rpc-stubgen/src/commands/declarative.rs | 17 ++++++++++------- wasm-rpc-stubgen/src/model/wasm_rpc.rs | 14 +++++++------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index 30135176..ef44ccba 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -7,7 +7,7 @@ use crate::log::{ }; use crate::model::oam; use crate::model::wasm_rpc::{ - include_glob_patter_from_yaml_file, init_oam_app, Application, ComponentName, CustomCommand, + include_glob_patter_from_yaml_file, init_oam_app, Application, ComponentName, ExternalCommand, ProfileName, WasmComponent, DEFAULT_CONFIG_FILE_NAME, }; use crate::stub::{StubConfig, StubDefinition}; @@ -373,7 +373,7 @@ fn component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<()> { let _indent = LogIndent::new(); for build_step in &component_properties.build { - execute_custom_command(&ctx, component, &build_step)?; + execute_external_command(&ctx, component, &build_step)?; } } @@ -614,7 +614,7 @@ pub fn custom_command(config: Config, command: String) -> anyhow::Result<()> { let _indent = LogIndent::new(); for step in custom_command { - execute_custom_command(&ctx, component, step)?; + execute_external_command(&ctx, component, step)?; } } } @@ -1203,10 +1203,10 @@ fn copy_wit_sources(source: &Path, target: &Path) -> anyhow::Result<()> { Ok(()) } -fn execute_custom_command( +fn execute_external_command( ctx: &ApplicationContext, component: &WasmComponent, - command: &CustomCommand, + command: &ExternalCommand, ) -> anyhow::Result<()> { let build_dir = command .dir @@ -1220,7 +1220,7 @@ fn execute_custom_command( if is_up_to_date(ctx.config.skip_up_to_date_checks, || inputs, || outputs) { log_skipping_up_to_date(format!( - "executing command: {} in directory {}", + "executing external command: {} in directory {}", command.command.log_color_highlight(), build_dir.log_color_highlight() )); @@ -1230,7 +1230,10 @@ fn execute_custom_command( log_action( "Executing", - format!("command: {}", command.command.log_color_highlight()), + format!( + "external command: {}", + command.command.log_color_highlight() + ), ); let command_tokens = command.command.split(' ').collect::>(); diff --git a/wasm-rpc-stubgen/src/model/wasm_rpc.rs b/wasm-rpc-stubgen/src/model/wasm_rpc.rs index e12102f5..5884d34e 100644 --- a/wasm-rpc-stubgen/src/model/wasm_rpc.rs +++ b/wasm-rpc-stubgen/src/model/wasm_rpc.rs @@ -248,7 +248,7 @@ mod raw { } mod template { - use crate::model::wasm_rpc::{raw, CustomCommand, WasmComponentProperties}; + use crate::model::wasm_rpc::{raw, ExternalCommand, WasmComponentProperties}; use serde::Serialize; use std::collections::HashMap; @@ -318,14 +318,14 @@ mod template { } impl Template for raw::CustomCommand { - type Rendered = CustomCommand; + type Rendered = ExternalCommand; fn render( &self, env: &minijinja::Environment, ctx: &C, ) -> Result { - Ok(CustomCommand { + Ok(ExternalCommand { command: self.command.render(env, ctx)?, dir: self.dir.render(env, ctx)?, inputs: self.inputs.render(env, ctx)?, @@ -1587,7 +1587,7 @@ impl Application { } } -pub type CustomCommand = raw::CustomCommand; +pub type ExternalCommand = raw::CustomCommand; // NOTE: we do not use PathBuf here for fields, rather Strings, as values might be templated #[derive(Clone, Debug)] @@ -1643,15 +1643,15 @@ pub enum BuildStepsLookupResult<'a> { NoBuildSteps, NoBuildStepsForRequestedProfile, BuildSteps { - build_steps: &'a [CustomCommand], + build_steps: &'a [ExternalCommand], }, BuildStepsForDefaultProfile { profile: &'a str, - build_steps: &'a [CustomCommand], + build_steps: &'a [ExternalCommand], }, BuildStepsForRequestedProfile { profile: &'a str, - build_steps: &'a [CustomCommand], + build_steps: &'a [ExternalCommand], }, } From 036ff1e791583b292ff78d116d038eb4e1bcf34e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Fri, 22 Nov 2024 14:06:02 -0500 Subject: [PATCH 088/109] fix custom command listing --- wasm-rpc-stubgen/src/commands/declarative.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index ef44ccba..6a55b0fd 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -576,6 +576,11 @@ pub fn custom_command(config: Config, command: String) -> anyhow::Result<()> { let all_custom_commands = ctx.application.all_custom_commands(ctx.profile()); if !all_custom_commands.contains(&command) { if all_custom_commands.is_empty() { + bail!( + "Custom command {} not found, no custom command is available", + command.log_color_error_highlight(), + ); + } else { bail!( "Custom command {} not found, available custom commands: {}", command.log_color_error_highlight(), @@ -584,11 +589,6 @@ pub fn custom_command(config: Config, command: String) -> anyhow::Result<()> { .map(|s| s.log_color_highlight()) .join(", ") ); - } else { - bail!( - "Custom command {} not found, no custom command is available", - command.log_color_error_highlight(), - ); } } From c6905cc19e03ef4928bbae9f55bda22f3fa3c0ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Fri, 22 Nov 2024 14:09:37 -0500 Subject: [PATCH 089/109] cleanup --- wasm-rpc-stubgen/src/commands/declarative.rs | 2 +- wasm-rpc-stubgen/src/model/wasm_rpc.rs | 8 +------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index 6a55b0fd..8dcfc2e5 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -373,7 +373,7 @@ fn component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<()> { let _indent = LogIndent::new(); for build_step in &component_properties.build { - execute_external_command(&ctx, component, &build_step)?; + execute_external_command(ctx, component, build_step)?; } } diff --git a/wasm-rpc-stubgen/src/model/wasm_rpc.rs b/wasm-rpc-stubgen/src/model/wasm_rpc.rs index 5884d34e..36a875fa 100644 --- a/wasm-rpc-stubgen/src/model/wasm_rpc.rs +++ b/wasm-rpc-stubgen/src/model/wasm_rpc.rs @@ -1014,12 +1014,6 @@ impl Application { )); } - if wasm_builds.len() == 1 { - for (_template_name, _build) in &wasm_builds[0].component_templates { - // TODO: validate templates - } - } - wasm_builds.into_iter().next() } @@ -1400,7 +1394,7 @@ impl Application { default_profile, profiles, } => { - let profile = profile.unwrap_or(&default_profile); + let profile = profile.unwrap_or(default_profile); profiles.get(profile).unwrap_or_else(|| { profiles.get(default_profile).unwrap_or_else(|| { panic!( From 523eb1077b27281e7fee0c2534a2d28cae99183a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Fri, 22 Nov 2024 14:10:41 -0500 Subject: [PATCH 090/109] format --- wasm-rpc-stubgen/src/commands/declarative.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index 8dcfc2e5..8c350a5d 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -1220,7 +1220,7 @@ fn execute_external_command( if is_up_to_date(ctx.config.skip_up_to_date_checks, || inputs, || outputs) { log_skipping_up_to_date(format!( - "executing external command: {} in directory {}", + "executing external command '{}' in directory {}", command.command.log_color_highlight(), build_dir.log_color_highlight() )); @@ -1231,8 +1231,9 @@ fn execute_external_command( log_action( "Executing", format!( - "external command: {}", - command.command.log_color_highlight() + "external command '{}' in directory {}", + command.command.log_color_highlight(), + build_dir.log_color_highlight() ), ); From b0136c6b8accf6e23103a1e5107b9dabf16fbec3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Fri, 22 Nov 2024 14:45:09 -0500 Subject: [PATCH 091/109] cleanup --- wasm-rpc-stubgen/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wasm-rpc-stubgen/src/lib.rs b/wasm-rpc-stubgen/src/lib.rs index 1419037a..5f7fa568 100644 --- a/wasm-rpc-stubgen/src/lib.rs +++ b/wasm-rpc-stubgen/src/lib.rs @@ -202,7 +202,7 @@ pub enum App { Build(DeclarativeBuildArgs), /// Clean outputs Clean(DeclarativeBuildArgs), - ///x Run custom command + /// Run custom command CustomCommand(DeclarativeCustomCommand), } From 8ef65e7bb139661ab3b36bfafa65bf27447e73db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Mon, 25 Nov 2024 15:55:44 -0500 Subject: [PATCH 092/109] add new app manifest schema / delete leftover example --- wasm-rpc-stubgen/example/deps/dep1/dep1.wit | 30 -- wasm-rpc-stubgen/example/deps/dep2/dep2.wit | 5 - wasm-rpc-stubgen/example/main.wit | 77 ----- .../schema/golem-wasm-rpc.schema.json | 273 ++++++++++++++++++ 4 files changed, 273 insertions(+), 112 deletions(-) delete mode 100644 wasm-rpc-stubgen/example/deps/dep1/dep1.wit delete mode 100644 wasm-rpc-stubgen/example/deps/dep2/dep2.wit delete mode 100644 wasm-rpc-stubgen/example/main.wit create mode 100644 wasm-rpc-stubgen/schema/golem-wasm-rpc.schema.json diff --git a/wasm-rpc-stubgen/example/deps/dep1/dep1.wit b/wasm-rpc-stubgen/example/deps/dep1/dep1.wit deleted file mode 100644 index 120db332..00000000 --- a/wasm-rpc-stubgen/example/deps/dep1/dep1.wit +++ /dev/null @@ -1,30 +0,0 @@ -package test:dep1@0.1.0; - -interface iface2 { - use test:dep2/types@0.1.0.{out}; - - enum a-or-b { - a, - b, - } - - type alias = a-or-b; - - variant storage { - in-memory, - s3(string), - filesystem(filesystem-config) - } - - record filesystem-config { - path: string, - max-size: u64, - } - - g: func(in: a-or-b) -> out; - gg: func(in: a-or-b) -> tuple; - - h: func(s: storage); - hh: func(s: storage) -> a-or-b; - hhh: func(in: alias) -> storage; -} diff --git a/wasm-rpc-stubgen/example/deps/dep2/dep2.wit b/wasm-rpc-stubgen/example/deps/dep2/dep2.wit deleted file mode 100644 index 2de169cb..00000000 --- a/wasm-rpc-stubgen/example/deps/dep2/dep2.wit +++ /dev/null @@ -1,5 +0,0 @@ -package test:dep2@0.1.0; - -interface types { - type out = bool; -} diff --git a/wasm-rpc-stubgen/example/main.wit b/wasm-rpc-stubgen/example/main.wit deleted file mode 100644 index 04a7b5d2..00000000 --- a/wasm-rpc-stubgen/example/main.wit +++ /dev/null @@ -1,77 +0,0 @@ -package test:main; - -interface iface1 { - - flags permissions { - read, - write, - exec, - close - } - - record metadata { - name: string, - origin: string, - perms: permissions - } - - record point { - x: s32, - y: s32, - metadata: metadata - } - - type point-tuple = tuple; - - add: func(x: s32, y: s32) -> s64; - point-to-string: func(p: point) -> string; - points-to-strings: func(ps: list) -> list; - tuple-to-point: func(t: point-tuple, metadata: option) -> result; - - process-result: func(r: result); - new-point: func(x: s32, y: s32, metadata: metadata) -> point; - - get-metadata: func() -> option; - - record product-item { - product-id: string, - name: string, - price: float32, - quantity: u32, - } - - record order { - order-id: string, - items: list, - total: float32, - timestamp: u64, - } - - record order-confirmation { - order-id: string, - } - - variant checkout-result { - error(string), - success(order-confirmation), - } - - resource cart { - constructor(user-id: string); - add-item: func(item: product-item) -> (); - remove-item: func(product-id: string) -> (); - update-item-quantity: func(product-id: string, quantity: u32) -> (); - checkout: func() -> checkout-result; - get-cart-contents: func() -> list; - merge-with: func(other-cart: borrow) -> (); - merge: static func(cart1: borrow, cart2: borrow) -> cart; - } -} - -world api { - export iface1; - export test:dep1/iface2@0.1.0; - - export run: func(param: string); - -} diff --git a/wasm-rpc-stubgen/schema/golem-wasm-rpc.schema.json b/wasm-rpc-stubgen/schema/golem-wasm-rpc.schema.json new file mode 100644 index 00000000..9c077e35 --- /dev/null +++ b/wasm-rpc-stubgen/schema/golem-wasm-rpc.schema.json @@ -0,0 +1,273 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://golem.cloud/golem-wasm-rpc.schema.json", + "title": "Golem WASM RPC Application Manifest", + "description": "Golem WASM RPC Application Manifest.", + "type": "object", + "properties": { + "include": { + "type": "array", + "description": "Include paths or globs for searching for application manifest documents. Only allowed in root application manifest documents.", + "items": { + "type": "string" + } + }, + "tempDir": { + "type": "string", + "description": "Temporary directory used for generating and building WIT and WASM artifacts. Default location is golem-temp." + }, + "templates": { + "type": "object", + "description": "Component definition templates", + "additionalProperties": { + "$ref": "#/definitions/componentTemplate" + } + }, + "components": { + "type": "object", + "description": "Components by component names", + "additionalProperties": { + "$ref": "#/definitions/component" + } + }, + "dependencies": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/componentDependency" + } + } + }, + "additionalProperties": false, + "definitions": { + "componentTemplate": { + "oneOf": [ + { + "$ref": "#/definitions/componentProperties", + "additionalProperties": false + }, + { + "$ref": "#/definitions/componentProfiles", + "additionalProperties": false + } + ] + }, + "component": { + "description": "Component definition", + "oneOf": [ + { + "$ref": "#/definitions/componentPropertiesWithTemplateRef", + "additionalProperties": false + }, + { + "$ref": "#/definitions/componentProfilesWithTemplateRef", + "additionalProperties": false + } + ] + }, + "templateRef": { + "properties": { + "template": { + "type": "string", + "description": "Component template to be used for defining this component." + } + }, + "additionalProperties": false + }, + "componentProperties": { + "type": "object", + "properties": { + "sourceWit": { + "type": "string", + "description": "Source WIT directory for the user defined component WIT source(s)." + }, + "generatedWit": { + "type": "string", + "description": "Generated WIT directory created by the golem tooling, which handles exported interface extraction and includes resolved package and stub dependencies." + }, + "componentWasm": { + "type": "string", + "description": "File path for the built WASM component." + }, + "linkedWasm": { + "type": "string", + "description": "File path for the linked WASM component which is ready to be uploaded to Golem." + }, + "build": { + "type": "array", + "description": "Commands used for creating component WASM.", + "items": { + "$ref": "#/definitions/externalCommand" + } + }, + "customCommands": { + "type": "object", + "description": "User defined custom commands.", + "additionalProperties": { + "type": "array", + "items": { + "$ref": "#/definitions/externalCommand" + } + } + }, + "clean": { + "type": "array", + "description": "User defined extra paths used in the clean command.", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false + }, + "componentPropertiesWithTemplateRef": { + "type": "object", + "properties": { + "template": { + "type": "string", + "description": "Component template to be used for defining this component." + }, + "sourceWit": { + "type": "string", + "description": "Source WIT directory for the user defined component WIT source(s)." + }, + "generatedWit": { + "type": "string", + "description": "Generated WIT directory created by the golem tooling, which handles exported interface extraction and includes resolved package and stub dependencies." + }, + "componentWasm": { + "type": "string", + "description": "File path for the built WASM component." + }, + "linkedWasm": { + "type": "string", + "description": "File path for the linked WASM component which is ready to be uploaded to Golem." + }, + "build": { + "type": "array", + "description": "Commands used for creating component WASM.", + "items": { + "$ref": "#/definitions/externalCommand" + } + }, + "customCommands": { + "type": "object", + "description": "User defined custom commands.", + "additionalProperties": { + "type": "array", + "items": { + "$ref": "#/definitions/externalCommand" + } + } + }, + "clean": { + "type": "array", + "description": "User defined extra paths used in the clean command.", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false + }, + "componentProfiles": { + "type": "object", + "description": "Component definition profiles", + "properties": { + "profiles": { + "type": "object", + "description": "Component definition profiles", + "additionalProperties": { + "$ref": "#/definitions/componentProperties" + } + } + }, + "additionalProperties": false + }, + "componentProfilesWithTemplateRef": { + "type": "object", + "description": "Component definition profiles", + "properties": { + "template": { + "type": "string", + "description": "Component template to be used for defining this component." + }, + "profiles": { + "type": "object", + "description": "Component definition profiles", + "additionalProperties": { + "$ref": "#/definitions/componentProperties" + } + } + }, + "additionalProperties": false + }, + "externalCommand": { + "type": "object", + "description": "External command with optional inputs and outputs with up-to-date checks", + "properties": { + }, + "oneOf": [ + { + "properties": { + "command": { + "type": "string", + "description": "External command to execute" + } + }, + "additionalProperties": false, + "required": [ + "command" + ] + }, + { + "properties": { + "command": { + "type": "string", + "description": "External command to execute" + }, + "inputs": { + "type": "array", + "description": "Inputs (paths and globs) for the external command", + "items": { + "type": "string" + } + }, + "outputs": { + "type": "array", + "description": "Output (paths and globs) for the external command", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false, + "required": [ + "command", + "inputs", + "outputs" + ] + } + ] + }, + "componentDependency": { + "type": "object", + "description": "Component dependencies", + "oneOf": [ + { + "properties": { + "type": { + "const": "wasm-rpc", + "description": "WASM RPC dependency" + }, + "target": { + "type": "string", + "description": "Target component name." + } + }, + "required": ["type", "target"], + "additionalProperties": false + } + ] + } + } +} \ No newline at end of file From 14470dfcb58d2478b3baa6533e8e96ff2b3eba23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Sun, 1 Dec 2024 22:32:38 +0100 Subject: [PATCH 093/109] WIP: switching to new schema --- .../schema/golem-wasm-rpc.schema.json | 27 +- wasm-rpc-stubgen/src/commands/declarative.rs | 472 ++-- wasm-rpc-stubgen/src/lib.rs | 3 - wasm-rpc-stubgen/src/model/mod.rs | 1 - wasm-rpc-stubgen/src/model/oam.rs | 307 --- wasm-rpc-stubgen/src/model/wasm_rpc.rs | 2079 ++++++----------- wasm-rpc-stubgen/src/stub.rs | 2 +- wasm-rpc-stubgen/src/validation.rs | 16 +- wasm-rpc-stubgen/src/wit_resolve.rs | 120 +- wasm-rpc/src/lib.rs | 2 +- 10 files changed, 1062 insertions(+), 1967 deletions(-) delete mode 100644 wasm-rpc-stubgen/src/model/oam.rs diff --git a/wasm-rpc-stubgen/schema/golem-wasm-rpc.schema.json b/wasm-rpc-stubgen/schema/golem-wasm-rpc.schema.json index 9c077e35..a8186770 100644 --- a/wasm-rpc-stubgen/schema/golem-wasm-rpc.schema.json +++ b/wasm-rpc-stubgen/schema/golem-wasm-rpc.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft-07/schema#", "$id": "https://golem.cloud/golem-wasm-rpc.schema.json", "title": "Golem WASM RPC Application Manifest", "description": "Golem WASM RPC Application Manifest.", @@ -179,9 +179,17 @@ "additionalProperties": { "$ref": "#/definitions/componentProperties" } + }, + "defaultProfile": { + "type": "string", + "description": "Default profile" } }, - "additionalProperties": false + "additionalProperties": false, + "required": [ + "profiles", + "defaultProfile" + ] }, "componentProfilesWithTemplateRef": { "type": "object", @@ -197,9 +205,17 @@ "additionalProperties": { "$ref": "#/definitions/componentProperties" } + }, + "defaultProfile": { + "type": "string", + "description": "Default profile" } }, - "additionalProperties": false + "additionalProperties": false, + "required": [ + "profiles", + "defaultProfile" + ] }, "externalCommand": { "type": "object", @@ -264,7 +280,10 @@ "description": "Target component name." } }, - "required": ["type", "target"], + "required": [ + "type", + "target" + ], "additionalProperties": false } ] diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index 8c350a5d..8a8f645e 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -5,10 +5,9 @@ use crate::log::{ log_action, log_skipping_up_to_date, log_validated_action_result, log_warn_action, LogColorize, LogIndent, }; -use crate::model::oam; +use crate::model::wasm_rpc::raw::ExternalCommand; use crate::model::wasm_rpc::{ - include_glob_patter_from_yaml_file, init_oam_app, Application, ComponentName, ExternalCommand, - ProfileName, WasmComponent, DEFAULT_CONFIG_FILE_NAME, + includes_from_yaml_file, raw, Application, ComponentName, ProfileName, DEFAULT_CONFIG_FILE_NAME, }; use crate::stub::{StubConfig, StubDefinition}; use crate::validation::ValidatedResult; @@ -21,6 +20,7 @@ use crate::{commands, naming, WasmRpcOverride}; use anyhow::{anyhow, bail, Context, Error}; use colored::Colorize; use glob::glob; +use golem_wasm_rpc::WASM_RPC_VERSION; use itertools::Itertools; use std::cell::OnceCell; use std::cmp::Ordering; @@ -47,7 +47,7 @@ struct ApplicationContext { application: Application, wit: ResolvedWitApplication, common_wit_deps: OnceCell>, - component_base_output_wit_deps: HashMap, + component_generated_base_wit_deps: HashMap, } impl ApplicationContext { @@ -61,7 +61,7 @@ impl ApplicationContext { application, wit, common_wit_deps: OnceCell::new(), - component_base_output_wit_deps: HashMap::new(), + component_generated_base_wit_deps: HashMap::new(), } }) }), @@ -104,137 +104,133 @@ impl ApplicationContext { for component_name in ctx.application.component_names() { let selection = ctx .application - .component_effective_template_and_profile(component_name, ctx.profile()); + .component_effective_property_source(component_name, ctx.profile()); - match ( + let message = match ( selection.profile, selection.template_name, ctx.profile().is_some(), selection.is_requested_profile, ) { (None, None, false, _) => { - log_action( - "Selected", - format!( - "default build for {}", - component_name.as_str().log_color_highlight() - ), - ); + format!( + "default build for {}", + component_name.as_str().log_color_highlight() + ) } (None, None, true, _) => { - log_action( - "Selected", - format!( - "default build for {}, component has no profiles", - component_name.as_str().log_color_highlight() - ), - ); + format!( + "default build for {}, component has no profiles", + component_name.as_str().log_color_highlight() + ) } (None, Some(template), false, _) => { - log_action( - "Selected", - format!( - "default build for {} using template {}", - component_name.as_str().log_color_highlight(), - template.log_color_highlight() - ), - ); + format!( + "default build for {} using template {}{}", + component_name.as_str().log_color_highlight(), + template.as_str().log_color_highlight(), + if selection.any_template_overrides { + " with overrides" + } else { + "" + } + ) } (None, Some(template), true, _) => { - log_action( - "Selected", - format!( - "default build for {} using template {}, component has no profiles", - component_name.as_str().log_color_highlight(), - template.log_color_highlight() - ), - ); + format!( + "default build for {} using template {}{}, component has no profiles", + component_name.as_str().log_color_highlight(), + template.as_str().log_color_highlight(), + if selection.any_template_overrides { + " with overrides" + } else { + "" + } + ) } (Some(profile), None, false, false) => { - log_action( - "Selected", - format!( - "default profile {} for {}", - profile.as_str().log_color_highlight(), - component_name.as_str().log_color_highlight() - ), - ); + format!( + "default profile {} for {}", + profile.as_str().log_color_highlight(), + component_name.as_str().log_color_highlight() + ) } (Some(profile), None, true, false) => { - log_action( - "Selected", - format!( - "default profile {} for {}, component has no matching requested profile", - profile.as_str().log_color_highlight(), - component_name.as_str().log_color_highlight() - ), - ); + format!( + "default profile {} for {}, component has no matching requested profile", + profile.as_str().log_color_highlight(), + component_name.as_str().log_color_highlight() + ) } (Some(profile), Some(template), false, false) => { - log_action( - "Selected", - format!( - "default profile {} for {} using template {}", - profile.as_str().log_color_highlight(), - component_name.as_str().log_color_highlight(), - template.log_color_highlight() - ), - ); + format!( + "default profile {} for {} using template {}{}", + profile.as_str().log_color_highlight(), + component_name.as_str().log_color_highlight(), + template.as_str().log_color_highlight(), + if selection.any_template_overrides { + " with overrides" + } else { + "" + } + ) } (Some(profile), Some(template), true, false) => { - log_action( - "Selected", - format!( - "default profile {} for {} using template {}, component has no matching requested profile", - profile.as_str().log_color_highlight(), - component_name.as_str().log_color_highlight(), - template.log_color_highlight() - ), - ); + format!( + "default profile {} for {} using template {}{}, component has no matching requested profile", + profile.as_str().log_color_highlight(), + component_name.as_str().log_color_highlight(), + template.as_str().log_color_highlight(), + if selection.any_template_overrides { + " with overrides" + } else { + "" + } + ) } (Some(profile), None, false, true) => { - log_action( - "Selected", - format!( - "profile {} for {}", - profile.as_str().log_color_highlight(), - component_name.as_str().log_color_highlight() - ), - ); + format!( + "profile {} for {}", + profile.as_str().log_color_highlight(), + component_name.as_str().log_color_highlight() + ) } (Some(profile), None, true, true) => { - log_action( - "Selected", - format!( - "requested profile {} for {}", - profile.as_str().log_color_highlight(), - component_name.as_str().log_color_highlight() - ), - ); + format!( + "requested profile {} for {}", + profile.as_str().log_color_highlight(), + component_name.as_str().log_color_highlight() + ) } (Some(profile), Some(template), false, true) => { - log_action( - "Selected", - format!( - "profile {} for {} using template {}", - profile.as_str().log_color_highlight(), - component_name.as_str().log_color_highlight(), - template.log_color_highlight() - ), - ); + format!( + "profile {} for {} using template {}{}", + profile.as_str().log_color_highlight(), + component_name.as_str().log_color_highlight(), + template.as_str().log_color_highlight(), + if selection.any_template_overrides { + " with overrides" + } else { + "" + } + ) } (Some(profile), Some(template), true, true) => { - log_action( - "Selected", - format!( - "requested profile {} for {} using template {}", - profile.as_str().log_color_highlight(), - component_name.as_str().log_color_highlight(), - template.log_color_highlight() - ), - ); + format!( + "requested profile {} for {} using template {}{}", + profile.as_str().log_color_highlight(), + component_name.as_str().log_color_highlight(), + template.as_str().log_color_highlight(), + if selection.any_template_overrides { + " with overrides" + } else { + "" + } + ) } - } + }; + + log_action("Selected", message); } } @@ -277,32 +273,24 @@ impl ApplicationContext { ) -> anyhow::Result<&WitDepsResolver> { // Not using the entry API, so we can skip copying the component name if !self - .component_base_output_wit_deps + .component_generated_base_wit_deps .contains_key(component_name) { - self.component_base_output_wit_deps.insert( + self.component_generated_base_wit_deps.insert( component_name.clone(), WitDepsResolver::new(vec![self .application - .component_base_output_wit(component_name) + .component_generated_base_wit(component_name) .join(naming::wit::DEPS_DIR)])?, ); } Ok(self - .component_base_output_wit_deps + .component_generated_base_wit_deps .get(component_name) .unwrap()) } } -pub fn init(component_name: ComponentName) -> anyhow::Result<()> { - fs::write_str( - DEFAULT_CONFIG_FILE_NAME, - init_oam_app(component_name).to_yaml_string(), - ) - .context("Failed to init component application manifest") -} - pub async fn pre_component_build(config: Config) -> anyhow::Result<()> { let mut ctx = ApplicationContext::new(config)?; pre_component_build_ctx(&mut ctx).await @@ -314,7 +302,7 @@ async fn pre_component_build_ctx(ctx: &mut ApplicationContext) -> anyhow::Result { for component_name in ctx.wit.component_order_cloned() { - create_base_output_wit(ctx, &component_name)?; + create_generatde_base_wit(ctx, &component_name)?; } for component_name in &ctx.application.all_wasm_rpc_dependencies() { build_stub(ctx, component_name).await?; @@ -324,7 +312,7 @@ async fn pre_component_build_ctx(ctx: &mut ApplicationContext) -> anyhow::Result { let mut any_changed = false; for component_name in ctx.application.component_names() { - let changed = create_output_wit(ctx, component_name)?; + let changed = create_generated_wit(ctx, component_name)?; if changed { update_cargo_toml(ctx, component_name)?; } @@ -350,7 +338,7 @@ fn component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<()> { log_action("Building", "components"); let _indent = LogIndent::new(); - for (component_name, component) in ctx.application.components() { + for component_name in ctx.application.component_names() { let component_properties = ctx .application .component_properties(component_name, ctx.profile()); @@ -373,7 +361,7 @@ fn component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<()> { let _indent = LogIndent::new(); for build_step in &component_properties.build { - execute_external_command(ctx, component, build_step)?; + execute_external_command(ctx, component_name, build_step)?; } } @@ -389,25 +377,28 @@ async fn post_component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<() log_action("Executing", "post-component-build steps"); let _indent = LogIndent::new(); - for (component_name, component) in ctx.application.components() { - let input_wasm = ctx + for component_name in ctx.application.component_names() { + let source = ctx.application.component_source_dir(component_name); + let dependencies = ctx + .application + .component_wasm_rpc_dependencies(component_name); + let component_wasm = ctx .application - .component_input_wasm(component_name, ctx.profile()); - let output_wasm = ctx + .component_wasm(component_name, ctx.profile()); + let linked_wasm = ctx .application - .component_output_wasm(component_name, ctx.profile()); + .component_linked_wasm(component_name, ctx.profile()); if is_up_to_date( ctx.config.skip_up_to_date_checks, // We also include the component specification source, // so it triggers build in case deps are changed - || [component.source.clone(), input_wasm.clone()], - || [output_wasm.clone()], + || [source.to_path_buf(), component_wasm.clone()], + || [linked_wasm.clone()], ) { log_skipping_up_to_date(format!( "composing wasm rpc dependencies ({}) into {}", - component - .wasm_rpc_dependencies + dependencies .iter() .map(|s| s.as_str().log_color_highlight()) .join(", "), @@ -416,23 +407,22 @@ async fn post_component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<() continue; } - if component.wasm_rpc_dependencies.is_empty() { + if dependencies.is_empty() { log_action( "Copying", format!( "(without composing) {} to {}, no wasm rpc dependencies defined", - input_wasm.log_color_highlight(), - output_wasm.log_color_highlight(), + component_wasm.log_color_highlight(), + linked_wasm.log_color_highlight(), ), ); - fs::copy(&input_wasm, &output_wasm)?; + fs::copy(&component_wasm, &linked_wasm)?; } else { log_action( "Composing", format!( "wasm rpc dependencies ({}) into {}", - component - .wasm_rpc_dependencies + dependencies .iter() .map(|s| s.as_str().log_color_highlight()) .join(", "), @@ -441,19 +431,18 @@ async fn post_component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<() ); let _indent = LogIndent::new(); - let stub_wasms = component - .wasm_rpc_dependencies + let stub_wasms = dependencies .iter() .map(|dep| ctx.application.stub_wasm(dep)) .collect::>(); commands::composition::compose( ctx.application - .component_input_wasm(component_name, ctx.profile()) + .component_wasm(component_name, ctx.profile()) .as_path(), &stub_wasms, ctx.application - .component_output_wasm(component_name, ctx.profile()) + .component_linked_wasm(component_name, ctx.profile()) .as_path(), ) .await?; @@ -494,19 +483,19 @@ pub fn clean(config: Config) -> anyhow::Result<()> { }; let paths = { let mut paths = BTreeSet::<(&'static str, PathBuf)>::new(); - for (component_name, component) in app.components() { + for component_name in app.component_names() { for profile in &all_profiles { paths.insert(( - "output wit", - app.component_output_wit(component_name, profile.as_ref()), + "generated wit", + app.component_generated_wit(component_name, profile.as_ref()), )); paths.insert(( - "input wasm", - app.component_input_wasm(component_name, profile.as_ref()), + "component wasm", + app.component_wasm(component_name, profile.as_ref()), )); paths.insert(( - "output wasm", - app.component_output_wasm(component_name, profile.as_ref()), + "linked wasm", + app.component_linked_wasm(component_name, profile.as_ref()), )); let properties = &app.component_properties(component_name, profile.as_ref()); @@ -515,22 +504,24 @@ pub fn clean(config: Config) -> anyhow::Result<()> { let build_dir = build_step .dir .as_ref() - .map(|dir| component.source_dir().join(dir)) - .unwrap_or_else(|| component.source_dir().to_path_buf()); + .map(|dir| app.component_source_dir(component_name).join(dir)) + .unwrap_or_else(|| { + app.component_source_dir(component_name).to_path_buf() + }); paths.extend( - compile_and_collect_globs(&build_dir, &build_step.outputs)? + compile_and_collect_globs(&build_dir, &build_step.targets)? .into_iter() .map(|path| ("build output", path)), ); } - paths.extend( - properties - .clean - .iter() - .map(|path| ("clean target", component.source_dir().join(path))), - ); + paths.extend(properties.clean.iter().map(|path| { + ( + "clean target", + app.component_source_dir(component_name).join(path), + ) + })); } } paths @@ -564,7 +555,7 @@ pub fn clean(config: Config) -> anyhow::Result<()> { log_action("Cleaning", "application build dir"); let _indent = LogIndent::new(); - delete_path("application build dir", &app.build_dir())?; + delete_path("temp dir", app.temp_dir())?; } Ok(()) @@ -598,7 +589,7 @@ pub fn custom_command(config: Config, command: String) -> anyhow::Result<()> { ); let _indent = LogIndent::new(); - for (component_name, component) in ctx.application.components() { + for component_name in ctx.application.component_names() { let properties = &ctx .application .component_properties(component_name, ctx.profile()); @@ -614,7 +605,7 @@ pub fn custom_command(config: Config, command: String) -> anyhow::Result<()> { let _indent = LogIndent::new(); for step in custom_command { - execute_external_command(&ctx, component, step)?; + execute_external_command(&ctx, component_name, step)?; } } } @@ -645,8 +636,7 @@ fn load_app_validated(config: &Config) -> ValidatedResult { sources .into_iter() .map(|source| { - ValidatedResult::from_result(oam::ApplicationWithSource::from_yaml_file(source)) - .and_then(oam::ApplicationWithSource::validate) + ValidatedResult::from_result(raw::ApplicationWithSource::from_yaml_file(source)) }) .collect::>>() }); @@ -654,7 +644,7 @@ fn load_app_validated(config: &Config) -> ValidatedResult { log_action("Collecting", "components"); let _indent = LogIndent::new(); - let app = oam_apps.and_then(Application::from_oam_apps); + let app = oam_apps.and_then(Application::from_raw_apps); log_validated_action_result("Found", &app, |app| { if app.component_names().next().is_none() { @@ -679,36 +669,22 @@ fn collect_sources(mode: &ApplicationSourceMode) -> ValidatedResult let sources = match mode { ApplicationSourceMode::Automatic => match find_main_source() { Some(source) => { + // TODO: save original current dir and use it as a component filter std::env::set_current_dir(PathExtra::new(&source).parent().unwrap()) .expect("Failed to set current dir for config parent"); - match include_glob_patter_from_yaml_file(source.as_path()) { - Some(pattern) => ValidatedResult::from_result( - glob(pattern.as_str()) - .map_err(|err| { - format!( - "Failed to compile glob pattern: {}, source: {}, error: {}", - pattern, - source.display(), - err - ) - }) - .and_then(|matches| { - matches.collect::, _>>().map_err(|err| { - format!( - "Failed to resolve glob pattern: {}, source: {}, error: {}", - pattern, - source.display(), - err - ) - }) - }), - ) + let includes = includes_from_yaml_file(source.as_path()); + if includes.is_empty() { + ValidatedResult::Ok(vec![source]) + } else { + ValidatedResult::from_result(compile_and_collect_globs( + source.as_path(), + &includes, + )) .map(|mut sources| { sources.insert(0, source); sources - }), - None => ValidatedResult::Ok(vec![source]), + }) } } None => ValidatedResult::from_error("No config file found!".to_string()), @@ -889,25 +865,25 @@ fn compile_and_collect_globs(root_dir: &Path, globs: &[String]) -> Result Result { - let component_input_wit = ctx + let component_source_wit = ctx .application - .component_input_wit(component_name, ctx.profile()); - let component_base_output_wit = ctx.application.component_base_output_wit(component_name); - let gen_dir_done_marker = GeneratedDirDoneMarker::new(&component_base_output_wit); + .component_source_wit(component_name, ctx.profile()); + let component_generated_base_wit = ctx.application.component_generated_base_wit(component_name); + let gen_dir_done_marker = GeneratedDirDoneMarker::new(&component_generated_base_wit); if is_up_to_date( ctx.config.skip_up_to_date_checks || !gen_dir_done_marker.is_done() || !ctx.wit.is_dep_graph_up_to_date(component_name)?, - || [component_input_wit.clone()], - || [component_base_output_wit.clone()], + || [component_source_wit.clone()], + || [component_generated_base_wit.clone()], ) { log_skipping_up_to_date(format!( - "creating base output wit directory for {}", + "creating generated base wit directory for {}", component_name.as_str().log_color_highlight() )); Ok(false) @@ -915,18 +891,22 @@ fn create_base_output_wit( log_action( "Creating", format!( - "base output wit directory for {}", + "generated base wit directory for {}", component_name.as_str().log_color_highlight(), ), ); let _indent = LogIndent::new(); - delete_path("base output wit directory", &component_base_output_wit)?; - copy_wit_sources(&component_input_wit, &component_base_output_wit)?; + delete_path( + "generated base wit directory", + &component_generated_base_wit, + )?; + copy_wit_sources(&component_source_wit, &component_generated_base_wit)?; { - let missing_package_deps = - ctx.wit.missing_generic_input_package_deps(component_name)?; + let missing_package_deps = ctx + .wit + .missing_generic_source_package_deps(component_name)?; if !missing_package_deps.is_empty() { log_action("Adding", "package deps"); @@ -935,7 +915,7 @@ fn create_base_output_wit( ctx.common_wit_deps()? .add_packages_with_transitive_deps_to_wit_dir( &missing_package_deps, - &component_base_output_wit, + &component_generated_base_wit, )?; } } @@ -953,7 +933,7 @@ fn create_base_output_wit( ctx.component_base_output_wit_deps(dep_component_name)? .add_packages_with_transitive_deps_to_wit_dir( &[dep_interface_package_name.clone()], - &component_base_output_wit, + &component_generated_base_wit, )?; } } @@ -964,13 +944,13 @@ fn create_base_output_wit( "Extracting", format!( "main interface package from {} to {}", - component_input_wit.log_color_highlight(), - component_base_output_wit.log_color_highlight() + component_source_wit.log_color_highlight(), + component_generated_base_wit.log_color_highlight() ), ); let _indent = LogIndent::new(); - extract_main_interface_as_wit_dep(&component_base_output_wit)?; + extract_main_interface_as_wit_dep(&component_generated_base_wit)?; } gen_dir_done_marker.mark_as_done()?; @@ -979,25 +959,25 @@ fn create_base_output_wit( } } -fn create_output_wit( +fn create_generated_wit( ctx: &ApplicationContext, component_name: &ComponentName, ) -> Result { - let component_base_output_wit = ctx.application.component_base_output_wit(component_name); - let component_output_wit = ctx + let component_generated_base_wit = ctx.application.component_generated_base_wit(component_name); + let component_generated_wit = ctx .application - .component_output_wit(component_name, ctx.profile()); - let gen_dir_done_marker = GeneratedDirDoneMarker::new(&component_output_wit); + .component_generated_wit(component_name, ctx.profile()); + let gen_dir_done_marker = GeneratedDirDoneMarker::new(&component_generated_wit); if is_up_to_date( ctx.config.skip_up_to_date_checks || !gen_dir_done_marker.is_done() || !ctx.wit.is_dep_graph_up_to_date(component_name)?, - || [component_base_output_wit.clone()], - || [component_output_wit.clone()], + || [component_generated_base_wit.clone()], + || [component_generated_wit.clone()], ) { log_skipping_up_to_date(format!( - "creating output wit directory for {}", + "creating generated wit directory for {}", component_name.as_str().log_color_highlight() )); Ok(false) @@ -1005,14 +985,14 @@ fn create_output_wit( log_action( "Creating", format!( - "output wit directory for {}", + "generated wit directory for {}", component_name.as_str().log_color_highlight(), ), ); let _indent = LogIndent::new(); - delete_path("output wit directory", &component_output_wit)?; - copy_wit_sources(&component_base_output_wit, &component_output_wit)?; + delete_path("generated wit directory", &component_generated_wit)?; + copy_wit_sources(&component_generated_base_wit, &component_generated_wit)?; add_stub_deps(ctx, component_name)?; gen_dir_done_marker.mark_as_done()?; @@ -1025,24 +1005,24 @@ fn update_cargo_toml( ctx: &ApplicationContext, component_name: &ComponentName, ) -> anyhow::Result<()> { - let component_input_wit = PathExtra::new( + let component_source_wit = PathExtra::new( ctx.application - .component_input_wit(component_name, ctx.profile()), + .component_source_wit(component_name, ctx.profile()), ); - let component_input_wit_parent = component_input_wit.parent().with_context(|| { + let component_source_wit_parent = component_source_wit.parent().with_context(|| { anyhow!( "Failed to get parent for component {}", component_name.as_str().log_color_highlight() ) })?; - let cargo_toml = component_input_wit_parent.join("Cargo.toml"); + let cargo_toml = component_source_wit_parent.join("Cargo.toml"); if cargo_toml.exists() { regenerate_cargo_package_component( &cargo_toml, &ctx.application - .component_output_wit(component_name, ctx.profile()), - ctx.application.stub_world(component_name), + .component_source_wit(component_name, ctx.profile()), + None, )? } @@ -1056,13 +1036,13 @@ async fn build_stub( let target_root = ctx.application.stub_temp_build_dir(component_name); let stub_def = StubDefinition::new(StubConfig { - source_wit_root: ctx.application.component_base_output_wit(component_name), + source_wit_root: ctx.application.component_generated_base_wit(component_name), target_root: target_root.clone(), - selected_world: ctx.application.stub_world(component_name), - stub_crate_version: ctx.application.stub_crate_version(component_name), + selected_world: None, + stub_crate_version: WASM_RPC_VERSION.to_string(), wasm_rpc_override: WasmRpcOverride { - wasm_rpc_path_override: ctx.application.stub_wasm_rpc_path(component_name), - wasm_rpc_version_override: ctx.application.stub_wasm_rpc_version(component_name), + wasm_rpc_path_override: None, + wasm_rpc_version_override: None, }, extract_source_interface_package: false, seal_cargo_workspace: true, @@ -1070,7 +1050,7 @@ async fn build_stub( .context("Failed to gather information for the stub generator")?; let stub_dep_package_ids = stub_def.stub_dep_package_ids(); - let stub_inputs: Vec = stub_def + let stub_sources: Vec = stub_def .packages_with_wit_sources() .flat_map(|(package_id, _, sources)| { (stub_dep_package_ids.contains(&package_id) || package_id == stub_def.source_package_id) @@ -1085,7 +1065,7 @@ async fn build_stub( if is_up_to_date( ctx.config.skip_up_to_date_checks || !gen_dir_done_marker.is_done(), - || stub_inputs, + || stub_sources, || [stub_wit.clone(), stub_wasm.clone()], ) { log_skipping_up_to_date(format!( @@ -1124,8 +1104,10 @@ async fn build_stub( } fn add_stub_deps(ctx: &ApplicationContext, component_name: &ComponentName) -> Result { - let component = ctx.application.component(component_name); - if component.wasm_rpc_dependencies.is_empty() { + let dependencies = ctx + .application + .component_wasm_rpc_dependencies(component_name); + if dependencies.is_empty() { Ok(false) } else { log_action( @@ -1138,7 +1120,7 @@ fn add_stub_deps(ctx: &ApplicationContext, component_name: &ComponentName) -> Re let _indent = LogIndent::new(); - for dep_component_name in &component.wasm_rpc_dependencies { + for dep_component_name in dependencies { log_action( "Adding", format!( @@ -1153,7 +1135,7 @@ fn add_stub_deps(ctx: &ApplicationContext, component_name: &ComponentName) -> Re stub_wit_root: ctx.application.stub_wit(dep_component_name), dest_wit_root: ctx .application - .component_output_wit(component_name, ctx.profile()), + .component_generated_wit(component_name, ctx.profile()), update_cargo_toml: UpdateCargoToml::NoUpdate, })? } @@ -1175,7 +1157,7 @@ fn copy_wit_sources(source: &Path, target: &Path) -> anyhow::Result<()> { let dir_content = fs_extra::dir::get_dir_content(source).with_context(|| { anyhow!( - "Failed to read component input wit directory entries for {}", + "Failed to read component source wit directory entries for {}", source.log_color_highlight() ) })?; @@ -1205,20 +1187,28 @@ fn copy_wit_sources(source: &Path, target: &Path) -> anyhow::Result<()> { fn execute_external_command( ctx: &ApplicationContext, - component: &WasmComponent, + component_name: &ComponentName, command: &ExternalCommand, ) -> anyhow::Result<()> { let build_dir = command .dir .as_ref() - .map(|dir| component.source_dir().join(dir)) - .unwrap_or_else(|| component.source_dir().to_path_buf()); + .map(|dir| { + ctx.application + .component_source_dir(component_name) + .join(dir) + }) + .unwrap_or_else(|| { + ctx.application + .component_source_dir(component_name) + .to_path_buf() + }); - if !command.inputs.is_empty() && !command.outputs.is_empty() { - let inputs = compile_and_collect_globs(&build_dir, &command.inputs)?; - let outputs = compile_and_collect_globs(&build_dir, &command.outputs)?; + if !command.sources.is_empty() && !command.targets.is_empty() { + let sources = compile_and_collect_globs(&build_dir, &command.sources)?; + let targets = compile_and_collect_globs(&build_dir, &command.targets)?; - if is_up_to_date(ctx.config.skip_up_to_date_checks, || inputs, || outputs) { + if is_up_to_date(ctx.config.skip_up_to_date_checks, || sources, || targets) { log_skipping_up_to_date(format!( "executing external command '{}' in directory {}", command.command.log_color_highlight(), diff --git a/wasm-rpc-stubgen/src/lib.rs b/wasm-rpc-stubgen/src/lib.rs index 5f7fa568..a83ddee6 100644 --- a/wasm-rpc-stubgen/src/lib.rs +++ b/wasm-rpc-stubgen/src/lib.rs @@ -190,8 +190,6 @@ pub struct InitializeWorkspaceArgs { #[derive(Subcommand, Debug)] pub enum App { - /// Creates application manifest for component - Init(DeclarativeInitArgs), /// Runs the pre-component-build steps (stub generation and adding wit dependencies) PreComponentBuild(DeclarativeBuildArgs), /// Runs component build steps @@ -301,7 +299,6 @@ pub fn initialize_workspace( pub async fn run_declarative_command(command: App) -> anyhow::Result<()> { match command { - App::Init(args) => commands::declarative::init(args.component_name.into()), App::PreComponentBuild(args) => { commands::declarative::pre_component_build(dec_build_args_to_config(args)).await } diff --git a/wasm-rpc-stubgen/src/model/mod.rs b/wasm-rpc-stubgen/src/model/mod.rs index 6e669204..72f864ac 100644 --- a/wasm-rpc-stubgen/src/model/mod.rs +++ b/wasm-rpc-stubgen/src/model/mod.rs @@ -1,3 +1,2 @@ -pub mod oam; pub mod unknown_properties; pub mod wasm_rpc; diff --git a/wasm-rpc-stubgen/src/model/oam.rs b/wasm-rpc-stubgen/src/model/oam.rs deleted file mode 100644 index 7c0e913c..00000000 --- a/wasm-rpc-stubgen/src/model/oam.rs +++ /dev/null @@ -1,307 +0,0 @@ -use crate::fs; -use crate::log::LogColorize; -use crate::validation::{ValidatedResult, ValidationBuilder}; -use itertools::Itertools; -use serde::de::DeserializeOwned; -use serde::{Deserialize, Serialize}; -use std::collections::{BTreeMap, BTreeSet}; -use std::path::PathBuf; - -pub const API_VERSION_V1BETA1: &str = "core.oam.dev/v1beta1"; -pub const KIND_APPLICATION: &str = "Application"; - -#[derive(Clone, Debug)] -pub struct ApplicationWithSource { - pub source: PathBuf, - pub application: Application, -} - -impl ApplicationWithSource { - pub fn from_yaml_file(file: PathBuf) -> anyhow::Result { - Ok(Self::from_yaml_string( - file.clone(), - fs::read_to_string(file)?, - )?) - } - - pub fn from_yaml_string(source: PathBuf, string: String) -> serde_yaml::Result { - Ok(Self { - source, - application: Application::from_yaml_str(string.as_str())?, - }) - } - - pub fn source_as_string(&self) -> String { - self.source.to_string_lossy().to_string() - } - - // NOTE: unlike the wasm_rpc model, here validation is optional separate step, so we can access the "raw" data - pub fn validate(self) -> ValidatedResult { - let mut validation = ValidationBuilder::new(); - validation.push_context("source", self.source_as_string()); - - if self.application.api_version != API_VERSION_V1BETA1 { - validation.add_warn(format!( - "Expected apiVersion: {}, got: {}", - API_VERSION_V1BETA1.log_color_highlight(), - self.application.api_version.log_color_error_highlight() - )) - } - - if self.application.kind != KIND_APPLICATION { - validation.add_error(format!( - "Expected kind: {}, got: {}", - KIND_APPLICATION.log_color_highlight(), - self.application.kind.log_color_error_highlight() - )) - } - - self.application - .spec - .components - .iter() - .map(|component| &component.name) - .counts() - .into_iter() - .filter(|(_, count)| *count > 1) - .for_each(|(component_name, count)| { - validation.add_warn(format!( - "Component specified multiple times component: {}, count: {}", - component_name, count - )); - }); - - validation.pop_context(); - validation.build(self) - } -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct Application { - #[serde(rename = "apiVersion")] - pub api_version: String, - pub kind: String, - pub metadata: Metadata, - pub spec: Spec, -} - -impl Application { - pub fn new(name: String) -> Self { - Self { - api_version: API_VERSION_V1BETA1.to_string(), - kind: KIND_APPLICATION.to_string(), - metadata: Metadata { - name, - annotations: Default::default(), - labels: Default::default(), - }, - spec: Spec { components: vec![] }, - } - } - - pub fn from_yaml_str(yaml: &str) -> serde_yaml::Result { - serde_yaml::from_str(yaml) - } - - pub fn to_yaml_string(&self) -> String { - serde_yaml::to_string(self).expect("Failed to serialize Application as YAML") - } -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct Metadata { - pub name: String, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub annotations: BTreeMap, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub labels: BTreeMap, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct Spec { - pub components: Vec, -} - -impl Spec { - pub fn extract_components_by_type( - &mut self, - component_types: &BTreeSet<&'static str>, - ) -> BTreeMap<&'static str, Vec> { - let mut components = Vec::::new(); - - std::mem::swap(&mut components, &mut self.components); - - let mut matching_components = BTreeMap::<&'static str, Vec>::new(); - let mut remaining_components = Vec::::new(); - - for component in components { - if let Some(component_type) = component_types.get(component.component_type.as_str()) { - matching_components - .entry(component_type) - .or_default() - .push(component) - } else { - remaining_components.push(component) - } - } - - std::mem::swap(&mut remaining_components, &mut self.components); - - matching_components - } -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct Component { - pub name: String, - #[serde(rename = "type")] - pub component_type: String, - pub properties: serde_json::Value, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub traits: Vec, -} - -pub trait TypedComponentProperties: Serialize + DeserializeOwned { - fn component_type() -> &'static str; -} - -impl Component { - pub fn typed_properties(&self) -> Result { - if self.component_type != T::component_type() { - panic!( - "Component type mismatch in clone_properties_as, self: {}, requested: {}", - self.component_type, - T::component_type() - ); - } - serde_json::from_value(self.properties.clone()) - } - - pub fn set_typed_properties(&mut self, properties: T) { - self.component_type = T::component_type().to_string(); - self.properties = serde_json::to_value(properties).expect("Failed to serialize properties"); - } - - pub fn extract_traits_by_type( - &mut self, - trait_types: &BTreeSet<&'static str>, - ) -> BTreeMap<&'static str, Vec> { - let mut component_traits = Vec::::new(); - - std::mem::swap(&mut component_traits, &mut self.traits); - - let mut matching_traits = BTreeMap::<&'static str, Vec>::new(); - let mut remaining_traits = Vec::::new(); - - for component_trait in component_traits { - if let Some(trait_type) = trait_types.get(component_trait.trait_type.as_str()) { - matching_traits - .entry(trait_type) - .or_default() - .push(component_trait); - } else { - remaining_traits.push(component_trait); - } - } - - std::mem::swap(&mut remaining_traits, &mut self.traits); - - matching_traits - } - - pub fn add_typed_trait(&mut self, properties: T) { - self.traits.push(Trait { - trait_type: T::trait_type().to_string(), - properties: serde_json::to_value(properties).expect("Failed to serialize typed trait"), - }); - } -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct Trait { - #[serde(rename = "type")] - pub trait_type: String, - pub properties: serde_json::Value, -} - -pub trait TypedTraitProperties: Serialize + DeserializeOwned { - fn trait_type() -> &'static str; - - fn from_generic_trait(value: Trait) -> Result { - if value.trait_type != Self::trait_type() { - panic!( - "Trait type mismatch in TryFrom, value: {}, typed: {}", - value.trait_type, - Self::trait_type() - ) - } - serde_json::from_value(value.properties) - } -} - -#[cfg(test)] -mod tests { - use test_r::test; - - use super::*; - use assert2::assert; - - #[test] - fn deserialize_example_application() { - let application: Application = serde_yaml::from_str( - r#" -apiVersion: core.oam.dev/v1beta1 -metadata: - name: "App name" -kind: Application -spec: - components: - - name: component-one - type: test-component-type - properties: - testComponentProperty: aaa - traits: - - type: test-trait-type-1 - properties: - testProperty: bbb - - type: test-trait-type-2 - properties: - testTraitProperty: ccc -"#, - ) - .unwrap(); - - assert!(application.api_version == API_VERSION_V1BETA1); - assert!(application.kind == KIND_APPLICATION); - assert!(application.metadata.name == "App name"); - assert!(application.spec.components.len() == 1); - - let component = &application.spec.components[0]; - - assert!(component.name == "component-one"); - assert!(component.component_type == "test-component-type"); - assert!(component.properties.is_object()); - - let properties = component.properties.as_object().unwrap(); - - assert!( - properties - .get_key_value("testComponentProperty") - .unwrap() - .1 - .as_str() - == Some("aaa") - ); - - assert!(component.traits.len() == 2); - - let component_trait = &component.traits[1]; - - assert!(component_trait.trait_type == "test-trait-type-2"); - assert!(component_trait.properties.is_object()); - - let properties = component_trait.properties.as_object().unwrap(); - - assert!(properties.get_key_value("testTraitProperty").unwrap().1 == "ccc"); - } -} diff --git a/wasm-rpc-stubgen/src/model/wasm_rpc.rs b/wasm-rpc-stubgen/src/model/wasm_rpc.rs index 36a875fa..a1c8de9f 100644 --- a/wasm-rpc-stubgen/src/model/wasm_rpc.rs +++ b/wasm-rpc-stubgen/src/model/wasm_rpc.rs @@ -1,21 +1,18 @@ use crate::log::LogColorize; -use crate::model::oam; -use crate::model::oam::TypedTraitProperties; -use crate::model::unknown_properties::HasUnknownProperties; use crate::model::wasm_rpc::template::Template; use crate::naming::wit::package_dep_dir_name_from_parser; use crate::validation::{ValidatedResult, ValidationBuilder}; use crate::{fs, naming}; -use golem_wasm_rpc::WASM_RPC_VERSION; use heck::{ ToKebabCase, ToLowerCamelCase, ToPascalCase, ToShoutyKebabCase, ToShoutySnakeCase, ToSnakeCase, ToTitleCase, ToTrainCase, ToUpperCamelCase, }; use itertools::Itertools; -use minijinja::Environment; +use serde::Serialize; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::fmt::Display; use std::fmt::Formatter; +use std::hash::Hash; use std::path::{Path, PathBuf}; use wit_parser::PackageName; @@ -44,7 +41,7 @@ impl From for ComponentName { impl From<&str> for ComponentName { fn from(value: &str) -> Self { - ComponentName(value.to_string()) + Self(value.to_string()) } } @@ -71,184 +68,189 @@ impl From for ProfileName { impl From<&str> for ProfileName { fn from(value: &str) -> Self { - ProfileName(value.to_string()) + Self(value.to_string()) } } -mod raw { - use crate::model::oam; - use crate::model::unknown_properties::{HasUnknownProperties, UnknownProperties}; - use serde::{Deserialize, Serialize}; - use std::collections::HashMap; - - pub const OAM_TRAIT_TYPE_WASM_RPC: &str = "wasm-rpc"; - - pub const OAM_COMPONENT_TYPE_WASM: &str = "wasm"; - pub const OAM_COMPONENT_TYPE_WASM_BUILD: &str = "wasm-build"; - pub const OAM_COMPONENT_TYPE_WASM_RPC_STUB_BUILD: &str = "wasm-rpc-stub-build"; +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct TemplateName(String); - #[derive(Clone, Debug, Serialize, Deserialize)] - pub struct CustomCommand { - pub command: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub dir: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub inputs: Vec, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub outputs: Vec, +impl TemplateName { + pub fn as_str(&self) -> &str { + &self.0 } +} - #[derive(Clone, Debug, Serialize, Deserialize)] - #[serde(rename_all = "camelCase")] - pub struct WasmComponentProperties { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub input_wit: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub output_wit: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub build: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub input_wasm: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub output_wasm: Option, - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub custom_commands: HashMap>, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub clean: Vec, - #[serde(flatten)] - pub unknown_properties: UnknownProperties, +impl Display for TemplateName { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) } +} - impl HasUnknownProperties for WasmComponentProperties { - fn unknown_properties(&self) -> &UnknownProperties { - &self.unknown_properties - } +impl From for TemplateName { + fn from(value: String) -> Self { + TemplateName(value) } +} - #[derive(Clone, Debug, Serialize, Deserialize)] - #[serde(rename_all = "camelCase")] - pub struct WasmComponentPropertiesComponentSpecific { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub component_template: Option, - - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub profiles: HashMap, +impl From<&str> for TemplateName { + fn from(value: &str) -> Self { + Self(value.to_string()) + } +} - #[serde(default, skip_serializing_if = "Option::is_none")] - pub default_profile: Option, +pub mod raw { + use crate::fs; + use serde::{Deserialize, Serialize}; + use std::collections::HashMap; + use std::path::PathBuf; - #[serde(flatten)] - pub properties: WasmComponentProperties, + #[derive(Clone, Debug)] + pub struct ApplicationWithSource { + pub source: PathBuf, + pub application: Application, } - impl HasUnknownProperties for WasmComponentPropertiesComponentSpecific { - fn unknown_properties(&self) -> &UnknownProperties { - &self.properties.unknown_properties + impl ApplicationWithSource { + pub fn from_yaml_file(file: PathBuf) -> anyhow::Result { + Ok(Self::from_yaml_string( + file.clone(), + fs::read_to_string(file)?, + )?) } - } - impl oam::TypedComponentProperties for WasmComponentPropertiesComponentSpecific { - fn component_type() -> &'static str { - OAM_COMPONENT_TYPE_WASM + pub fn from_yaml_string(source: PathBuf, string: String) -> serde_yaml::Result { + Ok(Self { + source, + application: Application::from_yaml_str(string.as_str())?, + }) + } + + pub fn source_as_string(&self) -> String { + self.source.to_string_lossy().to_string() } } #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] - pub struct WasmComponentPropertiesTemplate { - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub profiles: HashMap, - + pub struct Application { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub include: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] - pub default_profile: Option, - - #[serde(flatten)] - pub properties: WasmComponentProperties, + pub temp_dir: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub wit_deps: Vec, + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub templates: HashMap, + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub components: HashMap, + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub dependencies: HashMap, } - impl HasUnknownProperties for WasmComponentPropertiesTemplate { - fn unknown_properties(&self) -> &UnknownProperties { - &self.properties.unknown_properties + impl Application { + pub fn from_yaml_str(yaml: &str) -> serde_yaml::Result { + serde_yaml::from_str(yaml) + } + + pub fn to_yaml_string(&self) -> String { + serde_yaml::to_string(self).expect("Failed to serialize Application as YAML") } } #[derive(Clone, Debug, Serialize, Deserialize)] - pub struct WasmRpcTraitProperties { - #[serde(rename = "componentName")] - pub component_name: String, + #[serde(rename_all = "camelCase")] + pub struct ComponentTemplate { #[serde(flatten)] - pub unknown_properties: UnknownProperties, - } - - impl HasUnknownProperties for WasmRpcTraitProperties { - fn unknown_properties(&self) -> &UnknownProperties { - &self.unknown_properties - } + pub component_properties: ComponentProperties, + pub profiles: HashMap, + pub default_profile: Option, } - impl oam::TypedTraitProperties for WasmRpcTraitProperties { - fn trait_type() -> &'static str { - OAM_TRAIT_TYPE_WASM_RPC - } + #[derive(Clone, Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct Component { + pub template: Option, + #[serde(flatten)] + pub component_properties: ComponentProperties, + pub profiles: HashMap, + pub default_profile: Option, } #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] - pub struct WasmComponentBuildProperties { + pub struct ComponentProperties { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_wit: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub generated_wit: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub include: Option, + pub component_wasm: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub build_dir: Option, + pub linked_wasm: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub wit_deps: Vec, + pub build: Vec, #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub component_templates: HashMap, - #[serde(flatten)] - pub unknown_properties: UnknownProperties, + pub custom_commands: HashMap>, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub clean: Vec, } - impl oam::TypedComponentProperties for WasmComponentBuildProperties { - fn component_type() -> &'static str { - OAM_COMPONENT_TYPE_WASM_BUILD - } - } + impl ComponentProperties { + pub fn defined_property_names(&self) -> Vec<&str> { + let mut vec = Vec::<&str>::new(); + + if self.source_wit.is_some() { + vec.push("sourceWit"); + } + + if self.generated_wit.is_some() { + vec.push("generatedWit"); + } + + if self.component_wasm.is_some() { + vec.push("componentWasm"); + } + + if self.linked_wasm.is_some() { + vec.push("linkedWasm"); + } + + if !self.build.is_empty() { + vec.push("build"); + } - impl HasUnknownProperties for WasmComponentBuildProperties { - fn unknown_properties(&self) -> &UnknownProperties { - &self.unknown_properties + if !self.custom_commands.is_empty() { + vec.push("customCommands"); + } + + vec } } #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] - pub struct ComponentStubBuildProperties { - pub component_name: Option, - pub build_dir: Option, - pub wasm: Option, - pub wit: Option, - pub world: Option, - pub crate_version: Option, - pub wasm_rpc_path: Option, - pub wasm_rpc_version: Option, - #[serde(flatten)] - pub unknown_properties: UnknownProperties, - } - - impl oam::TypedComponentProperties for ComponentStubBuildProperties { - fn component_type() -> &'static str { - OAM_COMPONENT_TYPE_WASM_RPC_STUB_BUILD - } + pub struct ExternalCommand { + pub command: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub dir: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub sources: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub targets: Vec, } - impl HasUnknownProperties for ComponentStubBuildProperties { - fn unknown_properties(&self) -> &UnknownProperties { - &self.unknown_properties - } + #[derive(Clone, Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct Dependency { + #[serde(rename = "type")] + pub type_: String, + pub target: Option, } } mod template { - use crate::model::wasm_rpc::{raw, ExternalCommand, WasmComponentProperties}; + use crate::model::wasm_rpc::raw; use serde::Serialize; use std::collections::HashMap; @@ -317,37 +319,37 @@ mod template { } } - impl Template for raw::CustomCommand { - type Rendered = ExternalCommand; + impl Template for raw::ExternalCommand { + type Rendered = raw::ExternalCommand; fn render( &self, env: &minijinja::Environment, ctx: &C, ) -> Result { - Ok(ExternalCommand { + Ok(raw::ExternalCommand { command: self.command.render(env, ctx)?, dir: self.dir.render(env, ctx)?, - inputs: self.inputs.render(env, ctx)?, - outputs: self.outputs.render(env, ctx)?, + sources: self.sources.render(env, ctx)?, + targets: self.targets.render(env, ctx)?, }) } } - impl Template for WasmComponentProperties { - type Rendered = WasmComponentProperties; + impl Template for raw::ComponentProperties { + type Rendered = raw::ComponentProperties; fn render( &self, env: &minijinja::Environment, ctx: &C, ) -> Result { - Ok(WasmComponentProperties { + Ok(raw::ComponentProperties { + source_wit: self.source_wit.render(env, ctx)?, + generated_wit: self.generated_wit.render(env, ctx)?, + component_wasm: self.component_wasm.render(env, ctx)?, + linked_wasm: self.linked_wasm.render(env, ctx)?, build: self.build.render(env, ctx)?, - input_wit: self.input_wit.render(env, ctx)?, - output_wit: self.output_wit.render(env, ctx)?, - input_wasm: self.input_wasm.render(env, ctx)?, - output_wasm: self.output_wasm.render(env, ctx)?, custom_commands: self.custom_commands.render(env, ctx)?, clean: self.clean.render(env, ctx)?, }) @@ -355,874 +357,516 @@ mod template { } } -pub fn init_oam_app(_component_name: ComponentName) -> oam::Application { - // TODO: let's do it as part of https://github.com/golemcloud/wasm-rpc/issues/89 - todo!() -} - -// This a lenient non-validating peek for the include build property, -// as that is used early, during source collection -pub fn include_glob_patter_from_yaml_file(source: &Path) -> Option { +pub fn includes_from_yaml_file(source: &Path) -> Vec { fs::read_to_string(source) .ok() - .and_then(|source| oam::Application::from_yaml_str(source.as_str()).ok()) - .and_then(|mut oam_app| { - let mut includes = oam_app - .spec - .extract_components_by_type(&BTreeSet::from([raw::OAM_COMPONENT_TYPE_WASM_BUILD])) - .remove(raw::OAM_COMPONENT_TYPE_WASM_BUILD) - .unwrap_or_default() - .into_iter() - .filter_map(|component| { - component - .typed_properties::() - .ok() - .and_then(|properties| properties.include) - }); - - match includes.next() { - Some(include) => { - // Only return it if it's unique (if not it will cause validation errors later) - includes.next().is_none().then_some(include) - } - None => None, - } - }) + .and_then(|source| raw::Application::from_yaml_str(source.as_str()).ok()) + .map(|app| app.include) + .unwrap_or_default() } #[derive(Clone, Debug)] -enum RenderedWasmComponentProperties { - Single { - properties: WasmComponentProperties, +pub enum ResolvedComponentProperties { + Properties { + template_name: Option, + any_template_overrides: bool, + properties: ComponentProperties, }, Profiles { + template_name: Option, + any_template_overrides: HashMap, default_profile: ProfileName, - profiles: HashMap, + profiles: HashMap, }, } -pub struct ComponentEffectiveTemplateAndProfile<'a> { - pub template_name: Option<&'a str>, +pub struct ComponentEffectivePropertySource<'a> { + pub template_name: Option<&'a TemplateName>, pub profile: Option<&'a ProfileName>, pub is_requested_profile: bool, + pub any_template_overrides: bool, } #[derive(Clone, Debug)] pub struct Application { - common_wasm_build: Option, - common_wasm_rpc_stub_build: Option, - wasm_rpc_stub_builds_by_name: BTreeMap, - wasm_components_by_name: BTreeMap, - rendered_wasm_component_properties: HashMap, + temp_dir: Option, + wit_deps: Vec, + components: BTreeMap, + dependencies: BTreeMap>, + no_dependencies: BTreeSet, } impl Application { - pub fn from_oam_apps(oam_apps: Vec) -> ValidatedResult { + pub fn from_raw_apps(apps: Vec) -> ValidatedResult { let mut validation = ValidationBuilder::new(); - let (all_components, all_wasm_builds, all_wasm_rpc_stub_builds) = { - let mut all_components = Vec::::new(); - let mut all_wasm_builds = Vec::::new(); - let mut all_wasm_rpc_stub_builds = Vec::::new(); - - for mut oam_app in oam_apps { - let (components, wasm_build, wasm_rpc_stub_build) = - Self::extract_and_convert_oam_components(&mut validation, &mut oam_app); - all_components.extend(components); - all_wasm_builds.extend(wasm_build); - all_wasm_rpc_stub_builds.extend(wasm_rpc_stub_build); - } - - (all_components, all_wasm_builds, all_wasm_rpc_stub_builds) - }; - - let wasm_components_by_name = Self::validate_components(&mut validation, all_components); - - let (common_wasm_rpc_stub_build, wasm_rpc_stub_builds_by_name) = - Self::validate_wasm_rpc_stub_builds( - &mut validation, - &wasm_components_by_name, - all_wasm_rpc_stub_builds, - ); - - let common_wasm_build = Self::validate_wasm_builds(&mut validation, all_wasm_builds); + let mut include = Vec::::new(); + let mut include_sources = Vec::::new(); - let rendered_wasm_component_properties = Self::render_templates( - &mut validation, - common_wasm_build.as_ref(), - &wasm_components_by_name, - ); + let mut temp_dir: Option = None; + let mut temp_dir_sources = Vec::::new(); - validation.build(Self { - common_wasm_build, - common_wasm_rpc_stub_build, - wasm_rpc_stub_builds_by_name, - wasm_components_by_name, - rendered_wasm_component_properties, - }) - } + let mut wit_deps = Vec::::new(); + let mut wit_deps_sources = Vec::::new(); - fn extract_and_convert_oam_components( - validation: &mut ValidationBuilder, - oam_app: &mut oam::ApplicationWithSource, - ) -> (Vec, Vec, Vec) { - validation.push_context("source", oam_app.source_as_string()); - - let mut components_by_type = - oam_app - .application - .spec - .extract_components_by_type(&BTreeSet::from([ - raw::OAM_COMPONENT_TYPE_WASM, - raw::OAM_COMPONENT_TYPE_WASM_BUILD, - raw::OAM_COMPONENT_TYPE_WASM_RPC_STUB_BUILD, - ])); - - let wasm_components = Self::convert_components( - &oam_app.source, - validation, - &mut components_by_type, - raw::OAM_COMPONENT_TYPE_WASM, - Self::convert_wasm_component, - ); - - let wasm_builds = Self::convert_components( - &oam_app.source, - validation, - &mut components_by_type, - raw::OAM_COMPONENT_TYPE_WASM_BUILD, - Self::convert_wasm_build, - ); - - let wasm_rpc_stub_builds = Self::convert_components( - &oam_app.source, - validation, - &mut components_by_type, - raw::OAM_COMPONENT_TYPE_WASM_RPC_STUB_BUILD, - Self::convert_wasm_rpc_stub_build, - ); - - validation.add_warns(&oam_app.application.spec.components, |component| { - Some(( - vec![ - ("component name", component.name.clone()), - ("component type", component.component_type.clone()), - ], - "Unknown component-type".to_string(), - )) - }); + let mut templates = HashMap::::new(); + let mut template_sources = HashMap::>::new(); - validation.pop_context(); + let mut dependencies = BTreeMap::>::new(); + let mut dependency_sources = + HashMap::>>::new(); - (wasm_components, wasm_builds, wasm_rpc_stub_builds) - } + let mut components = HashMap::::new(); + let mut component_sources = HashMap::>::new(); - fn convert_components( - source: &Path, - validation: &mut ValidationBuilder, - components_by_type: &mut BTreeMap<&'static str, Vec>, - component_type: &str, - convert: F, - ) -> Vec - where - F: Fn(&Path, &mut ValidationBuilder, oam::Component) -> Option, - { - components_by_type - .remove(component_type) - .unwrap_or_default() - .into_iter() - .filter_map(|component| { - validation.push_context("component name", component.name.clone()); - validation.push_context("component type", component.component_type.clone()); - let result = convert(source, validation, component); - validation.pop_context(); - validation.pop_context(); - result - }) - .collect() - } + for app in apps { + validation.push_context("source", app.source.to_string_lossy().to_string()); - fn convert_wasm_component( - source: &Path, - validation: &mut ValidationBuilder, - mut component: oam::Component, - ) -> Option { - let properties = - component.typed_properties::(); - - if let Some(err) = properties.as_ref().err() { - validation.add_error(format!("Failed to get component properties: {}", err)) - } - - let wasm_rpc_traits = component - .extract_traits_by_type(&BTreeSet::from([raw::OAM_TRAIT_TYPE_WASM_RPC])) - .remove(raw::OAM_TRAIT_TYPE_WASM_RPC) - .unwrap_or_default(); - - let mut wasm_rpc_dependencies = Vec::::new(); - for wasm_rpc in wasm_rpc_traits { - validation.push_context("trait type", wasm_rpc.trait_type.clone()); - - match raw::WasmRpcTraitProperties::from_generic_trait(wasm_rpc) { - Ok(wasm_rpc) => { - wasm_rpc.add_unknown_property_warns( - || vec![("dep component name", wasm_rpc.component_name.clone())], - validation, - ); - wasm_rpc_dependencies.push(wasm_rpc.component_name.into()) + if let Some(dir) = app.application.temp_dir { + temp_dir_sources.push(app.source.to_path_buf()); + if temp_dir.is_none() { + temp_dir = Some(dir); } - Err(err) => validation - .add_error(format!("Failed to get wasm-rpc trait properties: {}", err)), } - validation.pop_context(); - } - - let non_unique_wasm_rpc_dependencies = wasm_rpc_dependencies - .iter() - .counts() - .into_iter() - .filter(|(_, count)| *count > 1); - validation.add_warns( - non_unique_wasm_rpc_dependencies, - |(dep_component_name, count)| { - Some(( - vec![], - format!( - "WASM RPC dependency specified multiple times for component: {}, count: {}", - dep_component_name, count - ), - )) - }, - ); - - validation.add_warns(component.traits, |component_trait| { - Some(( - vec![], - format!( - "Unknown trait for wasm component, trait type: {}", - component_trait.trait_type - ), - )) - }); - - let wasm_rpc_dependencies = wasm_rpc_dependencies - .into_iter() - .unique() - .sorted() - .collect::>(); - - match (properties, validation.has_any_errors()) { - (Ok(properties), false) => { - properties.add_unknown_property_warns(Vec::new, validation); - - let properties = Self::convert_wasm_component_properties( - validation, - Some(properties.component_template), - properties.profiles, - properties.default_profile, - properties.properties, - ); - - properties.map(|properties| WasmComponent { - name: component.name.into(), - source: source.to_path_buf(), - properties, - wasm_rpc_dependencies, - }) + if !app.application.include.is_empty() { + include_sources.push(app.source.to_path_buf()); + if include.is_empty() { + include = app.application.include; + } } - _ => None, - } - } - - fn convert_wasm_build( - source: &Path, - validation: &mut ValidationBuilder, - component: oam::Component, - ) -> Option { - let result = match component.typed_properties::() { - Ok(properties) => { - properties.add_unknown_property_warns(Vec::new, validation); - - let component_templates = properties - .component_templates - .into_iter() - .filter_map(|(template_name, template)| { - validation.push_context("template", template_name.clone()); - - template.add_unknown_property_warns(Vec::new, validation); - - let component_source = Self::convert_wasm_component_properties( - validation, - None, - template.profiles, - template.default_profile, - template.properties, - ); - - validation.pop_context(); - - component_source.map(|component_source| (template_name, component_source)) - }) - .collect::>(); - - let wasm_rpc_stub_build = WasmBuild { - source: source.to_path_buf(), - name: component.name, - build_dir: properties.build_dir.map(|s| s.into()), - component_templates, - wit_deps: properties.wit_deps.into_iter().map(|s| s.into()).collect(), - }; - Some(wasm_rpc_stub_build) - } - Err(err) => { - validation.add_error(format!("Failed to get wasm build properties: {}", err)); - None + if app.application.wit_deps.is_empty() { + wit_deps_sources.push(app.source.to_path_buf()); + if wit_deps.is_empty() { + wit_deps = app.application.wit_deps; // TODO: resolve from source? + } } - }; - validation.add_warns(component.traits, |component_trait| { - Some(( - vec![], - format!( - "Unknown trait for wasm build, trait type: {}", - component_trait.trait_type - ), - )) - }); - - result - } + for (template_name, template) in app.application.templates { + validation.push_context("template", template_name.clone()); + + let mut invalid_template = false; + if template.profiles.is_empty() { + if template.default_profile.is_some() { + validation.add_error(format!( + "When {} is not defined then {} should not be defined", + "profiles".log_color_highlight(), + "defaultProfile".log_color_highlight() + )); + invalid_template = true; + } + } else { + let defined_property_names = + template.component_properties.defined_property_names(); + if !defined_property_names.is_empty() { + for property_name in defined_property_names { + validation.add_error(format!( + "When {} is defined then {} should not be defined", + "profiles".log_color_highlight(), + property_name.log_color_highlight() + )); + invalid_template = true; + } + } - fn convert_wasm_component_properties( - validation: &mut ValidationBuilder, - component_template: Option>, - profiles: HashMap, - default_profile: Option, - properties: raw::WasmComponentProperties, - ) -> Option { - fn unwrap_validated_properties( - properties: raw::WasmComponentProperties, - ) -> WasmComponentProperties { - WasmComponentProperties { - build: properties.build, - input_wit: properties.input_wit.unwrap(), - output_wit: properties.output_wit.unwrap(), - input_wasm: properties.input_wasm.unwrap(), - output_wasm: properties.output_wasm.unwrap(), - custom_commands: properties.custom_commands, - clean: properties.clean, - } - } + if template.default_profile.is_none() { + validation.add_error(format!( + "When {} is defined then {} is mandatory", + "profiles".log_color_highlight(), + "defaultProfile".log_color_highlight() + )); + invalid_template = true; + } + } - let property_ignored_unless_defined_message = { - if component_template.is_some() { - format!( - "{} or {}", - "componentTemplate".log_color_highlight(), - "profile".log_color_highlight() - ) - } else { - "profile".log_color_highlight().to_string() - } - }; + let template_name = TemplateName::from(template_name); + if template_sources.contains_key(&template_name) { + template_sources + .get_mut(&template_name) + .unwrap() + .push(app.source.to_path_buf()); + } else { + template_sources.insert(template_name.clone(), vec![app.source.to_path_buf()]); + } + if !templates.contains_key(&template_name) && !invalid_template { + templates.insert(template_name, template); + } - for build_step in &properties.build { - let has_inputs = !build_step.inputs.is_empty(); - let has_outputs = !build_step.outputs.is_empty(); - - if (has_inputs && !has_outputs) || (!has_inputs && has_outputs) { - validation.push_context("command", build_step.command.clone()); - validation.add_warn(format!( - "Using {} and {} only has effect when both defined", - "inputs".log_color_highlight(), - "outputs".log_color_highlight() - )); validation.pop_context(); } - } - - if !profiles.is_empty() && !properties.build.is_empty() { - validation.add_warn(format!( - "If {} is defined then {} will be ignored", - "profiles".log_color_highlight(), - "build".log_color_highlight() - )); - } - if !profiles.is_empty() && default_profile.is_none() { - validation.add_error(format!( - "If {} is defined then {} also have to be defined", - "profiles".log_color_highlight(), - "defaultProfile".log_color_highlight() - )); - } - - if profiles.is_empty() && default_profile.is_some() { - validation.add_error(format!( - "If {} is defined then {} also have to be defined", - "defaultProfile".log_color_highlight(), - "profiles".log_color_highlight(), - )); - } + for (component_name, component) in app.application.components { + let component_name = ComponentName::from(component_name); - if let Some(default_profile) = &default_profile { - if !profiles.contains_key(default_profile) { - validation.add_error(format!( - "The defined {} ({}) if not found in {}", - "defaultProfile".log_color_highlight(), - default_profile.log_color_error_highlight(), - "profiles".log_color_highlight(), - )); - } - } - - match component_template { - Some(Some(template_name)) => { - for (property_defined, property_name) in [ - (properties.input_wit.is_some(), "inputWit"), - (properties.output_wit.is_some(), "outputWit"), - (!properties.build.is_empty(), "build"), - (properties.input_wasm.is_some(), "inputWasm"), - (properties.output_wasm.is_some(), "outputWasm"), - (!profiles.is_empty(), "profiles"), - (default_profile.is_some(), "defaultProfile"), - ] { - if property_defined { - validation.add_warn(format!( - "Component property {} is ignored when {} is defined", - property_name.log_color_error_highlight(), - "componentTemplate".log_color_highlight(), - )) - } + if !component_sources.contains_key(&component_name) { + component_sources.insert(component_name.clone(), Vec::new()); } + component_sources + .get_mut(&component_name) + .unwrap() + .push(app.source.to_path_buf()); - Some(WasmComponentPropertySource::Template { template_name }) + components.insert(component_name, (app.source.to_path_buf(), component)); } - _ => { - if profiles.is_empty() { - let mut any_mandatory_property_missing = false; - for (property, property_name) in [ - (&properties.input_wit, "inputWit"), - (&properties.output_wit, "outputWit"), - (&properties.input_wasm, "inputWasm"), - (&properties.output_wasm, "outputWasm"), - ] { - if property.is_none() { - any_mandatory_property_missing = true; - validation.add_error(format!( - "Component property {} must be defined, unless {} is defined", - property_name.log_color_error_highlight(), - property_ignored_unless_defined_message - )) - } - } - (!any_mandatory_property_missing).then_some( - WasmComponentPropertySource::Explicit { - properties: unwrap_validated_properties(properties), - }, - ) - } else { - default_profile.map(|default_profile| - WasmComponentPropertySource::ExplicitWithProfiles { - default_profile: default_profile.into(), - profiles: profiles - .into_iter() - .filter_map(|(profile_name, properties)| { - validation.push_context("profile", profile_name.clone()); - - properties.add_unknown_property_warns(Vec::new, validation); - - let mut any_mandatory_property_missing = false; - for (property, property_name) in [ - (&properties.input_wit, "inputWit"), - (&properties.output_wit, "outputWit"), - (&properties.input_wasm, "inputWasm"), - (&properties.output_wasm, "outputWasm"), - ] { - if property.is_none() { - any_mandatory_property_missing = true; - validation.add_error( - format!( - "Component property {} for profile {} must be defined, unless {} is defined", - property_name.log_color_error_highlight(), - profile_name.log_color_highlight(), - property_ignored_unless_defined_message, - ) - ); - } - } + for (component_name, dependency) in app.application.dependencies { + let component_name = ComponentName::from(component_name); + validation.push_context("component", component_name.to_string()); - validation.pop_context(); + if dependency.type_ == "wasm-rpc" { + match dependency.target { + Some(target) => { + let target_component_name = ComponentName::from(target); - (!any_mandatory_property_missing).then_some( - ( - profile_name.into(), - unwrap_validated_properties(properties), - ) - ) - }) - .collect(), - }) - } - } - } - } - - fn convert_wasm_rpc_stub_build( - source: &Path, - validation: &mut ValidationBuilder, - component: oam::Component, - ) -> Option { - let result = match component.typed_properties::() { - Ok(properties) => { - properties.add_unknown_property_warns(Vec::new, validation); - - let wasm_rpc_stub_build = WasmRpcStubBuild { - source: source.to_path_buf(), - name: component.name, - component_name: properties.component_name.map(Into::into), - build_dir: properties.build_dir.map(|s| s.into()), - wasm: properties.wasm.map(|s| s.into()), - wit: properties.wit.map(|s| s.into()), - world: properties.world, - crate_version: properties.crate_version, - wasm_rpc_path: properties.wasm_rpc_path, - wasm_rpc_version: properties.wasm_rpc_version, - }; - - if wasm_rpc_stub_build.build_dir.is_some() && wasm_rpc_stub_build.wasm.is_some() { - validation.add_warn( - "Both buildDir and wasm fields are defined, wasm takes precedence" - .to_string(), - ); - } + if !dependencies.contains_key(&component_name) { + dependencies.insert(component_name.clone(), BTreeSet::new()); + } + dependencies + .get_mut(&component_name) + .unwrap() + .insert(target_component_name.clone()); - if wasm_rpc_stub_build.build_dir.is_some() && wasm_rpc_stub_build.wit.is_some() { - validation.add_warn( - "Both buildDir and wit fields are defined, wit takes precedence" - .to_string(), - ); + if !dependency_sources.contains_key(&component_name) { + dependency_sources.insert(component_name.clone(), HashMap::new()); + } + let dependency_sources = + dependency_sources.get_mut(&component_name).unwrap(); + if !dependency_sources.contains_key(&target_component_name) { + dependency_sources + .insert(target_component_name.clone(), Vec::new()); + } + dependency_sources + .get_mut(&target_component_name) + .unwrap() + .push(app.source.to_path_buf()); + } + None => validation.add_error(format!( + "Missing {} field for component wasm-rpc dependency", + "target".log_color_error_highlight() + )), + } + } else { + validation.add_error(format!( + "Unknown component dependency type: {}", + dependency.type_.log_color_error_highlight() + )); } - if wasm_rpc_stub_build.component_name.is_some() - && wasm_rpc_stub_build.wasm.is_some() - { - validation.add_warn( - "In common (without component name) wasm rpc stub build the wasm field has no effect".to_string(), - ); - } + validation.pop_context(); + } - if wasm_rpc_stub_build.component_name.is_some() && wasm_rpc_stub_build.wit.is_some() - { - validation.add_warn( - "In common (without component name) wasm rpc stub build the wit field has no effect".to_string(), - ); - } + validation.pop_context(); + } - Some(wasm_rpc_stub_build) - } - Err(err) => { + for (property_name, sources) in [ + ("include", include_sources), + ("tempDir", temp_dir_sources), + ("witDeps", wit_deps_sources), + ] { + if !sources.is_empty() { validation.add_error(format!( - "Failed to get wasm rpc stub build properties: {}", - err - )); - None + "Property {} is defined in multiple sources: {}", + property_name.log_color_highlight(), + sources + .into_iter() + .map(|s| s.log_color_highlight()) + .join(", ") + )) } - }; + } - validation.add_warns(component.traits, |component_trait| { + let non_unique_templates = template_sources + .into_iter() + .filter(|(_, template)| !template.is_empty()); + + validation.add_errors(non_unique_templates, |(template_name, sources)| { Some(( vec![], format!( - "Unknown trait for wasm rpc stub build, trait type: {}", - component_trait.trait_type + "Template {} defined multiple times in sources: {}", + template_name.as_str().log_color_highlight(), + sources + .into_iter() + .map(|s| s.log_color_highlight()) + .join(", ") ), )) }); - result - } - - pub fn validate_components( - validation: &mut ValidationBuilder, - components: Vec, - ) -> BTreeMap { - let (wasm_components_by_name, sources) = { - let mut wasm_components_by_name = BTreeMap::::new(); - let mut sources = BTreeMap::>::new(); - for component in components { - sources - .entry(component.name.clone()) - .and_modify(|sources| sources.push(component.source_as_string())) - .or_insert_with(|| vec![component.source_as_string()]); - wasm_components_by_name.insert(component.name.clone(), component); - } - (wasm_components_by_name, sources) - }; + let non_unique_components = component_sources + .into_iter() + .filter(|(_, sources)| !sources.is_empty()); - let non_unique_components = sources.into_iter().filter(|(_, sources)| sources.len() > 1); - validation.add_errors(non_unique_components, |(component_name, sources)| { + validation.add_errors(non_unique_components, |(template_name, sources)| { Some(( - vec![("component name", component_name.0)], + vec![], format!( - "Component is specified multiple times in sources: {}", - sources.iter().map(|s| s.log_color_highlight()).join(", ") + "Component {} defined multiple times in sources: {}", + template_name.as_str().log_color_highlight(), + sources + .into_iter() + .map(|s| s.log_color_highlight()) + .join(", ") ), )) }); - for (component_name, component) in &wasm_components_by_name { - validation.push_context("source", component.source_as_string()); - - validation.add_errors(&component.wasm_rpc_dependencies, |dep_component_name| { - (!wasm_components_by_name.contains_key(dep_component_name)).then(|| { - ( - vec![], - format!( - "Component {} references unknown component {} as dependency", - component_name.as_str().log_color_highlight(), - dep_component_name.as_str().log_color_error_highlight(), - ), - ) - }) - }); - - validation.pop_context(); - } - - wasm_components_by_name - } - - fn validate_wasm_builds( - validation: &mut ValidationBuilder, - wasm_builds: Vec, - ) -> Option { - if wasm_builds.len() > 1 { - validation.add_error(format!( - "Component Build is specified multiple times in sources: {}", - wasm_builds - .iter() - .map(|c| format!( - "{} in {}", - c.name.log_color_highlight(), - c.source.log_color_highlight() - )) - .join(", ") - )); - } - - wasm_builds.into_iter().next() - } + for (component_name, dependency_sources) in dependency_sources { + for (target_component_name, dependency_sources) in dependency_sources { + if dependency_sources.len() > 1 { + validation.push_context("component", component_name.to_string()); + validation.push_context("target", target_component_name.to_string()); + + validation.add_warn(format!( + "WASM-RPC dependency is defined multiple times, sources: {}", + dependency_sources + .into_iter() + .map(|s| s.log_color_highlight()) + .join(", ") + )); - fn validate_wasm_rpc_stub_builds( - validation: &mut ValidationBuilder, - wasm_components_by_name: &BTreeMap, - wasm_rpc_stub_builds: Vec, - ) -> ( - Option, - BTreeMap, - ) { - let ( - common_wasm_rpc_stub_builds, - wasm_rpc_stub_builds_by_component_name, - common_sources, - sources, - ) = { - let mut common_wasm_rpc_stub_builds = Vec::::new(); - let mut wasm_rpc_stub_builds_by_component_name = - BTreeMap::::new(); - - let mut common_sources = Vec::::new(); - let mut by_name_sources = BTreeMap::>::new(); - - for wasm_rpc_stub_build in wasm_rpc_stub_builds { - match &wasm_rpc_stub_build.component_name { - Some(component_name) => { - by_name_sources - .entry(component_name.clone()) - .and_modify(|sources| { - sources.push(wasm_rpc_stub_build.source_as_string()) - }) - .or_insert_with(|| vec![wasm_rpc_stub_build.source_as_string()]); - wasm_rpc_stub_builds_by_component_name - .insert(component_name.clone(), wasm_rpc_stub_build); - } - None => { - common_sources.push(wasm_rpc_stub_build.source_as_string()); - common_wasm_rpc_stub_builds.push(wasm_rpc_stub_build) - } + validation.pop_context(); + validation.pop_context(); } } + } - ( - common_wasm_rpc_stub_builds, - wasm_rpc_stub_builds_by_component_name, - common_sources, - by_name_sources, - ) - }; - - let non_unique_wasm_rpc_stub_builds = - sources.into_iter().filter(|(_, sources)| sources.len() > 1); - - validation.add_errors( - non_unique_wasm_rpc_stub_builds, - |(component_name, sources)| { - Some(( - vec![("component name", component_name.0)], - format!( - "Wasm rpc stub build is specified multiple times in sources: {}", - sources.iter().map(|s| s.log_color_highlight()).join(", ") - ), - )) - }, - ); + let components = { + let template_env = Self::template_env(); + + let mut resolved_components = BTreeMap::::new(); + for (component_name, (source, mut component)) in components { + validation.push_context("source", source.to_string_lossy().to_string()); + validation.push_context("component", component_name.to_string()); + + let template_with_name = match component.template { + Some(template_name) => { + let template_name = TemplateName::from(template_name); + match templates.get(&template_name) { + Some(template) => Some(Some((template_name, template))), + None => { + validation.add_error(format!( + "Component references unknown template: {}", + template_name.as_str().log_color_error_highlight() + )); + None + } + } + } + None => Some(None), + }; - if common_sources.len() > 1 { - validation.add_error( - format!( - "Common (without component name) wasm rpc build is specified multiple times in sources: {}", - common_sources.iter().map(|s| s.log_color_highlight()).join(", "), - ) - ) - } + if let Some(template_with_name) = template_with_name { + let component_properties = match template_with_name { + Some((template_name, template)) => { + let mut incompatible_overrides = false; + + let defined_property_names = + component.component_properties.defined_property_names(); + + if !template.profiles.is_empty() && !defined_property_names.is_empty() { + incompatible_overrides = true; + for property_name in defined_property_names { + validation.add_error( + format!( + "Property {} cannot be used, as the component uses a template with profiles", + property_name.log_color_highlight() + ) + ); + } + } - validation.add_errors( - &wasm_rpc_stub_builds_by_component_name, - |(component_name, wasm_rpc_stub_build)| { - (!wasm_components_by_name.contains_key(component_name)).then(|| { - ( - vec![("source", wasm_rpc_stub_build.source_as_string())], - format!( - "Wasm rpc stub build {} references unknown component {}", - wasm_rpc_stub_build.name.log_color_highlight(), - component_name.as_str().log_color_error_highlight(), - ), - ) - }) - }, - ); + for profile_name in component.profiles.keys() { + if !template.profiles.contains_key(profile_name) { + incompatible_overrides = true; + validation.add_error( + format!( + "Profile {} cannot be used, as the component uses template {} with the following profiles: {}", + profile_name.log_color_highlight(), + template_name.as_str().log_color_highlight(), + component.profiles.keys().map(|s| s.log_color_highlight()).join(", ") + ) + ); + } + } - ( - common_wasm_rpc_stub_builds.into_iter().next(), - wasm_rpc_stub_builds_by_component_name, - ) - } + if incompatible_overrides { + None + } else { + let template_context = minijinja::context! { componentName => component_name.as_str() }; + + if template.profiles.is_empty() { + let rendered_template_properties = + ComponentProperties::from_template( + &template_env, + &template_context, + &template.component_properties, + ); - fn render_templates( - validation: &mut ValidationBuilder, - common_wasm_build: Option<&WasmBuild>, - wasm_components_by_name: &BTreeMap, - ) -> HashMap { - let env = Self::template_env(); - let templates = common_wasm_build.map(|build| &build.component_templates); - let mut rendered = HashMap::::new(); - - for (component_name, component) in wasm_components_by_name { - let WasmComponentPropertySource::Template { template_name } = &component.properties - else { - continue; - }; - - let template_context = minijinja::context! { componentName => component_name.as_str() }; - - match templates.and_then(|templates| templates.get(template_name)) { - Some(template) => match template { - WasmComponentPropertySource::Explicit { properties } => match properties - .render(&env, &template_context) - { - Ok(properties) => { - rendered.insert( - component_name.clone(), - RenderedWasmComponentProperties::Single { properties }, - ); - } - Err(err) => { - validation.push_context("component name", component_name.to_string()); - validation.push_context("template name", template_name.to_string()); - - validation - .add_error(format!("Failed to render component template: {}", err)); - validation.pop_context(); - validation.pop_context(); - } - }, - WasmComponentPropertySource::ExplicitWithProfiles { - default_profile, - profiles, - } => { - let profiles = profiles - .iter() - .filter_map(|(profile_name, template)| { - match template.render(&env, &template_context) { - Ok(properties) => { - Some((profile_name.as_str().into(), properties)) + match rendered_template_properties { + Ok(rendered_template_properties) => { + let (properties, any_template_overrides) = + rendered_template_properties.merge_with_overrides( + component.component_properties, + ); + Some(ResolvedComponentProperties::Properties { + template_name: Some(template_name), + any_template_overrides, + properties, + }) + } + Err(err) => { + validation.add_error(format!( + "Failed to render template {}, error: {}", + template_name.as_str().log_color_highlight(), + err.to_string().log_color_error_highlight() + )); + None + } } - Err(err) => { - validation.push_context( - "component name", - component_name.to_string(), - ); - validation.push_context( - "template name", - template_name.to_string(), - ); - validation.push_context( - "template profile name", - profile_name.to_string(), - ); - validation.add_error(format!( - "Failed to render component template: {}", - err - )); - validation.pop_context(); - validation.pop_context(); - validation.pop_context(); - None + } else { + let mut any_template_overrides = + HashMap::::new(); + let mut profiles = + HashMap::::new(); + let mut any_template_render_error = false; + + for (profile_name, template_component_properties) in + &template.profiles + { + let rendered_template_properties = + ComponentProperties::from_template( + &template_env, + &template_context, + template_component_properties, + ); + match rendered_template_properties { + Ok(rendered_template_properties) => { + let (properties, any_overrides) = { + if let Some(component_properties) = + component.profiles.remove(profile_name) + { + rendered_template_properties + .merge_with_overrides( + component_properties, + ) + } else { + (rendered_template_properties, false) + } + }; + + any_template_overrides.insert( + profile_name.clone().into(), + any_overrides, + ); + profiles.insert( + profile_name.clone().into(), + properties, + ); + } + Err(err) => { + validation.add_error(format!( + "Failed to render template {}, error: {}", + template_name.as_str().log_color_highlight(), + err.to_string().log_color_error_highlight() + )); + any_template_render_error = true + } + } } + + (!any_template_render_error).then(|| { + ResolvedComponentProperties::Profiles { + template_name: Some(template_name), + any_template_overrides, + default_profile: template + .default_profile + .clone() + .expect("Missing template default profile") + .into(), + profiles, + } + }) } - }) - .collect(); - rendered.insert( - component_name.clone(), - RenderedWasmComponentProperties::Profiles { - default_profile: default_profile.as_str().into(), - profiles, - }, - ); - } - WasmComponentPropertySource::Template { .. } => { - panic!("Unexpected template property source for template"); - } - }, - None => { - validation.add_error(format!( - "Component template {} not found, {}", - template_name.log_color_error_highlight(), - match templates { - Some(templates) if !templates.is_empty() => { - format!( - "available templates: {}", - templates.keys().map(|s| s.log_color_highlight()).join(", ") - ) } - _ => { - "no templates are defined".to_string() + } + None => { + if component.profiles.is_empty() { + if component.default_profile.is_some() { + validation.add_error(format!( + "When {} is not defined then {} should not be defined", + "profiles".log_color_highlight(), + "defaultProfile".log_color_highlight() + )); + None + } else { + Some(ResolvedComponentProperties::Properties { + template_name: None, + any_template_overrides: false, + properties: ComponentProperties::from( + component.component_properties, + ), + }) + } + } else if component.default_profile.is_none() { + validation.add_error(format!( + "When {} is defined then {} is mandatory", + "profiles".log_color_highlight(), + "defaultProfile".log_color_highlight() + )); + None + } else { + Some(ResolvedComponentProperties::Profiles { + template_name: None, + any_template_overrides: component + .profiles + .keys() + .map(|profile_name| { + (ProfileName::from(profile_name.clone()), false) + }) + .collect(), + default_profile: component.default_profile.unwrap().into(), + profiles: component + .profiles + .into_iter() + .map(|(profile_name, properties)| { + ( + ProfileName::from(profile_name), + ComponentProperties::from(properties), + ) + }) + .collect(), + }) } } - )); - validation.pop_context(); + }; + + if let Some(properties) = component_properties { + // TODO: validate (build steps too) + + resolved_components + .insert(component_name.clone(), Component { source, properties }); + } } + + validation.pop_context(); + validation.pop_context(); } - } - rendered + resolved_components + }; + + validation.build(Self { + temp_dir, + wit_deps, + components, + dependencies, + no_dependencies: BTreeSet::new(), + }) } - fn template_env<'a>() -> Environment<'a> { + fn template_env<'a>() -> minijinja::Environment<'a> { let mut env = minijinja::Environment::new(); env.add_filter("to_snake_case", |str: &str| str.to_snake_case()); @@ -1244,26 +888,16 @@ impl Application { env } - pub fn components(&self) -> impl Iterator { - self.wasm_components_by_name.iter() - } - pub fn component_names(&self) -> impl Iterator { - self.wasm_components_by_name.keys() + self.components.keys() } pub fn wit_deps(&self) -> Vec { - self.common_wasm_build - .as_ref() - .map(|wasm_build| wasm_build.wit_deps.clone()) - .unwrap_or_default() + self.wit_deps.iter().map(PathBuf::from).collect() } pub fn all_wasm_rpc_dependencies(&self) -> BTreeSet { - self.wasm_components_by_name - .iter() - .flat_map(|(_, component)| component.wasm_rpc_dependencies.iter().cloned()) - .collect() + self.dependencies.values().flatten().cloned().collect() } pub fn all_profiles(&self) -> BTreeSet { @@ -1283,101 +917,84 @@ impl Application { .collect() } - pub fn build_dir(&self) -> PathBuf { - self.common_wasm_build - .as_ref() - .and_then(|build| build.build_dir.clone()) - .unwrap_or_else(|| PathBuf::from("build")) + pub fn temp_dir(&self) -> &Path { + match self.temp_dir.as_ref() { + Some(temp_dir) => Path::new(temp_dir), + None => Path::new("golem-temp"), + } } - pub fn component(&self, component_name: &ComponentName) -> &WasmComponent { - self.wasm_components_by_name + fn component(&self, component_name: &ComponentName) -> &Component { + self.components .get(component_name) .unwrap_or_else(|| panic!("Component not found: {}", component_name)) } + pub fn component_source_dir(&self, component_name: &ComponentName) -> &Path { + self.component(component_name).source_dir() + } + + pub fn component_wasm_rpc_dependencies( + &self, + component_name: &ComponentName, + ) -> &BTreeSet { + self.dependencies + .get(component_name) + .unwrap_or(&self.no_dependencies) + } + fn component_profiles(&self, component_name: &ComponentName) -> HashSet { match &self.component(component_name).properties { - WasmComponentPropertySource::Explicit { .. } => HashSet::new(), - WasmComponentPropertySource::ExplicitWithProfiles { profiles, .. } => { + ResolvedComponentProperties::Properties { .. } => HashSet::new(), + ResolvedComponentProperties::Profiles { profiles, .. } => { profiles.keys().cloned().collect() } - WasmComponentPropertySource::Template { .. } => self - .rendered_wasm_component_properties - .get(component_name) - .and_then(|properties| match properties { - RenderedWasmComponentProperties::Single { .. } => None, - RenderedWasmComponentProperties::Profiles { profiles, .. } => { - Some(profiles.keys().cloned().collect()) - } - }) - .unwrap_or_default(), } } - pub fn component_effective_template_and_profile<'a>( + pub fn component_effective_property_source<'a>( &'a self, component_name: &ComponentName, profile: Option<&'a ProfileName>, - ) -> ComponentEffectiveTemplateAndProfile<'a> { + ) -> ComponentEffectivePropertySource<'a> { match &self.component(component_name).properties { - WasmComponentPropertySource::Explicit { .. } => ComponentEffectiveTemplateAndProfile { - template_name: None, + ResolvedComponentProperties::Properties { + template_name, + any_template_overrides, + .. + } => ComponentEffectivePropertySource { + template_name: template_name.as_ref(), profile: None, is_requested_profile: false, + any_template_overrides: *any_template_overrides, }, - WasmComponentPropertySource::ExplicitWithProfiles { + ResolvedComponentProperties::Profiles { + template_name, + any_template_overrides, default_profile, profiles, - } => match profile { - Some(profile) if profiles.contains_key(profile) => { - ComponentEffectiveTemplateAndProfile { - template_name: None, - profile: Some(profile), - is_requested_profile: true, - } - } - _ => ComponentEffectiveTemplateAndProfile { - template_name: None, - profile: Some(default_profile), - is_requested_profile: false, - }, - }, - WasmComponentPropertySource::Template { template_name } => { - let properties = self - .rendered_wasm_component_properties - .get(component_name) - .unwrap_or_else(|| { - panic!( - "Rendered component properties not found for {}", - component_name - ) - }); - match properties { - RenderedWasmComponentProperties::Single { .. } => { - ComponentEffectiveTemplateAndProfile { - template_name: Some(template_name), - profile: None, - is_requested_profile: false, - } - } - RenderedWasmComponentProperties::Profiles { - default_profile, - profiles, - } => match profile { - Some(profile) if profiles.contains_key(profile) => { - ComponentEffectiveTemplateAndProfile { - template_name: Some(template_name), - profile: Some(profile), - is_requested_profile: true, - } + } => { + let effective_profile = profile + .map(|profile| { + if profiles.contains_key(profile) { + profile + } else { + default_profile } - _ => ComponentEffectiveTemplateAndProfile { - template_name: Some(template_name), - profile: Some(default_profile), - is_requested_profile: false, - }, - }, + }) + .unwrap_or_else(|| default_profile); + + let is_requested_profile = Some(&effective_profile) == profile.as_ref(); + + let any_template_overrides = any_template_overrides + .get(effective_profile) + .cloned() + .unwrap_or_default(); + ComponentEffectivePropertySource { + template_name: template_name.as_ref(), + profile, + is_requested_profile, + any_template_overrides, } } } @@ -1387,55 +1004,30 @@ impl Application { &self, component_name: &ComponentName, profile: Option<&ProfileName>, - ) -> &WasmComponentProperties { + ) -> &ComponentProperties { match &self.component(component_name).properties { - WasmComponentPropertySource::Explicit { properties } => properties, - WasmComponentPropertySource::ExplicitWithProfiles { + ResolvedComponentProperties::Properties { properties, .. } => properties, + ResolvedComponentProperties::Profiles { default_profile, profiles, - } => { - let profile = profile.unwrap_or(default_profile); - profiles.get(profile).unwrap_or_else(|| { - profiles.get(default_profile).unwrap_or_else(|| { - panic!( - "Default profile {} for {} not found", - default_profile, component_name - ) - }) - }) - } - WasmComponentPropertySource::Template { .. } => { - let properties = self - .rendered_wasm_component_properties - .get(component_name) - .unwrap_or_else(|| { - panic!( - "Rendered component properties not found for {}", - component_name - ) - }); - match properties { - RenderedWasmComponentProperties::Single { properties } => properties, - RenderedWasmComponentProperties::Profiles { - default_profile, - profiles, - } => { - let profile = profile.unwrap_or(default_profile); - profiles.get(profile).unwrap_or_else(|| { - profiles.get(default_profile).unwrap_or_else(|| { - panic!( - "Default (rendered) profile {} for {} not found", - default_profile, component_name - ) - }) + .. + } => profiles + .get( + profile + .map(|profile| { + if profiles.contains_key(profile) { + profile + } else { + default_profile + } }) - } - } - } + .unwrap_or_else(|| default_profile), + ) + .unwrap(), } } - pub fn component_input_wit( + pub fn component_source_wit( &self, component_name: &ComponentName, profile: Option<&ProfileName>, @@ -1443,29 +1035,29 @@ impl Application { let component = self.component(component_name); component.source_dir().join( self.component_properties(component_name, profile) - .input_wit + .source_wit .clone(), ) } - pub fn component_base_output_wit(&self, component_name: &ComponentName) -> PathBuf { - self.build_dir() - .join("base_output_wit") + pub fn component_generated_base_wit(&self, component_name: &ComponentName) -> PathBuf { + self.temp_dir() + .join("generated-base-wit") .join(component_name.as_str()) } - pub fn component_base_output_wit_interface_package_dir( + pub fn component_generated_base_wit_interface_package_dir( &self, component_name: &ComponentName, interface_package_name: &PackageName, ) -> PathBuf { - self.component_base_output_wit(component_name) + self.component_generated_base_wit(component_name) .join(naming::wit::DEPS_DIR) .join(package_dep_dir_name_from_parser(interface_package_name)) .join(naming::wit::INTERFACE_WIT_FILE_NAME) } - pub fn component_output_wit( + pub fn component_generated_wit( &self, component_name: &ComponentName, profile: Option<&ProfileName>, @@ -1473,12 +1065,12 @@ impl Application { let component = self.component(component_name); component.source_dir().join( self.component_properties(component_name, profile) - .output_wit + .generated_wit .clone(), ) } - pub fn component_input_wasm( + pub fn component_wasm( &self, component_name: &ComponentName, profile: Option<&ProfileName>, @@ -1486,347 +1078,142 @@ impl Application { let component = self.component(component_name); component.source_dir().join( self.component_properties(component_name, profile) - .input_wasm + .component_wasm .clone(), ) } - pub fn component_output_wasm( + pub fn component_linked_wasm( &self, component_name: &ComponentName, profile: Option<&ProfileName>, ) -> PathBuf { - let component = self.component(component_name); - component.source_dir().join( + self.component_source_dir(component_name).join( self.component_properties(component_name, profile) - .output_wasm - .clone(), + .linked_wasm + .as_ref() + .cloned() + .map(PathBuf::from) + .unwrap_or_else(|| { + self.temp_dir() + .join("linked-wasm") + .join(format!("{}.wasm", component_name.as_str())) + }), ) } - pub fn stub_world(&self, component_name: &ComponentName) -> Option { - self.stub_gen_property(component_name, |build| build.world.clone()) - .flatten() - } - - pub fn stub_crate_version(&self, component_name: &ComponentName) -> String { - self.stub_gen_property(component_name, |build| build.crate_version.clone()) - .flatten() - .unwrap_or_else(|| WASM_RPC_VERSION.to_string()) - } - - pub fn stub_wasm_rpc_path(&self, component_name: &ComponentName) -> Option { - self.stub_gen_property(component_name, |build| build.wasm_rpc_path.clone()) - .flatten() - } - - pub fn stub_wasm_rpc_version(&self, component_name: &ComponentName) -> Option { - self.stub_gen_property(component_name, |build| build.wasm_rpc_version.clone()) - .flatten() - } - - pub fn stub_build_dir(&self, component_name: &ComponentName) -> PathBuf { - self.stub_gen_property(component_name, |build| { - build - .build_dir - .as_ref() - .map(|build_dir| build.source_dir().join(build_dir)) - }) - .flatten() - .unwrap_or_else(|| self.build_dir()) - .join("stub") + fn stub_build_dir(&self) -> PathBuf { + self.temp_dir().join("stub") } pub fn stub_temp_build_dir(&self, component_name: &ComponentName) -> PathBuf { - self.stub_build_dir(component_name) + self.stub_build_dir() .join(component_name.as_str()) .join("temp-build") } pub fn stub_wasm(&self, component_name: &ComponentName) -> PathBuf { - self.wasm_rpc_stub_builds_by_name - .get(component_name) - .and_then(|build| { - build - .wasm - .as_ref() - .map(|wasm| build.source_dir().join(wasm)) - }) - .unwrap_or_else(|| { - self.stub_build_dir(component_name) - .join(component_name.as_str()) - .join("stub.wasm") - }) + self.stub_build_dir() + .join(component_name.as_str()) + .join("stub.wasm") } pub fn stub_wit(&self, component_name: &ComponentName) -> PathBuf { - self.wasm_rpc_stub_builds_by_name - .get(component_name) - .and_then(|build| build.wit.as_ref().map(|wit| build.source_dir().join(wit))) - .unwrap_or_else(|| { - self.stub_build_dir(component_name) - .join(component_name.as_str()) - .join(naming::wit::WIT_DIR) - }) - } - - fn stub_gen_property(&self, component_name: &ComponentName, get_property: F) -> Option - where - F: Fn(&WasmRpcStubBuild) -> T, - { - self.wasm_rpc_stub_builds_by_name - .get(component_name) - .map(&get_property) - .or_else(|| self.common_wasm_rpc_stub_build.as_ref().map(get_property)) + self.stub_build_dir() + .join(component_name.as_str()) + .join(naming::wit::WIT_DIR) } } -pub type ExternalCommand = raw::CustomCommand; - -// NOTE: we do not use PathBuf here for fields, rather Strings, as values might be templated #[derive(Clone, Debug)] -pub struct WasmComponentProperties { - pub build: Vec, - pub input_wit: String, - pub output_wit: String, - pub input_wasm: String, - pub output_wasm: String, - pub custom_commands: HashMap>, - pub clean: Vec, -} - -#[derive(Clone, Debug)] -pub enum WasmComponentPropertySource { - Explicit { - properties: WasmComponentProperties, - }, - ExplicitWithProfiles { - default_profile: ProfileName, - profiles: HashMap, - }, - Template { - template_name: String, - }, -} - -#[derive(Clone, Debug)] -pub struct WasmComponent { - pub name: ComponentName, +pub struct Component { pub source: PathBuf, - pub properties: WasmComponentPropertySource, - pub wasm_rpc_dependencies: Vec, + pub properties: ResolvedComponentProperties, } -impl WasmComponent { - pub fn source_as_string(&self) -> String { - self.source.to_string_lossy().to_string() - } - +impl Component { pub fn source_dir(&self) -> &Path { self.source.parent().unwrap_or_else(|| { panic!( - "Failed to get parent for component {}, source: {}", - self.name, + "Failed to get parent for component, source: {}", self.source.display() ) }) } } -pub enum BuildStepsLookupResult<'a> { - NoBuildSteps, - NoBuildStepsForRequestedProfile, - BuildSteps { - build_steps: &'a [ExternalCommand], - }, - BuildStepsForDefaultProfile { - profile: &'a str, - build_steps: &'a [ExternalCommand], - }, - BuildStepsForRequestedProfile { - profile: &'a str, - build_steps: &'a [ExternalCommand], - }, -} - -#[derive(Clone, Debug)] -pub struct WasmBuild { - source: PathBuf, - name: String, - build_dir: Option, - component_templates: HashMap, - wit_deps: Vec, -} - #[derive(Clone, Debug)] -pub struct WasmRpcStubBuild { - source: PathBuf, - name: String, - component_name: Option, - build_dir: Option, - wasm: Option, - wit: Option, - world: Option, - crate_version: Option, - wasm_rpc_path: Option, - wasm_rpc_version: Option, +pub struct ComponentProperties { + pub source_wit: String, + pub generated_wit: String, + pub component_wasm: String, + pub linked_wasm: Option, + pub build: Vec, + pub custom_commands: HashMap>, + pub clean: Vec, } -impl WasmRpcStubBuild { - pub fn source_as_string(&self) -> String { - self.source.to_string_lossy().to_string() - } - - pub fn source_dir(&self) -> &Path { - self.source - .parent() - .expect("Failed to get parent for source") +impl ComponentProperties { + fn from_template( + env: &minijinja::Environment, + ctx: &C, + template_properties: &raw::ComponentProperties, + ) -> anyhow::Result { + Ok(ComponentProperties::from( + template_properties.render(env, ctx)?, + )) } -} - -#[cfg(test)] -mod tests { - use test_r::test; - - use super::*; - use crate::model::oam; - use assert2::assert; - #[test] - fn load_app_with_warns() { - let oam_app: oam::ApplicationWithSource = oam_app_one(); - let (app, warns, errors) = Application::from_oam_apps(vec![oam_app]).into_product(); + fn merge_with_overrides(mut self, overrides: raw::ComponentProperties) -> (Self, bool) { + let mut any_overrides = false; - assert!(app.is_some()); - let app = app.unwrap(); - - println!("Warns:\n{}", warns.join("\n")); - println!("Errors:\n{}", errors.join("\n")); - - assert!(app.wasm_components_by_name.len() == 3); - assert!(warns.len() == 3); - assert!(errors.len() == 0); - - let (component_name, component) = app.wasm_components_by_name.iter().next().unwrap(); - - assert!(component_name.as_str() == "component-one"); - assert!(component.name.as_str() == "component-one"); - // TODO: - /* - assert!(component.input_wit.to_string_lossy() == "input_wit"); - assert!(component.output_wit.to_string_lossy() == "output_wit"); - assert!(component.input_wasm.to_string_lossy() == "out/in.wasm"); - assert!(component.output_wasm.to_string_lossy() == "out/out.wasm"); - */ - assert!(component.wasm_rpc_dependencies.len() == 2); - - assert!(component.wasm_rpc_dependencies[0].as_str() == "component-three"); - assert!(component.wasm_rpc_dependencies[1].as_str() == "component-two"); - } + if let Some(source_wit) = overrides.source_wit { + self.source_wit = source_wit; + any_overrides = true; + } - #[test] - fn load_app_with_warns_and_errors() { - let oam_app: oam::ApplicationWithSource = oam_app_two(); - let (_app, warns, errors) = Application::from_oam_apps(vec![oam_app]).into_product(); + if let Some(generated_wit) = overrides.generated_wit { + self.generated_wit = generated_wit; + any_overrides = true; + } - println!("Warns:\n{}", warns.join("\n")); - println!("Errors:\n{}", errors.join("\n")); + if let Some(component_wasm) = overrides.component_wasm { + self.component_wasm = component_wasm; + any_overrides = true; + } - assert!(errors.len() == 2); + if overrides.linked_wasm.is_some() { + self.linked_wasm = overrides.linked_wasm; + any_overrides = true; + } - assert!(errors[0].contains("component-one")); - assert!(errors[0].contains("component-three")); - assert!(errors[0].contains("test-oam-app-two.yaml")); + if !overrides.build.is_empty() { + self.build = overrides.build; + any_overrides = true; + } - assert!(errors[1].contains("component-one")); - assert!(errors[1].contains("component-two")); - assert!(errors[1].contains("test-oam-app-two.yaml")); - } + for (custom_command_name, custom_command) in overrides.custom_commands { + if self.custom_commands.contains_key(&custom_command_name) { + any_overrides = true; + } + self.custom_commands + .insert(custom_command_name, custom_command); + } - fn oam_app_one() -> oam::ApplicationWithSource { - oam::ApplicationWithSource::from_yaml_string( - "test-oam-app-one.yaml".into(), - r#" -apiVersion: core.oam.dev/v1beta1 -metadata: - name: "App name" -kind: Application -spec: - components: - - name: component-one - type: wasm - properties: - inputWit: input_wit - outputWit: output_wit - inputWasm: out/in.wasm - outputWasm: out/out.wasm - testUnknownProp: test - traits: - - type: wasm-rpc - properties: - componentName: component-two - - type: wasm-rpc - properties: - componentName: component-three - testUnknownProp: test - - type: unknown-trait - properties: - testUnknownProp: test - - name: component-two - type: wasm - properties: - inputWit: input_wit - outputWit: output_wit - inputWasm: out/in.wasm - outputWasm: out/out.wasm - - name: component-three - type: wasm - properties: - inputWit: input_wit - outputWit: output_wit - inputWasm: out/in.wasm - outputWasm: out/out.wasm -"# - .to_string(), - ) - .unwrap() + (self, any_overrides) } +} - fn oam_app_two() -> oam::ApplicationWithSource { - oam::ApplicationWithSource::from_yaml_string( - "test-oam-app-two.yaml".into(), - r#" -apiVersion: core.oam.dev/v1beta1 -metadata: - name: "App name" -kind: Application -spec: - components: - - name: component-one - type: wasm - properties: - inputWit: input-wit - outputWit: input-wit - inputWasm: out/in.wasm - outputWasm: out/out.wasm - testUnknownProp: test - traits: - - type: wasm-rpc - properties: - componentName: component-two - - type: wasm-rpc - properties: - componentName: component-three - testUnknownProp: test - - type: unknown-trait - properties: - testUnknownProp: test - - name: component-one - type: unknown-component-type - properties: -"# - .to_string(), - ) - .unwrap() +impl From for ComponentProperties { + fn from(value: raw::ComponentProperties) -> Self { + Self { + source_wit: value.source_wit.unwrap_or_default(), + generated_wit: value.generated_wit.unwrap_or_default(), + component_wasm: value.component_wasm.unwrap_or_default(), + linked_wasm: value.linked_wasm, + build: value.build, + custom_commands: value.custom_commands, + clean: value.clean, + } } } diff --git a/wasm-rpc-stubgen/src/stub.rs b/wasm-rpc-stubgen/src/stub.rs index 22f4c2f0..87880073 100644 --- a/wasm-rpc-stubgen/src/stub.rs +++ b/wasm-rpc-stubgen/src/stub.rs @@ -336,7 +336,7 @@ impl StubDefinition { fn partition_world_items<'a>( &'a self, world_items: &'a IndexMap, - ) -> WorldItemsByType { + ) -> WorldItemsByType<'a> { let mut types = Vec::<(String, TypeId)>::new(); let mut functions = Vec::new(); let mut interfaces = Vec::new(); diff --git a/wasm-rpc-stubgen/src/validation.rs b/wasm-rpc-stubgen/src/validation.rs index a5ccec92..5a3ecb00 100644 --- a/wasm-rpc-stubgen/src/validation.rs +++ b/wasm-rpc-stubgen/src/validation.rs @@ -249,25 +249,25 @@ impl ValidationBuilder { self.warns.push(warn); } - pub fn add_errors(&mut self, elems: C, context_and_error: F) + pub fn add_errors(&mut self, elems: E, context_and_error: CE) where - C: IntoIterator, - F: Fn(T) -> Option<(Vec<(&'static str, String)>, String)>, + E: IntoIterator, + CE: Fn(T) -> Option<(Vec<(&'static str, String)>, String)>, { self.add_all(elems, context_and_error, Self::add_error); } - pub fn add_warns(&mut self, elems: C, context_and_error: F) + pub fn add_warns(&mut self, elems: E, context_and_error: CE) where - C: IntoIterator, - F: Fn(T) -> Option<(Vec<(&'static str, String)>, String)>, + E: IntoIterator, + CE: Fn(T) -> Option<(Vec<(&'static str, String)>, String)>, { self.add_all(elems, context_and_error, Self::add_warn); } - fn add_all(&mut self, elems: C, context_and_error: CE, add: A) + fn add_all(&mut self, elems: E, context_and_error: CE, add: A) where - C: IntoIterator, + E: IntoIterator, CE: Fn(T) -> Option<(Vec<(&'static str, String)>, String)>, A: Fn(&mut Self, String), { diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index f193799d..3ffd2d87 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -165,12 +165,12 @@ fn collect_package_sources( pub struct ResolvedWitComponent { main_package_name: PackageName, - resolved_output_wit_dir: Option, + resolved_generated_wit_dir: Option, app_component_deps: HashSet, - input_referenced_package_deps: HashSet, - input_contained_package_deps: HashSet, - input_component_deps: BTreeSet, // NOTE: BTree for making dep sorting deterministic - output_component_deps: Option>, + source_referenced_package_deps: HashSet, + source_contained_package_deps: HashSet, + source_component_deps: BTreeSet, // NOTE: BTree for making dep sorting deterministic + generated_component_deps: Option>, } pub struct ResolvedWitApplication { @@ -200,7 +200,7 @@ impl ResolvedWitApplication { resolved_app.validate_package_names(&mut validation); resolved_app.collect_component_deps(app, &mut validation); - resolved_app.sort_components_by_input_deps(&mut validation); + resolved_app.sort_components_by_source_deps(&mut validation); validation.build(resolved_app) } @@ -262,42 +262,42 @@ impl ResolvedWitApplication { app: &Application, profile: Option<&ProfileName>, ) { - for (component_name, component) in app.components() { + for component_name in app.component_names() { validation.push_context("component name", component_name.to_string()); - let input_wit_dir = app.component_input_wit(component_name, profile); - let output_wit_dir = app.component_output_wit(component_name, profile); + let source_wit_dir = app.component_source_wit(component_name, profile); + let generated_wit_dir = app.component_generated_wit(component_name, profile); log_action( "Resolving", format!( "component wit dirs for {} ({}, {})", component_name.as_str().log_color_highlight(), - input_wit_dir.log_color_highlight(), - output_wit_dir.log_color_highlight(), + source_wit_dir.log_color_highlight(), + generated_wit_dir.log_color_highlight(), ), ); let resolved_component = (|| -> anyhow::Result { - let unresolved_input_package_group = - UnresolvedPackageGroup::parse_dir(&input_wit_dir).with_context(|| { + let unresolved_source_package_group = + UnresolvedPackageGroup::parse_dir(&source_wit_dir).with_context(|| { anyhow!( - "Failed to parse component {} main package in input wit dir {}", + "Failed to parse component {} main package in source wit dir {}", component_name.as_str().log_color_error_highlight(), - input_wit_dir.log_color_highlight(), + source_wit_dir.log_color_highlight(), ) })?; - let input_referenced_package_deps = unresolved_input_package_group + let source_referenced_package_deps = unresolved_source_package_group .main .foreign_deps .keys() .cloned() .collect(); - let input_contained_package_deps = { + let source_contained_package_deps = { let deps_path = - Path::new(&app.component_properties(component_name, profile).input_wit) + Path::new(&app.component_properties(component_name, profile).source_wit) .join("deps"); if !deps_path.exists() { HashSet::new() @@ -309,29 +309,33 @@ impl ResolvedWitApplication { } }; - let main_package_name = unresolved_input_package_group.main.name.clone(); + let main_package_name = unresolved_source_package_group.main.name.clone(); - let resolved_output_wit_dir = ResolvedWitDir::new(&output_wit_dir).ok(); - let output_has_same_main_package_name = resolved_output_wit_dir + let resolved_generated_wit_dir = ResolvedWitDir::new(&generated_wit_dir).ok(); + let generated_has_same_main_package_name = resolved_generated_wit_dir .as_ref() .map(|wit| wit.main_package()) .transpose()? - .map(|output_main_package| main_package_name == output_main_package.name) + .map(|generated_main_package| main_package_name == generated_main_package.name) .unwrap_or_default(); - let resolved_output_wit_dir = output_has_same_main_package_name - .then_some(resolved_output_wit_dir) + let resolved_generated_wit_dir = generated_has_same_main_package_name + .then_some(resolved_generated_wit_dir) .flatten(); - let app_component_deps = component.wasm_rpc_dependencies.iter().cloned().collect(); + let app_component_deps = app + .component_wasm_rpc_dependencies(component_name) + .iter() + .cloned() + .collect(); Ok(ResolvedWitComponent { main_package_name, - resolved_output_wit_dir, + resolved_generated_wit_dir, app_component_deps, - input_referenced_package_deps, - input_contained_package_deps, - input_component_deps: Default::default(), - output_component_deps: Default::default(), + source_referenced_package_deps, + source_contained_package_deps, + source_component_deps: Default::default(), + generated_component_deps: Default::default(), }) })(); @@ -371,41 +375,47 @@ impl ResolvedWitApplication { ( component_deps( &self.interface_package_to_component, - &component.input_referenced_package_deps, + &component.source_referenced_package_deps, ), - component.resolved_output_wit_dir.as_ref().map(|wit_dir| { - component_deps( - &self.stub_package_to_component, - wit_dir.resolve.package_names.keys(), - ) - }), + component + .resolved_generated_wit_dir + .as_ref() + .map(|wit_dir| { + component_deps( + &self.stub_package_to_component, + wit_dir.resolve.package_names.keys(), + ) + }), ), ); } - for (component_name, (input_deps, output_deps)) in deps { + for (component_name, (source_deps, generated_deps)) in deps { let component = self.components.get_mut(&component_name).unwrap(); - component.input_component_deps = input_deps; - component.output_component_deps = output_deps; + component.source_component_deps = source_deps; + component.generated_component_deps = generated_deps; } for (component_name, component) in &self.components { let main_deps: HashSet = component_deps( &self.package_to_component, - &component.input_referenced_package_deps, + &component.source_referenced_package_deps, ); let stub_deps: HashSet = component_deps( &self.stub_package_to_component, - &component.input_referenced_package_deps, + &component.source_referenced_package_deps, ); if !main_deps.is_empty() || !stub_deps.is_empty() { validation.push_context("component name", component_name.to_string()); validation.push_context("package name", component.main_package_name.to_string()); { - let component = app.component(component_name); - validation - .push_context("source", component.source.to_string_lossy().to_string()); + validation.push_context( + "source", + app.component_source_dir(component_name) + .to_string_lossy() + .to_string(), + ); } for dep_component_name in main_deps { @@ -454,7 +464,7 @@ impl ResolvedWitApplication { } } - fn sort_components_by_input_deps(&mut self, validation: &mut ValidationBuilder) { + fn sort_components_by_source_deps(&mut self, validation: &mut ValidationBuilder) { struct Visit<'a> { resolved_app: &'a ResolvedWitApplication, component_names_by_id: Vec<&'a ComponentName>, @@ -514,7 +524,7 @@ impl ResolvedWitApplication { self.path.push(component_id); self.visiting.insert(component_id); - for dep_component_name in &component.input_component_deps { + for dep_component_name in &component.source_component_deps { if !self.visit( self.component_names_to_id[dep_component_name], dep_component_name, @@ -568,17 +578,17 @@ impl ResolvedWitApplication { // NOTE: Intended to be used for non-component wit package deps, so it does not include // component interface packages, as those are added from stubs - pub fn missing_generic_input_package_deps( + pub fn missing_generic_source_package_deps( &self, component_name: &ComponentName, ) -> anyhow::Result> { let component = self.component(component_name)?; Ok(component - .input_referenced_package_deps + .source_referenced_package_deps .iter() .filter(|&package_name| { !component - .input_contained_package_deps + .source_contained_package_deps .contains(package_name) && !self .interface_package_to_component @@ -594,7 +604,7 @@ impl ResolvedWitApplication { ) -> anyhow::Result> { let component = self.component(component_name)?; Ok(component - .input_referenced_package_deps + .source_referenced_package_deps .iter() .filter_map(|package_name| { match self.interface_package_to_component.get(package_name) { @@ -612,8 +622,8 @@ impl ResolvedWitApplication { // application model vs in wit dependencies pub fn is_dep_graph_up_to_date(&self, component_name: &ComponentName) -> anyhow::Result { let component = self.component(component_name)?; - Ok(match &component.output_component_deps { - Some(output_deps) => &component.app_component_deps == output_deps, + Ok(match &component.generated_component_deps { + Some(generated_deps) => &component.app_component_deps == generated_deps, None => false, }) } @@ -627,8 +637,8 @@ impl ResolvedWitApplication { ) -> anyhow::Result { let component = self.component(component_name)?; - Ok(match &component.output_component_deps { - Some(output_deps) => output_deps.contains(dep_component_name), + Ok(match &component.generated_component_deps { + Some(generated_deps) => generated_deps.contains(dep_component_name), None => false, }) } diff --git a/wasm-rpc/src/lib.rs b/wasm-rpc/src/lib.rs index 8a7a1750..11237879 100644 --- a/wasm-rpc/src/lib.rs +++ b/wasm-rpc/src/lib.rs @@ -132,7 +132,7 @@ pub use type_annotated_value::*; pub use text::{type_annotated_value_from_str, type_annotated_value_to_string}; #[cfg(feature = "arbitrary")] -impl<'a> arbitrary::Arbitrary<'a> for Uri { +impl arbitrary::Arbitrary<'_> for Uri { fn arbitrary(u: &mut arbitrary::Unstructured) -> arbitrary::Result { let uri = u.arbitrary::()?; Ok(Uri { value: uri }) From e402e892adf73d32ac1c97526d4202b5327feec6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Sun, 1 Dec 2024 22:41:01 +0100 Subject: [PATCH 094/109] model cleanups --- wasm-rpc-stubgen/src/commands/declarative.rs | 9 +- .../src/model/{wasm_rpc.rs => app.rs} | 281 +----------------- wasm-rpc-stubgen/src/model/app_raw.rs | 147 +++++++++ wasm-rpc-stubgen/src/model/mod.rs | 5 +- wasm-rpc-stubgen/src/model/template.rs | 105 +++++++ .../src/model/unknown_properties.rs | 40 --- wasm-rpc-stubgen/src/wit_resolve.rs | 2 +- 7 files changed, 273 insertions(+), 316 deletions(-) rename wasm-rpc-stubgen/src/model/{wasm_rpc.rs => app.rs} (80%) create mode 100644 wasm-rpc-stubgen/src/model/app_raw.rs create mode 100644 wasm-rpc-stubgen/src/model/template.rs delete mode 100644 wasm-rpc-stubgen/src/model/unknown_properties.rs diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/declarative.rs index 8a8f645e..e787a0eb 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/declarative.rs @@ -5,10 +5,11 @@ use crate::log::{ log_action, log_skipping_up_to_date, log_validated_action_result, log_warn_action, LogColorize, LogIndent, }; -use crate::model::wasm_rpc::raw::ExternalCommand; -use crate::model::wasm_rpc::{ - includes_from_yaml_file, raw, Application, ComponentName, ProfileName, DEFAULT_CONFIG_FILE_NAME, +use crate::model::app::{ + includes_from_yaml_file, Application, ComponentName, ProfileName, DEFAULT_CONFIG_FILE_NAME, }; +use crate::model::app_raw; +use crate::model::app_raw::ExternalCommand; use crate::stub::{StubConfig, StubDefinition}; use crate::validation::ValidatedResult; use crate::wit_generate::{ @@ -636,7 +637,7 @@ fn load_app_validated(config: &Config) -> ValidatedResult { sources .into_iter() .map(|source| { - ValidatedResult::from_result(raw::ApplicationWithSource::from_yaml_file(source)) + ValidatedResult::from_result(app_raw::ApplicationWithSource::from_yaml_file(source)) }) .collect::>>() }); diff --git a/wasm-rpc-stubgen/src/model/wasm_rpc.rs b/wasm-rpc-stubgen/src/model/app.rs similarity index 80% rename from wasm-rpc-stubgen/src/model/wasm_rpc.rs rename to wasm-rpc-stubgen/src/model/app.rs index a1c8de9f..26d2d6aa 100644 --- a/wasm-rpc-stubgen/src/model/wasm_rpc.rs +++ b/wasm-rpc-stubgen/src/model/app.rs @@ -1,5 +1,6 @@ use crate::log::LogColorize; -use crate::model::wasm_rpc::template::Template; +use crate::model::app_raw; +use crate::model::template::Template; use crate::naming::wit::package_dep_dir_name_from_parser; use crate::validation::{ValidatedResult, ValidationBuilder}; use crate::{fs, naming}; @@ -99,268 +100,10 @@ impl From<&str> for TemplateName { } } -pub mod raw { - use crate::fs; - use serde::{Deserialize, Serialize}; - use std::collections::HashMap; - use std::path::PathBuf; - - #[derive(Clone, Debug)] - pub struct ApplicationWithSource { - pub source: PathBuf, - pub application: Application, - } - - impl ApplicationWithSource { - pub fn from_yaml_file(file: PathBuf) -> anyhow::Result { - Ok(Self::from_yaml_string( - file.clone(), - fs::read_to_string(file)?, - )?) - } - - pub fn from_yaml_string(source: PathBuf, string: String) -> serde_yaml::Result { - Ok(Self { - source, - application: Application::from_yaml_str(string.as_str())?, - }) - } - - pub fn source_as_string(&self) -> String { - self.source.to_string_lossy().to_string() - } - } - - #[derive(Clone, Debug, Serialize, Deserialize)] - #[serde(rename_all = "camelCase")] - pub struct Application { - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub include: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub temp_dir: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub wit_deps: Vec, - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub templates: HashMap, - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub components: HashMap, - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub dependencies: HashMap, - } - - impl Application { - pub fn from_yaml_str(yaml: &str) -> serde_yaml::Result { - serde_yaml::from_str(yaml) - } - - pub fn to_yaml_string(&self) -> String { - serde_yaml::to_string(self).expect("Failed to serialize Application as YAML") - } - } - - #[derive(Clone, Debug, Serialize, Deserialize)] - #[serde(rename_all = "camelCase")] - pub struct ComponentTemplate { - #[serde(flatten)] - pub component_properties: ComponentProperties, - pub profiles: HashMap, - pub default_profile: Option, - } - - #[derive(Clone, Debug, Serialize, Deserialize)] - #[serde(rename_all = "camelCase")] - pub struct Component { - pub template: Option, - #[serde(flatten)] - pub component_properties: ComponentProperties, - pub profiles: HashMap, - pub default_profile: Option, - } - - #[derive(Clone, Debug, Serialize, Deserialize)] - #[serde(rename_all = "camelCase")] - pub struct ComponentProperties { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub source_wit: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub generated_wit: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub component_wasm: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub linked_wasm: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub build: Vec, - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub custom_commands: HashMap>, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub clean: Vec, - } - - impl ComponentProperties { - pub fn defined_property_names(&self) -> Vec<&str> { - let mut vec = Vec::<&str>::new(); - - if self.source_wit.is_some() { - vec.push("sourceWit"); - } - - if self.generated_wit.is_some() { - vec.push("generatedWit"); - } - - if self.component_wasm.is_some() { - vec.push("componentWasm"); - } - - if self.linked_wasm.is_some() { - vec.push("linkedWasm"); - } - - if !self.build.is_empty() { - vec.push("build"); - } - - if !self.custom_commands.is_empty() { - vec.push("customCommands"); - } - - vec - } - } - - #[derive(Clone, Debug, Serialize, Deserialize)] - #[serde(rename_all = "camelCase")] - pub struct ExternalCommand { - pub command: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub dir: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub sources: Vec, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub targets: Vec, - } - - #[derive(Clone, Debug, Serialize, Deserialize)] - #[serde(rename_all = "camelCase")] - pub struct Dependency { - #[serde(rename = "type")] - pub type_: String, - pub target: Option, - } -} - -mod template { - use crate::model::wasm_rpc::raw; - use serde::Serialize; - use std::collections::HashMap; - - pub trait Template { - type Rendered; - - fn render( - &self, - env: &minijinja::Environment, - ctx: &C, - ) -> Result; - } - - impl Template for String { - type Rendered = String; - - fn render( - &self, - env: &minijinja::Environment, - ctx: &C, - ) -> Result { - env.render_str(self, ctx) - } - } - - impl> Template for Option { - type Rendered = Option; - - fn render( - &self, - env: &minijinja::Environment, - ctx: &C, - ) -> Result { - match self { - Some(template) => Ok(Some(template.render(env, ctx)?)), - None => Ok(None), - } - } - } - - impl> Template for Vec { - type Rendered = Vec; - - fn render( - &self, - env: &minijinja::Environment, - ctx: &C, - ) -> Result { - self.iter().map(|elem| elem.render(env, ctx)).collect() - } - } - - impl> Template for HashMap { - type Rendered = HashMap; - - fn render( - &self, - env: &minijinja::Environment, - ctx: &C, - ) -> Result { - let mut rendered = HashMap::::new(); - for (key, template) in self { - rendered.insert(key.clone(), template.render(env, ctx)?); - } - Ok(rendered) - } - } - - impl Template for raw::ExternalCommand { - type Rendered = raw::ExternalCommand; - - fn render( - &self, - env: &minijinja::Environment, - ctx: &C, - ) -> Result { - Ok(raw::ExternalCommand { - command: self.command.render(env, ctx)?, - dir: self.dir.render(env, ctx)?, - sources: self.sources.render(env, ctx)?, - targets: self.targets.render(env, ctx)?, - }) - } - } - - impl Template for raw::ComponentProperties { - type Rendered = raw::ComponentProperties; - - fn render( - &self, - env: &minijinja::Environment, - ctx: &C, - ) -> Result { - Ok(raw::ComponentProperties { - source_wit: self.source_wit.render(env, ctx)?, - generated_wit: self.generated_wit.render(env, ctx)?, - component_wasm: self.component_wasm.render(env, ctx)?, - linked_wasm: self.linked_wasm.render(env, ctx)?, - build: self.build.render(env, ctx)?, - custom_commands: self.custom_commands.render(env, ctx)?, - clean: self.clean.render(env, ctx)?, - }) - } - } -} - pub fn includes_from_yaml_file(source: &Path) -> Vec { fs::read_to_string(source) .ok() - .and_then(|source| raw::Application::from_yaml_str(source.as_str()).ok()) + .and_then(|source| app_raw::Application::from_yaml_str(source.as_str()).ok()) .map(|app| app.include) .unwrap_or_default() } @@ -397,7 +140,7 @@ pub struct Application { } impl Application { - pub fn from_raw_apps(apps: Vec) -> ValidatedResult { + pub fn from_raw_apps(apps: Vec) -> ValidatedResult { let mut validation = ValidationBuilder::new(); let mut include = Vec::::new(); @@ -409,14 +152,14 @@ impl Application { let mut wit_deps = Vec::::new(); let mut wit_deps_sources = Vec::::new(); - let mut templates = HashMap::::new(); + let mut templates = HashMap::::new(); let mut template_sources = HashMap::>::new(); let mut dependencies = BTreeMap::>::new(); let mut dependency_sources = HashMap::>>::new(); - let mut components = HashMap::::new(); + let mut components = HashMap::::new(); let mut component_sources = HashMap::>::new(); for app in apps { @@ -1148,8 +891,8 @@ pub struct ComponentProperties { pub generated_wit: String, pub component_wasm: String, pub linked_wasm: Option, - pub build: Vec, - pub custom_commands: HashMap>, + pub build: Vec, + pub custom_commands: HashMap>, pub clean: Vec, } @@ -1157,14 +900,14 @@ impl ComponentProperties { fn from_template( env: &minijinja::Environment, ctx: &C, - template_properties: &raw::ComponentProperties, + template_properties: &app_raw::ComponentProperties, ) -> anyhow::Result { Ok(ComponentProperties::from( template_properties.render(env, ctx)?, )) } - fn merge_with_overrides(mut self, overrides: raw::ComponentProperties) -> (Self, bool) { + fn merge_with_overrides(mut self, overrides: app_raw::ComponentProperties) -> (Self, bool) { let mut any_overrides = false; if let Some(source_wit) = overrides.source_wit { @@ -1204,8 +947,8 @@ impl ComponentProperties { } } -impl From for ComponentProperties { - fn from(value: raw::ComponentProperties) -> Self { +impl From for ComponentProperties { + fn from(value: app_raw::ComponentProperties) -> Self { Self { source_wit: value.source_wit.unwrap_or_default(), generated_wit: value.generated_wit.unwrap_or_default(), diff --git a/wasm-rpc-stubgen/src/model/app_raw.rs b/wasm-rpc-stubgen/src/model/app_raw.rs new file mode 100644 index 00000000..55e3ade0 --- /dev/null +++ b/wasm-rpc-stubgen/src/model/app_raw.rs @@ -0,0 +1,147 @@ +use crate::fs; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::path::PathBuf; + +#[derive(Clone, Debug)] +pub struct ApplicationWithSource { + pub source: PathBuf, + pub application: Application, +} + +impl ApplicationWithSource { + pub fn from_yaml_file(file: PathBuf) -> anyhow::Result { + Ok(Self::from_yaml_string( + file.clone(), + fs::read_to_string(file)?, + )?) + } + + pub fn from_yaml_string(source: PathBuf, string: String) -> serde_yaml::Result { + Ok(Self { + source, + application: Application::from_yaml_str(string.as_str())?, + }) + } + + pub fn source_as_string(&self) -> String { + self.source.to_string_lossy().to_string() + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Application { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub include: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub temp_dir: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub wit_deps: Vec, + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub templates: HashMap, + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub components: HashMap, + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub dependencies: HashMap, +} + +impl Application { + pub fn from_yaml_str(yaml: &str) -> serde_yaml::Result { + serde_yaml::from_str(yaml) + } + + pub fn to_yaml_string(&self) -> String { + serde_yaml::to_string(self).expect("Failed to serialize Application as YAML") + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ComponentTemplate { + #[serde(flatten)] + pub component_properties: ComponentProperties, + pub profiles: HashMap, + pub default_profile: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Component { + pub template: Option, + #[serde(flatten)] + pub component_properties: ComponentProperties, + pub profiles: HashMap, + pub default_profile: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ComponentProperties { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_wit: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub generated_wit: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub component_wasm: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub linked_wasm: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub build: Vec, + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub custom_commands: HashMap>, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub clean: Vec, +} + +impl ComponentProperties { + pub fn defined_property_names(&self) -> Vec<&str> { + let mut vec = Vec::<&str>::new(); + + if self.source_wit.is_some() { + vec.push("sourceWit"); + } + + if self.generated_wit.is_some() { + vec.push("generatedWit"); + } + + if self.component_wasm.is_some() { + vec.push("componentWasm"); + } + + if self.linked_wasm.is_some() { + vec.push("linkedWasm"); + } + + if !self.build.is_empty() { + vec.push("build"); + } + + if !self.custom_commands.is_empty() { + vec.push("customCommands"); + } + + vec + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExternalCommand { + pub command: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub dir: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub sources: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub targets: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Dependency { + #[serde(rename = "type")] + pub type_: String, + pub target: Option, +} diff --git a/wasm-rpc-stubgen/src/model/mod.rs b/wasm-rpc-stubgen/src/model/mod.rs index 72f864ac..3b59e2e5 100644 --- a/wasm-rpc-stubgen/src/model/mod.rs +++ b/wasm-rpc-stubgen/src/model/mod.rs @@ -1,2 +1,3 @@ -pub mod unknown_properties; -pub mod wasm_rpc; +pub mod app; +pub mod app_raw; +pub mod template; diff --git a/wasm-rpc-stubgen/src/model/template.rs b/wasm-rpc-stubgen/src/model/template.rs new file mode 100644 index 00000000..3d9d6550 --- /dev/null +++ b/wasm-rpc-stubgen/src/model/template.rs @@ -0,0 +1,105 @@ +use crate::model::app_raw; +use serde::Serialize; +use std::collections::HashMap; + +pub trait Template { + type Rendered; + + fn render( + &self, + env: &minijinja::Environment, + ctx: &C, + ) -> Result; +} + +impl Template for String { + type Rendered = String; + + fn render( + &self, + env: &minijinja::Environment, + ctx: &C, + ) -> Result { + env.render_str(self, ctx) + } +} + +impl> Template for Option { + type Rendered = Option; + + fn render( + &self, + env: &minijinja::Environment, + ctx: &C, + ) -> Result { + match self { + Some(template) => Ok(Some(template.render(env, ctx)?)), + None => Ok(None), + } + } +} + +impl> Template for Vec { + type Rendered = Vec; + + fn render( + &self, + env: &minijinja::Environment, + ctx: &C, + ) -> Result { + self.iter().map(|elem| elem.render(env, ctx)).collect() + } +} + +impl> Template for HashMap { + type Rendered = HashMap; + + fn render( + &self, + env: &minijinja::Environment, + ctx: &C, + ) -> Result { + let mut rendered = HashMap::::new(); + for (key, template) in self { + rendered.insert(key.clone(), template.render(env, ctx)?); + } + Ok(rendered) + } +} + +impl Template for app_raw::ExternalCommand { + type Rendered = app_raw::ExternalCommand; + + fn render( + &self, + env: &minijinja::Environment, + ctx: &C, + ) -> Result { + Ok(app_raw::ExternalCommand { + command: self.command.render(env, ctx)?, + dir: self.dir.render(env, ctx)?, + sources: self.sources.render(env, ctx)?, + targets: self.targets.render(env, ctx)?, + }) + } +} + +impl Template for app_raw::ComponentProperties { + type Rendered = app_raw::ComponentProperties; + + fn render( + &self, + env: &minijinja::Environment, + ctx: &C, + ) -> Result { + Ok(app_raw::ComponentProperties { + source_wit: self.source_wit.render(env, ctx)?, + generated_wit: self.generated_wit.render(env, ctx)?, + component_wasm: self.component_wasm.render(env, ctx)?, + linked_wasm: self.linked_wasm.render(env, ctx)?, + build: self.build.render(env, ctx)?, + custom_commands: self.custom_commands.render(env, ctx)?, + clean: self.clean.render(env, ctx)?, + }) + } +} diff --git a/wasm-rpc-stubgen/src/model/unknown_properties.rs b/wasm-rpc-stubgen/src/model/unknown_properties.rs deleted file mode 100644 index 02be77ba..00000000 --- a/wasm-rpc-stubgen/src/model/unknown_properties.rs +++ /dev/null @@ -1,40 +0,0 @@ -use crate::log::LogColorize; -use crate::validation::ValidationBuilder; -use std::collections::BTreeMap; - -pub type UnknownProperties = BTreeMap; - -pub trait HasUnknownProperties { - fn unknown_properties(&self) -> &UnknownProperties; - - fn add_unknown_property_warns(&self, context: F, validation_builder: &mut ValidationBuilder) - where - F: FnOnce() -> Vec<(&'static str, String)>, - { - let unknown_properties = self.unknown_properties(); - if unknown_properties.is_empty() { - return; - } - - let context = context(); - let context_count = context.len(); - - for context in context { - validation_builder.push_context(context.0, context.1); - } - - validation_builder.add_warns(self.unknown_properties().keys(), |unknown_property_name| { - Some(( - vec![], - format!( - "Unknown property: {}", - unknown_property_name.log_color_highlight() - ), - )) - }); - - for _ in 0..context_count { - validation_builder.pop_context(); - } - } -} diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index 3ffd2d87..5decebf7 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -1,6 +1,6 @@ use crate::fs::PathExtra; use crate::log::{log_action, LogColorize, LogIndent}; -use crate::model::wasm_rpc::{Application, ComponentName, ProfileName}; +use crate::model::app::{Application, ComponentName, ProfileName}; use crate::validation::{ValidatedResult, ValidationBuilder}; use crate::{fs, naming}; use anyhow::{anyhow, bail, Context, Error}; From 009622ab16d0be6259626d9d4be1c88c2b99bcd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Mon, 2 Dec 2024 00:05:17 +0100 Subject: [PATCH 095/109] cleanups / fixes --- .../src/commands/{declarative.rs => app.rs} | 30 ++++--- wasm-rpc-stubgen/src/commands/mod.rs | 2 +- wasm-rpc-stubgen/src/lib.rs | 38 ++++---- wasm-rpc-stubgen/src/main.rs | 2 +- wasm-rpc-stubgen/src/model/app.rs | 89 ++++++++++--------- wasm-rpc-stubgen/src/model/app_raw.rs | 29 +++--- wasm-rpc-stubgen/src/validation.rs | 10 +-- 7 files changed, 109 insertions(+), 91 deletions(-) rename wasm-rpc-stubgen/src/commands/{declarative.rs => app.rs} (98%) diff --git a/wasm-rpc-stubgen/src/commands/declarative.rs b/wasm-rpc-stubgen/src/commands/app.rs similarity index 98% rename from wasm-rpc-stubgen/src/commands/declarative.rs rename to wasm-rpc-stubgen/src/commands/app.rs index e787a0eb..cf375deb 100644 --- a/wasm-rpc-stubgen/src/commands/declarative.rs +++ b/wasm-rpc-stubgen/src/commands/app.rs @@ -20,7 +20,7 @@ use crate::wit_resolve::{ResolvedWitApplication, WitDepsResolver}; use crate::{commands, naming, WasmRpcOverride}; use anyhow::{anyhow, bail, Context, Error}; use colored::Colorize; -use glob::glob; +use glob::{glob_with, MatchOptions}; use golem_wasm_rpc::WASM_RPC_VERSION; use itertools::Itertools; use std::cell::OnceCell; @@ -671,21 +671,20 @@ fn collect_sources(mode: &ApplicationSourceMode) -> ValidatedResult ApplicationSourceMode::Automatic => match find_main_source() { Some(source) => { // TODO: save original current dir and use it as a component filter - std::env::set_current_dir(PathExtra::new(&source).parent().unwrap()) + let source_ext = PathExtra::new(&source); + let source_dir = source_ext.parent().unwrap(); + std::env::set_current_dir(source_dir) .expect("Failed to set current dir for config parent"); let includes = includes_from_yaml_file(source.as_path()); if includes.is_empty() { ValidatedResult::Ok(vec![source]) } else { - ValidatedResult::from_result(compile_and_collect_globs( - source.as_path(), - &includes, - )) - .map(|mut sources| { - sources.insert(0, source); - sources - }) + ValidatedResult::from_result(compile_and_collect_globs(source_dir, &includes)) + .map(|mut sources| { + sources.insert(0, source); + sources + }) } } None => ValidatedResult::from_error("No config file found!".to_string()), @@ -852,8 +851,15 @@ fn compile_and_collect_globs(root_dir: &Path, globs: &[String]) -> Result, _>>() .map_err(|err| anyhow!(err)) diff --git a/wasm-rpc-stubgen/src/commands/mod.rs b/wasm-rpc-stubgen/src/commands/mod.rs index b91cb5c3..573e0f24 100644 --- a/wasm-rpc-stubgen/src/commands/mod.rs +++ b/wasm-rpc-stubgen/src/commands/mod.rs @@ -22,4 +22,4 @@ pub mod dependencies; pub mod composition; /// Declarative subcommands -pub mod declarative; +pub mod app; diff --git a/wasm-rpc-stubgen/src/lib.rs b/wasm-rpc-stubgen/src/lib.rs index a83ddee6..76ddc820 100644 --- a/wasm-rpc-stubgen/src/lib.rs +++ b/wasm-rpc-stubgen/src/lib.rs @@ -191,15 +191,15 @@ pub struct InitializeWorkspaceArgs { #[derive(Subcommand, Debug)] pub enum App { /// Runs the pre-component-build steps (stub generation and adding wit dependencies) - PreComponentBuild(DeclarativeBuildArgs), + PreComponentBuild(AppBuildArgs), /// Runs component build steps - ComponentBuild(DeclarativeBuildArgs), + ComponentBuild(AppBuildArgs), /// Runs the post-component-build steps (composing stubs) - PostComponentBuild(DeclarativeBuildArgs), + PostComponentBuild(AppBuildArgs), /// Runs all build steps (pre-component, component, post-component) - Build(DeclarativeBuildArgs), + Build(AppBuildArgs), /// Clean outputs - Clean(DeclarativeBuildArgs), + Clean(AppBuildArgs), /// Run custom command CustomCommand(DeclarativeCustomCommand), } @@ -213,7 +213,7 @@ pub struct DeclarativeInitArgs { #[derive(clap::Args, Debug)] #[command(version, about, long_about = None)] -pub struct DeclarativeBuildArgs { +pub struct AppBuildArgs { /// List of application manifests, can be defined multiple times #[clap(long, short)] pub app: Vec, @@ -229,7 +229,7 @@ pub struct DeclarativeBuildArgs { #[command(version, about, long_about = None)] pub struct DeclarativeCustomCommand { #[clap(flatten)] - args: DeclarativeBuildArgs, + args: AppBuildArgs, #[arg(value_name = "custom command")] command: String, } @@ -297,32 +297,30 @@ pub fn initialize_workspace( ) } -pub async fn run_declarative_command(command: App) -> anyhow::Result<()> { +pub async fn run_app_command(command: App) -> anyhow::Result<()> { match command { App::PreComponentBuild(args) => { - commands::declarative::pre_component_build(dec_build_args_to_config(args)).await - } - App::ComponentBuild(args) => { - commands::declarative::component_build(dec_build_args_to_config(args)) + commands::app::pre_component_build(app_args_to_config(args)).await } + App::ComponentBuild(args) => commands::app::component_build(app_args_to_config(args)), App::PostComponentBuild(args) => { - commands::declarative::post_component_build(dec_build_args_to_config(args)).await + commands::app::post_component_build(app_args_to_config(args)).await } - App::Build(args) => commands::declarative::build(dec_build_args_to_config(args)).await, - App::Clean(args) => commands::declarative::clean(dec_build_args_to_config(args)), + App::Build(args) => commands::app::build(app_args_to_config(args)).await, + App::Clean(args) => commands::app::clean(app_args_to_config(args)), App::CustomCommand(args) => { - commands::declarative::custom_command(dec_build_args_to_config(args.args), args.command) + commands::app::custom_command(app_args_to_config(args.args), args.command) } } } -fn dec_build_args_to_config(args: DeclarativeBuildArgs) -> commands::declarative::Config { - commands::declarative::Config { +fn app_args_to_config(args: AppBuildArgs) -> commands::app::Config { + commands::app::Config { app_resolve_mode: { if args.app.is_empty() { - commands::declarative::ApplicationSourceMode::Automatic + commands::app::ApplicationSourceMode::Automatic } else { - commands::declarative::ApplicationSourceMode::Explicit(args.app) + commands::app::ApplicationSourceMode::Explicit(args.app) } }, skip_up_to_date_checks: args.force_build, diff --git a/wasm-rpc-stubgen/src/main.rs b/wasm-rpc-stubgen/src/main.rs index df388c1d..2fc19ad0 100644 --- a/wasm-rpc-stubgen/src/main.rs +++ b/wasm-rpc-stubgen/src/main.rs @@ -32,7 +32,7 @@ async fn main() -> ExitCode { initialize_workspace(init_workspace_args, "wasm-rpc-stubgen", &[]) } #[cfg(feature = "unstable-dec-dep")] - Command::App { subcommand } => run_declarative_command(subcommand).await, + Command::App { subcommand } => run_app_command(subcommand).await, }; match result { diff --git a/wasm-rpc-stubgen/src/model/app.rs b/wasm-rpc-stubgen/src/model/app.rs index 26d2d6aa..4e4b4c5b 100644 --- a/wasm-rpc-stubgen/src/model/app.rs +++ b/wasm-rpc-stubgen/src/model/app.rs @@ -104,7 +104,13 @@ pub fn includes_from_yaml_file(source: &Path) -> Vec { fs::read_to_string(source) .ok() .and_then(|source| app_raw::Application::from_yaml_str(source.as_str()).ok()) - .map(|app| app.include) + .map(|app| { + if app.includes.is_empty() { + vec!["**/golem.yaml".to_string()] + } else { + app.includes + } + }) .unwrap_or_default() } @@ -172,14 +178,14 @@ impl Application { } } - if !app.application.include.is_empty() { + if !app.application.includes.is_empty() { include_sources.push(app.source.to_path_buf()); if include.is_empty() { - include = app.application.include; + include = app.application.includes; } } - if app.application.wit_deps.is_empty() { + if !app.application.wit_deps.is_empty() { wit_deps_sources.push(app.source.to_path_buf()); if wit_deps.is_empty() { wit_deps = app.application.wit_deps; // TODO: resolve from source? @@ -253,47 +259,50 @@ impl Application { components.insert(component_name, (app.source.to_path_buf(), component)); } - for (component_name, dependency) in app.application.dependencies { + for (component_name, component_dependencies) in app.application.dependencies { let component_name = ComponentName::from(component_name); validation.push_context("component", component_name.to_string()); - if dependency.type_ == "wasm-rpc" { - match dependency.target { - Some(target) => { - let target_component_name = ComponentName::from(target); - - if !dependencies.contains_key(&component_name) { - dependencies.insert(component_name.clone(), BTreeSet::new()); - } - dependencies - .get_mut(&component_name) - .unwrap() - .insert(target_component_name.clone()); + for dependency in component_dependencies { + if dependency.type_ == "wasm-rpc" { + match dependency.target { + Some(target) => { + let target_component_name = ComponentName::from(target); - if !dependency_sources.contains_key(&component_name) { - dependency_sources.insert(component_name.clone(), HashMap::new()); - } - let dependency_sources = - dependency_sources.get_mut(&component_name).unwrap(); - if !dependency_sources.contains_key(&target_component_name) { + if !dependencies.contains_key(&component_name) { + dependencies.insert(component_name.clone(), BTreeSet::new()); + } + dependencies + .get_mut(&component_name) + .unwrap() + .insert(target_component_name.clone()); + + if !dependency_sources.contains_key(&component_name) { + dependency_sources + .insert(component_name.clone(), HashMap::new()); + } + let dependency_sources = + dependency_sources.get_mut(&component_name).unwrap(); + if !dependency_sources.contains_key(&target_component_name) { + dependency_sources + .insert(target_component_name.clone(), Vec::new()); + } dependency_sources - .insert(target_component_name.clone(), Vec::new()); + .get_mut(&target_component_name) + .unwrap() + .push(app.source.to_path_buf()); } - dependency_sources - .get_mut(&target_component_name) - .unwrap() - .push(app.source.to_path_buf()); + None => validation.add_error(format!( + "Missing {} field for component wasm-rpc dependency", + "target".log_color_error_highlight() + )), } - None => validation.add_error(format!( - "Missing {} field for component wasm-rpc dependency", - "target".log_color_error_highlight() - )), + } else { + validation.add_error(format!( + "Unknown component dependency type: {}", + dependency.type_.log_color_error_highlight() + )); } - } else { - validation.add_error(format!( - "Unknown component dependency type: {}", - dependency.type_.log_color_error_highlight() - )); } validation.pop_context(); @@ -307,7 +316,7 @@ impl Application { ("tempDir", temp_dir_sources), ("witDeps", wit_deps_sources), ] { - if !sources.is_empty() { + if sources.len() > 1 { validation.add_error(format!( "Property {} is defined in multiple sources: {}", property_name.log_color_highlight(), @@ -321,7 +330,7 @@ impl Application { let non_unique_templates = template_sources .into_iter() - .filter(|(_, template)| !template.is_empty()); + .filter(|(_, sources)| sources.len() > 1); validation.add_errors(non_unique_templates, |(template_name, sources)| { Some(( @@ -339,7 +348,7 @@ impl Application { let non_unique_components = component_sources .into_iter() - .filter(|(_, sources)| !sources.is_empty()); + .filter(|(_, sources)| sources.len() > 1); validation.add_errors(non_unique_components, |(template_name, sources)| { Some(( diff --git a/wasm-rpc-stubgen/src/model/app_raw.rs b/wasm-rpc-stubgen/src/model/app_raw.rs index 55e3ade0..1c2f38ef 100644 --- a/wasm-rpc-stubgen/src/model/app_raw.rs +++ b/wasm-rpc-stubgen/src/model/app_raw.rs @@ -1,4 +1,6 @@ use crate::fs; +use crate::log::LogColorize; +use anyhow::{anyhow, Context}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::path::PathBuf; @@ -11,10 +13,8 @@ pub struct ApplicationWithSource { impl ApplicationWithSource { pub fn from_yaml_file(file: PathBuf) -> anyhow::Result { - Ok(Self::from_yaml_string( - file.clone(), - fs::read_to_string(file)?, - )?) + Self::from_yaml_string(file.clone(), fs::read_to_string(file.clone())?) + .with_context(|| anyhow!("Failed to load source {}", file.log_color_highlight())) } pub fn from_yaml_string(source: PathBuf, string: String) -> serde_yaml::Result { @@ -30,10 +30,10 @@ impl ApplicationWithSource { } #[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct Application { #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub include: Vec, + pub includes: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] pub temp_dir: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] @@ -43,7 +43,7 @@ pub struct Application { #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub components: HashMap, #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub dependencies: HashMap, + pub dependencies: HashMap>, } impl Application { @@ -57,26 +57,31 @@ impl Application { } #[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct ComponentTemplate { #[serde(flatten)] pub component_properties: ComponentProperties, + #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub profiles: HashMap, + #[serde(default, skip_serializing_if = "Option::is_none")] pub default_profile: Option, } #[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct Component { + #[serde(default, skip_serializing_if = "Option::is_none")] pub template: Option, #[serde(flatten)] pub component_properties: ComponentProperties, + #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub profiles: HashMap, + #[serde(default, skip_serializing_if = "Option::is_none")] pub default_profile: Option, } #[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct ComponentProperties { #[serde(default, skip_serializing_if = "Option::is_none")] pub source_wit: Option, @@ -127,7 +132,7 @@ impl ComponentProperties { } #[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct ExternalCommand { pub command: String, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -139,7 +144,7 @@ pub struct ExternalCommand { } #[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct Dependency { #[serde(rename = "type")] pub type_: String, diff --git a/wasm-rpc-stubgen/src/validation.rs b/wasm-rpc-stubgen/src/validation.rs index 5a3ecb00..c158f63f 100644 --- a/wasm-rpc-stubgen/src/validation.rs +++ b/wasm-rpc-stubgen/src/validation.rs @@ -1,6 +1,6 @@ use crate::log::LogColorize; use itertools::Itertools; -use std::fmt::Display; +use std::fmt::Debug; pub struct ValidationContext { pub name: &'static str, @@ -22,7 +22,7 @@ impl ValidatedResult { } } - pub fn from_result(result: Result) -> Self { + pub fn from_result(result: Result) -> Self { ValidatedResult::Ok(result).flatten() } @@ -120,16 +120,16 @@ impl ValidatedResult> { pub fn flatten(self) -> ValidatedResult where - E: Display, + E: Debug, { match self { ValidatedResult::Ok(value) => match value { Ok(value) => ValidatedResult::Ok(value), - Err(err) => ValidatedResult::WarnsAndErrors(vec![], vec![format!("{}", err)]), + Err(err) => ValidatedResult::WarnsAndErrors(vec![], vec![format!("{:?}", err)]), }, ValidatedResult::OkWithWarns(value, warns) => match value { Ok(value) => ValidatedResult::OkWithWarns(value, warns), - Err(err) => ValidatedResult::WarnsAndErrors(warns, vec![format!("{}", err)]), + Err(err) => ValidatedResult::WarnsAndErrors(warns, vec![format!("{:?}", err)]), }, ValidatedResult::WarnsAndErrors(warns, errors) => { ValidatedResult::WarnsAndErrors(warns, errors) From 07c97f389b7fd6ae1e77957ac9c9ced26145c48b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Mon, 2 Dec 2024 16:48:45 +0100 Subject: [PATCH 096/109] fix regenerate_cargo_package_component --- wasm-rpc-stubgen/src/commands/app.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wasm-rpc-stubgen/src/commands/app.rs b/wasm-rpc-stubgen/src/commands/app.rs index cf375deb..876f7372 100644 --- a/wasm-rpc-stubgen/src/commands/app.rs +++ b/wasm-rpc-stubgen/src/commands/app.rs @@ -1028,7 +1028,7 @@ fn update_cargo_toml( regenerate_cargo_package_component( &cargo_toml, &ctx.application - .component_source_wit(component_name, ctx.profile()), + .component_generated_wit(component_name, ctx.profile()), None, )? } From 3ff05338fae57d86a1e52a558fb3e906bbb9e134 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Mon, 2 Dec 2024 22:06:19 +0100 Subject: [PATCH 097/109] cleanups / offline mode --- wasm-rpc-stubgen/Cargo.toml | 3 - wasm-rpc-stubgen/src/commands/app.rs | 36 ++++-- wasm-rpc-stubgen/src/commands/generate.rs | 6 +- wasm-rpc-stubgen/src/compilation.rs | 10 +- wasm-rpc-stubgen/src/lib.rs | 109 +++++++++++------- wasm-rpc-stubgen/src/main.rs | 1 - wasm-rpc-stubgen/src/model/app.rs | 10 ++ wasm-rpc-stubgen/src/wit_generate.rs | 4 +- .../tests-integration/tests/compose.rs | 2 +- .../tests-integration/tests/stub_wasm.rs | 2 +- 10 files changed, 120 insertions(+), 63 deletions(-) diff --git a/wasm-rpc-stubgen/Cargo.toml b/wasm-rpc-stubgen/Cargo.toml index d7980652..ebdf170d 100644 --- a/wasm-rpc-stubgen/Cargo.toml +++ b/wasm-rpc-stubgen/Cargo.toml @@ -7,9 +7,6 @@ homepage = "https://golem.cloud" repository = "https://github.com/golemcloud/wasm-rpc" description = "Golem WASM RPC stub generator" -[features] -unstable-dec-dep = [] - [lib] name = "golem_wasm_rpc_stubgen" path = "src/lib.rs" diff --git a/wasm-rpc-stubgen/src/commands/app.rs b/wasm-rpc-stubgen/src/commands/app.rs index 876f7372..83ccf129 100644 --- a/wasm-rpc-stubgen/src/commands/app.rs +++ b/wasm-rpc-stubgen/src/commands/app.rs @@ -25,7 +25,7 @@ use golem_wasm_rpc::WASM_RPC_VERSION; use itertools::Itertools; use std::cell::OnceCell; use std::cmp::Ordering; -use std::collections::{BTreeSet, HashMap, HashSet}; +use std::collections::{BTreeSet, HashMap}; use std::path::{Path, PathBuf}; use std::process::Command; use std::time::SystemTime; @@ -35,6 +35,7 @@ pub struct Config { pub app_resolve_mode: ApplicationSourceMode, pub skip_up_to_date_checks: bool, pub profile: Option, + pub offline: bool, } #[derive(Debug, Clone)] @@ -235,6 +236,10 @@ impl ApplicationContext { } } + if ctx.config.offline { + log_action("Selected", "offline mode"); + } + Ok(ctx) } @@ -315,6 +320,7 @@ async fn pre_component_build_ctx(ctx: &mut ApplicationContext) -> anyhow::Result for component_name in ctx.application.component_names() { let changed = create_generated_wit(ctx, component_name)?; if changed { + // TODO: if this fails, it won't be retried, add done file for this update_cargo_toml(ctx, component_name)?; } any_changed |= changed; @@ -473,15 +479,7 @@ pub fn clean(config: Config) -> anyhow::Result<()> { log_action("Cleaning", "components"); let _indent = LogIndent::new(); - let all_profiles = { - let mut profiles = app - .all_profiles() - .into_iter() - .map(Some) - .collect::>(); - profiles.insert(None); - profiles - }; + let all_profiles = app.all_option_profiles(); let paths = { let mut paths = BTreeSet::<(&'static str, PathBuf)>::new(); for component_name in app.component_names() { @@ -562,6 +560,22 @@ pub fn clean(config: Config) -> anyhow::Result<()> { Ok(()) } +pub fn available_custom_commands(config: Config) -> anyhow::Result> { + let app = to_anyhow( + "Failed to load application manifest(s), see problems above", + load_app_validated(&config), + )?; + + let all_profiles = app.all_option_profiles(); + + let mut commands = BTreeSet::::new(); + for profile in &all_profiles { + commands.extend(app.all_custom_commands(profile.as_ref())) + } + + Ok(commands) +} + pub fn custom_command(config: Config, command: String) -> anyhow::Result<()> { let ctx = ApplicationContext::new(config)?; @@ -1100,7 +1114,7 @@ async fn build_stub( ); fs::create_dir_all(&target_root)?; - commands::generate::build(&stub_def, &stub_wasm, &stub_wit).await?; + commands::generate::build(&stub_def, &stub_wasm, &stub_wit, ctx.config.offline).await?; gen_dir_done_marker.mark_as_done()?; diff --git a/wasm-rpc-stubgen/src/commands/generate.rs b/wasm-rpc-stubgen/src/commands/generate.rs index c13bc803..70fb843b 100644 --- a/wasm-rpc-stubgen/src/commands/generate.rs +++ b/wasm-rpc-stubgen/src/commands/generate.rs @@ -37,8 +37,9 @@ pub async fn build( stub_def: &StubDefinition, dest_wasm: &Path, dest_wit_root: &Path, + offline: bool, ) -> anyhow::Result<()> { - let wasm_path = generate_and_build_stub(stub_def).await?; + let wasm_path = generate_and_build_stub(stub_def, offline).await?; fs::copy(wasm_path, dest_wasm).context("Failed to copy the WASM file to the destination")?; fs::create_dir_all(dest_wit_root).context("Failed to create the target WIT root directory")?; @@ -53,7 +54,7 @@ pub async fn build( Ok(()) } -pub async fn generate_and_build_stub(stub_def: &StubDefinition) -> anyhow::Result { +pub async fn generate_and_build_stub(stub_def: &StubDefinition, offline: bool) -> anyhow::Result { let _ = generate_stub_wit_dir(stub_def)?; generate_cargo_toml(stub_def).context("Failed to generate the Cargo.toml file")?; generate_stub_source(stub_def).context("Failed to generate the stub Rust source")?; @@ -69,6 +70,7 @@ pub async fn generate_and_build_stub(stub_def: &StubDefinition) -> anyhow::Resul stub_def.config.target_root.log_color_error_highlight() ) })?, + offline, ) .await .context("Failed to compile the generated stub")?; diff --git a/wasm-rpc-stubgen/src/compilation.rs b/wasm-rpc-stubgen/src/compilation.rs index 20d296eb..a6cacd5e 100644 --- a/wasm-rpc-stubgen/src/compilation.rs +++ b/wasm-rpc-stubgen/src/compilation.rs @@ -17,13 +17,14 @@ use cargo_component::{load_component_metadata, load_metadata, run_cargo_command} use cargo_component_core::terminal::{Color, Terminal, Verbosity}; use std::path::Path; -pub async fn compile(root: &Path) -> anyhow::Result<()> { +pub async fn compile(root: &Path, offline: bool) -> anyhow::Result<()> { let current_dir = std::env::current_dir()?; std::env::set_current_dir(root)?; let cargo_args = CargoArguments { release: true, manifest_path: Some(root.join("Cargo.toml")), + offline, ..Default::default() }; @@ -32,6 +33,11 @@ pub async fn compile(root: &Path) -> anyhow::Result<()> { let metadata = load_metadata(cargo_args.manifest_path.as_deref())?; let packages = load_component_metadata(&metadata, cargo_args.packages.iter(), cargo_args.workspace)?; + + let mut spawn_args = vec!["build".to_string(), "--release".to_string()]; + if offline { + spawn_args.push("--offline".to_string()); + } run_cargo_command( &config, @@ -39,7 +45,7 @@ pub async fn compile(root: &Path) -> anyhow::Result<()> { &packages, Some("build"), &cargo_args, - &["build".to_string(), "--release".to_string()], + &spawn_args, ) .await?; diff --git a/wasm-rpc-stubgen/src/lib.rs b/wasm-rpc-stubgen/src/lib.rs index 76ddc820..bf7303e8 100644 --- a/wasm-rpc-stubgen/src/lib.rs +++ b/wasm-rpc-stubgen/src/lib.rs @@ -27,10 +27,14 @@ pub mod wit_encode; pub mod wit_generate; pub mod wit_resolve; +use crate::commands::app::ApplicationSourceMode; +use crate::log::LogColorize; use crate::stub::{StubConfig, StubDefinition}; use crate::wit_generate::UpdateCargoToml; use anyhow::Context; use clap::{Parser, Subcommand}; +use colored::Colorize; +use itertools::Itertools; use std::path::PathBuf; use tempfile::TempDir; @@ -51,8 +55,8 @@ pub enum Command { /// Initializes a Golem-specific cargo-make configuration in a Cargo workspace for automatically /// generating stubs and composing results. InitializeWorkspace(InitializeWorkspaceArgs), - /// Build components and stubs with application manifests - #[cfg(feature = "unstable-dec-dep")] + /// Build components with application manifests + #[clap(about = app_about(), long_about = None)] App { #[command(subcommand)] subcommand: App, @@ -190,25 +194,13 @@ pub struct InitializeWorkspaceArgs { #[derive(Subcommand, Debug)] pub enum App { - /// Runs the pre-component-build steps (stub generation and adding wit dependencies) - PreComponentBuild(AppBuildArgs), /// Runs component build steps - ComponentBuild(AppBuildArgs), - /// Runs the post-component-build steps (composing stubs) - PostComponentBuild(AppBuildArgs), - /// Runs all build steps (pre-component, component, post-component) Build(AppBuildArgs), /// Clean outputs - Clean(AppBuildArgs), + Clean(AppCleanArgs), /// Run custom command - CustomCommand(DeclarativeCustomCommand), -} - -#[derive(clap::Args, Debug)] -#[command(version, about, long_about = None)] -pub struct DeclarativeInitArgs { - #[clap(long, short, required = true)] - pub component_name: String, + #[clap(external_subcommand)] + CustomCommand(Vec), } #[derive(clap::Args, Debug)] @@ -223,11 +215,22 @@ pub struct AppBuildArgs { /// Selects a build profile #[clap(long, short)] pub profile: Option, + /// When set to true will use offline mode where applicable (e.g. stub cargo builds), defaults to false + #[clap(long, short, default_value = "false")] + pub offline: bool, } #[derive(clap::Args, Debug)] #[command(version, about, long_about = None)] -pub struct DeclarativeCustomCommand { +pub struct AppCleanArgs { + /// List of application manifests, can be defined multiple times + #[clap(long, short)] + pub app: Vec, +} + +#[derive(clap::Args, Debug)] +#[command(version, about, long_about = None)] +pub struct AppCustomCommand { #[clap(flatten)] args: AppBuildArgs, #[arg(value_name = "custom command")] @@ -264,7 +267,7 @@ pub async fn build(args: BuildArgs) -> anyhow::Result<()> { }) .context("Failed to gather information for the stub generator")?; - commands::generate::build(&stub_def, &args.dest_wasm, &args.dest_wit_root).await + commands::generate::build(&stub_def, &args.dest_wasm, &args.dest_wit_root, false).await } pub fn add_stub_dependency(args: AddStubDependencyArgs) -> anyhow::Result<()> { @@ -299,31 +302,59 @@ pub fn initialize_workspace( pub async fn run_app_command(command: App) -> anyhow::Result<()> { match command { - App::PreComponentBuild(args) => { - commands::app::pre_component_build(app_args_to_config(args)).await + App::Build(args) => { + commands::app::build(commands::app::Config { + app_resolve_mode: app_manifest_sources_to_resolve_mode(args.app), + skip_up_to_date_checks: args.force_build, + profile: args.profile.map(|profile| profile.into()), + offline: args.offline, + }) + .await } - App::ComponentBuild(args) => commands::app::component_build(app_args_to_config(args)), - App::PostComponentBuild(args) => { - commands::app::post_component_build(app_args_to_config(args)).await + App::Clean(args) => commands::app::clean(commands::app::Config { + app_resolve_mode: app_manifest_sources_to_resolve_mode(args.app), + skip_up_to_date_checks: false, + profile: None, + offline: false, + }), + App::CustomCommand(_args) => { + // TODO: parse app manifest / profile args + // commands::app::custom_command(app_args_to_config(args.args), args.command) + Ok(()) } - App::Build(args) => commands::app::build(app_args_to_config(args)).await, - App::Clean(args) => commands::app::clean(app_args_to_config(args)), - App::CustomCommand(args) => { - commands::app::custom_command(app_args_to_config(args.args), args.command) + } +} + +// TODO: currently this is added on top of the derived help, also, it should by grouped by profiles (also handling overlaps) +fn app_about() -> String { + let commands = commands::app::available_custom_commands(commands::app::Config { + app_resolve_mode: ApplicationSourceMode::Automatic, + skip_up_to_date_checks: false, + profile: None, + offline: false, + }); + + match commands { + Ok(commands) if !commands.is_empty() => { + format!( + "\n\n{}\n{}", + "Custom commands:".bold().underline(), + commands + .into_iter() + .map(|cmd| format!(" {}", cmd.log_color_highlight())) + .join("\n") + ) } + _ => "".to_string(), } } -fn app_args_to_config(args: AppBuildArgs) -> commands::app::Config { - commands::app::Config { - app_resolve_mode: { - if args.app.is_empty() { - commands::app::ApplicationSourceMode::Automatic - } else { - commands::app::ApplicationSourceMode::Explicit(args.app) - } - }, - skip_up_to_date_checks: args.force_build, - profile: args.profile.map(|profile| profile.into()), +fn app_manifest_sources_to_resolve_mode( + sources: Vec, +) -> commands::app::ApplicationSourceMode { + if sources.is_empty() { + commands::app::ApplicationSourceMode::Automatic + } else { + commands::app::ApplicationSourceMode::Explicit(sources) } } diff --git a/wasm-rpc-stubgen/src/main.rs b/wasm-rpc-stubgen/src/main.rs index 2fc19ad0..7cb66591 100644 --- a/wasm-rpc-stubgen/src/main.rs +++ b/wasm-rpc-stubgen/src/main.rs @@ -31,7 +31,6 @@ async fn main() -> ExitCode { Command::InitializeWorkspace(init_workspace_args) => { initialize_workspace(init_workspace_args, "wasm-rpc-stubgen", &[]) } - #[cfg(feature = "unstable-dec-dep")] Command::App { subcommand } => run_app_command(subcommand).await, }; diff --git a/wasm-rpc-stubgen/src/model/app.rs b/wasm-rpc-stubgen/src/model/app.rs index 4e4b4c5b..84ad27f3 100644 --- a/wasm-rpc-stubgen/src/model/app.rs +++ b/wasm-rpc-stubgen/src/model/app.rs @@ -658,6 +658,16 @@ impl Application { .collect() } + pub fn all_option_profiles(&self) -> BTreeSet> { + let mut profiles = self + .component_names() + .flat_map(|component_name| self.component_profiles(component_name)) + .map(Some) + .collect::>(); + profiles.insert(None); + profiles + } + pub fn all_custom_commands(&self, profile: Option<&ProfileName>) -> BTreeSet { self.component_names() .flat_map(|component_name| { diff --git a/wasm-rpc-stubgen/src/wit_generate.rs b/wasm-rpc-stubgen/src/wit_generate.rs index 01d5243d..dd39d057 100644 --- a/wasm-rpc-stubgen/src/wit_generate.rs +++ b/wasm-rpc-stubgen/src/wit_generate.rs @@ -426,9 +426,7 @@ pub fn add_stub_as_dependency_to_wit_dir(config: AddStubAsDepConfig) -> anyhow:: if let Some(target_parent) = config.dest_wit_root.parent() { let target_cargo_toml = target_parent.join("Cargo.toml"); if target_cargo_toml.exists() && target_cargo_toml.is_file() { - if config.update_cargo_toml == UpdateCargoToml::NoUpdate { - eprintln!("Warning: the newly copied dependencies have to be added to {}. Use the --update-cargo-toml flag to update it automatically.", target_cargo_toml.display()); - } else { + if config.update_cargo_toml != UpdateCargoToml::NoUpdate { cargo::is_cargo_component_toml(&target_cargo_toml).context(format!( "The file {target_cargo_toml:?} is not a valid cargo-component project" ))?; diff --git a/wasm-rpc-stubgen/tests-integration/tests/compose.rs b/wasm-rpc-stubgen/tests-integration/tests/compose.rs index 808e7b8f..c14c6449 100644 --- a/wasm-rpc-stubgen/tests-integration/tests/compose.rs +++ b/wasm-rpc-stubgen/tests-integration/tests/compose.rs @@ -85,7 +85,7 @@ async fn init_stub(name: &str) -> (TempDir, TempDir, PathBuf) { seal_cargo_workspace: true, }) .unwrap(); - let wasm_path = generate_and_build_stub(&def).await.unwrap(); + let wasm_path = generate_and_build_stub(&def, false).await.unwrap(); (source_dir, stub_dir, wasm_path) } diff --git a/wasm-rpc-stubgen/tests-integration/tests/stub_wasm.rs b/wasm-rpc-stubgen/tests-integration/tests/stub_wasm.rs index 739f1a4b..0fbc4430 100644 --- a/wasm-rpc-stubgen/tests-integration/tests/stub_wasm.rs +++ b/wasm-rpc-stubgen/tests-integration/tests/stub_wasm.rs @@ -59,7 +59,7 @@ async fn all_wit_types() { }) .unwrap(); - let wasm_path = generate_and_build_stub(&def).await.unwrap(); + let wasm_path = generate_and_build_stub(&def, false).await.unwrap(); let stub_bytes = std::fs::read(wasm_path).unwrap(); let stub_component = Component::::from_bytes(&stub_bytes).unwrap(); From 6d960a02575b10e6273338e3540c63c2a3548ee5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 3 Dec 2024 08:22:30 +0100 Subject: [PATCH 098/109] format / use official wasm-tools --- Cargo.lock | 27 ++++++++++++----------- wasm-rpc-stubgen/Cargo.toml | 4 ++-- wasm-rpc-stubgen/src/commands/generate.rs | 5 ++++- wasm-rpc-stubgen/src/compilation.rs | 2 +- wasm-rpc-stubgen/src/wit_generate.rs | 2 +- 5 files changed, 22 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9815011e..aa3dd4a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" @@ -1942,7 +1942,7 @@ dependencies = [ "walkdir", "wit-bindgen-rust 0.26.0", "wit-encoder", - "wit-parser 0.219.1", + "wit-parser 0.221.2", ] [[package]] @@ -5223,7 +5223,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2b1b95711b3ad655656a341e301cc64e33cbee94de9a99a1c5a2ab88efab79d" dependencies = [ "leb128", - "wasmparser 0.219.1 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmparser 0.219.1", ] [[package]] @@ -5403,12 +5403,11 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.219.1" -source = "git+https://github.com/golemcloud/wasm-tools.git#3458fe1e6f965420e69e1e09885d864dc49b343a" +version = "0.221.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9845c470a2e10b61dd42c385839cdd6496363ed63b5c9e420b5488b77bd22083" dependencies = [ - "ahash", "bitflags 2.6.0", - "hashbrown 0.14.5", "indexmap 2.6.0", "semver", ] @@ -6192,14 +6191,15 @@ dependencies = [ [[package]] name = "wit-encoder" -version = "0.219.1" -source = "git+https://github.com/golemcloud/wasm-tools.git#3458fe1e6f965420e69e1e09885d864dc49b343a" +version = "0.221.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c2921dd7e71ae11b6e28b33d42cc0eed4fa6ad3fe3ed1f9c5df80dacfec6a28" dependencies = [ "id-arena", "pretty_assertions", "semver", "serde 1.0.210", - "wit-parser 0.219.1", + "wit-parser 0.221.2", ] [[package]] @@ -6258,8 +6258,9 @@ dependencies = [ [[package]] name = "wit-parser" -version = "0.219.1" -source = "git+https://github.com/golemcloud/wasm-tools.git#3458fe1e6f965420e69e1e09885d864dc49b343a" +version = "0.221.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbe1538eea6ea5ddbe5defd0dc82539ad7ba751e1631e9185d24a931f0a5adc8" dependencies = [ "anyhow", "id-arena", @@ -6270,7 +6271,7 @@ dependencies = [ "serde_derive", "serde_json", "unicode-xid", - "wasmparser 0.219.1 (git+https://github.com/golemcloud/wasm-tools.git)", + "wasmparser 0.221.2", ] [[package]] diff --git a/wasm-rpc-stubgen/Cargo.toml b/wasm-rpc-stubgen/Cargo.toml index ebdf170d..6b94b12b 100644 --- a/wasm-rpc-stubgen/Cargo.toml +++ b/wasm-rpc-stubgen/Cargo.toml @@ -60,8 +60,8 @@ toml = "0.8.14" wac-graph = "=0.6.0" walkdir = "2.5.0" wit-bindgen-rust = "=0.26.0" -wit-encoder = { git = "https://github.com/golemcloud/wasm-tools.git" } -wit-parser = { git = "https://github.com/golemcloud/wasm-tools.git" } +wit-encoder = "=0.221.2" +wit-parser = "=0.221.2" [dev-dependencies] diff --git a/wasm-rpc-stubgen/src/commands/generate.rs b/wasm-rpc-stubgen/src/commands/generate.rs index 70fb843b..ed16b899 100644 --- a/wasm-rpc-stubgen/src/commands/generate.rs +++ b/wasm-rpc-stubgen/src/commands/generate.rs @@ -54,7 +54,10 @@ pub async fn build( Ok(()) } -pub async fn generate_and_build_stub(stub_def: &StubDefinition, offline: bool) -> anyhow::Result { +pub async fn generate_and_build_stub( + stub_def: &StubDefinition, + offline: bool, +) -> anyhow::Result { let _ = generate_stub_wit_dir(stub_def)?; generate_cargo_toml(stub_def).context("Failed to generate the Cargo.toml file")?; generate_stub_source(stub_def).context("Failed to generate the stub Rust source")?; diff --git a/wasm-rpc-stubgen/src/compilation.rs b/wasm-rpc-stubgen/src/compilation.rs index a6cacd5e..3b0ca445 100644 --- a/wasm-rpc-stubgen/src/compilation.rs +++ b/wasm-rpc-stubgen/src/compilation.rs @@ -33,7 +33,7 @@ pub async fn compile(root: &Path, offline: bool) -> anyhow::Result<()> { let metadata = load_metadata(cargo_args.manifest_path.as_deref())?; let packages = load_component_metadata(&metadata, cargo_args.packages.iter(), cargo_args.workspace)?; - + let mut spawn_args = vec!["build".to_string(), "--release".to_string()]; if offline { spawn_args.push("--offline".to_string()); diff --git a/wasm-rpc-stubgen/src/wit_generate.rs b/wasm-rpc-stubgen/src/wit_generate.rs index dd39d057..05202bd5 100644 --- a/wasm-rpc-stubgen/src/wit_generate.rs +++ b/wasm-rpc-stubgen/src/wit_generate.rs @@ -426,7 +426,7 @@ pub fn add_stub_as_dependency_to_wit_dir(config: AddStubAsDepConfig) -> anyhow:: if let Some(target_parent) = config.dest_wit_root.parent() { let target_cargo_toml = target_parent.join("Cargo.toml"); if target_cargo_toml.exists() && target_cargo_toml.is_file() { - if config.update_cargo_toml != UpdateCargoToml::NoUpdate { + if config.update_cargo_toml != UpdateCargoToml::NoUpdate { cargo::is_cargo_component_toml(&target_cargo_toml).context(format!( "The file {target_cargo_toml:?} is not a valid cargo-component project" ))?; From ee5d22be4554e4809e210f1327b17bf49457fe76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 3 Dec 2024 08:48:30 +0100 Subject: [PATCH 099/109] bump wac-graph --- wasm-rpc-stubgen/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wasm-rpc-stubgen/Cargo.toml b/wasm-rpc-stubgen/Cargo.toml index 6b94b12b..04e9d26c 100644 --- a/wasm-rpc-stubgen/Cargo.toml +++ b/wasm-rpc-stubgen/Cargo.toml @@ -57,7 +57,7 @@ syn = "2.0.66" tempfile = { workspace = true } tokio = { workspace = true } toml = "0.8.14" -wac-graph = "=0.6.0" +wac-graph = "=0.6.1" walkdir = "2.5.0" wit-bindgen-rust = "=0.26.0" wit-encoder = "=0.221.2" From 986771e5d57d9c3b41861b40b34ffcf5d082a49e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 3 Dec 2024 09:09:33 +0100 Subject: [PATCH 100/109] bump wac-graph, add missing lock --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index aa3dd4a9..c1a0a08c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4873,9 +4873,9 @@ checksum = "9dcc60c0624df774c82a0ef104151231d37da4962957d691c011c852b2473314" [[package]] name = "wac-graph" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f708c892ce0ebc06de9915f3da2da9b4e482a8b7d417fa447263b110d0a244" +checksum = "d94268a683b67ae20210565b5f91e106fe05034c36b931e739fe90377ed80b98" dependencies = [ "anyhow", "id-arena", @@ -4892,9 +4892,9 @@ dependencies = [ [[package]] name = "wac-types" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96fe715180f72ab776d90e8c4f47f8e4297e0e61ab263567a31f73c77d45d8d" +checksum = "f5028a15e266f4c8fed48beb95aebb76af5232dcd554fd849a305a4e5cce1563" dependencies = [ "anyhow", "id-arena", From 8a0a6676b27d7c3b56d332e17460e0c43f8a598f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 3 Dec 2024 10:49:11 +0100 Subject: [PATCH 101/109] property extensions --- wasm-rpc-stubgen/src/commands/app.rs | 87 +++++++++------ wasm-rpc-stubgen/src/lib.rs | 12 +- wasm-rpc-stubgen/src/main.rs | 5 +- wasm-rpc-stubgen/src/model/app.rs | 147 +++++++++++++++++-------- wasm-rpc-stubgen/src/model/app_raw.rs | 4 + wasm-rpc-stubgen/src/model/template.rs | 51 ++++++++- wasm-rpc-stubgen/src/wit_resolve.rs | 17 ++- 7 files changed, 236 insertions(+), 87 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/app.rs b/wasm-rpc-stubgen/src/commands/app.rs index 83ccf129..4202c33d 100644 --- a/wasm-rpc-stubgen/src/commands/app.rs +++ b/wasm-rpc-stubgen/src/commands/app.rs @@ -6,7 +6,8 @@ use crate::log::{ LogIndent, }; use crate::model::app::{ - includes_from_yaml_file, Application, ComponentName, ProfileName, DEFAULT_CONFIG_FILE_NAME, + includes_from_yaml_file, Application, ComponentName, ComponentPropertiesExtensions, + ProfileName, DEFAULT_CONFIG_FILE_NAME, }; use crate::model::app_raw; use crate::model::app_raw::ExternalCommand; @@ -26,16 +27,18 @@ use itertools::Itertools; use std::cell::OnceCell; use std::cmp::Ordering; use std::collections::{BTreeSet, HashMap}; +use std::marker::PhantomData; use std::path::{Path, PathBuf}; use std::process::Command; use std::time::SystemTime; use walkdir::WalkDir; -pub struct Config { +pub struct Config { pub app_resolve_mode: ApplicationSourceMode, pub skip_up_to_date_checks: bool, pub profile: Option, pub offline: bool, + pub extensions: PhantomData, } #[derive(Debug, Clone)] @@ -44,16 +47,16 @@ pub enum ApplicationSourceMode { Explicit(Vec), } -struct ApplicationContext { - config: Config, - application: Application, +struct ApplicationContext { + config: Config, + application: Application, wit: ResolvedWitApplication, common_wit_deps: OnceCell>, component_generated_base_wit_deps: HashMap, } -impl ApplicationContext { - fn new(config: Config) -> anyhow::Result { +impl ApplicationContext { + fn new(config: Config) -> anyhow::Result> { let ctx = to_anyhow( "Failed to create application context, see problems above", load_app_validated(&config).and_then(|application| { @@ -297,18 +300,22 @@ impl ApplicationContext { } } -pub async fn pre_component_build(config: Config) -> anyhow::Result<()> { +pub async fn pre_component_build( + config: Config, +) -> anyhow::Result<()> { let mut ctx = ApplicationContext::new(config)?; pre_component_build_ctx(&mut ctx).await } -async fn pre_component_build_ctx(ctx: &mut ApplicationContext) -> anyhow::Result<()> { +async fn pre_component_build_ctx( + ctx: &mut ApplicationContext, +) -> anyhow::Result<()> { log_action("Executing", "pre-component-build steps"); let _indent = LogIndent::new(); { for component_name in ctx.wit.component_order_cloned() { - create_generatde_base_wit(ctx, &component_name)?; + create_generated_base_wit(ctx, &component_name)?; } for component_name in &ctx.application.all_wasm_rpc_dependencies() { build_stub(ctx, component_name).await?; @@ -333,12 +340,16 @@ async fn pre_component_build_ctx(ctx: &mut ApplicationContext) -> anyhow::Result Ok(()) } -pub fn component_build(config: Config) -> anyhow::Result<()> { +pub fn component_build( + config: Config, +) -> anyhow::Result<()> { let ctx = ApplicationContext::new(config)?; component_build_ctx(&ctx) } -fn component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<()> { +fn component_build_ctx( + ctx: &ApplicationContext, +) -> anyhow::Result<()> { log_action("Executing", "component-build steps"); let _indent = LogIndent::new(); @@ -375,12 +386,16 @@ fn component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<()> { Ok(()) } -pub async fn post_component_build(config: Config) -> anyhow::Result<()> { +pub async fn post_component_build( + config: Config, +) -> anyhow::Result<()> { let ctx = ApplicationContext::new(config)?; post_component_build_ctx(&ctx).await } -async fn post_component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<()> { +async fn post_component_build_ctx( + ctx: &ApplicationContext, +) -> anyhow::Result<()> { log_action("Executing", "post-component-build steps"); let _indent = LogIndent::new(); @@ -459,8 +474,8 @@ async fn post_component_build_ctx(ctx: &ApplicationContext) -> anyhow::Result<() Ok(()) } -pub async fn build(config: Config) -> anyhow::Result<()> { - let mut ctx = ApplicationContext::new(config)?; +pub async fn build(config: Config) -> anyhow::Result<()> { + let mut ctx = ApplicationContext::::new(config)?; pre_component_build_ctx(&mut ctx).await?; component_build_ctx(&ctx)?; @@ -469,7 +484,7 @@ pub async fn build(config: Config) -> anyhow::Result<()> { Ok(()) } -pub fn clean(config: Config) -> anyhow::Result<()> { +pub fn clean(config: Config) -> anyhow::Result<()> { let app = to_anyhow( "Failed to load application manifest(s), see problems above", load_app_validated(&config), @@ -560,7 +575,9 @@ pub fn clean(config: Config) -> anyhow::Result<()> { Ok(()) } -pub fn available_custom_commands(config: Config) -> anyhow::Result> { +pub fn available_custom_commands( + config: Config, +) -> anyhow::Result> { let app = to_anyhow( "Failed to load application manifest(s), see problems above", load_app_validated(&config), @@ -576,7 +593,10 @@ pub fn available_custom_commands(config: Config) -> anyhow::Result anyhow::Result<()> { +pub fn custom_command( + config: Config, + command: String, +) -> anyhow::Result<()> { let ctx = ApplicationContext::new(config)?; let all_custom_commands = ctx.application.all_custom_commands(ctx.profile()); @@ -645,7 +665,9 @@ fn delete_path(context: &str, path: &Path) -> anyhow::Result<()> { Ok(()) } -fn load_app_validated(config: &Config) -> ValidatedResult { +fn load_app_validated( + config: &Config, +) -> ValidatedResult> { let sources = collect_sources(&config.app_resolve_mode); let oam_apps = sources.and_then(|sources| { sources @@ -886,8 +908,8 @@ fn compile_and_collect_globs(root_dir: &Path, globs: &[String]) -> Result( + ctx: &mut ApplicationContext, component_name: &ComponentName, ) -> Result { let component_source_wit = ctx @@ -980,8 +1002,8 @@ fn create_generatde_base_wit( } } -fn create_generated_wit( - ctx: &ApplicationContext, +fn create_generated_wit( + ctx: &ApplicationContext, component_name: &ComponentName, ) -> Result { let component_generated_base_wit = ctx.application.component_generated_base_wit(component_name); @@ -1022,8 +1044,8 @@ fn create_generated_wit( } } -fn update_cargo_toml( - ctx: &ApplicationContext, +fn update_cargo_toml( + ctx: &ApplicationContext, component_name: &ComponentName, ) -> anyhow::Result<()> { let component_source_wit = PathExtra::new( @@ -1050,8 +1072,8 @@ fn update_cargo_toml( Ok(()) } -async fn build_stub( - ctx: &ApplicationContext, +async fn build_stub( + ctx: &ApplicationContext, component_name: &ComponentName, ) -> anyhow::Result { let target_root = ctx.application.stub_temp_build_dir(component_name); @@ -1124,7 +1146,10 @@ async fn build_stub( } } -fn add_stub_deps(ctx: &ApplicationContext, component_name: &ComponentName) -> Result { +fn add_stub_deps( + ctx: &ApplicationContext, + component_name: &ComponentName, +) -> Result { let dependencies = ctx .application .component_wasm_rpc_dependencies(component_name); @@ -1206,8 +1231,8 @@ fn copy_wit_sources(source: &Path, target: &Path) -> anyhow::Result<()> { Ok(()) } -fn execute_external_command( - ctx: &ApplicationContext, +fn execute_external_command( + ctx: &ApplicationContext, component_name: &ComponentName, command: &ExternalCommand, ) -> anyhow::Result<()> { diff --git a/wasm-rpc-stubgen/src/lib.rs b/wasm-rpc-stubgen/src/lib.rs index bf7303e8..62dc5d35 100644 --- a/wasm-rpc-stubgen/src/lib.rs +++ b/wasm-rpc-stubgen/src/lib.rs @@ -27,14 +27,15 @@ pub mod wit_encode; pub mod wit_generate; pub mod wit_resolve; -use crate::commands::app::ApplicationSourceMode; use crate::log::LogColorize; +use crate::model::app::{ComponentPropertiesExtensions, ComponentPropertiesExtensionsAny}; use crate::stub::{StubConfig, StubDefinition}; use crate::wit_generate::UpdateCargoToml; use anyhow::Context; use clap::{Parser, Subcommand}; use colored::Colorize; use itertools::Itertools; +use std::marker::PhantomData; use std::path::PathBuf; use tempfile::TempDir; @@ -300,7 +301,9 @@ pub fn initialize_workspace( ) } -pub async fn run_app_command(command: App) -> anyhow::Result<()> { +pub async fn run_app_command( + command: App, +) -> anyhow::Result<()> { match command { App::Build(args) => { commands::app::build(commands::app::Config { @@ -308,6 +311,7 @@ pub async fn run_app_command(command: App) -> anyhow::Result<()> { skip_up_to_date_checks: args.force_build, profile: args.profile.map(|profile| profile.into()), offline: args.offline, + extensions: PhantomData::, }) .await } @@ -316,6 +320,7 @@ pub async fn run_app_command(command: App) -> anyhow::Result<()> { skip_up_to_date_checks: false, profile: None, offline: false, + extensions: PhantomData::, }), App::CustomCommand(_args) => { // TODO: parse app manifest / profile args @@ -328,10 +333,11 @@ pub async fn run_app_command(command: App) -> anyhow::Result<()> { // TODO: currently this is added on top of the derived help, also, it should by grouped by profiles (also handling overlaps) fn app_about() -> String { let commands = commands::app::available_custom_commands(commands::app::Config { - app_resolve_mode: ApplicationSourceMode::Automatic, + app_resolve_mode: commands::app::ApplicationSourceMode::Automatic, skip_up_to_date_checks: false, profile: None, offline: false, + extensions: PhantomData::, }); match commands { diff --git a/wasm-rpc-stubgen/src/main.rs b/wasm-rpc-stubgen/src/main.rs index 7cb66591..7784d031 100644 --- a/wasm-rpc-stubgen/src/main.rs +++ b/wasm-rpc-stubgen/src/main.rs @@ -14,6 +14,7 @@ use clap::Parser; use colored::Colorize; +use golem_wasm_rpc_stubgen::model::app::ComponentPropertiesExtensionsNone; use golem_wasm_rpc_stubgen::*; use std::process::ExitCode; @@ -31,7 +32,9 @@ async fn main() -> ExitCode { Command::InitializeWorkspace(init_workspace_args) => { initialize_workspace(init_workspace_args, "wasm-rpc-stubgen", &[]) } - Command::App { subcommand } => run_app_command(subcommand).await, + Command::App { subcommand } => { + run_app_command::(subcommand).await + } }; match result { diff --git a/wasm-rpc-stubgen/src/model/app.rs b/wasm-rpc-stubgen/src/model/app.rs index 84ad27f3..203eeab9 100644 --- a/wasm-rpc-stubgen/src/model/app.rs +++ b/wasm-rpc-stubgen/src/model/app.rs @@ -9,7 +9,7 @@ use heck::{ ToTitleCase, ToTrainCase, ToUpperCamelCase, }; use itertools::Itertools; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::fmt::Display; use std::fmt::Formatter; @@ -115,17 +115,17 @@ pub fn includes_from_yaml_file(source: &Path) -> Vec { } #[derive(Clone, Debug)] -pub enum ResolvedComponentProperties { +pub enum ResolvedComponentProperties { Properties { template_name: Option, any_template_overrides: bool, - properties: ComponentProperties, + properties: ComponentProperties, }, Profiles { template_name: Option, any_template_overrides: HashMap, default_profile: ProfileName, - profiles: HashMap, + profiles: HashMap>, }, } @@ -137,15 +137,15 @@ pub struct ComponentEffectivePropertySource<'a> { } #[derive(Clone, Debug)] -pub struct Application { +pub struct Application { temp_dir: Option, wit_deps: Vec, - components: BTreeMap, + components: BTreeMap>, dependencies: BTreeMap>, no_dependencies: BTreeSet, } -impl Application { +impl Application { pub fn from_raw_apps(apps: Vec) -> ValidatedResult { let mut validation = ValidationBuilder::new(); @@ -387,7 +387,7 @@ impl Application { let components = { let template_env = Self::template_env(); - let mut resolved_components = BTreeMap::::new(); + let mut resolved_components = BTreeMap::>::new(); for (component_name, (source, mut component)) in components { validation.push_context("source", source.to_string_lossy().to_string()); validation.push_context("component", component_name.to_string()); @@ -450,7 +450,7 @@ impl Application { if template.profiles.is_empty() { let rendered_template_properties = - ComponentProperties::from_template( + ComponentProperties::from_raw_template( &template_env, &template_context, &template.component_properties, @@ -481,14 +481,14 @@ impl Application { let mut any_template_overrides = HashMap::::new(); let mut profiles = - HashMap::::new(); + HashMap::>::new(); let mut any_template_render_error = false; for (profile_name, template_component_properties) in &template.profiles { let rendered_template_properties = - ComponentProperties::from_template( + ComponentProperties::from_raw_template( &template_env, &template_context, template_component_properties, @@ -553,13 +553,23 @@ impl Application { )); None } else { - Some(ResolvedComponentProperties::Properties { - template_name: None, - any_template_overrides: false, - properties: ComponentProperties::from( - component.component_properties, - ), - }) + let properties = ComponentProperties::::from_raw( + component.component_properties, + ); + + match properties { + Ok(properties) => { + Some(ResolvedComponentProperties::Properties { + template_name: None, + any_template_overrides: false, + properties, + }) + } + Err(err) => { + validation.add_error(format!("{:?}", err)); + None + } + } } } else if component.default_profile.is_none() { validation.add_error(format!( @@ -582,11 +592,17 @@ impl Application { profiles: component .profiles .into_iter() - .map(|(profile_name, properties)| { - ( - ProfileName::from(profile_name), - ComponentProperties::from(properties), - ) + .filter_map(|(profile_name, properties)| { + match ComponentProperties::::from_raw(properties) { + Ok(properties) => Some(( + ProfileName::from(profile_name), + properties, + )), + Err(err) => { + validation.add_error(format!("{:?}", err)); + None + } + } }) .collect(), }) @@ -595,7 +611,7 @@ impl Application { }; if let Some(properties) = component_properties { - // TODO: validate (build steps too) + // TODO: validate (build steps and extensions too) resolved_components .insert(component_name.clone(), Component { source, properties }); @@ -686,7 +702,7 @@ impl Application { } } - fn component(&self, component_name: &ComponentName) -> &Component { + fn component(&self, component_name: &ComponentName) -> &Component { self.components .get(component_name) .unwrap_or_else(|| panic!("Component not found: {}", component_name)) @@ -766,7 +782,7 @@ impl Application { &self, component_name: &ComponentName, profile: Option<&ProfileName>, - ) -> &ComponentProperties { + ) -> &ComponentProperties { match &self.component(component_name).properties { ResolvedComponentProperties::Properties { properties, .. } => properties, ResolvedComponentProperties::Profiles { @@ -888,12 +904,12 @@ impl Application { } #[derive(Clone, Debug)] -pub struct Component { +pub struct Component { pub source: PathBuf, - pub properties: ResolvedComponentProperties, + pub properties: ResolvedComponentProperties, } -impl Component { +impl Component { pub fn source_dir(&self) -> &Path { self.source.parent().unwrap_or_else(|| { panic!( @@ -905,7 +921,7 @@ impl Component { } #[derive(Clone, Debug)] -pub struct ComponentProperties { +pub struct ComponentProperties { pub source_wit: String, pub generated_wit: String, pub component_wasm: String, @@ -913,17 +929,29 @@ pub struct ComponentProperties { pub build: Vec, pub custom_commands: HashMap>, pub clean: Vec, + pub extensions: CPE, } -impl ComponentProperties { - fn from_template( +impl ComponentProperties { + fn from_raw(raw: app_raw::ComponentProperties) -> anyhow::Result { + Ok(Self { + source_wit: raw.source_wit.unwrap_or_default(), + generated_wit: raw.generated_wit.unwrap_or_default(), + component_wasm: raw.component_wasm.unwrap_or_default(), + linked_wasm: raw.linked_wasm, + build: raw.build, + custom_commands: raw.custom_commands, + clean: raw.clean, + extensions: CPE::from_serde_json(serde_json::Value::Object(raw.extensions))?, + }) + } + + fn from_raw_template( env: &minijinja::Environment, ctx: &C, template_properties: &app_raw::ComponentProperties, ) -> anyhow::Result { - Ok(ComponentProperties::from( - template_properties.render(env, ctx)?, - )) + ComponentProperties::from_raw(template_properties.render(env, ctx)?) } fn merge_with_overrides(mut self, overrides: app_raw::ComponentProperties) -> (Self, bool) { @@ -966,16 +994,43 @@ impl ComponentProperties { } } -impl From for ComponentProperties { - fn from(value: app_raw::ComponentProperties) -> Self { - Self { - source_wit: value.source_wit.unwrap_or_default(), - generated_wit: value.generated_wit.unwrap_or_default(), - component_wasm: value.component_wasm.unwrap_or_default(), - linked_wasm: value.linked_wasm, - build: value.build, - custom_commands: value.custom_commands, - clean: value.clean, - } +pub trait ComponentPropertiesExtensions { + fn from_serde_json(extensions: serde_json::Value) -> serde_json::Result + where + Self: Sized; + + fn validate(&self, validation: &mut ValidationBuilder); +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ComponentPropertiesExtensionsNone {} + +impl ComponentPropertiesExtensions for ComponentPropertiesExtensionsNone { + fn from_serde_json(extensions: serde_json::Value) -> serde_json::Result + where + Self: Sized, + { + serde_json::from_value(extensions) + } + + fn validate(&self, _validation: &mut ValidationBuilder) { + // NOP + } +} + +#[derive(Debug, Clone)] +pub struct ComponentPropertiesExtensionsAny; + +impl ComponentPropertiesExtensions for ComponentPropertiesExtensionsAny { + fn from_serde_json(_extensions: serde_json::Value) -> serde_json::Result + where + Self: Sized, + { + Ok(ComponentPropertiesExtensionsAny) + } + + fn validate(&self, _validation: &mut ValidationBuilder) { + // NOP } } diff --git a/wasm-rpc-stubgen/src/model/app_raw.rs b/wasm-rpc-stubgen/src/model/app_raw.rs index 1c2f38ef..bdffbf72 100644 --- a/wasm-rpc-stubgen/src/model/app_raw.rs +++ b/wasm-rpc-stubgen/src/model/app_raw.rs @@ -97,6 +97,8 @@ pub struct ComponentProperties { pub custom_commands: HashMap>, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub clean: Vec, + #[serde(flatten)] + pub extensions: serde_json::Map, } impl ComponentProperties { @@ -127,6 +129,8 @@ impl ComponentProperties { vec.push("customCommands"); } + self.extensions.keys().for_each(|name| vec.push(name)); + vec } } diff --git a/wasm-rpc-stubgen/src/model/template.rs b/wasm-rpc-stubgen/src/model/template.rs index 3d9d6550..b1ee6719 100644 --- a/wasm-rpc-stubgen/src/model/template.rs +++ b/wasm-rpc-stubgen/src/model/template.rs @@ -59,7 +59,7 @@ impl> Template for HashMap { env: &minijinja::Environment, ctx: &C, ) -> Result { - let mut rendered = HashMap::::new(); + let mut rendered = HashMap::::with_capacity(self.len()); for (key, template) in self { rendered.insert(key.clone(), template.render(env, ctx)?); } @@ -84,6 +84,54 @@ impl Template for app_raw::ExternalCommand { } } +impl Template for serde_json::Value { + type Rendered = serde_json::Value; + + fn render( + &self, + env: &minijinja::Environment, + ctx: &C, + ) -> Result { + Ok(match self { + value @ serde_json::Value::Null => value.clone(), + value @ serde_json::Value::Bool(_) => value.clone(), + value @ serde_json::Value::Number(_) => value.clone(), + value @ serde_json::Value::String(_) => value.clone(), + serde_json::Value::Array(elems) => { + let mut rendered_elems = Vec::::with_capacity(elems.len()); + for template in elems { + rendered_elems.push(template.render(env, ctx)?); + } + serde_json::Value::Array(rendered_elems) + } + serde_json::Value::Object(props) => { + let mut rendered_props = + serde_json::Map::::with_capacity(props.len()); + for (name, template) in props { + rendered_props.insert(name.clone(), template.render(env, ctx)?); + } + serde_json::Value::Object(rendered_props) + } + }) + } +} + +impl Template for serde_json::Map { + type Rendered = serde_json::Map; + + fn render( + &self, + env: &minijinja::Environment, + ctx: &C, + ) -> Result { + let mut rendered = serde_json::Map::::with_capacity(self.len()); + for (key, template) in self { + rendered.insert(key.clone(), template.render(env, ctx)?); + } + Ok(rendered) + } +} + impl Template for app_raw::ComponentProperties { type Rendered = app_raw::ComponentProperties; @@ -100,6 +148,7 @@ impl Template for app_raw::ComponentProperties { build: self.build.render(env, ctx)?, custom_commands: self.custom_commands.render(env, ctx)?, clean: self.clean.render(env, ctx)?, + extensions: self.extensions.render(env, ctx)?, }) } } diff --git a/wasm-rpc-stubgen/src/wit_resolve.rs b/wasm-rpc-stubgen/src/wit_resolve.rs index 5decebf7..56a1fdfb 100644 --- a/wasm-rpc-stubgen/src/wit_resolve.rs +++ b/wasm-rpc-stubgen/src/wit_resolve.rs @@ -1,6 +1,6 @@ use crate::fs::PathExtra; use crate::log::{log_action, LogColorize, LogIndent}; -use crate::model::app::{Application, ComponentName, ProfileName}; +use crate::model::app::{Application, ComponentName, ComponentPropertiesExtensions, ProfileName}; use crate::validation::{ValidatedResult, ValidationBuilder}; use crate::{fs, naming}; use anyhow::{anyhow, bail, Context, Error}; @@ -182,7 +182,10 @@ pub struct ResolvedWitApplication { } impl ResolvedWitApplication { - pub fn new(app: &Application, profile: Option<&ProfileName>) -> ValidatedResult { + pub fn new( + app: &Application, + profile: Option<&ProfileName>, + ) -> ValidatedResult { log_action("Resolving", "application wit directories"); let _indent = LogIndent::new(); @@ -256,10 +259,10 @@ impl ResolvedWitApplication { self.components.insert(component_name, resolved_component); } - fn add_components_from_app( + fn add_components_from_app( &mut self, validation: &mut ValidationBuilder, - app: &Application, + app: &Application, profile: Option<&ProfileName>, ) { for component_name in app.component_names() { @@ -350,7 +353,11 @@ impl ResolvedWitApplication { } } - fn collect_component_deps(&mut self, app: &Application, validation: &mut ValidationBuilder) { + fn collect_component_deps( + &mut self, + app: &Application, + validation: &mut ValidationBuilder, + ) { fn component_deps< 'a, I: IntoIterator, From d5cf6f78b67a163f5e66d8b6ee340097d905e0f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 3 Dec 2024 10:52:54 +0100 Subject: [PATCH 102/109] pub ApplicationContext --- wasm-rpc-stubgen/src/commands/app.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wasm-rpc-stubgen/src/commands/app.rs b/wasm-rpc-stubgen/src/commands/app.rs index 4202c33d..af8e6c37 100644 --- a/wasm-rpc-stubgen/src/commands/app.rs +++ b/wasm-rpc-stubgen/src/commands/app.rs @@ -47,7 +47,7 @@ pub enum ApplicationSourceMode { Explicit(Vec), } -struct ApplicationContext { +pub struct ApplicationContext { config: Config, application: Application, wit: ResolvedWitApplication, From 019c49731cebec291f8dcd40f597ef1f62acf4c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 3 Dec 2024 12:41:09 +0100 Subject: [PATCH 103/109] component validation and extension conversion --- wasm-rpc-stubgen/src/model/app.rs | 137 ++++++++++++++++++++++++++---- 1 file changed, 121 insertions(+), 16 deletions(-) diff --git a/wasm-rpc-stubgen/src/model/app.rs b/wasm-rpc-stubgen/src/model/app.rs index 203eeab9..30245fe5 100644 --- a/wasm-rpc-stubgen/src/model/app.rs +++ b/wasm-rpc-stubgen/src/model/app.rs @@ -11,8 +11,8 @@ use heck::{ use itertools::Itertools; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; -use std::fmt::Display; use std::fmt::Formatter; +use std::fmt::{Debug, Display}; use std::hash::Hash; use std::path::{Path, PathBuf}; use wit_parser::PackageName; @@ -610,11 +610,110 @@ impl Application { } }; - if let Some(properties) = component_properties { - // TODO: validate (build steps and extensions too) + if let Some(mut properties) = component_properties { + fn validate_properties_and_convert_extensions< + CPE: ComponentPropertiesExtensions, + >( + validation: &mut ValidationBuilder, + properties: &mut ComponentProperties, + ) -> bool { + let mut any_error = false; + + for (name, value) in [ + ("sourceWit", &properties.source_wit), + ("generatedWit", &properties.source_wit), + ("componentWasm", &properties.source_wit), + ] { + if value.is_empty() { + validation.add_error(format!( + "Property {} is empty or undefined", + name.log_color_highlight() + )); + any_error = true; + } + } + + properties.extensions = + properties.extensions_raw.convert_and_validate(validation); + any_error |= properties.extensions.is_none(); + + any_error + } + + let any_error: bool = match &mut properties { + ResolvedComponentProperties::Properties { + template_name, + any_template_overrides, + properties, + } => { + template_name.iter().for_each(|template_name| { + validation.push_context("template", template_name.to_string()); + validation.push_context( + "overrides", + any_template_overrides.to_string(), + ); + }); + + let any_error = validate_properties_and_convert_extensions( + &mut validation, + properties, + ); + + if template_name.is_some() { + validation.pop_context(); + validation.pop_context(); + } + + any_error + } + ResolvedComponentProperties::Profiles { + template_name, + any_template_overrides, + profiles, + .. + } => { + template_name.iter().for_each(|template_name| { + validation.push_context("template", template_name.to_string()); + }); + + let mut any_error = false; + + for (profile_name, properties) in profiles { + validation.push_context("profile", profile_name.to_string()); + let any_template_overrides = + any_template_overrides.get(profile_name); + any_template_overrides.iter().for_each( + |any_template_overrides| { + validation.push_context( + "overrides", + any_template_overrides.to_string(), + ); + }, + ); - resolved_components - .insert(component_name.clone(), Component { source, properties }); + any_error |= validate_properties_and_convert_extensions( + &mut validation, + properties, + ); + + if any_template_overrides.is_some() { + validation.pop_context(); + } + validation.pop_context(); + } + + if template_name.is_some() { + validation.pop_context(); + } + + any_error + } + }; + + if !any_error { + resolved_components + .insert(component_name.clone(), Component { source, properties }); + } } } @@ -929,7 +1028,8 @@ pub struct ComponentProperties { pub build: Vec, pub custom_commands: HashMap>, pub clean: Vec, - pub extensions: CPE, + pub extensions_raw: CPE, + pub extensions: Option, } impl ComponentProperties { @@ -942,7 +1042,8 @@ impl ComponentProperties { build: raw.build, custom_commands: raw.custom_commands, clean: raw.clean, - extensions: CPE::from_serde_json(serde_json::Value::Object(raw.extensions))?, + extensions_raw: CPE::from_serde_json(serde_json::Value::Object(raw.extensions))?, + extensions: None, }) } @@ -994,12 +1095,12 @@ impl ComponentProperties { } } -pub trait ComponentPropertiesExtensions { - fn from_serde_json(extensions: serde_json::Value) -> serde_json::Result - where - Self: Sized; +pub trait ComponentPropertiesExtensions: Sized + Debug + Clone { + type Validated: Debug + Clone; - fn validate(&self, validation: &mut ValidationBuilder); + fn from_serde_json(extensions: serde_json::Value) -> serde_json::Result; + + fn convert_and_validate(&self, validation: &mut ValidationBuilder) -> Option; } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -1007,6 +1108,8 @@ pub trait ComponentPropertiesExtensions { pub struct ComponentPropertiesExtensionsNone {} impl ComponentPropertiesExtensions for ComponentPropertiesExtensionsNone { + type Validated = Self; + fn from_serde_json(extensions: serde_json::Value) -> serde_json::Result where Self: Sized, @@ -1014,8 +1117,8 @@ impl ComponentPropertiesExtensions for ComponentPropertiesExtensionsNone { serde_json::from_value(extensions) } - fn validate(&self, _validation: &mut ValidationBuilder) { - // NOP + fn convert_and_validate(&self, _validation: &mut ValidationBuilder) -> Option { + Some(ComponentPropertiesExtensionsNone {}) } } @@ -1023,6 +1126,8 @@ impl ComponentPropertiesExtensions for ComponentPropertiesExtensionsNone { pub struct ComponentPropertiesExtensionsAny; impl ComponentPropertiesExtensions for ComponentPropertiesExtensionsAny { + type Validated = Self; + fn from_serde_json(_extensions: serde_json::Value) -> serde_json::Result where Self: Sized, @@ -1030,7 +1135,7 @@ impl ComponentPropertiesExtensions for ComponentPropertiesExtensionsAny { Ok(ComponentPropertiesExtensionsAny) } - fn validate(&self, _validation: &mut ValidationBuilder) { - // NOP + fn convert_and_validate(&self, _validation: &mut ValidationBuilder) -> Option { + Some(ComponentPropertiesExtensionsAny) } } From 60c28edc12f0272f3d476db6e74df778e89be36e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 3 Dec 2024 12:56:24 +0100 Subject: [PATCH 104/109] cleanups --- wasm-rpc-stubgen/src/model/app.rs | 41 +++++++++++++++++++------------ 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/wasm-rpc-stubgen/src/model/app.rs b/wasm-rpc-stubgen/src/model/app.rs index 30245fe5..1ba313ec 100644 --- a/wasm-rpc-stubgen/src/model/app.rs +++ b/wasm-rpc-stubgen/src/model/app.rs @@ -633,8 +633,10 @@ impl Application { } } - properties.extensions = - properties.extensions_raw.convert_and_validate(validation); + properties.extensions = CPE::convert_and_validate( + validation, + properties.extensions_raw.take().unwrap(), + ); any_error |= properties.extensions.is_none(); any_error @@ -1028,8 +1030,10 @@ pub struct ComponentProperties { pub build: Vec, pub custom_commands: HashMap>, pub clean: Vec, - pub extensions_raw: CPE, - pub extensions: Option, + + // TODO: clean up: move extensions_raw to a temporary var and make extensions non optional + pub extensions_raw: Option, + pub extensions: Option, } impl ComponentProperties { @@ -1042,7 +1046,9 @@ impl ComponentProperties { build: raw.build, custom_commands: raw.custom_commands, clean: raw.clean, - extensions_raw: CPE::from_serde_json(serde_json::Value::Object(raw.extensions))?, + extensions_raw: Some(CPE::raw_from_serde_json(serde_json::Value::Object( + raw.extensions, + ))?), extensions: None, }) } @@ -1096,11 +1102,11 @@ impl ComponentProperties { } pub trait ComponentPropertiesExtensions: Sized + Debug + Clone { - type Validated: Debug + Clone; + type Raw: Debug + Clone; - fn from_serde_json(extensions: serde_json::Value) -> serde_json::Result; + fn raw_from_serde_json(extensions: serde_json::Value) -> serde_json::Result; - fn convert_and_validate(&self, validation: &mut ValidationBuilder) -> Option; + fn convert_and_validate(validation: &mut ValidationBuilder, raw: Self::Raw) -> Option; } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -1108,17 +1114,17 @@ pub trait ComponentPropertiesExtensions: Sized + Debug + Clone { pub struct ComponentPropertiesExtensionsNone {} impl ComponentPropertiesExtensions for ComponentPropertiesExtensionsNone { - type Validated = Self; + type Raw = Self; - fn from_serde_json(extensions: serde_json::Value) -> serde_json::Result + fn raw_from_serde_json(extensions: serde_json::Value) -> serde_json::Result where Self: Sized, { serde_json::from_value(extensions) } - fn convert_and_validate(&self, _validation: &mut ValidationBuilder) -> Option { - Some(ComponentPropertiesExtensionsNone {}) + fn convert_and_validate(_validation: &mut ValidationBuilder, raw: Self::Raw) -> Option { + Some(raw) } } @@ -1126,16 +1132,19 @@ impl ComponentPropertiesExtensions for ComponentPropertiesExtensionsNone { pub struct ComponentPropertiesExtensionsAny; impl ComponentPropertiesExtensions for ComponentPropertiesExtensionsAny { - type Validated = Self; + type Raw = Self; - fn from_serde_json(_extensions: serde_json::Value) -> serde_json::Result + fn raw_from_serde_json(_extensions: serde_json::Value) -> serde_json::Result where Self: Sized, { Ok(ComponentPropertiesExtensionsAny) } - fn convert_and_validate(&self, _validation: &mut ValidationBuilder) -> Option { - Some(ComponentPropertiesExtensionsAny) + fn convert_and_validate( + _validation: &mut ValidationBuilder, + raw: Self::Raw, + ) -> Option { + Some(raw) } } From 1bf1a58239265ca27297cef6817e23821500f1bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 3 Dec 2024 13:07:38 +0100 Subject: [PATCH 105/109] add source to extensions convert --- wasm-rpc-stubgen/src/model/app.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/wasm-rpc-stubgen/src/model/app.rs b/wasm-rpc-stubgen/src/model/app.rs index 1ba313ec..e1dd1443 100644 --- a/wasm-rpc-stubgen/src/model/app.rs +++ b/wasm-rpc-stubgen/src/model/app.rs @@ -614,6 +614,7 @@ impl Application { fn validate_properties_and_convert_extensions< CPE: ComponentPropertiesExtensions, >( + source: &Path, validation: &mut ValidationBuilder, properties: &mut ComponentProperties, ) -> bool { @@ -634,6 +635,7 @@ impl Application { } properties.extensions = CPE::convert_and_validate( + source, validation, properties.extensions_raw.take().unwrap(), ); @@ -657,6 +659,7 @@ impl Application { }); let any_error = validate_properties_and_convert_extensions( + &source, &mut validation, properties, ); @@ -694,6 +697,7 @@ impl Application { ); any_error |= validate_properties_and_convert_extensions( + &source, &mut validation, properties, ); @@ -1106,7 +1110,11 @@ pub trait ComponentPropertiesExtensions: Sized + Debug + Clone { fn raw_from_serde_json(extensions: serde_json::Value) -> serde_json::Result; - fn convert_and_validate(validation: &mut ValidationBuilder, raw: Self::Raw) -> Option; + fn convert_and_validate( + source: &Path, + validation: &mut ValidationBuilder, + raw: Self::Raw, + ) -> Option; } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -1123,7 +1131,11 @@ impl ComponentPropertiesExtensions for ComponentPropertiesExtensionsNone { serde_json::from_value(extensions) } - fn convert_and_validate(_validation: &mut ValidationBuilder, raw: Self::Raw) -> Option { + fn convert_and_validate( + _source: &Path, + _validation: &mut ValidationBuilder, + raw: Self::Raw, + ) -> Option { Some(raw) } } @@ -1142,6 +1154,7 @@ impl ComponentPropertiesExtensions for ComponentPropertiesExtensionsAny { } fn convert_and_validate( + _source: &Path, _validation: &mut ValidationBuilder, raw: Self::Raw, ) -> Option { From bec5871bc7a07400dc43ac3e353086308d7d6376 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 3 Dec 2024 13:18:44 +0100 Subject: [PATCH 106/109] clippy + expose app ctx new --- wasm-rpc-stubgen/src/commands/app.rs | 2 +- wasm-rpc-stubgen/src/model/template.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/wasm-rpc-stubgen/src/commands/app.rs b/wasm-rpc-stubgen/src/commands/app.rs index af8e6c37..7259a58f 100644 --- a/wasm-rpc-stubgen/src/commands/app.rs +++ b/wasm-rpc-stubgen/src/commands/app.rs @@ -56,7 +56,7 @@ pub struct ApplicationContext { } impl ApplicationContext { - fn new(config: Config) -> anyhow::Result> { + pub fn new(config: Config) -> anyhow::Result> { let ctx = to_anyhow( "Failed to create application context, see problems above", load_app_validated(&config).and_then(|application| { diff --git a/wasm-rpc-stubgen/src/model/template.rs b/wasm-rpc-stubgen/src/model/template.rs index b1ee6719..1fc0d39b 100644 --- a/wasm-rpc-stubgen/src/model/template.rs +++ b/wasm-rpc-stubgen/src/model/template.rs @@ -87,6 +87,7 @@ impl Template for app_raw::ExternalCommand { impl Template for serde_json::Value { type Rendered = serde_json::Value; + #[warn(clippy::only_used_in_recursion)] fn render( &self, env: &minijinja::Environment, From ef6f85572e646a3d06d8e6d50d751b5f215efd9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 3 Dec 2024 13:21:39 +0100 Subject: [PATCH 107/109] expose app ctx fields --- wasm-rpc-stubgen/src/commands/app.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wasm-rpc-stubgen/src/commands/app.rs b/wasm-rpc-stubgen/src/commands/app.rs index 7259a58f..595cab53 100644 --- a/wasm-rpc-stubgen/src/commands/app.rs +++ b/wasm-rpc-stubgen/src/commands/app.rs @@ -48,9 +48,9 @@ pub enum ApplicationSourceMode { } pub struct ApplicationContext { - config: Config, - application: Application, - wit: ResolvedWitApplication, + pub config: Config, + pub application: Application, + pub wit: ResolvedWitApplication, common_wit_deps: OnceCell>, component_generated_base_wit_deps: HashMap, } From d8000ba4cdebce0ca25c9ea850b5f99ff12ca14d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 3 Dec 2024 15:42:21 +0100 Subject: [PATCH 108/109] make app command optional, clippy, remove custom command "about" for now --- .github/workflows/ci.yaml | 2 ++ wasm-rpc-stubgen/Cargo.toml | 4 ++++ wasm-rpc-stubgen/src/lib.rs | 30 +------------------------- wasm-rpc-stubgen/src/main.rs | 5 ++++- wasm-rpc-stubgen/src/model/template.rs | 2 +- 5 files changed, 12 insertions(+), 31 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index dc9ef977..a0aaa474 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -54,6 +54,8 @@ jobs: run: cargo test --tests --package wasm-rpc-stubgen-tests-integration -- --test-threads=1 --format junit --logfile target/report.xml - name: Build in stub mode run: cargo component build -p golem-wasm-rpc --no-default-features --features stub + - name: Build stubgen without default features + run: cargo build -p golem-wasm-rpc-stubgen --no-default-features - name: Publish Test Report uses: mikepenz/action-junit-report@v4 if: success() || failure() # always run even if the previous step fails diff --git a/wasm-rpc-stubgen/Cargo.toml b/wasm-rpc-stubgen/Cargo.toml index 04e9d26c..0a732e28 100644 --- a/wasm-rpc-stubgen/Cargo.toml +++ b/wasm-rpc-stubgen/Cargo.toml @@ -7,6 +7,10 @@ homepage = "https://golem.cloud" repository = "https://github.com/golemcloud/wasm-rpc" description = "Golem WASM RPC stub generator" +[features] +default = ["app-command"] +app-command = [] + [lib] name = "golem_wasm_rpc_stubgen" path = "src/lib.rs" diff --git a/wasm-rpc-stubgen/src/lib.rs b/wasm-rpc-stubgen/src/lib.rs index 62dc5d35..b6c3ec5c 100644 --- a/wasm-rpc-stubgen/src/lib.rs +++ b/wasm-rpc-stubgen/src/lib.rs @@ -27,14 +27,11 @@ pub mod wit_encode; pub mod wit_generate; pub mod wit_resolve; -use crate::log::LogColorize; use crate::model::app::{ComponentPropertiesExtensions, ComponentPropertiesExtensionsAny}; use crate::stub::{StubConfig, StubDefinition}; use crate::wit_generate::UpdateCargoToml; use anyhow::Context; use clap::{Parser, Subcommand}; -use colored::Colorize; -use itertools::Itertools; use std::marker::PhantomData; use std::path::PathBuf; use tempfile::TempDir; @@ -57,7 +54,7 @@ pub enum Command { /// generating stubs and composing results. InitializeWorkspace(InitializeWorkspaceArgs), /// Build components with application manifests - #[clap(about = app_about(), long_about = None)] + #[cfg(feature = "app-command")] App { #[command(subcommand)] subcommand: App, @@ -330,31 +327,6 @@ pub async fn run_app_command( } } -// TODO: currently this is added on top of the derived help, also, it should by grouped by profiles (also handling overlaps) -fn app_about() -> String { - let commands = commands::app::available_custom_commands(commands::app::Config { - app_resolve_mode: commands::app::ApplicationSourceMode::Automatic, - skip_up_to_date_checks: false, - profile: None, - offline: false, - extensions: PhantomData::, - }); - - match commands { - Ok(commands) if !commands.is_empty() => { - format!( - "\n\n{}\n{}", - "Custom commands:".bold().underline(), - commands - .into_iter() - .map(|cmd| format!(" {}", cmd.log_color_highlight())) - .join("\n") - ) - } - _ => "".to_string(), - } -} - fn app_manifest_sources_to_resolve_mode( sources: Vec, ) -> commands::app::ApplicationSourceMode { diff --git a/wasm-rpc-stubgen/src/main.rs b/wasm-rpc-stubgen/src/main.rs index 7784d031..7e6694ec 100644 --- a/wasm-rpc-stubgen/src/main.rs +++ b/wasm-rpc-stubgen/src/main.rs @@ -14,10 +14,12 @@ use clap::Parser; use colored::Colorize; -use golem_wasm_rpc_stubgen::model::app::ComponentPropertiesExtensionsNone; use golem_wasm_rpc_stubgen::*; use std::process::ExitCode; +#[cfg(feature = "app-command")] +use golem_wasm_rpc_stubgen::model::app::ComponentPropertiesExtensionsNone; + #[tokio::main] async fn main() -> ExitCode { pretty_env_logger::init(); @@ -32,6 +34,7 @@ async fn main() -> ExitCode { Command::InitializeWorkspace(init_workspace_args) => { initialize_workspace(init_workspace_args, "wasm-rpc-stubgen", &[]) } + #[cfg(feature = "app-command")] Command::App { subcommand } => { run_app_command::(subcommand).await } diff --git a/wasm-rpc-stubgen/src/model/template.rs b/wasm-rpc-stubgen/src/model/template.rs index 1fc0d39b..3dc7741b 100644 --- a/wasm-rpc-stubgen/src/model/template.rs +++ b/wasm-rpc-stubgen/src/model/template.rs @@ -87,7 +87,7 @@ impl Template for app_raw::ExternalCommand { impl Template for serde_json::Value { type Rendered = serde_json::Value; - #[warn(clippy::only_used_in_recursion)] + #[allow(clippy::only_used_in_recursion)] fn render( &self, env: &minijinja::Environment, From 28344b81084da9bdeeb88ec8a4b6cc0e80e098db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 3 Dec 2024 16:03:05 +0100 Subject: [PATCH 109/109] bump fallback version (workaround for branch dep) --- wasm-rpc/src/version.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wasm-rpc/src/version.rs b/wasm-rpc/src/version.rs index 6aa64d85..454223ee 100644 --- a/wasm-rpc/src/version.rs +++ b/wasm-rpc/src/version.rs @@ -3,7 +3,7 @@ pub use git_version::git_version; macro_rules! lib_version { () => {{ let version = - crate::version::git_version!(args = ["--tags"], cargo_prefix = "", fallback = "0.0.0"); + crate::version::git_version!(args = ["--tags"], cargo_prefix = "", fallback = "1.0.0"); if !version.is_empty() && version.as_bytes()[0] == b'v' { unsafe { std::str::from_utf8_unchecked(std::slice::from_raw_parts(