From a69b41ad0b4b703b67bf301f68ba50bb71b87f78 Mon Sep 17 00:00:00 2001 From: Alexander Koz <888526+boozook@users.noreply.github.com> Date: Thu, 13 Jun 2024 17:56:18 +0400 Subject: [PATCH] Package new assets-builder' artefacts using meta tree (#381) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix assets-plan index - wrong for dev * package new assets-artifacts * fix "Assets artifacts not found" for crates without assets 🤦🏻‍♂️ * clean up --- Cargo.lock | 124 +-- cargo/Cargo.toml | 4 +- cargo/src/assets/cache.rs | 60 ++ cargo/src/assets/mod.rs | 1094 +++++++-------------------- cargo/src/assets/pdc.rs | 8 +- cargo/src/assets/plan.rs | 1046 +++++++++---------------- cargo/src/config.rs | 1 + cargo/src/main.rs | 12 +- cargo/src/package/mod.rs | 308 ++++---- cargo/src/proc/reader.rs | 8 +- cargo/src/utils/cargo/build_plan.rs | 8 +- cargo/src/utils/cargo/format.rs | 10 +- cargo/src/utils/cargo/meta_deps.rs | 107 ++- cargo/src/utils/cargo/metadata.rs | 4 +- cargo/src/utils/cargo/unit_graph.rs | 6 +- cargo/src/utils/mod.rs | 36 - 16 files changed, 990 insertions(+), 1846 deletions(-) create mode 100644 cargo/src/assets/cache.rs diff --git a/Cargo.lock b/Cargo.lock index 04780e82..88bec64a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -406,9 +406,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.72" +version = "0.3.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17c6a35df3749d2e8bb1b7b21a976d82b15548788d2735b9d82f329268f71a11" +checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" dependencies = [ "addr2line", "cc", @@ -658,7 +658,7 @@ dependencies = [ "tempfile", "time", "toml", - "toml_edit 0.22.14", + "toml_edit", "tracing", "tracing-chrome", "tracing-subscriber", @@ -726,7 +726,7 @@ dependencies = [ [[package]] name = "cargo-playdate" -version = "0.5.0-beta.4" +version = "0.5.0-beta.5" dependencies = [ "anstyle", "anyhow", @@ -755,10 +755,10 @@ dependencies = [ "serde_json", "target", "toml", - "toml_edit 0.22.14", + "toml_edit", "try-lazy-init", "walkdir", - "zip 1.1.4", + "zip", ] [[package]] @@ -908,15 +908,6 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cbd0f76e066e64fdc5631e3bb46381254deab9ef1158292f27c8c57e3bf3fe59" -[[package]] -name = "cmake" -version = "0.1.50" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" -dependencies = [ - "cc", -] - [[package]] name = "color-print" version = "0.3.6" @@ -1582,7 +1573,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" dependencies = [ "crc32fast", - "libz-ng-sys", "libz-sys", "miniz_oxide", ] @@ -2759,9 +2749,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.9.2" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f3935c160d00ac752e09787e6e6bfc26494c2183cc922f1bc678a60d4733bc2" +checksum = "d0e7a4dd27b9476dc40cb050d3632d3bba3a70ddbff012285f7f8559a1e7e545" [[package]] name = "httpdate" @@ -3308,16 +3298,6 @@ dependencies = [ "pkg-config", ] -[[package]] -name = "libz-ng-sys" -version = "1.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6409efc61b12687963e602df8ecf70e8ddacf95bc6576bcf16e3ac6328083c5" -dependencies = [ - "cmake", - "libc", -] - [[package]] name = "libz-sys" version = "1.1.18" @@ -3703,27 +3683,6 @@ dependencies = [ "libc", ] -[[package]] -name = "num_enum" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02339744ee7253741199f897151b38e72257d13802d4ee837285cc2990a90845" -dependencies = [ - "num_enum_derive", -] - -[[package]] -name = "num_enum_derive" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b" -dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn 2.0.66", -] - [[package]] name = "num_threads" version = "0.1.7" @@ -3752,9 +3711,9 @@ dependencies = [ [[package]] name = "object" -version = "0.35.0" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8ec7ab813848ba4522158d5517a6093db1ded27575b070f4177b8d12b41db5e" +checksum = "576dfe1fc8f9df304abb159d767a29d0476f7750fbf8aa7ad07816004a207434" dependencies = [ "memchr", ] @@ -4497,15 +4456,6 @@ dependencies = [ "elliptic-curve", ] -[[package]] -name = "proc-macro-crate" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" -dependencies = [ - "toml_edit 0.21.1", -] - [[package]] name = "proc-macro2" version = "1.0.85" @@ -5304,7 +5254,7 @@ dependencies = [ "symbolic-ppdb", "thiserror", "wasmparser", - "zip 2.1.1", + "zip", "zstd", ] @@ -5617,7 +5567,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.14", + "toml_edit", ] [[package]] @@ -5629,17 +5579,6 @@ dependencies = [ "serde", ] -[[package]] -name = "toml_edit" -version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" -dependencies = [ - "indexmap 2.2.6", - "toml_datetime", - "winnow 0.5.40", -] - [[package]] name = "toml_edit" version = "0.22.14" @@ -6524,6 +6463,20 @@ name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] [[package]] name = "zerovec" @@ -6549,9 +6502,9 @@ dependencies = [ [[package]] name = "zip" -version = "1.1.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cc23c04387f4da0374be4533ad1208cbb091d5c11d070dfef13676ad6497164" +checksum = "1dd56a4d5921bc2f99947ac5b3abe5f510b1be7376fdc5e9fce4a23c6a93e87c" dependencies = [ "aes", "arbitrary", @@ -6565,32 +6518,17 @@ dependencies = [ "hmac", "indexmap 2.2.6", "lzma-rs", - "num_enum", + "memchr", "pbkdf2", + "rand", "sha1", "thiserror", "time", + "zeroize", "zopfli", "zstd", ] -[[package]] -name = "zip" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd56a4d5921bc2f99947ac5b3abe5f510b1be7376fdc5e9fce4a23c6a93e87c" -dependencies = [ - "arbitrary", - "crc32fast", - "crossbeam-utils", - "displaydoc", - "flate2", - "indexmap 2.2.6", - "memchr", - "thiserror", - "zopfli", -] - [[package]] name = "zopfli" version = "0.8.1" diff --git a/cargo/Cargo.toml b/cargo/Cargo.toml index d845e5da..cd727bdd 100644 --- a/cargo/Cargo.toml +++ b/cargo/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cargo-playdate" -version = "0.5.0-beta.4" +version = "0.5.0-beta.5" readme = "README.md" description = "Build tool for neat yellow console." keywords = ["playdate", "build", "cargo", "plugin", "cargo-subcommand"] @@ -43,7 +43,7 @@ toml.workspace = true toml_edit = { version = "0.22", features = ["serde"] } regex.workspace = true byteorder = "1.5" -zip = { version = "1.1", features = ["time"] } +zip = { version = "2.1", features = ["time"] } walkdir = "2.5" anyhow = "1.0" diff --git a/cargo/src/assets/cache.rs b/cargo/src/assets/cache.rs new file mode 100644 index 00000000..4ab2b3d7 --- /dev/null +++ b/cargo/src/assets/cache.rs @@ -0,0 +1,60 @@ +use std::collections::BTreeMap; +use std::path::PathBuf; + +use cargo::CargoResult; +use playdate::assets::plan::BuildPlan; + +pub struct PlanCache { + pub difference: Difference, + pub serialized: Option, + pub path: PathBuf, +} + + +#[derive(Debug, Clone, Copy, serde::Serialize)] +pub enum Difference { + Same, + Different, + /// There is not cache file. + Missing, +} + +impl Difference { + pub fn is_same(&self) -> bool { matches!(self, Self::Same) } +} + + +#[must_use = "Cached plan must be used"] +pub fn plan_cache(path: PathBuf, plan: &BuildPlan<'_, '_>) -> CargoResult { + let mut serializable = plan.iter_flatten_meta().collect::>(); + serializable.sort(); + + #[derive(serde::Serialize)] + struct SerializablePlan<'t> { + items: &'t [(playdate::assets::plan::MappingKind, PathBuf, (PathBuf, Option))], + env: &'t BTreeMap, + } + + let serializable = SerializablePlan { items: &serializable, + env: plan.used_env_vars() }; + let json = serde_json::to_string(&serializable)?; + + let difference = if path.try_exists()? { + if std::fs::read_to_string(&path)? == json { + log::debug!("Cached plan is the same"); + Difference::Same + } else { + log::debug!("Cache mismatch, need diff & rebuild"); + Difference::Different + } + } else { + log::debug!("Cache mismatch, full rebuilding"); + Difference::Missing + }; + + let serialized = (!difference.is_same()).then_some(json); + + Ok(PlanCache { path, + difference, + serialized }) +} diff --git a/cargo/src/assets/mod.rs b/cargo/src/assets/mod.rs index 430f5d32..2f7eae6e 100644 --- a/cargo/src/assets/mod.rs +++ b/cargo/src/assets/mod.rs @@ -1,901 +1,371 @@ use std::borrow::Cow; -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::BTreeMap; use std::path::{PathBuf, Path}; +use cargo::CargoResult; +use cargo::core::{PackageId, Verbosity}; use anstyle::AnsiColor as Color; use anyhow::bail; -use cargo::CargoResult; -use cargo::core::{Package, PackageId, Verbosity}; -use playdate::metadata::source::MetadataSource; -use playdate::metadata::METADATA_FIELD; -use playdate::layout::Layout; -use crate::assets::plan::{AssetKind, CachedPlan}; +use playdate::assets::plan::BuildPlan; +use playdate::assets::BuildReport; +use playdate::layout::Layout as _; +use playdate::metadata::format::AssetsOptions; + use crate::config::Config; -use crate::layout::{PlaydateAssets, LayoutLockable, Layout as _, CrossTargetLayout}; -use crate::logger::LogErr; -use crate::utils::LazyBuildContext; use crate::utils::path::AsRelativeTo; -use self::plan::Metadata; +use crate::utils::cargo::meta_deps::{MetaDeps, RootNode}; +use crate::layout::{LayoutLockable, CrossTargetLayout}; +use crate::layout::{PlaydateAssets, Layout}; -mod plan; mod pdc; +mod plan; +mod cache; #[derive(Debug)] -pub struct AssetsArtifact { +pub struct Artifact { pub package_id: PackageId, pub layout: PlaydateAssets, - /// Cached metadata - pub metadata: Option, + pub kind: Kind, } -/// One artifact per package. -pub type AssetsArtifacts<'cfg> = HashMap<&'cfg Package, AssetsArtifact>; - - -pub mod proto { - use super::*; - - use plan::proto::RootKey; - use plan::Difference; - use playdate::assets::plan::BuildPlan; - use playdate::assets::BuildReport; - use playdate::layout::Layout as _; - use playdate::metadata::format::AssetsOptions; - - use crate::utils::cargo::meta_deps::{MetaDeps, RootNode}; - use crate::layout::{PlaydateAssets, Layout}; - - #[derive(Debug)] - pub struct AssetsArtifact { - pub package_id: PackageId, - pub layout: PlaydateAssets, - pub kind: AssetKind, - } - - - pub struct AssetsArtifactsNew<'t, 'cfg> { - artifacts: Vec, - index: BTreeMap>, - tree: &'t MetaDeps<'cfg>, - } +pub struct Artifacts<'t, 'cfg> { + artifacts: Vec, + index: BTreeMap>, + pub tree: &'t MetaDeps<'cfg>, +} - impl AssetsArtifactsNew<'_, '_> { - pub fn iter(&self) -> impl Iterator)> { - self.index - .iter() - .flat_map(|(key, index)| { - self.tree - .roots() - .iter() - .filter(|r| key.is_for(r)) - .map(|root| (root, index.as_slice())) - }) - .map(|(root, index)| { - let arts = index.iter().map(|i| &self.artifacts[*i]); - (root, arts) - }) - } +impl Artifacts<'_, '_> { + pub fn len(&self) -> usize { self.artifacts.len() } + pub fn artifacts(&self) -> &[Artifact] { &self.artifacts } + pub fn index(&self) -> &BTreeMap> { &self.index } + + pub fn iter(&self) -> impl Iterator)> { + self.index + .iter() + .flat_map(|(key, index)| { + self.tree + .roots() + .iter() + .filter(|r| key.is_for(r)) + .map(|root| (root, index.as_slice())) + }) + .map(|(root, index)| { + let arts = index.iter().map(|i| &self.artifacts[*i]); + (root, arts) + }) } +} - impl core::fmt::Debug for AssetsArtifactsNew<'_, '_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("AssetsArtifacts") - .field_with("artifacts", |f| { - self.artifacts - .iter() - .enumerate() - .collect::>() - .fmt(f) - }) - .field("index", &self.index) - .finish_non_exhaustive() - } +impl core::fmt::Debug for Artifacts<'_, '_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AssetsArtifacts") + .field_with("artifacts", |f| { + self.artifacts + .iter() + .enumerate() + .collect::>() + .fmt(f) + }) + .field("index", &self.index) + .finish_non_exhaustive() } +} - pub fn build_all<'t, 'cfg>(cfg: &Config<'cfg>, - tree: &'t MetaDeps<'cfg>) - -> CargoResult> { - // planning: - let plans = plan::proto::plan_all(cfg, tree)?; - - // validation: - if let Err(err) = plan::proto::merge_all_virtually(cfg, tree, &plans) && - !cfg.compile_options.build_config.keep_going - { - return Err(err.context("Assets validation failed")); - } - - // results: - let mut artifacts = AssetsArtifactsNew { artifacts: Vec::with_capacity(plans.plans.len()), - index: Default::default(), - tree }; - - - // checking cache, apply each plan: - for (index, plan) in plans.plans.into_iter().enumerate() { - let key = plans.index.iter().find(|(_, v)| **v == index).expect("key").0; - - log::debug!("#{index} build (dev:{}) {}", key.dev, key.id); - - - let global_layout = CrossTargetLayout::new(cfg, key.id, None)?; - let mut layout = global_layout.assets_layout(cfg); - - - // clean layout if needed: - if !cfg.dry_run && cfg.compile_options.build_config.force_rebuild { - if !matches!(cfg.workspace.gctx().shell().verbosity(), Verbosity::Quiet) { - cfg.log().status("Clean", format!("assets for {}", key.id)); - } - layout.clean()?; - } - - - let mut locked = layout.lock_mut(cfg.workspace.gctx())?; - locked.prepare()?; - - // path of build-plan file: - let path = if key.dev { - locked.as_inner().assets_plan_for_dev(cfg, &key.id) - } else { - locked.as_inner().assets_plan_for(cfg, &key.id) - }; - - let mut cache = plan_cache(path, &plan)?; - if cfg.compile_options.build_config.force_rebuild { - cache.difference = Difference::Missing; - } - - - let dest = if key.dev { - locked.as_inner().assets_dev() - } else { - locked.as_inner().assets() - }; +#[derive(Debug, Clone, Copy, serde::Serialize)] +pub enum Kind { + Package, + Dev, +} +impl Kind { + pub fn is_dev(&self) -> bool { matches!(self, Self::Dev) } +} - // kind of assets just for log: - let kind_prefix = key.dev.then_some("dev-").unwrap_or_default(); - // this one for assets: - let kind = if key.dev { - AssetKind::Dev - } else { - AssetKind::Package - }; - - - // build if needed: - if cache.difference.is_same() { - cfg.log().status( - "Skip", - format!( - "{} {kind_prefix}assets cache state is {:?}", - key.id, &cache.difference - ), - ); - } else { - cfg.log() - .status("Build", format!("{kind_prefix}assets for {}", key.id)); - cfg.log().verbose(|mut log| { - let dep_root = plan.crate_root(); - let dest = format!("destination: {:?}", dest.as_relative_to_root(cfg)); - log.status("", dest); - let src = format!("root: {:?}", dep_root.as_relative_to_root(cfg)); - log.status("", src); - }); - - - // Since we build each plan separately independently, the default options are sufficient. - // The options are needed further when merging assets into a package. - let dep_opts = Default::default(); - let report = apply(cache, plan, &dest, &dep_opts, cfg)?; - - - // print report: - for (x, (m, results)) in report.results.iter().enumerate() { - let results = results.iter().enumerate(); - let expr = m.exprs(); - let incs = m.sources(); - - for (y, res) in results { - let path = incs[y].target(); - let path = path.as_relative_to_root(cfg); - match res { - Ok(op) => { - cfg.log().verbose(|mut log| { - let msg = format!("asset [{x}:{y}] {}", path.display()); - log.status(format!("{op:?}"), msg) - }) - }, - Err(err) => { - use fs_extra::error::ErrorKind as FsExtraErrorKind; - - let error = match &err.kind { - FsExtraErrorKind::Io(err) => format!("IO: {err}"), - FsExtraErrorKind::StripPrefix(err) => format!("StripPrefix: {err}"), - FsExtraErrorKind::OsString(err) => format!("OsString: {err:?}"), - _ => err.to_string(), - }; - let message = format!( - "Asset [{x}:{y}], rule: '{} <- {} | {}', {error}", - expr.0.original(), - expr.1.original(), - path.display() - ); - - cfg.log().status_with_color("Error", message, Color::Red) - }, - }; - } - } - // TODO: log report.exclusions - - if report.has_errors() && !cfg.compile_options.build_config.keep_going { - use anyhow::Error; - - #[derive(Debug)] - pub struct Mapping(String); - impl std::error::Error for Mapping {} - impl std::fmt::Display for Mapping { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { self.0.fmt(f) } - } - - let err = report.results - .into_iter() - .filter_map(|(map, res)| { - if res.iter().any(|res| res.is_err()) { - let err = Mapping(map.pretty_print_compact()); - let err = res.into_iter() - .filter_map(|res| res.err()) - .fold(Error::new(err), Error::context); - Some(err) - } else { - None - } - }) - .fold(Error::msg("Assets build failed"), Error::context); - return Err(err); - } +pub fn build_all<'t, 'cfg>(cfg: &Config<'cfg>, tree: &'t MetaDeps<'cfg>) -> CargoResult> { + // planning: + let plans = plan::plan_all(cfg, tree)?; + // validation: + if let Err(err) = plan::merge_all_virtually(cfg, tree, &plans) && + !cfg.compile_options.build_config.keep_going + { + return Err(err.context("Assets validation failed")); + } - // finally build with pdc: - // if not disallowed explicitly - if cfg.skip_prebuild { - const REASON: &str = "as requested"; - let msg = format!("{kind_prefix}assets pre-build for {}, {REASON}.", key.id); - cfg.log().status("Skip", msg); - } else { - match pdc::build(cfg, &key.id, locked.as_inner(), kind) { - Ok(_) => { - let msg = format!("{kind_prefix}assets for {}", key.id); - cfg.log().status("Finished", msg); - }, - Err(err) => { - let msg = format!("build {kind_prefix}assets with pdc failed: {err}"); - cfg.log().status_with_color("Error", msg, Color::Red); - if !cfg.compile_options.build_config.keep_going { - bail!("Assets build failed."); - } - }, - } - } - } + // results: + let mut artifacts = Artifacts { artifacts: Vec::with_capacity(plans.plans.len()), + index: Default::default(), + tree }; - // Finale: + // checking cache, apply each plan: + for (index, plan) in plans.plans.into_iter().enumerate() { + let key = plans.index + .iter() + .find_map(|(k, i)| (*i == index).then_some(k)) + .ok_or_else(|| anyhow::anyhow!("No assets-plan key in plan #{index}"))?; - locked.unlock(); + log::debug!("#{index} build (dev:{}) {}", key.dev, key.id); - let art_index = artifacts.artifacts.len(); - artifacts.artifacts.push(AssetsArtifact { kind, - package_id: key.id, - layout: layout.clone() }); + let global_layout = CrossTargetLayout::new(cfg, key.id, None)?; + let mut layout = global_layout.assets_layout(cfg); - log::debug!( - "Assets artifact for {} at {:?}", - key.id, - crate::layout::Layout::dest(&layout).as_relative_to_root(cfg) - ); - for (r_key, index) in plans.targets.iter().filter(|(_, i)| i.contains(&index)) { - artifacts.index - .entry(r_key.to_owned()) - .or_insert(Vec::with_capacity(index.len())) - .push(art_index); + // clean layout if needed: + if !cfg.dry_run && cfg.compile_options.build_config.force_rebuild { + if !matches!(cfg.workspace.gctx().shell().verbosity(), Verbosity::Quiet) { + cfg.log().status("Clean", format!("assets for {}", key.id)); } + layout.clean()?; } - cfg.log_extra_verbose(|mut logger| { - artifacts.iter().for_each(|(root, arts)| { - use cargo::core::compiler::CompileKind; - let ct: Cow<_> = match root.node().unit().platform { - CompileKind::Host => "host".into(), - CompileKind::Target(kind) => kind.short_name().to_owned().into(), - }; - - let root = format!( - "{} {} of {} for {ct}", - root.node().target().kind().description(), - root.node().target().name, - root.node().package_id().name(), - ); - logger.status("Assets", format!("artifacts for {root}:")); - arts.for_each(|art| { - let dest = match art.kind { - AssetKind::Package => art.layout.assets(), - AssetKind::Dev => art.layout.assets_dev(), - }; - let msg = format!( - "[{:?}] {} - {:?}", - art.kind, - art.package_id.name(), - dest.as_relative_to_root(cfg) - ); - logger.status("", msg); - }); - }); - }); - - Ok(artifacts) - } - - - struct PlanCache { - pub difference: Difference, - pub serialized: Option, - pub path: PathBuf, - } + let mut locked = layout.lock_mut(cfg.workspace.gctx())?; + locked.prepare()?; + // path of build-plan file: + let path = if key.dev { + locked.as_inner().assets_plan_for_dev(cfg, &key.id) + } else { + locked.as_inner().assets_plan_for(cfg, &key.id) + }; - #[must_use = "Cached plan must be used"] - fn plan_cache(path: PathBuf, plan: &BuildPlan<'_, '_>) -> CargoResult { - let mut serializable = plan.iter_flatten_meta().collect::>(); - serializable.sort(); - - #[derive(serde::Serialize)] - struct SerializablePlan<'t> { - items: &'t [(playdate::assets::plan::MappingKind, PathBuf, (PathBuf, Option))], - env: &'t BTreeMap, + let mut cache = cache::plan_cache(path, &plan)?; + if cfg.compile_options.build_config.force_rebuild { + cache.difference = cache::Difference::Missing; } - let serializable = SerializablePlan { items: &serializable, - env: plan.used_env_vars() }; - let json = serde_json::to_string(&serializable)?; - let difference = if path.try_exists()? { - if std::fs::read_to_string(&path)? == json { - log::debug!("Cached plan is the same"); - Difference::Same - } else { - log::debug!("Cache mismatch, need diff & rebuild"); - Difference::Different - } + let dest = if key.dev { + locked.as_inner().assets_dev() } else { - log::debug!("Cache mismatch, full rebuilding"); - Difference::Missing + locked.as_inner().assets() }; - let serialized = (!difference.is_same()).then_some(json); - Ok(PlanCache { path, - difference, - serialized }) - } + // kind of assets just for log: + let kind_prefix = key.dev.then_some("dev-").unwrap_or_default(); + // this one for assets: + let kind = if key.dev { Kind::Dev } else { Kind::Package }; - fn apply<'l, 'r>(cache: PlanCache, - plan: BuildPlan<'l, 'r>, - dest: &Path, - options: &AssetsOptions, - config: &Config) - -> CargoResult> { - use crate::playdate::assets::apply_build_plan; - - let report = apply_build_plan(plan, dest, options)?; - // and finally save cache of just successfully applied plan: - // only if there is no errors - if !report.has_errors() { - if let Some(data) = cache.serialized.as_deref() { - log::trace!("writing cache to {:?}", cache.path); - std::fs::write(&cache.path, data)?; - config.log().verbose(|mut log| { - let path = cache.path.as_relative_to_root(config); - log.status("Cache", format_args!("saved to {}", path.display())); - }); - } else { - config.log().verbose(|mut log| { - log.status("Cache", "nothing to save"); - }); - } + // build if needed: + if cache.difference.is_same() { + cfg.log().status( + "Skip", + format!( + "{} {kind_prefix}assets cache state is {:?}", + key.id, &cache.difference + ), + ); } else { - config.log().verbose(|mut log| { - let message = "build has errors, so cache was not saved"; - log.status("Cache", message); - }); - } - - Ok(report) - } -} - - -pub fn build<'cfg>(config: &'cfg Config) -> CargoResult> { - let bcx = LazyBuildContext::new(config)?; - let mut artifacts = AssetsArtifacts::new(); - - for (package, targets, ..) in config.possible_targets()? { - let env = plan::LazyEnvBuilder::new_for(config, package); - let mut plans: HashMap<&Package, _> = Default::default(); - let global_layout = CrossTargetLayout::new(config, package.package_id(), None)?; - let mut layout = global_layout.assets_layout(config); - let mut options = HashMap::new(); - - if !config.dry_run && config.compile_options.build_config.force_rebuild { - if !matches!(config.workspace.gctx().shell().verbosity(), Verbosity::Quiet) { - config.log() - .status("Clean", format!("assets for {}", package.package_id())); - } - layout.clean()?; - } - - // primary top-level package - let target_pid = package.package_id(); - let has_dev = targets.iter() - .any(|t| t.is_example() || t.is_test() || t.is_bench()); - - - log::debug!("Inspecting dependencies tree for {}", package.package_id()); - let packages = deps_tree_metadata(package, &bcx, config)?; - - - // XXX: compare with beta-proto - #[cfg(debug_assertions)] - { - let tree = crate::utils::cargo::meta_deps::meta_deps(config)?; - - // planning: - let plans = plan::proto::plan_all(config, &tree)?; - - - for (p, _) in packages.iter() - .filter(|(_, m)| !m.assets().is_empty() || !m.dev_assets().is_empty()) - { - let id = p.package_id(); - assert!(plans.index.keys().any(|k| k.id == id), "not found: {id}"); - } - - // validation: - if let Err(err) = plan::proto::merge_all_virtually(config, &tree, &plans) && - !config.compile_options.build_config.keep_going - { - return Err(err.context("Assets validation failed")); - } - } - + cfg.log() + .status("Build", format!("{kind_prefix}assets for {}", key.id)); + cfg.log().verbose(|mut log| { + let dep_root = plan.crate_root(); + let dest = format!("destination: {:?}", dest.as_relative_to_root(cfg)); + log.status("", dest); + let src = format!("root: {:?}", dep_root.as_relative_to_root(cfg)); + log.status("", src); + }); + + + // Since we build each plan separately independently, the default options are sufficient. + // The options are needed further when merging assets into a package. + let dep_opts = Default::default(); + let report = apply(cache, plan, &dest, &dep_opts, cfg)?; + + + // print report: + for (x, (m, results)) in report.results.iter().enumerate() { + let results = results.iter().enumerate(); + let expr = m.exprs(); + let incs = m.sources(); + + for (y, res) in results { + let path = incs[y].target(); + let path = path.as_relative_to_root(cfg); + match res { + Ok(op) => { + cfg.log().verbose(|mut log| { + let msg = format!("asset [{x}:{y}] {}", path.display()); + log.status(format!("{op:?}"), msg) + }) + }, + Err(err) => { + use fs_extra::error::ErrorKind as FsExtraErrorKind; - // TODO: list deps in the plan - - for (package, metadata) in packages { - let locked = layout.lock_mut(config.workspace.gctx())?; - let dev = has_dev && package.package_id() == target_pid; - let err_msg = |err| format!("{err}, caused when planning assets for {}.", package.package_id()); - - match plan::plan_for(config, package, &metadata, &env, &locked, dev) { - // nothing to pack: - Ok(plan) if plan.is_empty() => { - config.log() - .verbose(|mut log| log.status("Skip", format!("{} without plan", package.package_id()))) - // TODO: add clean assets task for `package`/`kind` - // Also remove old build-plan. - // Here and below for error case. - }, - - // report and continue: - Err(err) if config.compile_options.build_config.keep_going => { - let msg = format!("{} Continuing because `keep-going` is set.", err_msg(&err)); - config.log().error(msg) - }, - - // abort: - Err(err) => { - config.log().error(err_msg(&err)); - return Err(err); - }, - - // add plan to pack: - Ok(plan) => { - // TODO: Check main/dev is empty and add clean assets task for `package`/`kind` - // Also remove old build-plan. - - options.insert(package, metadata); - plans.insert(package, plan); - }, - } - } + let error = match &err.kind { + FsExtraErrorKind::Io(err) => format!("IO: {err}"), + FsExtraErrorKind::StripPrefix(err) => format!("StripPrefix: {err}"), + FsExtraErrorKind::OsString(err) => format!("OsString: {err:?}"), + _ => err.to_string(), + }; + let message = format!( + "Asset [{x}:{y}], rule: '{} <- {} | {}', {error}", + expr.0.original(), + expr.1.original(), + path.display() + ); - // report if needed: - if config.compile_options.build_config.emit_json() || config.compile_options.build_config.build_plan { - for (package, plan) in plans.iter() { - for (plan, kind) in plan.main - .as_ref() - .into_iter() - .map(|plan| (plan, AssetKind::Package)) - .chain(plan.dev.as_ref().into_iter().map(|plan| (plan, AssetKind::Dev))) - { - let message = plan.printable_serializable(package.package_id(), kind); - config.workspace.gctx().shell().print_json(&message)?; + cfg.log().status_with_color("Error", message, Color::Red) + }, + }; } } - } else { - config.workspace - .gctx() - .shell() - .verbose(|shell| { - for (package, plan) in plans.iter() { - for plan in plan.main - .as_ref() - .into_iter() - .chain(plan.dev.as_ref().into_iter()) - { - shell.status("Assets", format!("build plan for {}", package.package_id()))?; - plan.pretty_print(shell, config.workspace.root())?; - } - } - Ok(()) - }) - .log_err() - .ok(); - } - /* NOTE for future: how to resolve conflicts better: - - merge all plans, where - - resolve conflicts as it happening in the `build_plan()::re-mapping`: - e.g.: Mapping::* -> Mapping::ManyInto - */ - - { - // validate plans: - let mut has_errors = false; - let mut targets = HashMap::new(); - - let mut check_duplicates = |package_id: PackageId, target_kind: AssetKind, plan| { - for target in plan { - if let Some((pid, kind)) = targets.get::>(&target) { - has_errors = true; - let err_msg = |pid, kind| { - match kind { - AssetKind::Package => format!("{pid} in [assets]"), - AssetKind::Dev => format!("{pid} in [dev-assets]"), - } - }; - let a = err_msg(pid, *kind); - let b = err_msg(&package_id, target_kind); - let message = format!( - "Duplicate dev-asset destination: '{}':\n\t{a}\n\t{b}", - target.as_relative_to_root(config).display(), - ); - - config.log().error(message); - } else { - targets.insert(target, (package_id, target_kind)); - } - } - }; + // TODO: log report.exclusions + if report.has_errors() && !cfg.compile_options.build_config.keep_going { + use anyhow::Error; - for (package, plan) in plans.iter() { - let package_id = package.package_id(); - if let Some(plan) = plan.main.as_ref() { - check_duplicates(package_id, AssetKind::Package, plan.as_inner().targets()); - } - if package_id == target_pid { - if let Some(plan) = plan.dev.as_ref() { - check_duplicates(package_id, AssetKind::Dev, plan.as_inner().targets()); - } + #[derive(Debug)] + pub struct Mapping(String); + impl std::error::Error for Mapping {} + impl std::fmt::Display for Mapping { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { self.0.fmt(f) } } - } - if has_errors { - if config.compile_options.build_config.keep_going { - config.log() - .status("Drop", format!("assets for {}", package.package_id())); - continue; - } else { - bail!("Duplicated asset destination, stopping compilation."); - } + let err = report.results + .into_iter() + .filter_map(|(map, res)| { + if res.iter().any(|res| res.is_err()) { + let err = Mapping(map.pretty_print_compact()); + let err = res.into_iter() + .filter_map(|res| res.err()) + .fold(Error::new(err), Error::context); + Some(err) + } else { + None + } + }) + .fold(Error::msg("Assets build failed"), Error::context); + return Err(err); } - // TODO: Also check sources duplicates, but only warn. - } - - - // finally apply plans: - if !config.dry_run && !config.compile_options.build_config.build_plan && !plans.is_empty() { - let mut locked = layout.lock_mut(config.workspace.gctx())?; - locked.prepare()?; - - for (dependency, mut plan) in plans.into_iter() { - let dep_pkg_id = dependency.package_id(); - - let apply = |plan: CachedPlan, kind| -> CargoResult<()> { - let dest = match kind { - AssetKind::Package => locked.as_inner().assets(), - AssetKind::Dev => locked.as_inner().assets_dev(), - }; - let kind_prefix = match kind { - AssetKind::Package => "", - AssetKind::Dev => "dev-", - }; - - let dep_root = dependency.manifest_path().parent().unwrap(); - - config.log() - .status("Build", format!("{kind_prefix}assets for {}", dep_pkg_id)); - config.log().verbose(|mut log| { - let dest = format!("destination: {}", dest.as_relative_to_root(config).display()); - log.status("", dest); - let src = format!("root {}", dep_root.as_relative_to_root(config).display()); - log.status("", src); - if dep_root != plan.path { - let path = plan.plan.crate_root(); - let src = format!("root (plan) {}", path.as_relative_to_root(config).display()); - log.status("", src); - } - }); - - - // FIXME: use primary (top-level) assets-options, but not options of dependency! - let metadata = options.get(dependency).expect("Metadata is gone, impossible!"); - let report = plan.apply(&dest, &metadata.assets_options(), config)?; - - - // print report: - for (x, (m, results)) in report.results.iter().enumerate() { - let results = results.iter().enumerate(); - let expr = m.exprs(); - let incs = m.sources(); - - for (y, res) in results { - let path = incs[y].target(); - let path = path.as_relative_to_root(config); - match res { - Ok(op) => { - config.log().verbose(|mut log| { - let msg = format!("asset [{x}:{y}] {}", path.display()); - log.status(format!("{op:?}"), msg) - }) - }, - Err(err) => { - use fs_extra::error::ErrorKind as FsExtraErrorKind; - - let error = match &err.kind { - FsExtraErrorKind::Io(err) => format!("IO: {err}"), - FsExtraErrorKind::StripPrefix(err) => format!("StripPrefix: {err}"), - FsExtraErrorKind::OsString(err) => format!("OsString: {err:?}"), - _ => err.to_string(), - }; - let message = format!( - "Asset [{x}:{y}], rule: '{} <- {} | {}', {error}", - expr.0.original(), - expr.1.original(), - path.display() - ); - - config.log().status_with_color("Error", message, Color::Red) - }, - }; - } - } - - if report.has_errors() && !config.compile_options.build_config.keep_going { - bail!("Assets build failed."); - } - - - // finally build with pdc: - // if not disallowed explicitly - if config.skip_prebuild { - const REASON: &str = "as requested"; - let msg = format!("{kind_prefix}assets pre-build for {}, {REASON}.", dep_pkg_id); - config.log().status("Skip", msg); - } else { - match pdc::build(config, &dependency.package_id(), locked.as_inner(), kind) { - Ok(_) => { - let msg = format!("{kind_prefix}assets for {}", dep_pkg_id); - config.log().status("Finished", msg); - }, - Err(err) => { - let msg = format!("build {kind_prefix}assets with pdc failed: {err}"); - config.log().status_with_color("Error", msg, Color::Red); - if !config.compile_options.build_config.keep_going { - bail!("Assets build failed."); - } - }, - } - } - - Ok(()) - }; - - // main: - let mut main_cache_hit = false; - if dep_pkg_id == target_pid { - if let Some(plan) = plan.main.take() { - if plan.difference.is_same() { - config.log().status( - "Skip", - format!("{}, cache state is {:?}", dep_pkg_id, &plan.difference), - ); - main_cache_hit = true; - // continue; - } - - apply(plan, AssetKind::Package)?; - } - } - // dev: - if dep_pkg_id == target_pid { - if let Some(plan) = plan.dev.take() { - if main_cache_hit && plan.difference.is_same() { - config.log().status( - "Skip", - format!("{} (dev), cache state is {:?}", dep_pkg_id, &plan.difference), - ); - continue; + // finally build with pdc: + // if not disallowed explicitly + if cfg.skip_prebuild { + const REASON: &str = "as requested"; + let msg = format!("{kind_prefix}assets pre-build for {}, {REASON}.", key.id); + cfg.log().status("Skip", msg); + } else { + match pdc::build(cfg, &key.id, locked.as_inner(), kind) { + Ok(_) => { + let msg = format!("{kind_prefix}assets for {}", key.id); + cfg.log().status("Finished", msg); + }, + Err(err) => { + let msg = format!("build {kind_prefix}assets with pdc failed: {err}"); + cfg.log().status_with_color("Error", msg, Color::Red); + if !cfg.compile_options.build_config.keep_going { + bail!("Assets build failed."); } - - apply(plan, AssetKind::Dev)?; - } + }, } } - - locked.unlock(); - - log::debug!( - "Assets artifact for {} at {}", - package.package_id(), - crate::layout::Layout::dest(&layout).as_relative_to_root(config) - .display() - ); - let metadata = options.remove(package); - artifacts.insert( - package, - AssetsArtifact { package_id: package.package_id(), - layout, - metadata, }, - ); } - } - if config.compile_options.build_config.build_plan { - config.workspace.gctx().shell().out().flush()?; - config.workspace.gctx().shell().err().flush()?; - std::process::exit(0); - } + // Finale: - Ok(artifacts) -} + locked.unlock(); -fn deps_tree_metadata<'cfg: 'r, 't: 'r, 'r>(package: &'cfg Package, - bcx: &'t LazyBuildContext<'t, 'cfg>, - config: &Config<'_>) - -> CargoResult> { - let mut packages = HashMap::new(); - if let Some(metadata) = playdate_metadata(package) { - // if explicitly allowed collect deps => scan deps-tree - if metadata.assets_options().dependencies() { - log::debug!("inspecting deps-tree of {}", package.package_id()); + let art_index = artifacts.artifacts.len(); + artifacts.artifacts.push(Artifact { kind, + package_id: key.id, + layout: layout.clone() }); - packages.insert(package, metadata); + log::debug!( + "Assets artifact for {} at {:?}", + key.id, + crate::layout::Layout::dest(&layout).as_relative_to_root(cfg) + ); - let bcx = bcx.get()?; + for (r_key, index) in plans.targets.iter().filter(|(_, i)| i.contains(&index)) { + artifacts.index + .entry(r_key.to_owned()) + .or_insert(Vec::with_capacity(index.len())) + .push(art_index); + } + } - // TODO: Cache hash of bcx.unit_graph in the assets-build-plan - // find this package in roots: - let root = bcx.unit_graph - .keys() - .find(|u| u.pkg.package_id() == package.package_id()); - let mut dependencies = HashSet::new(); + cfg.log_extra_verbose(|mut logger| { + artifacts.iter().for_each(|(root, arts)| { + use cargo::core::compiler::CompileKind; + let ct: Cow<_> = match root.node().unit().platform { + CompileKind::Host => "host".into(), + CompileKind::Target(kind) => kind.short_name().to_owned().into(), + }; - if let Some(deps) = root.and_then(|root| bcx.unit_graph.get(root)) { - // find all dependencies: - dependencies.extend( - deps.iter() - .flat_map(|ud| bcx.unit_graph.get_key_value(&ud.unit).map(|(u, _)| u)), + let root = format!( + "{} {} of {} for {ct}", + root.node().target().kind().description(), + root.node().target().name, + root.node().package_id().name(), + ); + logger.status("Assets", format!("artifacts for {root}:")); + arts.for_each(|art| { + let dest = match art.kind { + Kind::Package => art.layout.assets(), + Kind::Dev => art.layout.assets_dev(), + }; + let msg = format!( + "[{:?}] {} - {:?}", + art.kind, + art.package_id.name(), + dest.as_relative_to_root(cfg) ); + logger.status("", msg); + }); + }); + }); - - let mut last_length = 0; - while last_length != dependencies.len() { - let pre_last_length = dependencies.len(); - - let next = dependencies.iter() - .flat_map(|u| { - bcx.unit_graph.get(u).into_iter().flat_map(|deps| { - deps.iter().flat_map(|ud| { - bcx.unit_graph - .get_key_value(&ud.unit) - .map(|(u, _)| u) - }) - }) - }) - .filter(|u| !dependencies.contains(u)) - .collect::>(); - dependencies.extend(next); - last_length = pre_last_length; - } - - // dedup dependencies, choose only highest versions - let mut id_ver = dependencies.iter() - .filter(|u| u.pkg.package_id() != package.package_id()) - .filter(|u| u.pkg.manifest().custom_metadata().is_some()) - .map(|u| { - let name = u.pkg.name(); - let source_id = u.pkg.package_id().source_id(); - let versions = dependencies.iter() - .filter(|u| { - u.pkg.name() == name && - u.pkg.package_id().source_id() == - source_id - }) - .map(|u| (u.pkg.version(), *u)) - .collect::>(); - ((name, source_id), versions) - }) - .collect::>(); - - id_ver.values_mut().for_each(|vers| { - vers.sort_by_key(|(v, _)| *v); - vers.dedup_by_key(|(v, _)| *v); - }); - - - dependencies.retain(|u| { - let key = (u.pkg.name(), u.pkg.package_id().source_id()); - id_ver.get(&key) - .and_then(|vec| vec.last()) - .map(|(_, u)| *u) - .filter(|u1| u == u1) - .is_some() - }); - } - - - let with_meta = dependencies.into_iter() - .inspect(|u| { - config.log().verbose(|mut log| { - log.status("Check", format!("{}", u.pkg.package_id())) - }) - }) - .filter_map(|u| playdate_metadata(&u.pkg).map(|m| (&u.pkg, m))); + Ok(artifacts) +} - packages.extend(with_meta); +fn apply<'l, 'r>(cache: cache::PlanCache, + plan: BuildPlan<'l, 'r>, + dest: &Path, + options: &AssetsOptions, + config: &Config) + -> CargoResult> { + use crate::playdate::assets::apply_build_plan; + + let report = apply_build_plan(plan, dest, options)?; + // and finally save cache of just successfully applied plan: + // only if there is no errors + if !report.has_errors() { + if let Some(data) = cache.serialized.as_deref() { + log::trace!("writing cache to {:?}", cache.path); + std::fs::write(&cache.path, data)?; + config.log().verbose(|mut log| { + let path = cache.path.as_relative_to_root(config); + log.status("Cache", format_args!("saved to {}", path.display())); + }); } else { - packages.insert(package, metadata); + config.log().verbose(|mut log| { + log.status("Cache", "nothing to save"); + }); } + } else { + config.log().verbose(|mut log| { + let message = "build has errors, so cache was not saved"; + log.status("Cache", message); + }); } - Ok(packages) -} - -pub fn playdate_metadata(package: &Package) -> Option { - package.manifest() - .custom_metadata() - .and_then(|m| m.as_table().map(|t| t.get(METADATA_FIELD))) - .flatten() - .and_then(|v| v.to_owned().try_into::().log_err().ok()) + Ok(report) } diff --git a/cargo/src/assets/pdc.rs b/cargo/src/assets/pdc.rs index 646f10cb..e3e2cd19 100644 --- a/cargo/src/assets/pdc.rs +++ b/cargo/src/assets/pdc.rs @@ -12,21 +12,21 @@ use crate::config::Config; use crate::layout::PlaydateAssets; use crate::proc::logging::cmd_logged; -use super::plan::AssetKind; +use super::Kind; pub fn build(config: &Config, package_id: &PackageId, layout: &PlaydateAssets, - kind: AssetKind) + kind: Kind) -> CargoResult<()> { let (src, build) = match kind { - AssetKind::Package => { + Kind::Package => { let src = layout.assets(); let build = layout.build(); (src, build) }, - AssetKind::Dev => { + Kind::Dev => { let src = layout.assets_dev(); let build = layout.build_dev(); std::fs::create_dir(&build).ok(); diff --git a/cargo/src/assets/plan.rs b/cargo/src/assets/plan.rs index 49a4df31..513da557 100644 --- a/cargo/src/assets/plan.rs +++ b/cargo/src/assets/plan.rs @@ -1,760 +1,418 @@ -use std::collections::HashMap; -use std::path::{PathBuf, Path}; -use anyhow::anyhow; -use cargo::core::Shell; -use cargo::core::{Package, PackageId}; -use playdate::assets::BuildReport; -use playdate::assets::apply_build_plan; -use playdate::config::Env; +use std::path::Path; +use std::borrow::Cow; +use std::collections::{BTreeMap, HashMap, HashSet}; + +use cargo::core::PackageId; +use cargo::util::CargoResult; + +use playdate::assets::plan::build_plan; +use playdate::assets::plan::BuildPlan; +use playdate::consts::SDK_ENV_VAR; +use playdate::manifest::PackageSource as _; use playdate::metadata::format::AssetsOptions; use playdate::metadata::source::MetadataSource as _; -use crate::config::Config; -use crate::utils::path::AsRelativeTo; -use playdate::consts::SDK_ENV_VAR; -use cargo::util::CargoResult; -pub use playdate::metadata::format::Metadata; -use playdate::assets::plan::BuildPlan as AssetsPlan; -use playdate::assets::plan::build_plan as assets_build_plan; -use try_lazy_init::Lazy; -use crate::layout::{PlaydateAssets, LayoutLock}; +use crate::utils::cargo::meta_deps::MetaDeps; +use crate::utils::cargo::meta_deps::{Node, RootNode}; +use crate::utils::path::AsRelativeTo; +use crate::config::Config; -pub struct LazyEnvBuilder<'a, 'cfg> { - config: &'a Config<'cfg>, - package_id: PackageId, - root: &'cfg Path, - env: Lazy, +pub struct AssetsPlans<'cfg> { + pub plans: Vec>, + /// per-dep ?dev plan + pub index: BTreeMap, + /// per-root plans to merge + pub targets: BTreeMap>, } -impl<'a, 'cfg> LazyEnvBuilder<'a, 'cfg> { - pub fn new(config: &'cfg Config, pkg_id: PackageId, root: &'cfg Path) -> Self { - Self { env: Lazy::new(), - package_id: pkg_id, - root, - config } - } +/// Target-agnostic package key. +#[derive(Debug, Clone, Copy, Hash, PartialEq, PartialOrd, Eq, Ord)] +pub struct Key { + pub id: PackageId, + pub dev: bool, +} - pub fn new_for(config: &'cfg Config, package: &'cfg Package) -> Self { - Self::new(config, package.package_id(), package.root()) +impl Key { + fn with_dev(&self, dev: bool) -> Self { + Self { id: self.id.to_owned(), + dev } } - - pub fn get(&'a self) -> CargoResult<&'a Env> { - self.env.try_get_or_create(move || { - let root = self.root.display().to_string(); - let vars = vec![ - ("CARGO_PKG_NAME", self.package_id.name().to_string()), - ("CARGO_MANIFEST_DIR", root.to_string()), - ]; - - let mut env = Env::try_from_iter(vars.into_iter()).map_err(|err| anyhow::anyhow!("{err}"))?; - - // add global environment: - for (k, v) in std::env::vars() { - if !env.vars.contains_key(&k) { - env.vars.insert(k, v); - } - } - - if let Some(path) = self.config.sdk_path.as_ref() { - env.vars.insert(SDK_ENV_VAR.into(), path.display().to_string()); - } - - Ok::<_, anyhow::Error>(env) - }) +} +impl From<&'_ Node<'_>> for Key { + fn from(node: &'_ Node<'_>) -> Self { + Key { id: node.package_id().to_owned(), + dev: node.target().is_dev() } } } -pub type LockedLayout<'t> = LayoutLock<&'t mut PlaydateAssets>; - - -pub mod proto { - use std::borrow::Cow; - use std::collections::{BTreeMap, HashMap, HashSet}; - use std::path::Path; - - use super::{PackageId, Config, CargoResult}; - use super::assets_build_plan; - - use playdate::assets::plan::BuildPlan; - use playdate::consts::SDK_ENV_VAR; - use playdate::manifest::PackageSource as _; - use playdate::metadata::format::AssetsOptions; - use playdate::metadata::source::MetadataSource as _; - - use crate::utils::cargo::meta_deps::MetaDeps; - use crate::utils::cargo::meta_deps::{Node, RootNode}; - use crate::utils::path::AsRelativeTo; - - - pub struct AssetsPlans<'cfg> { - pub plans: Vec>, - /// per-dep ?dev plan - pub index: BTreeMap, - /// per-root plans to merge - pub targets: BTreeMap>, +/// Target-agnostic root-package key with all dependencies. +#[derive(Debug, Clone, Hash, PartialEq, PartialOrd, Eq, Ord)] +pub struct RootKey { + /// Dependencies + id: Vec, + /// Primary target is dev + dev: bool, +} +impl From<&'_ RootNode<'_>> for RootKey { + fn from(root: &'_ RootNode<'_>) -> Self { + Self { dev: root.node().target().is_dev(), + id: root.deps().iter().map(|d| d.package_id().to_owned()).collect() } } - - /// Target-agnostic package key. - #[derive(Debug, Clone, Copy, Hash, PartialEq, PartialOrd, Eq, Ord)] - pub struct Key { - pub id: PackageId, - pub dev: bool, +} +impl RootKey { + pub fn dev(&self) -> bool { self.dev } + + pub fn is_for(&self, root: &'_ RootNode<'_>) -> bool { + root.node().target().is_dev() == self.dev && + root.deps() + .iter() + .enumerate() + .all(|(i, d)| self.id.get(i).filter(|k| *k == d.package_id()).is_some()) } +} - impl Key { - fn with_dev(&self, dev: bool) -> Self { - Self { id: self.id.to_owned(), - dev } - } - } - impl From<&'_ Node<'_>> for Key { - fn from(node: &'_ Node<'_>) -> Self { - Key { id: node.package_id().to_owned(), - dev: node.target().is_dev() } - } - } +pub fn plan_all<'cfg>(cfg: &Config<'cfg>, tree: &MetaDeps<'cfg>) -> CargoResult> { + // results: + let mut plans = AssetsPlans { plans: Vec::with_capacity(tree.roots().len() * 2), + index: BTreeMap::new(), + targets: BTreeMap::new() }; + + // prepare env: + let global_env: BTreeMap<_, _> = + std::env::vars().chain({ + cfg.sdk() + .map(|sdk| sdk.path()) + .ok() + .or(cfg.sdk_path.as_deref()) + .map(|p| (SDK_ENV_VAR.to_string(), p.display().to_string())) + .into_iter() + }) + .collect(); + let env = |id: &PackageId, root: &Path| { + use playdate::config::Env; + + let vars = [ + (Cow::from("CARGO_PKG_NAME"), Cow::from(id.name().as_str())), + (Cow::from("CARGO_MANIFEST_DIR"), root.display().to_string().into()), + ]; + + let iter = global_env.iter() + .map(|(k, v)| (Cow::Borrowed(k.as_str()), Cow::Borrowed(v.as_str()))) + .chain(vars); + + Env::try_from_iter(iter).map_err(|err| anyhow::anyhow!("{err}")) + }; - /// Target-agnostic root-package key with all dependencies. - #[derive(Debug, Clone, Hash, PartialEq, PartialOrd, Eq, Ord)] - pub struct RootKey { - /// Dependencies - id: Vec, - /// Primary target is dev - dev: bool, - } - impl From<&'_ RootNode<'_>> for RootKey { - fn from(root: &'_ RootNode<'_>) -> Self { - Self { dev: root.node().target().is_dev(), - id: root.deps().iter().map(|d| d.package_id().to_owned()).collect() } - } - } - impl RootKey { - pub fn dev(&self) -> bool { self.dev } - - pub fn is_for(&self, root: &'_ RootNode<'_>) -> bool { - root.node().target().is_dev() == self.dev && - root.deps() - .iter() - .enumerate() - .all(|(i, d)| self.id.get(i).filter(|k| *k == d.package_id()).is_some()) - } - } + // use target-agnostic selection of roots: + for root in tree.roots_compile_target_agnostic() { + let meta_source = root.as_source(); - pub fn plan_all<'cfg>(cfg: &Config<'cfg>, tree: &MetaDeps<'cfg>) -> CargoResult> { - // results: - let mut plans = AssetsPlans { plans: Vec::with_capacity(tree.roots().len() * 2), - index: BTreeMap::new(), - targets: BTreeMap::new() }; + let options = meta_source.assets_options(); - // prepare env: - let global_env: BTreeMap<_, _> = - std::env::vars().chain({ - cfg.sdk() - .map(|sdk| sdk.path()) - .ok() - .or(cfg.sdk_path.as_deref()) - .map(|p| (SDK_ENV_VAR.to_string(), p.display().to_string())) - .into_iter() - }) - .collect(); - let env = |id: &PackageId, root: &Path| { - use playdate::config::Env; + let root_is_dev = root.node().target().is_dev(); - let vars = [ - (Cow::from("CARGO_PKG_NAME"), Cow::from(id.name().as_str())), - (Cow::from("CARGO_MANIFEST_DIR"), root.display().to_string().into()), - ]; + log::debug!( + "planning for {} ({}) +dev:{root_is_dev}", + root.package_id(), + root.node().target().kind().description() + ); + log::debug!(" dependencies are allowed: {}", options.dependencies()); - let iter = global_env.iter() - .map(|(k, v)| (Cow::Borrowed(k.as_str()), Cow::Borrowed(v.as_str()))) - .chain(vars); - Env::try_from_iter(iter).map_err(|err| anyhow::anyhow!("{err}")) - }; + let plan_key = RootKey::from(root); + if plans.targets.contains_key(&plan_key) { + log::debug!(" skip: already done"); + continue; + } - // use target-agnostic selection of roots: - for root in tree.roots_compile_target_agnostic() { - let meta_source = root.as_source(); + let mut indices = Vec::::with_capacity(root.deps().len()); - let options = meta_source.assets_options(); + for dep in root.deps().iter().rev() { + log::debug!(" planning dep: {}", dep.package_id()); - let root_is_dev = root.node().target().is_dev(); + let crate_root = dep.manifest_path() + .and_then(|p| p.parent()) + .ok_or_else(|| anyhow::anyhow!("Unable to get crate root"))?; - log::debug!( - "planning for {} ({}) +dev:{root_is_dev}", - root.package_id(), - root.node().target().kind().description() - ); - log::debug!(" dependencies are allowed: {}", options.dependencies()); + let env = env(dep.package_id(), crate_root)?; + // dep_key is dev only if this it a primary target (root unit) and dev is requested: + let with_dev = root_is_dev && dep.package_id() == root.package_id(); + let dep_key = Key::from(dep).with_dev(with_dev); - let plan_key = RootKey::from(root); - if plans.targets.contains_key(&plan_key) { - log::debug!(" skip: already done"); - continue; - } + let plan_for = |plans: &mut AssetsPlans, indices: &mut Vec, key: Key| -> anyhow::Result<()> { + let source = dep.as_source(); + let dev_prefix = key.dev.then_some("dev-").unwrap_or_default(); + if let Some(assets) = source.metadata() + .map(|m| if key.dev { m.dev_assets() } else { m.assets() }) && + !assets.is_empty() + { + match build_plan(&env, assets, &options, Some(crate_root.into())) { + Ok(plan) => { + let pid = key.id; + let is_dev = key.dev; + let dev_index = plans.plans.len(); + let compile_target_agnostic = plan.compile_target_agnostic(); + plans.index.insert(key, dev_index); + plans.plans.push(plan); + indices.push(dev_index); - let mut indices = Vec::::with_capacity(root.deps().len()); - - for dep in root.deps().iter().rev() { - log::debug!(" planning dep: {}", dep.package_id()); - - let crate_root = dep.manifest_path() - .and_then(|p| p.parent()) - .ok_or_else(|| anyhow::anyhow!("Unable to get crate root"))?; - - let env = env(dep.package_id(), crate_root)?; - - // dep_key is dev only if this it a primary target (root unit) and dev is requested: - let with_dev = root_is_dev && dep.package_id() == root.package_id(); - let dep_key = Key::from(dep).with_dev(with_dev); - - - let plan_for = - |plans: &mut AssetsPlans, indices: &mut Vec, key: Key, dev: bool| -> anyhow::Result<()> { - let source = dep.as_source(); - let name_log = dev.then_some("dev-").unwrap_or_default(); - if let Some(assets) = source.metadata() - .map(|m| if dev { m.dev_assets() } else { m.assets() }) && - !assets.is_empty() - { - match assets_build_plan(&env, assets, &options, Some(crate_root.into())) { - Ok(plan) => { - let pid = key.id; - let is_dev = key.dev; - let dev_index = plans.plans.len(); - let compile_target_agnostic = plan.compile_target_agnostic(); - plans.index.insert(key, dev_index); - plans.plans.push(plan); - indices.push(dev_index); - - log::debug!(" done: +#{dev_index} (dev:{is_dev})"); - cfg.log().verbose(|mut log| { - log.status("Plan", format_args!("{name_log}assets for {pid} planned",)) - }); - - if !compile_target_agnostic { - cfg.log() - .error("Assets is not compile-target-agnostic, this is not supported"); - } - }, - Err(err) => { - cfg.log() - .error(format_args!("{err}, caused when planning {name_log}assets for {}", key.id)); - return cfg.compile_options - .build_config - .keep_going - .then_some(()) - .ok_or(err.into()); - }, - } - } else { + log::debug!(" done: +#{dev_index} (dev:{is_dev})"); cfg.log().verbose(|mut log| { - log.status( - "Skip", - format_args!( - "{name_log}assets for {} without plan, reason: empty", - key.id - ), - ) - }) - } - Ok(()) - }; - - if let Some(i) = plans.index.get(&dep_key) { - // we already have plan for this dep - log::debug!(" done (~#{i}) (dev:{})", dep_key.dev); - indices.push(*i); - } else if with_dev && let Some(base_index) = plans.index.get(&dep_key.with_dev(false)).copied() { - // we already have plan for this dep, but not for dev part - indices.push(base_index); - log::debug!(" done (~#{base_index}) (dev:{})", dep_key.dev); - - plan_for(&mut plans, &mut indices, dep_key, true)?; - } else { - // else just build a plan - plan_for(&mut plans, &mut indices, dep_key, false)?; - - // also for dev targets if needed - if with_dev { - plan_for(&mut plans, &mut indices, dep_key, true)?; - } - } - } + log.status("Plan", format_args!("{dev_prefix}assets for {pid} planned")) + }); - - plans.targets - .entry(plan_key) - .and_modify(|vec| vec.append(&mut indices)) - .or_insert(indices); - } - - - // report: - cfg.log() - .status("Assets", "planning complete for all requested targets"); - cfg.log_extra_verbose(|mut logger| { - for (k, v) in &plans.targets { - let dev = k.dev.then_some(" +dev").unwrap_or_default(); - let key = k.id.iter().map(|p| p.name()).collect::>().join(", "); - logger.status("Plans", format_args!("for{dev} {key}")); - for i in v { - let plan = &plans.plans[*i]; - logger.status("Plan", format_args!("#{i}:\n{plan:>10}")); - } - } - }); - - // check: - for root in tree.roots() { - let key = RootKey::from(root); - debug_assert!(plans.targets.contains_key(&key)); - } - - Ok(plans) - } - - - /// Try to merge virtually and validate. - /// Emits warnings and errors, returns errors-chain. - pub fn merge_all_virtually<'cfg>(cfg: &Config<'cfg>, - tree: &MetaDeps<'cfg>, - plans: &AssetsPlans<'cfg>) - -> CargoResult<()> { - // prepare context: - let mut root_package: HashMap<&RootKey, HashSet<&PackageId>> = HashMap::with_capacity(tree.roots().len()); - let mut root_options: HashMap<&PackageId, AssetsOptions> = HashMap::with_capacity(tree.roots().len()); - - plans.targets - .iter() - .flat_map(|(key, _)| { - tree.roots() - .iter() - .filter(|r| key.is_for(r)) - .map(move |r| (key, r)) - }) - .for_each(|(key, root)| { - root_package.entry(key) - .or_insert_with(|| HashSet::with_capacity(tree.roots().len())) - .insert(root.package_id()); - - if !root_options.contains_key(root.package_id()) { - let options = root.as_source().assets_options(); - root_options.insert(root.package_id(), options); - } - }); - - - // Buffered errors: - let mut overrides = Vec::new(); - - - // merge, analyse: - for (key, index) in plans.targets.iter() { - // Note, correct ordering in `index` guaranteed by the planner. - - // Need merge many into one: - let _many = index.len() > 1; - - for root_id in root_package.get(key).into_iter().flat_map(|set| set.iter()) { - use playdate::assets::plan::*; - - log::trace!("v-merging for {} (dev:{})", root_id.name(), key.dev); - - let options = &root_options[root_id]; - - let mut _plan: Vec = Default::default(); - - - let mut targets = BTreeMap::new(); - let mut sources = BTreeMap::new(); - - - for i in index { - let next = &plans.plans[*i]; - - for (kind, dst, src) in next.iter_flatten() { - let target: Cow<_> = match kind { - MappingKind::AsIs | MappingKind::ManyInto => dst, - MappingKind::Into => dst.join(src.file_name().expect("filename")), - }.into(); - - // Save for future check if we already have this source: - sources.entry(Cow::from(src)) - .or_insert_with(|| Vec::with_capacity(2)) - .push(*i); - - // Check if we already have this target: - targets.entry(target.clone()) - .or_insert_with(|| Vec::with_capacity(2)) - .push(*i); - - if let Some(past) = targets.get(&target) && - past.len() > 1 - { - let id = past.iter() - .flat_map(|x| plans.index.iter().find_map(|(key, i)| (i == x).then_some(key))) - .collect::>(); - debug_assert!(!id.is_empty()); - - let this = id.last().unwrap(); - let other = &id[..id.len() - 1]; - - let dev = this.dev.then_some("dev-").unwrap_or_default(); - let others = other.is_empty() - .then_some("by itself") - .map(Cow::from) - .unwrap_or_else(|| { - other.iter() - .map(|k| { - let dev = k.dev.then_some("dev-").unwrap_or_default(); - format!("{}'s '{dev}assets'", k.id.name()) - }) - .collect::>() - .join(", ") - .into() - }); - - let name = this.id.name(); - let root_name = root_id.name(); - let why = format!("but that's not allowed by the top-level crate {root_name}"); - let msg = format!("{name}'s `{dev}assets.{target:?}` overrides {others}, {why}"); - - if options.overwrite() { - cfg.log().warn(msg) - } else { - cfg.log().error(&msg); - overrides.push(msg); + if !compile_target_agnostic { + cfg.log() + .error("Assets is not compile-target-agnostic, this is not supported"); } - } + }, + Err(err) => { + cfg.log() + .error(format_args!("{err}, caused when planning {dev_prefix}assets for {}", key.id)); + return cfg.compile_options + .build_config + .keep_going + .then_some(()) + .ok_or(err.into()); + }, } + } else { + cfg.log().verbose(|mut log| { + log.status( + "Skip", + format_args!( + "{dev_prefix}assets for {} without plan, reason: empty", + key.id + ), + ) + }) } + Ok(()) + }; + + if let Some(i) = plans.index.get(&dep_key) { + // we already have plan for this dep + log::debug!(" done (~#{i}) (dev:{})", dep_key.dev); + indices.push(*i); + } else if with_dev && let Some(base_index) = plans.index.get(&dep_key.with_dev(false)).copied() { + // we already have plan for this dep, but not for dev part + indices.push(base_index); + log::debug!(" done (~#{base_index}) (dev:{})", false); + + plan_for(&mut plans, &mut indices, dep_key.with_dev(true))?; + } else { + // else just build a plan + plan_for(&mut plans, &mut indices, dep_key.with_dev(false))?; - // Check if we already have this source: - for (src, index) in sources { - if index.len() < 2 { - continue; - } - - let id = index.into_iter() - .flat_map(|x| plans.index.iter().find_map(|(key, i)| (*i == x).then_some(key))) - .collect::>(); - debug_assert!(!id.is_empty()); - - let src_rel = src.as_relative_to_root(cfg); - let others = id.is_empty() - .then_some("itself") - .map(Cow::from) - .unwrap_or_else(|| { - id.into_iter() - .map(|k| { - let dev = k.dev.then_some("dev-").unwrap_or_default(); - format!("{}'s '{dev}assets'", k.id.name()) - }) - .collect::>() - .join(", ") - .into() - }); - let msg = format!("asset {src_rel:?} used multiple times in {others}"); - cfg.log().warn(msg); + // also for dev targets if needed + if with_dev { + plan_for(&mut plans, &mut indices, dep_key.with_dev(true))?; } } } - { - use err::Override; - use anyhow::Error; - overrides.is_empty() - .then_some(()) - .ok_or_else(|| overrides.into_iter().fold(Error::new(Override), Error::context)) - } + plans.targets + .entry(plan_key) + .and_modify(|vec| vec.append(&mut indices)) + .or_insert(indices); } - mod err { - #[derive(Debug)] - pub struct Override; - impl std::error::Error for Override {} - impl std::fmt::Display for Override { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { "Not allowed override".fmt(f) } - } - } -} - - -/// Returns `None` if there is no `assets` metadata. -pub fn plan_for<'cfg, 'env, 'l>(config: &'cfg Config, - package: &'cfg Package, - metadata: &Metadata, - env: &'cfg LazyEnvBuilder<'env, 'cfg>, - layout: &'l LockedLayout<'l>, - with_dev: bool) - -> CargoResult> { - let opts = metadata.assets_options(); - - let has_dev_assets = with_dev && !metadata.dev_assets().is_empty(); - let is_empty = metadata.assets().is_empty() && !has_dev_assets; + // report: + cfg.log() + .status("Assets", "planning complete for all requested targets"); + cfg.log_extra_verbose(|mut logger| { + for (k, v) in &plans.targets { + let dev = k.dev.then_some(" +dev").unwrap_or_default(); + let key = k.id.iter().map(|p| p.name()).collect::>().join(", "); + logger.status("Plans", format_args!("for{dev} {key}")); + for i in v { + let plan = &plans.plans[*i]; + logger.status("Plan", format_args!("#{i}:\n{plan:>10}")); + } + } + }); - if is_empty { - return Ok(PackageAssetsPlan { main: None, - dev: None }); + // check: + for root in tree.roots() { + let key = RootKey::from(root); + debug_assert!(plans.targets.contains_key(&key)); } - let env = env.get()?; - let root = package.manifest_path() - .parent() - .ok_or(anyhow!("No parent of manifest-path"))?; - - let main = if !metadata.assets().is_empty() { - let plan = assets_build_plan(env, metadata.assets(), opts.as_ref(), Some(root.into()))?; - - // main-assets plan: - let path = layout.as_inner().assets_plan_for(config, &package.package_id()); - let mut cached = CachedPlan::new(path, plan)?; - if config.compile_options.build_config.force_rebuild { - cached.difference = Difference::Missing; - } - - Some(cached) - } else { - None - }; - - - // dev-assets plan: - let dev = if has_dev_assets && !metadata.dev_assets().is_empty() { - let assets = metadata.dev_assets(); - let dev_plan = assets_build_plan(env, assets, opts.as_ref(), Some(root.into()))?; - - let path = layout.as_inner() - .assets_plan_for_dev(config, &package.package_id()); - let mut dev_cached = CachedPlan::new(path, dev_plan)?; - - // Inheritance, if main is stale or missing - this one is too: - if let Some(main) = main.as_ref() { - if !matches!(main.difference, Difference::Same) { - dev_cached.difference = main.difference; - } - } - - dev_cached.into() - } else { - None - }; - - Ok(PackageAssetsPlan { main, dev }) + Ok(plans) } -#[derive(Debug)] -pub struct PackageAssetsPlan<'t, 'cfg> { - /// Main build-plan. - /// - /// Can be empty, so `None`. - pub main: Option>, - - /// Dev-assets build-plan. - /// - /// Inherited by main `plan`. - /// - /// Can be empty, so `None`. - pub dev: Option>, -} - -impl<'t, 'cfg> PackageAssetsPlan<'t, 'cfg> { - pub fn is_empty(&self) -> bool { self.main.is_none() && self.dev.is_none() } -} - - -#[derive(Debug)] -pub struct CachedPlan<'t, 'cfg> { - /// Inner build-plan - pub plan: AssetsPlan<'t, 'cfg>, - - /// Path to the cache file - pub path: PathBuf, - - /// State of the cache - pub difference: Difference, - - serialized: Option, -} - - -impl<'t, 'cfg> CachedPlan<'t, 'cfg> { - #[must_use = "Cached plan must be used"] - fn new(path: PathBuf, plan: AssetsPlan<'t, 'cfg>) -> CargoResult { - let mut serializable = plan.iter_flatten_meta().collect::>(); - serializable.sort_by_key(|(_, _, (p, _))| p.to_string_lossy().to_string()); - let json = serde_json::to_string(&serializable)?; - - let difference = if path.try_exists()? { - if std::fs::read_to_string(&path)? == json { - log::debug!("Cached plan is the same"); - Difference::Same - } else { - log::debug!("Cache mismatch, need diff & rebuild"); - Difference::Different - } - } else { - log::debug!("Cache mismatch, full rebuilding"); - Difference::Missing - }; - - let serialized = (!difference.is_same()).then_some(json); - - Ok(Self { plan, - path, - difference, - serialized }) - } - - - pub fn apply(self, - dest: &Path, - options: &AssetsOptions, - config: &Config) - -> CargoResult> { - let cache = self.serialized; - let report = apply_build_plan(self.plan, dest, options)?; - // and finally save cache of just successfully applied plan: - // only if there is no errors - if !report.has_errors() { - if let Some(data) = &cache { - std::fs::write(&self.path, data)?; - config.log().verbose(|mut log| { - let path = self.path.as_relative_to_root(config); - log.status("Cache", format!("saved to {}", path.display())); - }); - } else { - config.log().verbose(|mut log| { - log.status("Cache", "nothing to save"); - }); - } - } else { - config.log().verbose(|mut log| { - let message = "build has errors, so cache was not saved"; - log.status("Cache", message); - }); - } - - Ok(report) - } - - - pub fn printable_serializable(&self, source: PackageId, kind: AssetKind) -> SerializablePlan<'_, 't, 'cfg> { - SerializablePlan { package: source, - plan: &self.plan, - difference: &self.difference, - path: &self.path, - kind } - } - - - pub fn as_inner(&self) -> &AssetsPlan<'t, 'cfg> { &self.plan } - - - pub fn pretty_print(&self, shell: &mut Shell, root: &Path) -> CargoResult<()> { - use playdate::assets::plan::*; - use playdate::assets::resolver::*; - - let title = |(left, right): &(Expr, Expr)| format!("rule: {} = {}", left.original(), right.original()); - let row_columns = |target: &Path, source: &Path| { - (format!(" {}", target.as_relative_to(&root).display()), - source.as_relative_to(&root).display().to_string()) - }; - - let mut sections = HashMap::new(); - for mapping in self.as_inner().as_inner().iter() { - match mapping { - Mapping::AsIs(inc, exprs) => { - sections.insert(title(exprs), vec![row_columns(&inc.target(), &inc.source())]); - }, - Mapping::Into(inc, exprs) => { - sections.insert(title(exprs), vec![row_columns(&inc.target(), &inc.source())]); - }, - Mapping::ManyInto { sources, - target, - exprs, - .. } => { - let mut rows = Vec::new(); - for inc in sources.iter() { - rows.push(row_columns(&target.join(inc.target()), &inc.source())); +/// Try to merge virtually and validate. +/// Emits warnings and errors, returns errors-chain. +pub fn merge_all_virtually<'cfg>(cfg: &Config<'cfg>, + tree: &MetaDeps<'cfg>, + plans: &AssetsPlans<'cfg>) + -> CargoResult<()> { + // prepare context: + let mut root_package: HashMap<&RootKey, HashSet<&PackageId>> = HashMap::with_capacity(tree.roots().len()); + let mut root_options: HashMap<&PackageId, AssetsOptions> = HashMap::with_capacity(tree.roots().len()); + + plans.targets + .iter() + .flat_map(|(key, _)| { + tree.roots() + .iter() + .filter(|r| key.is_for(r)) + .map(move |r| (key, r)) + }) + .for_each(|(key, root)| { + root_package.entry(key) + .or_insert_with(|| HashSet::with_capacity(tree.roots().len())) + .insert(root.package_id()); + + if !root_options.contains_key(root.package_id()) { + let options = root.as_source().assets_options(); + root_options.insert(root.package_id(), options); + } + }); + + + // Buffered errors: + let mut overrides = Vec::new(); + + + // merge, analyse: + for (key, index) in plans.targets.iter() { + // Note, correct ordering in `index` guaranteed by the planner. + + // Need merge many into one: + let _many = index.len() > 1; + + for root_id in root_package.get(key).into_iter().flat_map(|set| set.iter()) { + use playdate::assets::plan::*; + + log::trace!("v-merging for {} (dev:{})", root_id.name(), key.dev); + + let options = &root_options[root_id]; + + let mut _plan: Vec = Default::default(); + + + let mut targets = BTreeMap::new(); + let mut sources = BTreeMap::new(); + + + for i in index { + let next = &plans.plans[*i]; + + for (kind, dst, src) in next.iter_flatten() { + let target: Cow<_> = match kind { + MappingKind::AsIs | MappingKind::ManyInto => dst, + MappingKind::Into => dst.join(src.file_name().expect("filename")), + }.into(); + + // Save for future check if we already have this source: + sources.entry(Cow::from(src)) + .or_insert_with(|| Vec::with_capacity(2)) + .push(*i); + + // Check if we already have this target: + targets.entry(target.clone()) + .or_insert_with(|| Vec::with_capacity(2)) + .push(*i); + + if let Some(past) = targets.get(&target) && + past.len() > 1 + { + let id = past.iter() + .flat_map(|x| plans.index.iter().find_map(|(key, i)| (i == x).then_some(key))) + .collect::>(); + debug_assert!(!id.is_empty()); + + let this = id.last().unwrap(); + let other = &id[..id.len() - 1]; + + let dev = this.dev.then_some("dev-").unwrap_or_default(); + let others = other.is_empty() + .then_some("by itself") + .map(Cow::from) + .unwrap_or_else(|| { + other.iter() + .map(|k| { + let dev = k.dev.then_some("dev-").unwrap_or_default(); + format!("{}'s '{dev}assets'", k.id.name()) + }) + .collect::>() + .join(", ") + .into() + }); + + let name = this.id.name(); + let root_name = root_id.name(); + let msg = format!("{name}'s `{dev}assets.{target:?}` overrides {others}"); + + if options.overwrite() { + cfg.log().warn(msg) + } else { + let why = format!("but that's not allowed by the top-level crate {root_name}"); + cfg.log().error(format_args!("{msg}, {why}")); + overrides.push(msg); + } } - sections.insert(title(exprs), rows); - }, - } - } - - // calc max len for left column: - let mut max_len = 0; - for (_, rows) in sections.iter() { - for (left, _) in rows.iter() { - max_len = left.len().max(max_len); + } } - } - // add padding to left column: - for (_, rows) in sections.iter_mut() { - for (left, _) in rows.iter_mut() { - let diff = max_len - left.len(); - left.push_str(&" ".repeat(diff)); - } - } + // Check if we already have this source: + for (src, index) in sources { + if index.len() < 2 { + continue; + } - // print: - for (title, rows) in sections.iter_mut() { - shell.status("", title)?; - for (left, right) in rows.iter_mut() { - shell.status("", format!("{left} <- {right}"))?; + let id = index.into_iter() + .flat_map(|x| plans.index.iter().find_map(|(key, i)| (*i == x).then_some(key))) + .collect::>(); + debug_assert!(!id.is_empty()); + + let src_rel = src.as_relative_to_root(cfg); + let others = id.is_empty() + .then_some("itself") + .map(Cow::from) + .unwrap_or_else(|| { + id.into_iter() + .map(|k| { + let dev = k.dev.then_some("dev-").unwrap_or_default(); + format!("{}'s '{dev}assets'", k.id.name()) + }) + .collect::>() + .join(", ") + .into() + }); + let msg = format!("asset {src_rel:?} used multiple times in {others}"); + cfg.log().warn(msg); } } - - Ok(()) } -} -#[derive(Debug, Clone, Copy, serde::Serialize)] -pub enum Difference { - Same, - Different, - /// There is not cache file. - Missing, -} - -impl Difference { - /// Needs rebuild - pub fn is_same(&self) -> bool { matches!(self, Self::Same) } + { + use err::Override; + use anyhow::Error; + overrides.is_empty() + .then_some(()) + .ok_or_else(|| overrides.into_iter().fold(Error::new(Override), Error::context)) + } } -#[derive(Debug, serde::Serialize)] -pub struct SerializablePlan<'p, 't, 'cfg> { - package: PackageId, - - kind: AssetKind, - - #[serde(rename = "assets")] - plan: &'p AssetsPlan<'t, 'cfg>, - - #[serde(rename = "cache")] - difference: &'p Difference, - - #[serde(rename = "plan")] - path: &'p Path, -} - -#[derive(Debug, Clone, Copy, serde::Serialize)] -pub enum AssetKind { - Package, - Dev, +mod err { + #[derive(Debug)] + pub struct Override; + impl std::error::Error for Override {} + impl std::fmt::Display for Override { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { "Not allowed override".fmt(f) } + } } diff --git a/cargo/src/config.rs b/cargo/src/config.rs index 33874c6f..9b54103b 100644 --- a/cargo/src/config.rs +++ b/cargo/src/config.rs @@ -166,6 +166,7 @@ impl<'cfg> Config<'cfg> { }) } + #[deprecated = "corrupts cargo build cache"] pub fn build_plan(&self) -> CargoResult<&crate::utils::cargo::build_plan::format::BuildPlan> { self.build_plan .try_get_or_create(|| crate::utils::cargo::build_plan::build_plan(self)) diff --git a/cargo/src/main.rs b/cargo/src/main.rs index 4a42619b..8768bd53 100644 --- a/cargo/src/main.rs +++ b/cargo/src/main.rs @@ -65,7 +65,7 @@ fn execute(config: &Config) -> CargoResult<()> { match config.cmd { cli::cmd::Cmd::Assets => { let deps_tree = crate::utils::cargo::meta_deps::meta_deps(config)?; - assets::proto::build_all(config, &deps_tree)?; + assets::build_all(config, &deps_tree)?; }, cli::cmd::Cmd::Build => { @@ -81,7 +81,8 @@ fn execute(config: &Config) -> CargoResult<()> { }, cli::cmd::Cmd::Package => { - let assets = assets::build(config)?; + let deps_tree = crate::utils::cargo::meta_deps::meta_deps(config)?; + let assets = assets::build_all(config, &deps_tree)?; let products = build::build(config)?; log::debug!("assets artifacts: {}", assets.len()); @@ -145,8 +146,10 @@ fn execute(config: &Config) -> CargoResult<()> { bail!("Nothing found to run"); } + let deps_tree = crate::utils::cargo::meta_deps::meta_deps(config)?; + // build requested package(s): - let assets = assets::build(config)?; + let assets = assets::build_all(config, &deps_tree)?; let mut products = build::build(config)?; // filter products with expected: @@ -179,8 +182,7 @@ fn execute(config: &Config) -> CargoResult<()> { } let package = packages.first().unwrap(); - config.log() - .build_finished(true, Some(package.package.package_id())); + config.log().build_finished(true, Some(package.package_id)); { diff --git a/cargo/src/package/mod.rs b/cargo/src/package/mod.rs index 43ccc09e..0a5964bc 100644 --- a/cargo/src/package/mod.rs +++ b/cargo/src/package/mod.rs @@ -7,6 +7,7 @@ use std::process::Command; use anyhow::anyhow; use anyhow::bail; +use cargo::core::PackageId; use cargo::CargoResult; use cargo::core::Package; use cargo::core::compiler::CompileKind; @@ -20,13 +21,11 @@ use playdate::layout::Layout; use playdate::layout::Name; use playdate::manifest::format::ManifestFmt; use playdate::manifest::PackageSource; -use playdate::metadata::format::Metadata; use playdate::metadata::validation::Validate; use playdate::metadata::validation::ValidateCrate; -use crate::assets::AssetsArtifact; -use crate::assets::AssetsArtifacts; -use crate::assets::playdate_metadata; +use crate::assets::Artifacts as AssetsArtifacts; +use crate::assets::Artifact as AssetsArtifact; use crate::build::BuildProduct; use crate::config::Config; use crate::layout::CrossTargetLayout; @@ -34,25 +33,28 @@ use crate::layout::LayoutLockable; use crate::layout::ForTargetLayout; use crate::proc::logging::cmd_logged; use crate::proc::reader::format::ArtifactProfile; +use crate::utils::cargo::meta_deps::RootNode; use crate::utils::path::AsRelativeTo; mod ar; #[derive(Debug)] -pub struct Product<'p> { - pub package: &'p Package, +pub struct Product { + pub package_id: PackageId, pub crate_types: Vec, pub targets: Vec, + /// Build-product name pub name: String, + /// Path of produced artifact - pdx-dir or zip-file pub path: PathBuf, } -pub fn build_all<'b>(config: &'_ Config, - assets: AssetsArtifacts<'_>, - products: Vec>) - -> CargoResult>> { +pub fn build_all(config: &'_ Config, + assets: AssetsArtifacts<'_, '_>, + products: Vec>) + -> CargoResult> { let products: Vec = products.into_iter().flat_map(TryInto::try_into).collect(); let mut targets = HashMap::<_, Vec<_>>::new(); @@ -68,17 +70,63 @@ pub fn build_all<'b>(config: &'_ Config, let mut results = Vec::new(); for ((package, _), mut products) in targets { + let package_id = package.package_id(); + + log::debug!( + "Looking for assets artifacts for ({}) {}::{} for {}:", + &products[0].src_ct, + package_id, + products[0].name, + match &products[0].ck { + CompileKind::Host => "host", + CompileKind::Target(ref kind) => kind.short_name(), + } + ); + + let (root, assets) = assets.iter() + .find_map(|(r, arts)| { + let unit = r.node().unit(); + (unit.package_id == package_id && + unit.platform == products[0].ck && + unit.target.crate_types.contains(&products[0].src_ct) && + unit.target.name == products[0].name) + .then_some((r, Some(arts))) + }) + .or_else(|| { + assets.tree + .roots() + .iter() + .find(|r| { + let unit = r.node().unit(); + unit.package_id == package_id && + unit.platform == products[0].ck && + unit.target.crate_types.contains(&products[0].src_ct) && + unit.target.name == products[0].name + }) + .map(|r| (r, None)) + }) + .ok_or_else(|| { + anyhow!( + "Root not found for ({}) {}::{} for {}", + &products[0].src_ct, + package_id, + products[0].name, + match products[0].ck { + CompileKind::Host => "host", + CompileKind::Target(ref kind) => kind.short_name(), + } + ) + })?; + match products.len() { 0 => unreachable!("impossible len=0"), 1 => { - let assets = assets.get(package); let product = products.pop().unwrap(); - let result = package_single_target(config, product, assets)?; + let result = package_single_target(config, product, root, assets)?; results.push(result); }, _ => { - let assets = assets.get(package); - let result = package_multi_target(config, package, products, assets)?; + let result = package_multi_target(config, package, products, root, assets)?; results.push(result); }, } @@ -88,10 +136,11 @@ pub fn build_all<'b>(config: &'_ Config, } -fn package_single_target<'p>(config: &Config, - product: SuccessfulBuildProduct<'p>, - assets: Option<&AssetsArtifact>) - -> CargoResult> { +fn package_single_target<'p, 'art>(config: &Config, + product: SuccessfulBuildProduct<'p>, + root: &RootNode<'_>, + assets: Option>) + -> CargoResult { let presentable_name = product.presentable_name(); config.log().status( "Packaging", @@ -101,18 +150,18 @@ fn package_single_target<'p>(config: &Config, ), ); + + if config.compile_options.build_config.force_rebuild { + // TODO: clean entire product.layout.build() if force rebuild requested + } + + if let Some(assets) = assets { - assert_eq!(assets.package_id, product.package.package_id()); - log::debug!("Preparing assets for packaging {}", product.presentable_name()); - - prepare_assets( - config, - assets, - product.example, - product.layout.build(), - true, - product.layout.root(), - )?; + for art in assets { + log::debug!("Packaging assets {:?} {}", art.kind, product.presentable_name()); + + prepare_assets(config, art, product.layout.build(), true, product.layout.root())?; + } } // manifest: @@ -122,7 +171,7 @@ fn package_single_target<'p>(config: &Config, config, &product.layout, product.package, - assets, + root.as_source(), cargo_target, product.example, )?; @@ -137,7 +186,7 @@ fn package_single_target<'p>(config: &Config, } let result = Product { name: product.name, - package: product.package, + package_id: product.package.package_id(), crate_types: vec![product.src_ct.clone()], targets: vec![product.ck], path: artifact.to_path_buf() }; @@ -155,11 +204,12 @@ fn package_single_target<'p>(config: &Config, /// So one executable and one or two dylibs. /// /// __Can't mix macos dylib with linux dylib in a one package.__ -fn package_multi_target<'p>(config: &Config, - package: &'p Package, - products: Vec, - assets: Option<&AssetsArtifact>) - -> CargoResult> { +fn package_multi_target<'p, 'art>(config: &Config, + package: &'p Package, + products: Vec, + root: &RootNode<'_>, + assets: Option>) + -> CargoResult { let src_cts = products.iter() .map(|p| format!("{}", p.src_ct)) .collect::>() @@ -239,15 +289,14 @@ fn package_multi_target<'p>(config: &Config, let mut layout = CrossTargetLayout::new(config, package.package_id(), Some(layout_target_name))?.lock(config.workspace .gctx())?; - if let Some(assets) = assets { - debug_assert_eq!( - layout.as_ref().assets_layout(config).root(), - assets.layout.root(), - "wrong layout root" - ); - } crate::layout::Layout::prepare(&mut layout.as_mut())?; + + if config.compile_options.build_config.force_rebuild { + // TODO: clean entire product.layout.build() if force rebuild requested + } + + let mut dev = Default::default(); for product in &products { log::debug!("Preparing binaries for packaging {}", product.presentable_name()); @@ -263,16 +312,25 @@ fn package_multi_target<'p>(config: &Config, // Then the same as for single-product package: if let Some(assets) = assets { - log::debug!("Preparing assets for packaging {}", assets.package_id.name()); - assert_eq!(package.package_id(), assets.package_id, "package must be same"); - prepare_assets( - config, - assets, - dev.is_some(), - layout.build(), - true, - layout.as_inner().target(), - )?; + let mut has_dev = false; + for artifact in assets { + log::debug!( + "Packaging assets {:?} {}", + artifact.kind, + artifact.package_id.name() + ); + + debug_assert_eq!( + layout.as_ref().assets_layout(config).root(), + artifact.layout.root(), + "wrong layout root" + ); + + has_dev = has_dev || artifact.kind.is_dev(); + + prepare_assets(config, artifact, layout.build(), true, layout.as_inner().target())?; + } + assert_eq!(dev.is_some(), has_dev); } // manifest: @@ -283,7 +341,7 @@ fn package_multi_target<'p>(config: &Config, config, &layout, package, - assets, + root.as_source(), cargo_target, products[0].example, )?; @@ -302,7 +360,7 @@ fn package_multi_target<'p>(config: &Config, soft_link_checked(&artifact, &link, true, product.layout.root())?; } - let result = Product { package, + let result = Product { package_id: package.package_id(), name: products[0].name.clone(), crate_types: products.iter().map(|p| p.src_ct.clone()).collect(), targets: products.iter().map(|p| p.ck).collect(), @@ -314,7 +372,7 @@ fn package_multi_target<'p>(config: &Config, fn build_manifest(config: &Config, layout: &Layout, package: &Package, - assets: Option<&AssetsArtifact>, + source: impl PackageSource, cargo_target: Option>, dev: bool) -> CargoResult<()> { @@ -332,42 +390,24 @@ fn build_manifest(config: &Config, config.log().warn(msg); } }; - let log_src_problem = |problem: Problem| { - let msg = "Manifest validation".to_string(); - log_problem(&msg, problem) - }; - let log_meta_problem = |problem: Problem| { - let msg = "Metadata validation".to_string(); - log_problem(&msg, problem) - }; + let log_src_problem = |problem: Problem| log_problem("Manifest validation", problem); + let log_meta_problem = |problem: Problem| log_problem("Metadata validation", problem); - let validate = |src: &ManifestSource| { - if let Some(target) = &cargo_target { - src.validate_for(target) - .into_iter() - // .filter(Problem::is_err) - .for_each(log_meta_problem); - } else { - src.validate() - .into_iter() - // .filter(Problem::is_err) - .for_each(log_meta_problem); - } - }; - let manifest = if let Some(metadata) = assets.and_then(|a| a.metadata.as_ref()) { - let source = ManifestSource::new(package, metadata.into()); - // This validation not needed at this step. May be earlier: - validate(&source); - source.manifest_override_or_crate(cargo_target.as_deref(), dev) + // validate the source: + if let Some(target) = &cargo_target { + source.validate_for(target) + .into_iter() + // .filter(Problem::is_err) + .for_each(log_meta_problem); } else { - let metadata = playdate_metadata(package); - let source = ManifestSource::new(package, metadata.as_ref()); - // This validation not needed at this step. May be earlier: - validate(&source); - source.manifest_override_or_crate(cargo_target.as_deref(), dev) - }; + source.validate() + .into_iter() + // .filter(Problem::is_err) + .for_each(log_meta_problem); + } + let manifest = source.manifest_override_or_crate(cargo_target.as_deref(), dev); // validation, lints manifest.validate().into_iter().for_each(log_src_problem); @@ -423,7 +463,6 @@ fn execute_pdc<'l, Layout: playdate::layout::Layout>(config: &Config, fn prepare_assets>(config: &Config, assets: &AssetsArtifact, - dev: bool, dst: Dst, overwrite: bool, root: impl AsRef) @@ -432,6 +471,8 @@ fn prepare_assets>(config: &Config, // - `assets.layout.build` points to pre-built assets, but that can fail // - `assets.layout.assets` points to original assets, so use it as fallback + let dev = assets.kind.is_dev(); + let filter_hidden = |path: &PathBuf| { if let Some(filename) = path.file_name() { !filename.starts_with(".") @@ -467,25 +508,15 @@ fn prepare_assets>(config: &Config, Ok(()) }; + let (assets_src, assets_build) = if dev { + (assets.layout.assets_dev(), assets.layout.build_dev()) + } else { + (assets.layout.assets(), assets.layout.build()) + }; - // Main assets: - let files: Vec<_> = select_files_in(&assets.layout.build(), &assets.layout.assets())?; + let files: Vec<_> = select_files_in(&assets_build, &assets_src)?; link_assets(files)?; - // Dev assets: - if dev { - let assets_dev = assets.layout.assets_dev(); - if assets_dev.exists() { - let files: Vec<_> = select_files_in(&assets.layout.build_dev(), &assets_dev)?; - link_assets(files)?; - } else { - // That's OK, dev-assets can be missing, we doesn't create dir without need. - log::debug!( - "Asset (dev) not found at {}", - assets_dev.as_relative_to_root(config).display() - ); - } - } Ok(()) } @@ -546,70 +577,3 @@ impl<'cfg> TryFrom> for SuccessfulBuildProduct<'cfg> { } } } - - -struct ManifestSource<'cfg, 'm> { - package: &'cfg Package, - authors: Vec<&'cfg str>, - bins: Vec<&'cfg str>, - examples: Vec<&'cfg str>, - metadata: Option<&'m Metadata>, -} - -impl<'cfg, 'm> ManifestSource<'cfg, 'm> { - fn new(package: &'cfg Package, metadata: Option<&'m Metadata>) -> Self { - log::debug!("new manifest-source for {}", package.package_id()); - - let mut bins = Vec::new(); - let mut examples = Vec::new(); - - package.targets() - .iter() - .inspect(|t| log::trace!("target: {} ({:?})", t.description_named(), t.crate_name())) - .filter(|t| !t.is_custom_build()) - .for_each(|t| { - if t.is_bin() { - bins.push(t.name()); - log::debug!("+bin: {}", t.description_named()); - } else if t.is_example() { - examples.push(t.name()); - log::debug!("+example: {}", t.description_named()); - } - }); - - Self { authors: package.manifest() - .metadata() - .authors - .iter() - .map(|s| s.as_str()) - .collect(), - bins, - examples, - package, - metadata } - } -} - -impl<'cfg> PackageSource for ManifestSource<'cfg, '_> { - type Authors = [&'cfg str]; - type Metadata = Metadata; - - fn name(&self) -> Cow { self.package.name().as_str().into() } - fn authors(&self) -> &[&'cfg str] { &self.authors } - fn version(&self) -> Cow { self.package.version().to_string().into() } - fn description(&self) -> Option> { - self.package - .manifest() - .metadata() - .description - .as_deref() - .map(|s: &str| s.into()) - } - - fn bins(&self) -> &[&str] { &self.bins } - fn examples(&self) -> &[&str] { &self.examples } - - fn metadata(&self) -> Option<&Self::Metadata> { self.metadata } - - fn manifest_path(&self) -> Cow { Cow::Borrowed(self.package.manifest_path()) } -} diff --git a/cargo/src/proc/reader.rs b/cargo/src/proc/reader.rs index c25b9be6..07371005 100644 --- a/cargo/src/proc/reader.rs +++ b/cargo/src/proc/reader.rs @@ -126,8 +126,8 @@ pub mod format { use serde::Serialize; use serde::Deserialize; use cargo::core::PackageId; - use crate::utils::cargo::build_plan::format::deserialize_crate_types; - use crate::utils::cargo::format::deserialize_package_id; + use crate::utils::cargo::build_plan::format::de_crate_types; + use crate::utils::cargo::format::de_package_id_or_spec; pub use crate::utils::cargo::format::TargetKind; @@ -161,7 +161,7 @@ pub mod format { #[derive(Debug, Serialize, Deserialize)] pub struct Artifact { - #[serde(deserialize_with = "deserialize_package_id")] + #[serde(deserialize_with = "de_package_id_or_spec")] pub package_id: PackageId, pub manifest_path: PathBuf, pub target: SerializedTarget, @@ -185,7 +185,7 @@ pub mod format { pub kind: TargetKind, /// Corresponds to `--crate-type` compiler attribute. /// See - #[serde(deserialize_with = "deserialize_crate_types")] + #[serde(deserialize_with = "de_crate_types")] pub crate_types: Vec, pub name: InternedString, pub src_path: Option, diff --git a/cargo/src/utils/cargo/build_plan.rs b/cargo/src/utils/cargo/build_plan.rs index 9826dd2a..58eef49c 100644 --- a/cargo/src/utils/cargo/build_plan.rs +++ b/cargo/src/utils/cargo/build_plan.rs @@ -8,6 +8,7 @@ use crate::proc::read_cargo_json; use self::format::TargetKind; +#[deprecated = "corrupts cargo build cache"] pub fn build_plan(cfg: &Config) -> CargoResult { let mut cargo = cargo_proxy_cmd(cfg, &Cmd::Build)?; @@ -20,6 +21,7 @@ pub fn build_plan(cfg: &Config) -> CargoResult { impl format::BuildPlan { + #[deprecated] pub fn build_package_invocations<'plan: 'i, 'p: 'i, 'i>( &'plan self, package: &'p PackageId) @@ -34,7 +36,7 @@ impl format::BuildPlan { } -#[allow(dead_code)] +#[derive(Debug, Clone, Copy)] pub enum TargetKindWild { Lib, Bin, @@ -71,6 +73,7 @@ pub mod format { #[derive(Debug, Serialize, Deserialize)] + #[deprecated = "Do not use, corrupts cargo build cache"] pub struct BuildPlan { /// Program invocations needed to build the target (along with dependency information). pub invocations: Vec, @@ -79,6 +82,7 @@ pub mod format { } /// A tool invocation. + #[deprecated = "Do not use, corrupts cargo build cache"] #[derive(Debug, Serialize, Deserialize, PartialEq)] pub struct Invocation { /// The package this invocation is building a part of. @@ -89,7 +93,7 @@ pub mod format { pub target_kind: TargetKind, /// Whether the files created by this invocation are for the host or target system. #[serde(serialize_with = "CompileKind::serialize")] - #[serde(deserialize_with = "deserialize_compile_kind")] + #[serde(deserialize_with = "de_compile_kind")] pub kind: CompileKind, #[serde(serialize_with = "CompileMode::serialize")] #[serde(deserialize_with = "CompileModeProxy::deserialize")] diff --git a/cargo/src/utils/cargo/format.rs b/cargo/src/utils/cargo/format.rs index 1d8d820a..80df6c81 100644 --- a/cargo/src/utils/cargo/format.rs +++ b/cargo/src/utils/cargo/format.rs @@ -89,7 +89,7 @@ impl Serialize for TargetKind { } -pub fn deserialize_package_ids<'de, D>(deserializer: D) -> Result, D::Error> +pub fn de_package_id_or_specs<'de, D>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de> { let items = Vec::::deserialize(deserializer)?; let mut ids = Vec::with_capacity(items.len()); @@ -99,7 +99,7 @@ pub fn deserialize_package_ids<'de, D>(deserializer: D) -> Result Ok(ids) } -pub fn deserialize_package_id<'de, D>(deserializer: D) -> Result +pub fn de_package_id_or_spec<'de, D>(deserializer: D) -> Result where D: Deserializer<'de> { string_to_package_id(String::deserialize(deserializer)?) } @@ -159,7 +159,7 @@ pub fn string_to_package_id(mut line: String) -> Result } -pub fn deserialize_crate_types<'de, D>(deserializer: D) -> Result, D::Error> +pub fn de_crate_types<'de, D>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de> { let kinds = Vec::<&str>::deserialize(deserializer)?; let kinds = kinds.into_iter() @@ -169,7 +169,7 @@ pub fn deserialize_crate_types<'de, D>(deserializer: D) -> Result } -pub fn deserialize_compile_kind<'de, D>(deserializer: D) -> Result +pub fn de_compile_kind<'de, D>(deserializer: D) -> Result where D: Deserializer<'de> { let res = if let Some(s) = Option::<&str>::deserialize(deserializer)? { let target = CompileTarget::new(s).map_err(serde::de::Error::custom)?; @@ -188,7 +188,7 @@ mod tests { #[derive(Debug, Serialize, Deserialize)] pub struct PackageIdWrapped { - #[serde(deserialize_with = "super::deserialize_package_id")] + #[serde(deserialize_with = "super::de_package_id_or_spec")] pub package_id: PackageId, } diff --git a/cargo/src/utils/cargo/meta_deps.rs b/cargo/src/utils/cargo/meta_deps.rs index 167714e0..394d54bb 100644 --- a/cargo/src/utils/cargo/meta_deps.rs +++ b/cargo/src/utils/cargo/meta_deps.rs @@ -1,4 +1,5 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::fmt::Write; use std::path::Path; use cargo::core::compiler::CompileMode; @@ -33,6 +34,10 @@ pub fn meta_deps<'cfg>(cfg: &'cfg Config<'cfg>) -> CargoResult> { pub struct MetaDeps<'cfg> { units: &'cfg UnitGraph, meta: &'cfg CargoMetadataPd, + + /// Root units filtered, + /// only those matches: [`CompileMode::Build`] + /// and [`TargetKind::Lib`] `|` [`TargetKind::Bin`] `|` [`TargetKind::Example`] roots: Vec>, } @@ -55,6 +60,25 @@ impl<'t> RootNode<'t> { pub fn deps(&self) -> &[Node<'t>] { &self.deps } } +impl std::fmt::Display for RootNode<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.node.fmt(f)?; + + if !self.deps.is_empty() { + f.write_fmt(format_args!(" with {} deps", self.deps.len()))?; + + if f.alternate() { + f.write_char(':')?; + for dep in self.deps().iter() { + f.write_str("\n ")?; + dep.fmt(f)?; + } + } + } + Ok(()) + } +} + #[derive(Debug, Clone, Copy)] pub struct Node<'cfg> { @@ -72,6 +96,57 @@ impl<'t> Node<'t> { pub fn manifest_path(&self) -> Option<&'t Path> { self.meta.as_ref().map(|m| m.manifest_path.as_path()) } } +impl std::fmt::Display for Node<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + use cargo::core::compiler::CompileKind; + + f.write_fmt(format_args!( + "{} {} of {}", + self.target().kind().description(), + self.target().name, + self.package_id().name() + ))?; + + if f.alternate() { + f.write_str(" for")?; + match self.unit().platform { + CompileKind::Host => f.write_str(" host"), + CompileKind::Target(kind) => f.write_fmt(format_args!(" {}", kind.short_name())), + }?; + + let meta = self.meta + .and_then(|p| p.metadata.as_ref()) + .and_then(|m| m.inner.as_ref()); + if let Some(meta) = meta { + f.write_str(" (meta")?; + + let assets = !meta.assets().is_empty(); + let assets_dev = !meta.dev_assets().is_empty(); + + if assets || assets_dev { + f.write_str(": ")? + } + + if assets { + f.write_str("assets")? + } + + if assets && assets_dev { + f.write_str(", ")? + } + + if assets_dev { + f.write_str("dev")? + } + + f.write_char(')')?; + } + } + + Ok(()) + } +} + impl<'t> MetaDeps<'t> { pub fn new(units: &'t UnitGraph, meta: &'t CargoMetadataPd) -> Self { @@ -215,7 +290,7 @@ impl<'t> MetaDeps<'t> { .filter(|m| !m.assets().is_empty() || (root_is_dev && !m.dev_assets().is_empty())) .is_some() { - log::trace!( + log::debug!( "add root too because it has assets for {}", root.node.unit.target.name ); @@ -224,15 +299,10 @@ impl<'t> MetaDeps<'t> { } - deps.iter().enumerate().for_each(|(i, n)| { - log::trace!( - "{i}: {}::{} ({:?}), meta: {}", - root.package_id().name(), - root.node.unit.target.name, - root.node.unit.target.kind, - n.meta.is_some() - ); - }); + log::debug!("ready: {root:#}, deps:"); + deps.iter() + .enumerate() + .for_each(|(i, n)| log::debug!("{i}: {n:#}")); log::debug!( "Total finally deps for {}::{} ({:?}) : {}", @@ -249,6 +319,8 @@ impl<'t> MetaDeps<'t> { } + /// Filtered root units, contains only those with mode is [build][CompileMode::Build] + /// and kind matches [`TargetKind::Lib`] `|` [`TargetKind::Bin`] `|` [`TargetKind::Example`]. pub fn roots(&self) -> &[RootNode<'t>] { self.roots.as_slice() } @@ -283,10 +355,21 @@ impl<'t> MetaDeps<'t> { } - pub fn root_for(&self, id: &PackageId, tk: &TargetKindWild, tname: &str) -> CargoResult<&RootNode<'t>> { + pub fn root_for(&self, + id: &PackageId, + tk: cargo::core::TargetKind, + tname: &str) + -> CargoResult<&RootNode<'t>> { + self.roots + .iter() + .find(|n| n.package_id() == id && tk == n.node.unit.target.kind() && n.node.unit.target.name == tname) + .ok_or_else(|| anyhow::anyhow!("Root not found for {id}::{tname}")) + } + + pub fn root_for_wild(&self, id: &PackageId, tk: TargetKindWild, tname: &str) -> CargoResult<&RootNode<'t>> { self.roots .iter() - .find(|n| n.package_id() == id && *tk == n.node.unit.target.kind && n.node.unit.target.name == tname) + .find(|n| n.package_id() == id && tk == n.node.unit.target.kind && n.node.unit.target.name == tname) .ok_or_else(|| anyhow::anyhow!("Root not found for {id}::{tname}")) } diff --git a/cargo/src/utils/cargo/metadata.rs b/cargo/src/utils/cargo/metadata.rs index 45fdf432..8d4d30a8 100644 --- a/cargo/src/utils/cargo/metadata.rs +++ b/cargo/src/utils/cargo/metadata.rs @@ -130,7 +130,7 @@ pub mod format { pub source: Option, pub req: semver::VersionReq, #[serde(serialize_with = "DepKind::serialize")] - #[serde(deserialize_with = "deserialize_dep_kind")] + #[serde(deserialize_with = "de_dep_kind")] pub kind: DepKind, pub optional: bool, @@ -145,7 +145,7 @@ pub mod format { pub dep_kinds: serde_json::Value, } - pub fn deserialize_dep_kind<'de, D>(deserializer: D) -> Result + pub fn de_dep_kind<'de, D>(deserializer: D) -> Result where D: Deserializer<'de> { let kind = Option::<&str>::deserialize(deserializer)?; let kind = match kind { diff --git a/cargo/src/utils/cargo/unit_graph.rs b/cargo/src/utils/cargo/unit_graph.rs index d5dcb351..dc505484 100644 --- a/cargo/src/utils/cargo/unit_graph.rs +++ b/cargo/src/utils/cargo/unit_graph.rs @@ -90,11 +90,11 @@ pub mod format { #[derive(Debug, Deserialize, Eq, Hash, PartialEq, PartialOrd, Ord)] pub struct Unit { - #[serde(deserialize_with = "deserialize_package_id", alias = "pkg_id")] + #[serde(deserialize_with = "de_package_id_or_spec", alias = "pkg_id")] pub package_id: PackageId, pub target: UnitTarget, #[serde(serialize_with = "CompileKind::serialize")] - #[serde(deserialize_with = "deserialize_compile_kind")] + #[serde(deserialize_with = "de_compile_kind")] pub platform: CompileKind, #[serde(serialize_with = "CompileMode::serialize")] #[serde(deserialize_with = "CompileModeProxy::deserialize")] @@ -108,7 +108,7 @@ pub mod format { #[derive(Debug, Deserialize, Eq, Hash, PartialEq, PartialOrd, Ord)] pub struct UnitTarget { pub(crate) kind: TargetKind, - #[serde(deserialize_with = "deserialize_crate_types")] + #[serde(deserialize_with = "de_crate_types")] pub crate_types: Vec, pub name: String, pub src_path: String, diff --git a/cargo/src/utils/mod.rs b/cargo/src/utils/mod.rs index a674417f..7cc557e4 100644 --- a/cargo/src/utils/mod.rs +++ b/cargo/src/utils/mod.rs @@ -1,41 +1,5 @@ -use ::cargo::CargoResult; -use ::cargo::core::Workspace; -use ::cargo::core::compiler::BuildContext; -use ::cargo::core::compiler::UnitInterner; -use ::cargo::ops::CompileOptions; -use ::cargo::ops::create_bcx; -use ::cargo::util::command_prelude::CompileMode; -use try_lazy_init::Lazy; - -use crate::config::Config; - pub mod rustc; pub mod cargo; pub mod workspace; pub mod path; pub mod logging; - - -#[deprecated(since = "0.5", - note = "TODO: use crate::utils::cargo:: unit_graph with metadata instead")] -pub struct LazyBuildContext<'a, 'cfg> { - workspace: &'cfg Workspace<'cfg>, - bcx: Lazy>, - interner: UnitInterner, - options: CompileOptions, -} - -impl<'a, 'cfg> LazyBuildContext<'a, 'cfg> { - pub fn new(config: &'cfg Config) -> CargoResult { - let options = CompileOptions::new(config.workspace.gctx(), CompileMode::Check { test: false })?; - Ok(Self { bcx: Lazy::new(), - interner: UnitInterner::new(), - workspace: &config.workspace, - options }) - } - - pub fn get(&'a self) -> CargoResult<&BuildContext<'a, 'cfg>> { - self.bcx - .try_get_or_create(move || create_bcx(self.workspace, &self.options, &self.interner)) - } -}