diff --git a/.reuse/dep5 b/.reuse/dep5 index 2c6628539..b9ad3ed8b 100644 --- a/.reuse/dep5 +++ b/.reuse/dep5 @@ -6,6 +6,7 @@ Source: http://sel4.systems Files: Cargo.lock support/*.json + hacking/cargo-manifest-management/Cargo.lock hacking/nix/scope/capdl-tool/capDL-tool.nix hacking/nix/scope/capdl-tool/base-compat-0-11-2.nix hacking/nix/scope/capdl-tool/base-compat-batteries-0-11-2.nix diff --git a/crates/sel4-capdl-initializer/with-embedded-spec/Cargo.toml b/crates/sel4-capdl-initializer/with-embedded-spec/Cargo.toml index 9f95dc99b..27e9ac60e 100644 --- a/crates/sel4-capdl-initializer/with-embedded-spec/Cargo.toml +++ b/crates/sel4-capdl-initializer/with-embedded-spec/Cargo.toml @@ -34,7 +34,5 @@ path = "../../sel4-root-task" default-features = false features = ["single-threaded"] -[build-dependencies] - [build-dependencies.sel4-capdl-initializer-with-embedded-spec-embedded-spec-validate] path = "embedded-spec/validate" diff --git a/crates/sel4/config/generic/types/Cargo.toml b/crates/sel4/config/generic/types/Cargo.toml index 03712a69e..7458702b7 100644 --- a/crates/sel4/config/generic/types/Cargo.toml +++ b/crates/sel4/config/generic/types/Cargo.toml @@ -16,8 +16,6 @@ authors = ["Nick Spinale "] edition = "2021" license = "BSD-2-Clause" -[dependencies] - [dependencies.serde] version = "1.0.147" default-features = false diff --git a/hacking/cargo-manifest-management/.gitignore b/hacking/cargo-manifest-management/.gitignore new file mode 100644 index 000000000..3846a5860 --- /dev/null +++ b/hacking/cargo-manifest-management/.gitignore @@ -0,0 +1,7 @@ +# +# Copyright 2023, Colias Group, LLC +# +# SPDX-License-Identifier: BSD-2-Clause +# + +/target/ diff --git a/hacking/cargo-manifest-management/Cargo.lock b/hacking/cargo-manifest-management/Cargo.lock new file mode 100644 index 000000000..993844c84 --- /dev/null +++ b/hacking/cargo-manifest-management/Cargo.lock @@ -0,0 +1,337 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "anstream" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" + +[[package]] +name = "anstyle-parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" +dependencies = [ + "anstyle", + "windows-sys", +] + +[[package]] +name = "cargo-manifest-management" +version = "0.1.0" +dependencies = [ + "clap", + "either", + "serde", + "serde_json", + "similar", + "toml_edit", +] + +[[package]] +name = "clap" +version = "4.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d04704f56c2cde07f43e8e2c154b43f216dc5c92fc98ada720177362f953b956" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e231faeaca65ebd1ea3c737966bf858971cd38c3849107aa3ea7de90a804e45" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" + +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "hashbrown" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "indexmap" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + +[[package]] +name = "memchr" +version = "2.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" + +[[package]] +name = "proc-macro2" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + +[[package]] +name = "serde" +version = "1.0.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "similar" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aeaf503862c419d66959f5d7ca015337d864e9c49485d771b732e2a20453597" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "2.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "toml_datetime" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" + +[[package]] +name = "toml_edit" +version = "0.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "380f9e8120405471f7c9ad1860a713ef5ece6a670c7eae39225e477340f32fc4" +dependencies = [ + "indexmap", + "toml_datetime", + "winnow", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "winnow" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3b801d0e0a6726477cc207f60162da452f3a95adb368399bef20a946e06f65c" +dependencies = [ + "memchr", +] diff --git a/hacking/cargo-manifest-management/Cargo.toml b/hacking/cargo-manifest-management/Cargo.toml new file mode 100644 index 000000000..0977582e4 --- /dev/null +++ b/hacking/cargo-manifest-management/Cargo.toml @@ -0,0 +1,24 @@ +# +# Copyright 2023, Colias Group, LLC +# +# SPDX-License-Identifier: BSD-2-Clause +# +# + +[package] +name = "cargo-manifest-management" +version = "0.1.0" +authors = ["Nick Spinale "] +edition = "2021" +license = "BSD-2-Clause" + +[dependencies] +clap = { version = "4.4.6", features = [ "derive" ] } +either = "1.9" +serde = { version = "1.0", features = [ "derive" ] } +serde_json = "1.0" +similar = "2.3" +toml_edit = "0.20.4" + +# exclude from top-level workspace +[workspace] diff --git a/hacking/cargo-manifest-management/Makefile b/hacking/cargo-manifest-management/Makefile index 0b47e7a74..cf70784fc 100644 --- a/hacking/cargo-manifest-management/Makefile +++ b/hacking/cargo-manifest-management/Makefile @@ -4,16 +4,20 @@ # SPDX-License-Identifier: BSD-2-Clause # -run_script := \ - script=$$(nix-build -A script --no-out-link) && $$script +plan := $$(nix-build -A planJSON --no-out-link) +run := cargo run -- --plan $(plan) .PHONY: none none: +.PHONY: clean +clean: + rm -rf target + .PHONY: update update: - $(run_script) + $(run) .PHONY: check check: - $(run_script) --just-check + $(run) --just-check diff --git a/hacking/cargo-manifest-management/default.nix b/hacking/cargo-manifest-management/default.nix index 9382f0484..0b84ab1ff 100644 --- a/hacking/cargo-manifest-management/default.nix +++ b/hacking/cargo-manifest-management/default.nix @@ -7,16 +7,48 @@ rec { pkgs = (import ../nix {}).pkgs.build; - utils = pkgs.callPackage ./utils {}; + inherit (pkgs) lib; - manifestScope = pkgs.callPackage ./manifest-scope.nix {}; + tool = import ./tool.nix { + inherit lib; + }; + + manifestScope = import ./manifest-scope.nix { + inherit lib; + }; manualManifests = import ./manual-manifests.nix; - workspace = pkgs.callPackage ./workspace.nix { - inherit utils manifestScope manualManifests; + workspace = tool { + inherit manifestScope manualManifests; workspaceRoot = toString ../..; + workspaceDirFilter = relativePathSegments: lib.head relativePathSegments == "crates"; }; - inherit (workspace) script; + inherit (workspace) plan; + + prettyJSON = name: value: + with pkgs; + runCommand name { + nativeBuildInputs = [ + jq + ]; + } '' + jq < ${builtins.toFile "plan.json" (builtins.toJSON value)} > $out + ''; + + planJSON = prettyJSON "plan.json" plan; + + # for debugging: + + test = prettyJSON "Crate.json" testValue; + + testValue = getManifestByPackageName "sel4"; + + getManifestByPackageName = name: lib.head + (lib.filter + (v: v.package.name or null == name) + (map + (v: v.manifest) + (lib.attrValues plan))); } diff --git a/hacking/cargo-manifest-management/execute_plan.py b/hacking/cargo-manifest-management/execute_plan.py deleted file mode 100644 index efbc84186..000000000 --- a/hacking/cargo-manifest-management/execute_plan.py +++ /dev/null @@ -1,77 +0,0 @@ -# -# Copyright 2023, Colias Group, LLC -# -# SPDX-License-Identifier: BSD-2-Clause -# - -import argparse -import difflib -import json -import subprocess -import sys -import toml -from pathlib import Path - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--plan', type=Path) - parser.add_argument('--just-check', default=False, action=argparse.BooleanOptionalAction) - args = parser.parse_args() - - with args.plan.open() as f: - plan = json.load(f) - - run(plan, args.just_check) - - -def run(plan, just_check): - for manifest_path, v in plan.items(): - manifest_path = Path(manifest_path) - src = Path(v['src']) - just_check_equivalence = v['justEnsureEquivalence'] - - if just_check: - if not manifest_path.is_file(): - die(f'{manifest_path} does not exist') - - if just_check_equivalence: - check_equivalence(manifest_path, src) - else: - diff_proc = subprocess.run(['diff', '-u', manifest_path, src], stdout=subprocess.PIPE) - if diff_proc.returncode != 0: - if just_check: - diff = diff_proc.stdout.decode('utf-8') - die(f'{manifest_path} differs from {src}:\n{diff}') - else: - if subprocess.run(['cp', '-vL', '--no-preserve=all', src, manifest_path]).returncode != 0: - die() - - -def check_equivalence(a_path, b_path): - def read(f): - return json.dumps(toml.load(f), indent=2, sort_keys=True).splitlines() - - with a_path.open() as f: - a = read(f) - - with b_path.open() as f: - b = read(f) - - if a != b: - print(f'{a_path} differs from {b_path}:', file=sys.stderr) - - for line in difflib.unified_diff(a, b, fromfile=a_path, tofile=b_path, lineterm=""): - print(line, file=sys.stderr) - - die() - - -def die(msg=''): - if msg: - print(f'error: {msg}', file=sys.stderr) - sys.exit(1) - - -if __name__ == '__main__': - main() diff --git a/hacking/cargo-manifest-management/src/cargo_policy.rs b/hacking/cargo-manifest-management/src/cargo_policy.rs new file mode 100644 index 000000000..a8d6de4a9 --- /dev/null +++ b/hacking/cargo-manifest-management/src/cargo_policy.rs @@ -0,0 +1,103 @@ +// +// Copyright 2023, Colias Group, LLC +// +// SPDX-License-Identifier: BSD-2-Clause +// + +use std::cmp::{Ordering, Reverse}; + +use crate::{PathSegment, Policy}; + +pub struct CargoPolicy; + +impl Policy for CargoPolicy { + fn compare_keys(path: &[PathSegment], a: &str, b: &str) -> Ordering { + let ranking = if path.len() == 0 { + Ranking { + front: &[ + "package", + "lib", + "bin", + "features", + "dependencies", + "dev-dependencies", + "build-dependencies", + "workspace", + "profile", + ], + back: &[], + } + } else if path.len() == 1 && path[0].is_table_key("package") { + Ranking { + front: &["name", "version"], + back: &["description"], + } + } else if path.len() >= 2 + && path[path.len() - 2] + .as_table_key() + .map(|k| k.ends_with("dependencies")) + .unwrap_or(false) + { + Ranking { + front: &[ + "path", + "git", + "branch", + "tag", + "rev", + "version", + "registry", + "default-features", + "features", + "optional", + ], + back: &[], + } + } else if path.len() == 2 && path[0].is_table_key("target") { + Ranking { + front: &["dependencies", "dev-dependencies", "build-dependencies"], + back: &[], + } + } else if path.len() == 1 && path[0].is_table_key("workspace") { + Ranking { + front: &[], + back: &["members", "exclude"], + } + } else { + Ranking { + front: &[], + back: &[], + } + }; + ranking.compare(a, b) + } + + fn is_always_table(path: &[PathSegment]) -> bool { + path.len() <= 1 + || (path.len() <= 3 && path[0].is_table_key("target")) + || (path.len() <= 3 && path[0].is_table_key("profile")) + } + + fn is_always_array_of_tables(path: &[PathSegment]) -> bool { + path.len() == 2 && path[1].is_table_key("bin") + } +} + +struct Ranking<'a> { + front: &'a [&'a str], + back: &'a [&'a str], +} + +impl<'a> Ranking<'a> { + fn order<'b>(&self, a: &'b str) -> (Reverse>>, Option, &'b str) { + ( + Reverse(self.front.iter().position(|s| s == &a).map(Reverse)), + self.back.iter().position(|s| s == &a), + a, + ) + } + + fn compare(&self, a: &str, b: &str) -> Ordering { + self.order(a).cmp(&self.order(b)) + } +} diff --git a/hacking/cargo-manifest-management/src/diff.rs b/hacking/cargo-manifest-management/src/diff.rs new file mode 100644 index 000000000..a6abc76d4 --- /dev/null +++ b/hacking/cargo-manifest-management/src/diff.rs @@ -0,0 +1,21 @@ +// +// Copyright 2023, Colias Group, LLC +// +// SPDX-License-Identifier: BSD-2-Clause +// + +use similar::{ChangeTag, TextDiff}; + +pub fn diff(a: &str, b: &str) -> String { + let mut s = String::new(); + let d = TextDiff::from_lines(a, b); + for change in d.iter_all_changes() { + let sign = match change.tag() { + ChangeTag::Delete => "-", + ChangeTag::Insert => "+", + ChangeTag::Equal => " ", + }; + s.push_str(&format!("{}{}", sign, change)) + } + s +} diff --git a/hacking/cargo-manifest-management/src/format.rs b/hacking/cargo-manifest-management/src/format.rs new file mode 100644 index 000000000..378373f00 --- /dev/null +++ b/hacking/cargo-manifest-management/src/format.rs @@ -0,0 +1,289 @@ +// +// Copyright 2023, Colias Group, LLC +// +// SPDX-License-Identifier: BSD-2-Clause +// + +use std::cmp::Ordering; +use std::collections::BTreeMap; +use std::marker::PhantomData; +use std::mem; + +use either::Either; +use serde_json::Value as JsonValue; +use toml_edit::{Array, ArrayOfTables, Document, Formatted, InlineTable, Item, Table, Value}; + +const MAX_WIDTH: usize = 100; + +pub trait Policy { + fn compare_keys(path: &[PathSegment], a: &str, b: &str) -> Ordering; + + fn is_always_table(path: &[PathSegment]) -> bool; + + fn is_always_array_of_tables(path: &[PathSegment]) -> bool; +} + +pub enum PathSegment { + TableKey(String), + ArrayIndex(usize), +} + +impl PathSegment { + pub fn as_table_key(&self) -> Option<&str> { + match self { + Self::TableKey(k) => Some(k), + _ => None, + } + } + + pub fn is_table_key(&self, key: &str) -> bool { + self.as_table_key().map(|k| k == key).unwrap_or(false) + } +} + +pub fn format(manifest: &JsonValue) -> Document { + let mut state = ValueTranslation { + current_path: vec![], + _phantom: PhantomData::

, + }; + let item = state.translate(manifest).into_item(); + let mut doc = Document::new(); + let _ = mem::replace(doc.as_item_mut(), item); + doc +} + +struct ValueTranslation

{ + current_path: Vec, + _phantom: PhantomData

, +} + +impl ValueTranslation

{ + fn current_key(&self) -> Option<&str> { + self.current_path.last()?.as_table_key() + } + + fn translate(&mut self, v: &JsonValue) -> Translated { + match v { + JsonValue::Bool(w) => Value::Boolean(Formatted::new(*w)).into(), + JsonValue::Number(w) => if let Some(x) = w.as_i64() { + Value::Integer(Formatted::new(x)) + } else if let Some(x) = w.as_f64() { + Value::Float(Formatted::new(x)) + } else { + panic!() + } + .into(), + JsonValue::String(w) => Value::String(Formatted::new(w.clone())).into(), + JsonValue::Array(w) => { + let mut children = vec![]; + for (i, x) in w.iter().enumerate() { + self.current_path.push(PathSegment::ArrayIndex(i)); + children.push(self.translate(x)); + self.current_path.pop(); + } + let homogenous_children = match TranslatedManyArray::homogenize(&children) { + TranslatedManyArray::Values(values) + if P::is_always_array_of_tables(&self.current_path) => + { + TranslatedManyArray::Tables(TranslatedManyArray::tables_from_values( + &values, + )) + } + x => x, + }; + match homogenous_children { + TranslatedManyArray::Values(values) => { + let mut array = Array::new(); + for v in values.into_iter() { + array.push(v); + } + let wrap = self + .current_key() + .map(|k| is_kv_too_long(k, &Value::Array(array.clone()))) + .unwrap_or(false); + if wrap { + array.iter_mut().for_each(|e| { + e.decor_mut().set_prefix("\n "); + }); + array.iter_mut().last().map(|e| { + e.decor_mut().set_suffix("\n"); + }); + } + Translated::Value(Value::Array(array)) + } + TranslatedManyArray::Tables(tables) => { + let mut array = ArrayOfTables::new(); + for v in tables.into_iter() { + array.push(v); + } + Translated::Item(Item::ArrayOfTables(array)) + } + } + } + JsonValue::Object(w) => { + let mut children = BTreeMap::new(); + for (k, x) in w.iter() { + self.current_path.push(PathSegment::TableKey(k.clone())); + children.insert(k.clone(), self.translate(x)); + self.current_path.pop(); + } + let homogenous_children = match TranslatedManyMap::homogenize(&children) { + TranslatedManyMap::Values(values) if P::is_always_table(&self.current_path) => { + TranslatedManyMap::Items(TranslatedManyMap::items_from_values(values)) + } + x => x, + }; + let inline_table_or_items = match homogenous_children { + TranslatedManyMap::Values(values) => { + let mut table = InlineTable::new(); + for (k, v) in values.clone().into_iter() { + table.insert(k, v); + } + table.sort_values_by(|k1, _v1, k2, _v2| { + P::compare_keys(&self.current_path, k1, k2) + }); + if !self + .current_key() + .map(|k| is_kv_too_long(k, &Value::InlineTable(table.clone()))) + .unwrap_or(false) + { + Either::Left(table) + } else { + Either::Right(TranslatedManyMap::items_from_values(values)) + } + } + TranslatedManyMap::Items(items) => Either::Right(items), + }; + match inline_table_or_items { + Either::Left(inline_table) => { + Translated::Value(Value::InlineTable(inline_table)) + } + Either::Right(items) => { + let mut table = Table::new(); + table.set_implicit(true); + for (k, v) in items.into_iter() { + table.insert(&k, v); + } + table.sort_values_by(|k1, _v1, k2, _v2| { + P::compare_keys(&self.current_path, k1, k2) + }); + Translated::Item(Item::Table(table)) + } + } + } + _ => panic!(), + } + } +} + +#[derive(Clone)] +enum Translated { + Item(Item), + Value(Value), +} + +impl From for Translated { + fn from(item: Item) -> Self { + Self::Item(item) + } +} + +impl From for Translated { + fn from(value: Value) -> Self { + Self::Value(value) + } +} + +impl Translated { + fn as_value(&self) -> Option<&Value> { + match self { + Self::Value(v) => Some(v), + _ => None, + } + } + + fn is_value(&self) -> bool { + self.as_value().is_some() + } + + fn into_item(self) -> Item { + match self { + Self::Value(value) => Item::Value(value), + Self::Item(item) => item, + } + } +} + +enum TranslatedManyMap { + Items(BTreeMap), + Values(BTreeMap), +} + +impl TranslatedManyMap { + fn homogenize(heterogeneous: &BTreeMap) -> Self { + if heterogeneous.values().all(Translated::is_value) { + TranslatedManyMap::Values( + heterogeneous + .iter() + .map(|(k, v)| (k.clone(), v.as_value().unwrap().clone())) + .collect(), + ) + } else { + TranslatedManyMap::Items( + heterogeneous + .iter() + .map(|(k, v)| (k.clone(), v.clone().into_item())) + .collect(), + ) + } + } + + fn items_from_values(values: BTreeMap) -> BTreeMap { + values + .into_iter() + .map(|(k, v)| (k, Item::Value(v))) + .collect() + } +} + +enum TranslatedManyArray { + Tables(Vec), + Values(Vec), +} + +impl TranslatedManyArray { + fn homogenize(heterogeneous: &[Translated]) -> Self { + if heterogeneous.iter().all(Translated::is_value) { + TranslatedManyArray::Values( + heterogeneous + .iter() + .map(|v| v.as_value().unwrap().clone()) + .collect(), + ) + } else { + TranslatedManyArray::Tables( + heterogeneous + .iter() + .map(|v| v.clone().into_item().as_table().unwrap().clone()) + .collect(), + ) + } + } + + fn tables_from_values(values: &[Value]) -> Vec
{ + values + .into_iter() + .map(|v| v.as_inline_table().unwrap().clone().into_table()) + .collect() + } +} + +fn is_kv_too_long(k: &str, v: &Value) -> bool { + let mut table = Table::new(); + table.insert(k, Item::Value(v.clone())); + let mut doc = Document::new(); + let _ = mem::replace(doc.as_item_mut(), Item::Table(table)); + let mut s = doc.to_string(); + assert_eq!(s.pop(), Some('\n')); + s.chars().count() > MAX_WIDTH +} diff --git a/hacking/cargo-manifest-management/src/main.rs b/hacking/cargo-manifest-management/src/main.rs new file mode 100644 index 000000000..ac550999a --- /dev/null +++ b/hacking/cargo-manifest-management/src/main.rs @@ -0,0 +1,47 @@ +// +// Copyright 2023, Colias Group, LLC +// +// SPDX-License-Identifier: BSD-2-Clause +// + +#![allow(dead_code)] + +use std::fs::File; +use std::io; +use std::path::PathBuf; + +use clap::Parser; + +mod cargo_policy; +mod diff; +mod format; +mod plan; + +use cargo_policy::CargoPolicy; +use diff::diff; +use format::{format, PathSegment, Policy}; +use plan::Plan; + +#[derive(Debug, Parser)] +struct Args { + #[arg(long)] + plan: PathBuf, + + #[arg(long)] + just_check: bool, +} + +fn main() { + let args = Args::parse(); + let plan_file = File::open(&args.plan).unwrap(); + let plan: Plan = serde_json::from_reader(plan_file).unwrap(); + plan.execute::(args.just_check); +} + +// for debugging: + +fn test_format() { + let json_value = serde_json::from_reader(io::stdin()).unwrap(); + let toml_doc = format::(&json_value); + print!("{}", toml_doc) +} diff --git a/hacking/cargo-manifest-management/src/plan.rs b/hacking/cargo-manifest-management/src/plan.rs new file mode 100644 index 000000000..c9b05705a --- /dev/null +++ b/hacking/cargo-manifest-management/src/plan.rs @@ -0,0 +1,91 @@ +// +// Copyright 2023, Colias Group, LLC +// +// SPDX-License-Identifier: BSD-2-Clause +// + +use std::collections::BTreeMap; +use std::fs; +use std::path::PathBuf; +use std::process; +use std::str; + +use serde::Deserialize; +use serde_json::Value as JsonValue; + +use crate::{diff, format, Policy}; + +#[derive(Debug, Deserialize)] +#[serde(transparent)] +pub struct Plan { + pub entries: BTreeMap, +} + +#[derive(Debug, Deserialize)] +pub struct Entry { + pub manifest: JsonValue, + pub frontmatter: Option, + pub just_ensure_equivalence: bool, +} + +impl Plan { + pub fn get_entry_by_package_name(&self, name: &str) -> Option<&Entry> { + self.entries + .values() + .filter(|entry| entry.get_package_name() == Some(name)) + .next() + } + + pub fn execute(&self, just_check: bool) { + for (path, entry) in self.entries.iter() { + assert!(!entry.just_ensure_equivalence); // TODO unimplemented + let rendered = entry.render::

(); + let mut write = true; + if path.is_file() { + let existing = fs::read(path).unwrap(); + let existing = str::from_utf8(&existing).unwrap(); + if existing == rendered { + write = false; + } else if just_check { + eprintln!("error: {} is out of date:", path.display()); + eprintln!("{}", diff(&rendered, existing)); + process::exit(1); + } + } else if just_check { + eprintln!("error: {} does not exist", path.display()); + process::exit(1); + } + if write { + eprintln!("writing {}", path.display()); + fs::write(path, rendered).unwrap(); + } + } + } +} + +impl Entry { + pub fn get_package_name(&self) -> Option<&str> { + Some( + self.manifest + .as_object() + .unwrap() + .get("package")? + .as_object() + .unwrap() + .get("name") + .unwrap() + .as_str() + .unwrap(), + ) + } + + fn render(&self) -> String { + let mut s = String::new(); + if let Some(frontmatter) = self.frontmatter.as_ref() { + s.push_str(frontmatter); + s.push('\n'); + } + s.push_str(&format::

(&self.manifest).to_string()); + s + } +} diff --git a/hacking/cargo-manifest-management/workspace.nix b/hacking/cargo-manifest-management/tool.nix similarity index 51% rename from hacking/cargo-manifest-management/workspace.nix rename to hacking/cargo-manifest-management/tool.nix index e3f89337f..402f5a45e 100644 --- a/hacking/cargo-manifest-management/workspace.nix +++ b/hacking/cargo-manifest-management/tool.nix @@ -4,41 +4,62 @@ # SPDX-License-Identifier: BSD-2-Clause # -{ lib, callPackage -, writeText, linkFarm, writeScript -, runtimeShell -, python3 - -, utils, manifestScope, manualManifests -, workspaceRoot -}: +{ lib }: let - inherit (utils) pathBetween scanDirForFilesWithName renderManifest; + parseSimplePath = path: + let + isAbsolute = lib.hasPrefix "/" path; + pathWithoutLeadingSlash = lib.substring (if isAbsolute then 1 else 0) (lib.stringLength path) path; + segments = if pathWithoutLeadingSlash == "" then [] else lib.splitString "/" pathWithoutLeadingSlash; + in + assert lib.all (seg: seg != "" && seg != "." && seg != "..") segments; + { + inherit isAbsolute segments; + }; - generatedManifestSources = + takeWhile = pred: let - dirFilter = relativePathSegments: lib.head relativePathSegments == "crates"; + f = acc: xs: + if xs == [] + then acc + else ( + let + x = lib.head xs; + xs' = lib.tail xs; + in + if pred x then f (acc ++ [x]) xs' else acc + ); in - scanDirForFilesWithName dirFilter "Cargo.nix" workspaceRoot; + f []; - genrateManifest = cargoNixAbsolutePath: + pathBetween = a: b: let - absolutePath = builtins.dirOf cargoNixAbsolutePath; - manifestExpr = callManifest { - inherit absolutePath; - f = import cargoNixAbsolutePath; - }; - parsed = parseManifestExpr manifestExpr; - inherit (parsed) manifestValue frontmatter justEnsureEquivalence; - in { - inherit absolutePath manifestValue frontmatter justEnsureEquivalence; - packageName = manifestValue.package.name or null; - packageVersion = manifestValue.package.version or null; - manifestTOML = renderManifest { - inherit manifestValue frontmatter; - }; - }; + aParsed = parseSimplePath a; + bParsed = parseSimplePath b; + in + assert aParsed.isAbsolute == bParsed.isAbsolute; + let + commonPrefixLen = lib.length (takeWhile lib.id (lib.zipListsWith (x: y: x == y) aParsed.segments bParsed.segments)); + aDeviatingSegs = lib.drop commonPrefixLen aParsed.segments; + bDeviatingSegs = lib.drop commonPrefixLen bParsed.segments; + relSegs = lib.genList (lib.const "..") (lib.length aDeviatingSegs) ++ bDeviatingSegs; + in + lib.concatStringsSep "/" relSegs; + + scanDirForFilesWithName = dirFilter: fileName: + let + f = relativePathSegments: absolutePath: lib.concatLists (lib.mapAttrsToList (name: type: + let + childAbsolutePath = "${absolutePath}/${name}"; + childRelativePathSegments = relativePathSegments ++ [ name ]; + in { + "regular" = lib.optional (name == fileName) childAbsolutePath; + "directory" = lib.optionals (dirFilter childRelativePathSegments) (f childRelativePathSegments childAbsolutePath); + }."${type}" or [] + ) (builtins.readDir absolutePath)); + in + f []; parseManifestExpr = let @@ -61,6 +82,33 @@ let inherit manifestValue; inherit (elaboratedNix) frontmatter justEnsureEquivalence; }; +in + +{ workspaceRoot, workspaceDirFilter +, manifestScope, manualManifests +}: + +let + generatedManifestSources = + let + dirFilter = relativePathSegments: lib.head relativePathSegments == "crates"; + in + scanDirForFilesWithName workspaceDirFilter "Cargo.nix" workspaceRoot; + + genrateManifest = cargoNixAbsolutePath: + let + absolutePath = builtins.dirOf cargoNixAbsolutePath; + manifestExpr = callManifest { + inherit absolutePath; + f = import cargoNixAbsolutePath; + }; + parsed = parseManifestExpr manifestExpr; + inherit (parsed) manifestValue frontmatter justEnsureEquivalence; + in { + inherit absolutePath manifestValue frontmatter justEnsureEquivalence; + packageName = manifestValue.package.name or null; + packageVersion = manifestValue.package.version or null; + }; callManifest = { absolutePath, f }: let @@ -88,28 +136,13 @@ let plan = lib.listToAttrs (lib.forEach generatedManifestsList (manifest: { name = "${manifest.absolutePath}/Cargo.toml"; - value = { - src = manifest.manifestTOML; - inherit (manifest) justEnsureEquivalence; + value = with manifest; { + inherit frontmatter; + manifest = manifestValue; + just_ensure_equivalence = justEnsureEquivalence; }; })); - planJSON = writeText "plan.json" (builtins.toJSON plan); - - script = writeScript "execute-plan.sh" '' - #!${runtimeShell} - exec ${python3.withPackages (p: [ p.toml ])}/bin/python3 ${./execute_plan.py} --plan ${planJSON} "$@" - ''; - - # for manual inspection, useful for debugging this script - links = linkFarm "crates" - (lib.mapAttrs' - (absolutePath: v: lib.nameValuePair "root/${absolutePath}" v.src) - plan); - in { - inherit script; - - # for debugging - inherit generatedManifestsByPackageName links; + inherit plan; } diff --git a/hacking/cargo-manifest-management/utils/coarsely_stylize_manifest.py b/hacking/cargo-manifest-management/utils/coarsely_stylize_manifest.py deleted file mode 100644 index ece9f5384..000000000 --- a/hacking/cargo-manifest-management/utils/coarsely_stylize_manifest.py +++ /dev/null @@ -1,131 +0,0 @@ -# -# Copyright 2023, Colias Group, LLC -# -# SPDX-License-Identifier: BSD-2-Clause -# - -import sys -import json -import toml -from collections import OrderedDict - - -def main(): - raw = json.load(sys.stdin) - assert isinstance(raw, dict) - stylized = stylize([], raw) - toml.dump(stylized, sys.stdout, encoder=toml.TomlEncoder(preserve=True)) - - -def stylize(path, v): - if isinstance(v, dict): - if should_inline_table(path): - new = InlineTableOrderedDict() - else: - new = OrderedDict() - keys = list(v.keys()) - sort_table(path, keys) - for key in keys: - new[key] = stylize(path + [key], v[key]) - return new - else: - return v - - -class InlineTableOrderedDict(OrderedDict, toml.decoder.InlineTableDict): - pass - - -def should_inline_table(path): - if len(path) <= 1: - return False - if len(path) <= 3 and path[0] == 'target': - return False - if len(path) <= 3 and path[0] == 'profile': - return False - return True - - -# TODO: remove any bits of policy that rustfmt overrides -def sort_table(path, keys): - if len(path) == 0: - keys.sort( - key=key_using_ranking( - front=[ - 'package', - 'lib', - 'bin', - 'features', - 'dependencies', - 'dev-dependencies', - 'build-dependencies', - 'workspace', - 'profile', - ], - ), - ) - elif len(path) == 1 and path[0] == 'package': - keys.sort( - key=key_using_ranking( - front=[ - 'name', - 'version', - ], - back=[ - 'description', - ], - ), - ) - elif len(path) >= 2 and path[-2].endswith('dependencies'): - keys.sort( - key=key_using_ranking( - front=[ - 'path', - 'git', - 'branch', - 'tag', - 'rev', - 'version', - 'registry', - 'default-features', - 'features', - 'optional', - ], - ), - ) - elif len(path) == 2 and path[0] == 'target': - keys.sort( - key=key_using_ranking( - front=[ - 'dependencies', - 'dev-dependencies', - 'build-dependencies', - ], - ), - ) - elif len(path) == 1 and path[0] == 'workspace': - keys.sort( - key=key_using_ranking( - back=[ - 'members', - ], - ), - ) - - -def key_using_ranking(front=[], back=[]): - def key(v): - try: - front_rank = front.index(v) - except ValueError: - front_rank = len(front) - try: - back_rank = back.index(v) - except ValueError: - back_rank = -1 - return (front_rank, back_rank, v) - return key - - -if __name__ == '__main__': - main() diff --git a/hacking/cargo-manifest-management/utils/default.nix b/hacking/cargo-manifest-management/utils/default.nix deleted file mode 100644 index c70a0b604..000000000 --- a/hacking/cargo-manifest-management/utils/default.nix +++ /dev/null @@ -1,108 +0,0 @@ -# -# Copyright 2023, Colias Group, LLC -# -# SPDX-License-Identifier: BSD-2-Clause -# - -{ lib, callPackage -, runCommand -, python3, python3Packages -}: - -let - - parseSimplePath = path: - let - isAbsolute = lib.hasPrefix "/" path; - pathWithoutLeadingSlash = lib.substring (if isAbsolute then 1 else 0) (lib.stringLength path) path; - segments = if pathWithoutLeadingSlash == "" then [] else lib.splitString "/" pathWithoutLeadingSlash; - in - assert lib.all (seg: seg != "" && seg != "." && seg != "..") segments; - { - inherit isAbsolute segments; - }; - - takeWhile = pred: - let - f = acc: xs: - if xs == [] - then acc - else ( - let - x = lib.head xs; - xs' = lib.tail xs; - in - if pred x then f (acc ++ [x]) xs' else acc - ); - in - f []; - - pathBetween = a: b: - let - aParsed = parseSimplePath a; - bParsed = parseSimplePath b; - in - assert aParsed.isAbsolute == bParsed.isAbsolute; - let - commonPrefixLen = lib.length (takeWhile lib.id (lib.zipListsWith (x: y: x == y) aParsed.segments bParsed.segments)); - aDeviatingSegs = lib.drop commonPrefixLen aParsed.segments; - bDeviatingSegs = lib.drop commonPrefixLen bParsed.segments; - relSegs = lib.genList (lib.const "..") (lib.length aDeviatingSegs) ++ bDeviatingSegs; - in - lib.concatStringsSep "/" relSegs; - - scanDirForFilesWithName = dirFilter: fileName: - let - f = relativePathSegments: absolutePath: lib.concatLists (lib.mapAttrsToList (name: type: - let - childAbsolutePath = "${absolutePath}/${name}"; - childRelativePathSegments = relativePathSegments ++ [ name ]; - in { - "regular" = lib.optional (name == fileName) childAbsolutePath; - "directory" = lib.optionals (dirFilter childRelativePathSegments) (f childRelativePathSegments childAbsolutePath); - }."${type}" or [] - ) (builtins.readDir absolutePath)); - in - f []; - - renderManifest = { manifestValue, frontmatter ? null }: - let - frontmatterCatArg = lib.optionalString - (frontmatter != null) - (builtins.toFile "frontmatter.txt" "${frontmatter}\n"); - - # TODO figure out what's going on here - rustfmtLdHack = "LD_LIBRARY_PATH=${rustToolchainForRustfmtWithTOMLSupport}/lib"; - in - runCommand "Cargo.toml" { - nativeBuildInputs = [ - python3Packages.toml - rustfmtWithTOMLSupport - ]; - manifestJSON = builtins.toJSON manifestValue; - passAsFile = [ "manifestJSON" ]; - } '' - python3 ${./coarsely_stylize_manifest.py} < $manifestJSONPath > Cargo.toml - ${rustfmtLdHack} rustfmt --config format_cargo_toml=true Cargo.toml - cat ${frontmatterCatArg} Cargo.toml >> $out - ''; - - fenix = - let - rev = "0fbf5652c9596038c0f4784d3e93d1f1a543e24b"; - src = fetchTarball "https://github.com/nix-community/fenix/archive/${rev}.tar.gz"; - in import src {}; - - rustToolchainForRustfmtWithTOMLSupport = (fenix.toolchainOf { - channel = "nightly"; - date = "2023-07-01"; - sha256 = "sha256-pWd4tAHP4QWGC3CKWZzDjzYANxATC6CGRmKuP2ZZv5k="; - }).completeToolchain; - - rustfmtWithTOMLSupport = callPackage ./rustfmt-with-toml-support.nix { - rustToolchain = rustToolchainForRustfmtWithTOMLSupport; - }; - -in { - inherit pathBetween scanDirForFilesWithName renderManifest; -} diff --git a/hacking/cargo-manifest-management/utils/rustfmt-with-toml-support.nix b/hacking/cargo-manifest-management/utils/rustfmt-with-toml-support.nix deleted file mode 100644 index 9b561375d..000000000 --- a/hacking/cargo-manifest-management/utils/rustfmt-with-toml-support.nix +++ /dev/null @@ -1,61 +0,0 @@ -# -# Copyright 2023, Colias Group, LLC -# -# SPDX-License-Identifier: BSD-2-Clause -# - -{ stdenv, lib -, fetchFromGitHub -, rustToolchain -, rustPlatform -}: - -let - # src = fetchFromGitHub { - # owner = "xxchan"; - # repo = "rustfmt"; - # rev = "1ad83c1d48ac2f5717ea8ae398443510c95734b1"; - # hash = "sha256-YJ9qNpSnEmOEb45TZcs/HwnZRWOTIXKqvW+f65MtMVE="; - # }; - - src = fetchFromGitHub rec { - owner = "coliasgroup"; - repo = "rustfmt"; - rev = "742ef5f05bcb527461e685c830c364da0bd46193"; # branch format-cargo-manifest - hash = "sha256-uZT4qHx6NJaSbQMn2jjxS4wqjWgwi0Zr6qtJzsIH5pc="; - }; - - cargoDeps = rustPlatform.importCargoLock { - lockFile = "${src}/Cargo.lock"; - }; - - cargoConfig = "${cargoDeps}/.cargo/config"; - -in -stdenv.mkDerivation { - name = "rustfmt-with-toml-support"; - - inherit src; - - nativeBuildInputs = [ - rustToolchain - ]; - - dontConfigure = true; - dontInstall = true; - - patchPhase = '' - ln -s ${cargoDeps} cargo-vendor-dir - ln -s ${cargoDeps}/.cargo . - ''; - - buildPhase = '' - cargo build \ - -Z unstable-options \ - --offline \ - --frozen \ - --release \ - --out-dir $out/bin \ - -j $NIX_BUILD_CORES - ''; -}