From 87e6056e4185e19c98e649cebb03ac2bb845d3f7 Mon Sep 17 00:00:00 2001 From: Enrico Borba Date: Thu, 2 May 2024 17:26:38 +0200 Subject: [PATCH] dylib compilation and runtime loading (#127) Co-authored-by: tjjfvi --- Cargo.lock | 13 +++- Cargo.toml | 3 +- build.rs | 8 ++ cspell.json | 5 ++ src/compile.rs | 6 +- src/main.rs | 203 ++++++++++++++++++++++++++++++++++++++++++------- src/stdlib.rs | 15 +++- 7 files changed, 219 insertions(+), 34 deletions(-) create mode 100644 build.rs diff --git a/Cargo.lock b/Cargo.lock index b649ad4a..15271e8b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -295,12 +295,13 @@ checksum = "809e18805660d7b6b2e2b9f316a5099521b5998d5cba4dda11b5157a21aaef03" [[package]] name = "hvm-core" -version = "0.2.25" +version = "0.2.26" dependencies = [ "TSPL", "arrayvec", "clap", "insta", + "libloading", "nohash-hasher", "ordered-float", "parking_lot", @@ -336,6 +337,16 @@ version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +[[package]] +name = "libloading" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" +dependencies = [ + "cfg-if", + "windows-targets 0.52.3", +] + [[package]] name = "linked-hash-map" version = "0.5.6" diff --git a/Cargo.toml b/Cargo.toml index 9a951cc7..ea9511fd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hvm-core" -version = "0.2.25" +version = "0.2.26" edition = "2021" description = "HVM-Core is a massively parallel Interaction Combinator evaluator." license = "MIT" @@ -27,6 +27,7 @@ debug = "full" TSPL = "0.0.9" arrayvec = "0.7.4" clap = { version = "4.5.1", features = ["derive"], optional = true } +libloading = "0.8.3" nohash-hasher = { version = "0.2.0" } ordered-float = "4.2.0" parking_lot = "0.12.1" diff --git a/build.rs b/build.rs new file mode 100644 index 00000000..e1ea44ca --- /dev/null +++ b/build.rs @@ -0,0 +1,8 @@ +use std::{env, process::Command}; + +fn main() { + let rustc_version = Command::new(env::var("RUSTC").unwrap()).arg("--version").output().unwrap().stdout; + let rustc_version = String::from_utf8(rustc_version).unwrap(); + + println!("cargo::rustc-env=RUSTC_VERSION={rustc_version}"); +} diff --git a/cspell.json b/cspell.json index e598243e..53f320a8 100644 --- a/cspell.json +++ b/cspell.json @@ -14,6 +14,8 @@ "dereferencable", "dref", "dups", + "dylib", + "dylibs", "effectful", "fmts", "fuzzer", @@ -24,6 +26,8 @@ "ilog", "inlinees", "insta", + "libhvmc", + "libloading", "lldb", "lnet", "lnum", @@ -48,6 +52,7 @@ "rnet", "rnum", "rsplit", + "rustc", "rwts", "sigabrt", "skippable", diff --git a/src/compile.rs b/src/compile.rs index bc2f9401..ab2eadd1 100644 --- a/src/compile.rs +++ b/src/compile.rs @@ -70,7 +70,11 @@ fn _compile_host(host: &Host) -> Result { continue; } - let fields = refs.iter().map(|r| format!("def_{}", sanitize_name(r))).collect::>().join(", "); + let fields = refs + .iter() + .map(|r| format!("def_{rust_name}: def_{rust_name}.clone()", rust_name = sanitize_name(r))) + .collect::>() + .join(", "); writeln!( code, diff --git a/src/main.rs b/src/main.rs index 3e974608..b2bceb35 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,15 +5,19 @@ use hvmc::{ ast::{Book, Net, Tree}, host::Host, run::{DynNet, Mode, Trg}, - stdlib::create_host, + stdlib::{create_host, insert_stdlib}, transform::{TransformOpts, TransformPass, TransformPasses}, *, }; use parking_lot::Mutex; use std::{ - fs, io, - path::Path, + env::consts::{DLL_PREFIX, DLL_SUFFIX}, + ffi::OsStr, + fmt::Write, + fs::{self, File}, + io::{self, BufRead}, + path::{Path, PathBuf}, process::{self, Stdio}, str::FromStr, sync::Arc, @@ -26,24 +30,47 @@ fn main() { } if cfg!(feature = "_full_cli") { let cli = FullCli::parse(); + match cli.mode { - CliMode::Compile { file, transform_args, output } => { - let output = output.as_deref().or_else(|| file.strip_suffix(".hvmc")).unwrap_or_else(|| { + CliMode::Compile { file, dylib, transform_args, output } => { + let output = if let Some(output) = output { + output + } else if let Some("hvmc") = file.extension().and_then(OsStr::to_str) { + file.with_extension("") + } else { eprintln!("file missing `.hvmc` extension; explicitly specify an output path with `--output`."); + process::exit(1); - }); - let host = create_host(&load_book(&[file.clone()], &transform_args)); - compile_executable(output, host).unwrap(); + }; + + let host = create_host(&load_book(&[file], &transform_args)); + create_temp_hvm(host).unwrap(); + + if dylib { + prepare_temp_hvm_dylib().unwrap(); + compile_temp_hvm(&["--lib"]).unwrap(); + + fs::copy(format!(".hvm/target/release/{DLL_PREFIX}hvmc{DLL_SUFFIX}"), output).unwrap(); + } else { + compile_temp_hvm(&[]).unwrap(); + + fs::copy(".hvm/target/release/hvmc", output).unwrap(); + } } CliMode::Run { run_opts, mut transform_args, file, args } => { // Don't pre-reduce or prune the entry point transform_args.transform_opts.pre_reduce_skip.push(args.entry_point.clone()); transform_args.transform_opts.prune_entrypoints.push(args.entry_point.clone()); - let host = create_host(&load_book(&[file], &transform_args)); + + let host: Arc> = Default::default(); + load_dylibs(host.clone(), &run_opts.include); + insert_stdlib(host.clone()); + host.lock().insert_book(&load_book(&[file], &transform_args)); + run(host, run_opts, args); } CliMode::Reduce { run_opts, transform_args, files, exprs } => { - let host = create_host(&load_book(&files, &transform_args)); + let host = load_host(&files, &transform_args, &run_opts.include); let exprs: Vec<_> = exprs.iter().map(|x| Net::from_str(x).unwrap()).collect(); reduce_exprs(host, &exprs, &run_opts); } @@ -98,17 +125,22 @@ enum CliMode { /// Compile a hvm-core program into a Rust crate. Compile { /// hvm-core file to compile. - file: String, - #[arg(short = 'o', long = "output")] + file: PathBuf, + /// Compile this hvm-core file to a dynamic library. + /// + /// These can be included when running with the `--include` option. + #[arg(short, long)] + dylib: bool, /// Output path; defaults to the input file with `.hvmc` stripped. - output: Option, + #[arg(short, long)] + output: Option, #[command(flatten)] transform_args: TransformArgs, }, /// Run a program, optionally passing a list of arguments to it. Run { /// Name of the file to load. - file: String, + file: PathBuf, #[command(flatten)] args: RunArgs, #[command(flatten)] @@ -123,17 +155,17 @@ enum CliMode { /// which makes it possible to reference definitions from the file /// in the expression. Reduce { - #[arg(required = false)] /// Files to load before reducing the expressions. /// /// Multiple files will act as if they're concatenated together. - files: Vec, - #[arg(required = false, last = true)] + #[arg(required = false)] + files: Vec, /// Expressions to reduce. /// /// The normal form of each expression will be /// printed on a new line. This list must be separated from the file list /// with a double dash ('--'). + #[arg(required = false, last = true)] exprs: Vec, #[command(flatten)] run_opts: RuntimeOpts, @@ -146,7 +178,7 @@ enum CliMode { /// /// Multiple files will act as if they're concatenated together. #[arg(required = true)] - files: Vec, + files: Vec, #[command(flatten)] transform_args: TransformArgs, }, @@ -157,37 +189,41 @@ struct TransformArgs { /// Enables or disables transformation passes. #[arg(short = 'O', value_delimiter = ' ', action = clap::ArgAction::Append)] transform_passes: Vec, - #[command(flatten)] transform_opts: TransformOpts, } #[derive(Args, Clone, Debug)] struct RuntimeOpts { - #[arg(short = 's', long = "stats")] /// Show performance statistics. + #[arg(short, long = "stats")] show_stats: bool, - #[arg(short = '1', long = "single")] /// Single-core mode (no parallelism). + #[arg(short = '1', long = "single")] single_core: bool, - #[arg(short = 'l', long = "lazy")] /// Lazy mode. /// /// Lazy mode only expands references that are reachable /// by a walk from the root of the net. This leads to a dramatic slowdown, /// but allows running programs that would expand indefinitely otherwise. + #[arg(short, long = "lazy")] lazy_mode: bool, - #[arg(short = 'm', long = "memory", value_parser = util::parse_abbrev_number::)] /// How much memory to allocate on startup. /// /// Supports abbreviations such as '4G' or '400M'. + #[arg(short, long, value_parser = util::parse_abbrev_number::)] memory: Option, + /// Dynamic library hvm-core files to include. + /// + /// hvm-core files can be compiled as dylibs with the `--dylib` option. + #[arg(short, long, value_delimiter = ' ', action = clap::ArgAction::Append)] + include: Vec, } #[derive(Args, Clone, Debug)] struct RunArgs { - #[arg(short = 'e', default_value = "main")] /// Name of the definition that will get reduced. + #[arg(short, default_value = "main")] entry_point: String, /// List of arguments to pass to the program. /// @@ -209,7 +245,19 @@ fn run(host: Arc>, opts: RuntimeOpts, args: RunArgs) { reduce_exprs(host, &[net], &opts); } -fn load_book(files: &[String], transform_args: &TransformArgs) -> Book { +fn load_host( + files: &[PathBuf], + transform_args: &TransformArgs, + include: &[PathBuf], +) -> Arc> { + let host: Arc> = Default::default(); + load_dylibs(host.clone(), include); + insert_stdlib(host.clone()); + host.lock().insert_book(&load_book(files, transform_args)); + host +} + +fn load_book(files: &[PathBuf], transform_args: &TransformArgs) -> Book { let mut book = files .iter() .map(|name| { @@ -233,6 +281,47 @@ fn load_book(files: &[String], transform_args: &TransformArgs) -> Book { book } +fn load_dylibs(host: Arc>, include: &[PathBuf]) { + let current_dir = std::env::current_dir().unwrap(); + + for file in include { + unsafe { + let lib = if file.is_absolute() { + libloading::Library::new(file) + } else { + libloading::Library::new(current_dir.join(file)) + } + .expect("failed to load dylib"); + + let rust_version = + lib.get:: &'static str>(b"hvmc_dylib_v0__rust_version").expect("failed to load rust version"); + let rust_version = rust_version(); + if rust_version != env!("RUSTC_VERSION") { + eprintln!( + "warning: dylib {file:?} was compiled with rust version {rust_version}, but is being run with rust version {}", + env!("RUSTC_VERSION") + ); + } + + let hvmc_version = + lib.get:: &'static str>(b"hvmc_dylib_v0__hvmc_version").expect("failed to load hvmc version"); + let hvmc_version = hvmc_version(); + if hvmc_version != env!("CARGO_PKG_VERSION") { + eprintln!( + "warning: dylib {file:?} was compiled with hvmc version {hvmc_version}, but is being run with hvmc version {}", + env!("CARGO_PKG_VERSION") + ); + } + + let insert_into_host = + lib.get::(b"hvmc_dylib_v0__insert_host").expect("failed to load insert_host"); + insert_into_host(&mut host.lock()); + + std::mem::forget(lib); + } + } +} + fn reduce_exprs(host: Arc>, exprs: &[Net], opts: &RuntimeOpts) { let heap = run::Heap::new(opts.memory).expect("memory allocation failed"); for expr in exprs { @@ -276,7 +365,9 @@ fn pretty_num(n: u64) -> String { .collect() } -fn compile_executable(target: &str, host: Arc>) -> Result<(), io::Error> { +/// Copies the `hvm-core` source to a temporary `.hvm` directory. +/// Only a subset of `Cargo.toml` is included. +fn create_temp_hvm(host: Arc>) -> Result<(), io::Error> { let gen = compile::compile_host(&host.lock()); let outdir = ".hvm"; if Path::new(&outdir).exists() { @@ -358,17 +449,73 @@ fn compile_executable(target: &str, host: Arc>) -> Result<(), } } + Ok(()) +} + +/// Appends a function to `lib.rs` that will be dynamically loaded +/// by hvm-core when the generated dylib is included. +fn prepare_temp_hvm_dylib() -> Result<(), io::Error> { + insert_crate_type_cargo_toml()?; + + let mut lib = fs::read_to_string(".hvm/src/lib.rs")?; + + writeln!(lib).unwrap(); + writeln!( + lib, + r#" +#[no_mangle] +pub fn hvmc_dylib_v0__insert_host(host: &mut host::Host) {{ + gen::insert_into_host(host) +}} + +#[no_mangle] +pub fn hvmc_dylib_v0__hvmc_version() -> &'static str {{ + {hvmc_version:?} +}} + +#[no_mangle] +pub fn hvmc_dylib_v0__rust_version() -> &'static str {{ + {rust_version:?} +}} + "#, + hvmc_version = env!("CARGO_PKG_VERSION"), + rust_version = env!("RUSTC_VERSION"), + ) + .unwrap(); + + fs::write(".hvm/src/lib.rs", lib) +} + +/// Adds `crate_type = ["dylib"]` under the `[lib]` section of `Cargo.toml`. +fn insert_crate_type_cargo_toml() -> Result<(), io::Error> { + let mut cargo_toml = String::new(); + + let file = File::open(".hvm/Cargo.toml")?; + for line in io::BufReader::new(file).lines() { + let line = line?; + writeln!(cargo_toml, "{line}").unwrap(); + + if line == "[lib]" { + writeln!(cargo_toml, r#"crate_type = ["dylib"]"#).unwrap(); + } + } + + fs::write(".hvm/Cargo.toml", cargo_toml) +} + +/// Compiles the `.hvm` directory, appending the provided `args` to `cargo`. +fn compile_temp_hvm(args: &[&'static str]) -> Result<(), io::Error> { let output = process::Command::new("cargo") .current_dir(".hvm") .arg("build") .arg("--release") + .args(args) .stderr(Stdio::inherit()) .output()?; + if !output.status.success() { process::exit(1); } - fs::copy(".hvm/target/release/hvmc", target)?; - Ok(()) } diff --git a/src/stdlib.rs b/src/stdlib.rs index 39211932..79d66e3a 100644 --- a/src/stdlib.rs +++ b/src/stdlib.rs @@ -62,7 +62,18 @@ impl AsHostedDef for LogDef { #[cfg(feature = "std")] #[allow(clippy::absolute_paths)] pub fn create_host(book: &crate::ast::Book) -> Arc> { - let host = Arc::new(Mutex::new(Host::default())); + let host: Arc> = Default::default(); + + insert_stdlib(host.clone()); + host.lock().insert_book(book); + + host +} + +/// Insert `hvm-core`'s built-in definitions into a host. +#[cfg(feature = "std")] +#[allow(clippy::absolute_paths)] +pub fn insert_stdlib(host: Arc>) { host.lock().insert_def("HVM.log", unsafe { crate::stdlib::LogDef::new(host.clone(), { move |tree| { @@ -71,8 +82,6 @@ pub fn create_host(book: &crate::ast::Book) -> Arc> { }) }); host.lock().insert_def("HVM.black_box", DefRef::Static(unsafe { &*IDENTITY })); - host.lock().insert_book(book); - host } #[repr(transparent)]