Merge pull request #45 from CertainLach/gc-v2

Implement tracing Gc with rust-gc

bindings/jsonnet/Cargo.toml

@@ -10,6 +10,7 @@ publish = false
 [dependencies]
 jrsonnet-evaluator = { path = "../../crates/jrsonnet-evaluator", version = "0.3.8" }
 jrsonnet-parser = { path = "../../crates/jrsonnet-parser", version = "0.3.8" }
+jrsonnet-gc = { version = "0.4.2", features = ["derive"] }
 
 [lib]
 crate-type = ["cdylib"]

bindings/jsonnet/src/native.rs

@@ -1,8 +1,14 @@
-use jrsonnet_evaluator::{error::Error, native::NativeCallback, EvaluationState, Val};
+use jrsonnet_evaluator::{
+	error::{Error, LocError},
+	native::{NativeCallback, NativeCallbackHandler},
+	EvaluationState, Val,
+};
+use jrsonnet_gc::{unsafe_empty_trace, Finalize, Gc, Trace};
 use jrsonnet_parser::{Param, ParamsDesc};
 use std::{
 	ffi::{c_void, CStr},
 	os::raw::{c_char, c_int},
+	path::Path,
 	rc::Rc,
 };
 
@@ -12,6 +18,39 @@ type JsonnetNativeCallback = unsafe extern "C" fn(
 	success: *mut c_int,
 ) -> *mut Val;
 
+struct JsonnetNativeCallbackHandler {
+	ctx: *const c_void,
+	cb: JsonnetNativeCallback,
+}
+impl Finalize for JsonnetNativeCallbackHandler {}
+unsafe impl Trace for JsonnetNativeCallbackHandler {
+	unsafe_empty_trace!();
+}
+impl NativeCallbackHandler for JsonnetNativeCallbackHandler {
+	fn call(&self, _from: Option<Rc<Path>>, args: &[Val]) -> Result<Val, LocError> {
+		let mut n_args = Vec::new();
+		for a in args {
+			n_args.push(Some(Box::new(a.clone())));
+		}
+		n_args.push(None);
+		let mut success = 1;
+		let v = unsafe {
+			(self.cb)(
+				self.ctx,
+				&n_args as *const _ as *const *const Val,
+				&mut success,
+			)
+		};
+		let v = unsafe { *Box::from_raw(v) };
+		if success == 1 {
+			Ok(v)
+		} else {
+			let e = v.try_cast_str("native error").expect("error msg");
+			Err(Error::RuntimeError(e).into())
+		}
+	}
+}
+
 /// # Safety
 #[no_mangle]
 pub unsafe extern "C" fn jsonnet_native_callback(
@@ -35,21 +74,9 @@ pub unsafe extern "C" fn jsonnet_native_callback(
 
 	vm.add_native(
 		name,
-		Rc::new(NativeCallback::new(params, move |_caller, args| {
-			let mut n_args = Vec::new();
-			for a in args {
-				n_args.push(Some(Box::new(a.clone())));
-			}
-			n_args.push(None);
-			let mut success = 1;
-			let v = cb(ctx, &n_args as *const _ as *const *const Val, &mut success);
-			let v = *Box::from_raw(v);
-			if success == 1 {
-				Ok(v)
-			} else {
-				let e = v.try_cast_str("native error").expect("error msg");
-				Err(Error::RuntimeError(e).into())
-			}
-		})),
+		Gc::new(NativeCallback::new(
+			params,
+			Box::new(JsonnetNativeCallbackHandler { ctx, cb }),
+		)),
 	)
 }

bindings/jsonnet/src/val_make.rs

@@ -1,10 +1,10 @@
 //! Create values in VM
 
 use jrsonnet_evaluator::{ArrValue, EvaluationState, ObjValue, Val};
+use jrsonnet_gc::Gc;
 use std::{
 	ffi::CStr,
 	os::raw::{c_char, c_double, c_int},
-	rc::Rc,
 };
 
 /// # Safety
@@ -38,7 +38,7 @@ pub extern "C" fn jsonnet_json_make_null(_vm: &EvaluationState) -> *mut Val {
 
 #[no_mangle]
 pub extern "C" fn jsonnet_json_make_array(_vm: &EvaluationState) -> *mut Val {
-	Box::into_raw(Box::new(Val::Arr(ArrValue::Eager(Rc::new(Vec::new())))))
+	Box::into_raw(Box::new(Val::Arr(ArrValue::Eager(Gc::new(Vec::new())))))
 }
 
 #[no_mangle]

bindings/jsonnet/src/val_modify.rs

@@ -3,8 +3,9 @@
 //! In jrsonnet every value is immutable, and this code is probally broken
 
 use jrsonnet_evaluator::{ArrValue, EvaluationState, LazyBinding, LazyVal, ObjMember, Val};
+use jrsonnet_gc::Gc;
 use jrsonnet_parser::Visibility;
-use std::{ffi::CStr, os::raw::c_char, rc::Rc};
+use std::{ffi::CStr, os::raw::c_char};
 
 /// # Safety
 ///
@@ -22,7 +23,7 @@ pub unsafe extern "C" fn jsonnet_json_array_append(
 				new.push(item);
 			}
 			new.push(LazyVal::new_resolved(val.clone()));
-			*arr = Val::Arr(ArrValue::Lazy(Rc::new(new)));
+			*arr = Val::Arr(ArrValue::Lazy(Gc::new(new)));
 		}
 		_ => panic!("should receive array"),
 	}

cmds/jrsonnet/Cargo.toml

@@ -8,15 +8,14 @@ edition = "2018"
 publish = false
 
 [features]
-default = []
+default = ["mimalloc"]
 # Use mimalloc as allocator
-mimalloc = []
+mimalloc = ["mimallocator"]
 
 [dependencies]
 jrsonnet-evaluator = { path = "../../crates/jrsonnet-evaluator", version = "0.3.8" }
 jrsonnet-parser = { path = "../../crates/jrsonnet-parser", version = "0.3.8" }
 jrsonnet-cli = { path = "../../crates/jrsonnet-cli", version = "0.3.8" }
-# TODO: Fix mimalloc compile errors, and use them
 mimallocator = { version = "0.1.3", optional = true }
 thiserror = "1.0"
 

cmds/jrsonnet/src/main.rs

@@ -1,5 +1,5 @@
 use clap::{AppSettings, Clap, IntoApp};
-use jrsonnet_cli::{ConfigureState, GeneralOpts, InputOpts, ManifestOpts, OutputOpts};
+use jrsonnet_cli::{ConfigureState, GcOpts, GeneralOpts, InputOpts, ManifestOpts, OutputOpts};
 use jrsonnet_evaluator::{error::LocError, EvaluationState, ManifestFormat};
 use std::{
 	fs::{create_dir_all, File},
@@ -61,6 +61,8 @@ struct Opts {
 	output: OutputOpts,
 	#[clap(flatten)]
 	debug: DebugOpts,
+	#[clap(flatten)]
+	gc: GcOpts,
 }
 
 fn main() {
@@ -114,6 +116,7 @@ impl From<LocError> for Error {
 }
 
 fn main_catch(opts: Opts) -> bool {
+	let _printer = opts.gc.stats_printer();
 	let state = EvaluationState::default();
 	if let Err(e) = main_real(&state, opts) {
 		if let Error::Evaluation(e) = e {
@@ -127,6 +130,7 @@ fn main_catch(opts: Opts) -> bool {
 }
 
 fn main_real(state: &EvaluationState, opts: Opts) -> Result<(), Error> {
+	opts.gc.configure_global();
 	opts.general.configure(&state)?;
 	opts.manifest.configure(&state)?;
 

crates/jrsonnet-cli/Cargo.toml

@@ -10,6 +10,7 @@ publish = false
 [dependencies]
 jrsonnet-evaluator = { path = "../../crates/jrsonnet-evaluator", version = "0.3.6", features = ["explaining-traces"] }
 jrsonnet-parser = { path = "../../crates/jrsonnet-parser", version = "0.3.6" }
+jrsonnet-gc = { version = "0.4.2", features = ["derive", "unstable-config", "unstable-stats"] }
 
 [dependencies.clap]
 git = "https://github.com/clap-rs/clap"

crates/jrsonnet-cli/src/lib.rs

@@ -95,3 +95,55 @@ impl ConfigureState for GeneralOpts {
 		Ok(())
 	}
 }
+
+#[derive(Clap)]
+#[clap(help_heading = "GARBAGE COLLECTION")]
+pub struct GcOpts {
+	/// Min bytes allocated to start garbage collection
+	#[clap(long, default_value = "20000000")]
+	gc_initial_threshold: usize,
+	/// How much heap should grow after unsuccessful garbage collection
+	#[clap(long)]
+	gc_used_space_ratio: Option<f64>,
+	/// Do not skip gc on exit
+	#[clap(long)]
+	gc_collect_on_exit: bool,
+	/// Print gc stats before exit
+	#[clap(long)]
+	gc_print_stats: bool,
+	/// Force garbage collection before printing stats
+	/// Useful for checking for memory leaks
+	/// Does nothing useless --gc-print-stats is specified
+	#[clap(long)]
+	gc_collect_before_printing_stats: bool,
+}
+impl GcOpts {
+	pub fn stats_printer(&self) -> Option<GcStatsPrinter> {
+		self.gc_print_stats
+			.then(|| GcStatsPrinter(self.gc_collect_before_printing_stats))
+	}
+	pub fn configure_global(&self) {
+		jrsonnet_gc::configure(|config| {
+			config.leak_on_drop = !self.gc_collect_on_exit;
+			config.threshold = self.gc_initial_threshold;
+			if let Some(used_space_ratio) = self.gc_used_space_ratio {
+				config.used_space_ratio = used_space_ratio;
+			}
+		});
+	}
+}
+pub struct GcStatsPrinter(bool);
+impl Drop for GcStatsPrinter {
+	fn drop(&mut self) {
+		if self.0 {
+			jrsonnet_gc::force_collect()
+		}
+		eprintln!("=== GC STATS ===");
+		jrsonnet_gc::configure(|c| {
+			eprintln!("Final threshold: {:?}", c.threshold);
+		});
+		let stats = jrsonnet_gc::stats();
+		eprintln!("Collections performed: {}", stats.collections_performed);
+		eprintln!("Bytes still allocated: {}", stats.bytes_allocated);
+	}
+}

crates/jrsonnet-evaluator/Cargo.toml

@@ -30,13 +30,12 @@ jrsonnet-stdlib = { path = "../jrsonnet-stdlib", version = "0.3.8" }
 jrsonnet-types = { path = "../jrsonnet-types", version = "0.3.8" }
 pathdiff = "0.2.0"
 
-closure = "0.3.0"
-
 md5 = "0.7.0"
 base64 = "0.13.0"
 rustc-hash = "1.1.0"
 
 thiserror = "1.0"
+jrsonnet-gc = { version = "0.4.2", features = ["derive"] }
 
 [dependencies.anyhow]
 version = "1.0"

crates/jrsonnet-evaluator/README.md

@@ -6,9 +6,6 @@ Interpreter for parsed jsonnet tree
 
 jsonnet stdlib is embedded into evaluator, but there is different modes for this:
 
-- `codegenerated-stdlib`
-  - generates source code for reproducing stdlib AST ([Example](https://gist.githubusercontent.com/CertainLach/7b3149df556f3406f5e9368aaa9f32ec/raw/0c80d8ab9aa7b9288c6219a2779cb2ab37287669/a.rs))
-  - fastest on interpretation, slowest on compilation (it takes more than 5 minutes to optimize them by llvm)
 - `serialized-stdlib`
   - serializes standard library AST using serde
   - slower than `codegenerated-stdlib` at runtime, but have no compilation speed penality
@@ -23,8 +20,6 @@ Because of `codegenerated-stdlib` compilation slowdown, `serialized-stdlib` is u
 Can also be run via `cargo bench`
 
 ```markdown
-# codegenerated-stdlib
-test tests::bench_codegen   ... bench:     401,696 ns/iter (+/- 38,521)
 # serialized-stdlib
 test tests::bench_serialize ... bench:   1,763,999 ns/iter (+/- 76,211)
 # none
@@ -34,7 +29,3 @@ test tests::bench_parse     ... bench:   7,206,164 ns/iter (+/- 1,067,418)
 ## Intrinsics
 
 Some functions from stdlib are implemented as intrinsics
-
-### Intrinsic handling
-
-If indexed jsonnet object has field '__intrinsic_namespace__' of type 'string', then any not found field/method is resolved as `Val::Intrinsic(__intrinsic_namespace__, name)`