From 7cf528ad20a5929d1081b55ab7404e1eafd4c54b Mon Sep 17 00:00:00 2001 From: Tom Solberg Date: Mon, 25 Nov 2024 13:47:44 +0100 Subject: [PATCH 1/4] ensure nnef tars are deterministic --- nnef/src/framework.rs | 42 ++++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/nnef/src/framework.rs b/nnef/src/framework.rs index 09d51f759e..e81ba65b86 100644 --- a/nnef/src/framework.rs +++ b/nnef/src/framework.rs @@ -85,7 +85,9 @@ impl Nnef { pub fn write_to_tar(&self, model: &TypedModel, w: W) -> TractResult { let mut ar = tar::Builder::new(w); - self._write_to_tar(model, &mut ar, false)?; + let timestamp = + std::time::SystemTime::now().duration_since(std::time::SystemTime::UNIX_EPOCH).unwrap(); + self._write_to_tar(model, &mut ar, false, timestamp)?; ar.into_inner().context("Finalizing tar") } @@ -94,9 +96,17 @@ impl Nnef { model: &TypedModel, w: W, compress_nested_models: bool, + deterministic: bool, ) -> TractResult { let mut ar = tar::Builder::new(w); - self._write_to_tar(model, &mut ar, compress_nested_models)?; + let timestamp = if deterministic { + // 1 Jan 1980, MS-DOS epoch. Some tools have issues with 0 timestamps. + std::time::Duration::from_secs(315532800) + } else { + std::time::SystemTime::now().duration_since(std::time::SystemTime::UNIX_EPOCH).unwrap() + }; + + self._write_to_tar(model, &mut ar, compress_nested_models, timestamp)?; ar.into_inner().context("Finalizing tar") } @@ -105,6 +115,7 @@ impl Nnef { model: &TypedModel, ar: &mut Builder, compress_nested_models: bool, + timestamp: std::time::Duration, ) -> TractResult<()> { let proto_model = crate::ser::to_proto_model(self, model).context("Translating model to proto_model")?; @@ -113,21 +124,22 @@ impl Nnef { crate::ast::dump::Dumper::new(self, &mut graph_data) .document(&proto_model.doc) .context("Serializing graph.nnef")?; - let now = - std::time::SystemTime::now().duration_since(std::time::SystemTime::UNIX_EPOCH).unwrap(); let mut header = tar::Header::new_gnu(); header.set_path("graph.nnef").context("Setting graph.nnef path")?; header.set_size(graph_data.len() as u64); header.set_mode(0o644); - header.set_mtime(now.as_secs()); + header.set_mtime(timestamp.as_secs()); header.set_cksum(); ar.append(&header, &mut &*graph_data).context("Appending graph.nnef")?; - if let Some(quantization) = proto_model.quantization { + if let Some(mut quantization) = proto_model.quantization { let mut quant_data = vec![]; - for (name, format) in quantization.into_iter() { + let mut keys = quantization.keys().cloned().collect::>(); + keys.sort(); + for name in keys { + let format = quantization.remove(&name).unwrap(); write_quant_format( &mut quant_data, &name, @@ -140,12 +152,15 @@ impl Nnef { header.set_path("graph.quant").context("Setting graph.quant path")?; header.set_size(quant_data.len() as u64); header.set_mode(0o644); - header.set_mtime(now.as_secs()); + header.set_mtime(timestamp.as_secs()); header.set_cksum(); ar.append(&header, &mut &*quant_data).context("Appending graph.quant")?; } - for (label, t) in &proto_model.tensors { + let mut labels = proto_model.tensors.keys().cloned().collect::>(); + labels.sort(); + for label in labels { + let t = proto_model.tensors.get(&label).unwrap(); let mut label = label.0.to_string() + ".dat"; if label.starts_with('/') { label.insert(0, '.'); @@ -158,14 +173,17 @@ impl Nnef { let mut header = tar::Header::new_gnu(); header.set_size(data.len() as u64); header.set_mode(0o644); - header.set_mtime(now.as_secs()); + header.set_mtime(timestamp.as_secs()); header.set_cksum(); ar.append_data(&mut header, filename, &mut &*data) .with_context(|| format!("Appending tensor {filename:?}"))?; } - for (label, resource) in proto_model.resources.iter() { + let mut labels = proto_model.resources.keys().collect::>(); + labels.sort(); + for label in labels { + let resource = proto_model.resources.get(label).unwrap(); if let Some(typed_model_resource) = resource.downcast_ref::() { let mut submodel_data = vec![]; let mut filename = std::path::PathBuf::from_str(label)?; @@ -186,7 +204,7 @@ impl Nnef { let mut header = tar::Header::new_gnu(); header.set_size(submodel_data.len() as u64); header.set_mode(0o644); - header.set_mtime(now.as_secs()); + header.set_mtime(timestamp.as_secs()); header.set_cksum(); ar.append_data(&mut header, filename, &mut &*submodel_data) From 4fa9062b6a6961f8f56f16a0117c27981d9a8fe0 Mon Sep 17 00:00:00 2001 From: Tom Solberg Date: Mon, 25 Nov 2024 13:56:59 +0100 Subject: [PATCH 2/4] avoid allocation --- nnef/src/framework.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nnef/src/framework.rs b/nnef/src/framework.rs index e81ba65b86..1478b8a3c7 100644 --- a/nnef/src/framework.rs +++ b/nnef/src/framework.rs @@ -157,10 +157,10 @@ impl Nnef { ar.append(&header, &mut &*quant_data).context("Appending graph.quant")?; } - let mut labels = proto_model.tensors.keys().cloned().collect::>(); + let mut labels = proto_model.tensors.keys().collect::>(); labels.sort(); for label in labels { - let t = proto_model.tensors.get(&label).unwrap(); + let t = proto_model.tensors.get(label).unwrap(); let mut label = label.0.to_string() + ".dat"; if label.starts_with('/') { label.insert(0, '.'); From 815e02a9d0aabb86bdecffcb203ef588fe376101 Mon Sep 17 00:00:00 2001 From: Tom Solberg Date: Mon, 25 Nov 2024 14:08:38 +0100 Subject: [PATCH 3/4] fix cli --- cli/src/dump.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cli/src/dump.rs b/cli/src/dump.rs index 2a1eac4e05..0ecf64b839 100644 --- a/cli/src/dump.rs +++ b/cli/src/dump.rs @@ -1,5 +1,5 @@ -use crate::plan_options::plan_options_from_subcommand; use crate::params::SomeGraphDef; +use crate::plan_options::plan_options_from_subcommand; use crate::tensor::run_params_from_subcommand; use crate::Parameters; use fs_err as fs; @@ -150,7 +150,11 @@ pub fn handle( if sub_matches.is_present("tmp_mem_usage") { let plan_options = plan_options_from_subcommand(sub_matches)?; - annotations.track_tmp_memory_usage(model, |n| !n.op_is::(), plan_options.skip_order_opt_ram)?; + annotations.track_tmp_memory_usage( + model, + |n| !n.op_is::(), + plan_options.skip_order_opt_ram, + )?; } if let Some(asserts) = ¶ms.assertions.assert_output_facts { @@ -177,7 +181,7 @@ pub fn handle( rename_outputs(&mut typed, sub_matches)?; let file = fs::File::create(path)?; let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::default()); - nnef.write_to_tar_with_config(&typed, encoder, compress_submodels) + nnef.write_to_tar_with_config(&typed, encoder, compress_submodels, false) .context("Writing model to tgz")?; } else { bail!("Only typed model can be dumped") @@ -189,7 +193,7 @@ pub fn handle( if let Some(mut typed) = model.downcast_ref::().cloned() { rename_outputs(&mut typed, sub_matches)?; let file = fs::File::create(path)?; - nnef.write_to_tar_with_config(&typed, file, compress_submodels) + nnef.write_to_tar_with_config(&typed, file, compress_submodels, false) .context("Writing model to tar")?; } else { bail!("Only typed model can be dumped") From d2d03d74f9ce00ff2da8cefb81f7006f13879b8b Mon Sep 17 00:00:00 2001 From: Tom Solberg Date: Tue, 26 Nov 2024 11:01:05 +0100 Subject: [PATCH 4/4] add flag --- cli/src/dump.rs | 5 +++-- cli/src/main.rs | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/cli/src/dump.rs b/cli/src/dump.rs index 0ecf64b839..91cfcd204c 100644 --- a/cli/src/dump.rs +++ b/cli/src/dump.rs @@ -175,13 +175,14 @@ pub fn handle( } let compress_submodels = sub_matches.is_present("compress-submodels"); + let deterministic = sub_matches.is_present("nnef-deterministic"); if let Some(path) = sub_matches.value_of("nnef") { let nnef = super::nnef(matches); if let Some(mut typed) = model.downcast_ref::().cloned() { rename_outputs(&mut typed, sub_matches)?; let file = fs::File::create(path)?; let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::default()); - nnef.write_to_tar_with_config(&typed, encoder, compress_submodels, false) + nnef.write_to_tar_with_config(&typed, encoder, compress_submodels, deterministic) .context("Writing model to tgz")?; } else { bail!("Only typed model can be dumped") @@ -193,7 +194,7 @@ pub fn handle( if let Some(mut typed) = model.downcast_ref::().cloned() { rename_outputs(&mut typed, sub_matches)?; let file = fs::File::create(path)?; - nnef.write_to_tar_with_config(&typed, file, compress_submodels, false) + nnef.write_to_tar_with_config(&typed, file, compress_submodels, deterministic) .context("Writing model to tar")?; } else { bail!("Only typed model can be dumped") diff --git a/cli/src/main.rs b/cli/src/main.rs index 15b3f0a977..9e265a71f9 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -383,6 +383,11 @@ fn dump_subcommand<'a>() -> clap::Command<'a> { Arg::new("compress-submodels") .long("compress-submodels") .help("Compress submodels if any (as a .tgz file)"), + ) + .arg( + Arg::new("nnef-deterministic") + .long("nnef-deterministic") + .help("If provided, will try to make output .nnef.tar files deterministic"), ) .arg( Arg::new("nnef-graph")