validation: write playground stats to file for parsing

WIP.md

@@ -2,11 +2,12 @@
 
 - today:
 
+  - playground stats (should match accelsim)
   - convert, match and plot statistics
-  - rename crates and github repo
-  - publish to crates.io
   - record mem fetch latency in playground and box
+  - DONE: rename crates and github repo
   - DONE: write trait for tag array
+  - SKIP: publish to crates.io
 
 - TODO:
 

accelsim/a4000/gpgpusim.trace.config

@@ -0,0 +1,23 @@
+-trace_opcode_latency_initiation_int 2,2
+-trace_opcode_latency_initiation_sp 2,1
+-trace_opcode_latency_initiation_dp 64,64
+-trace_opcode_latency_initiation_sfu 21,8
+-trace_opcode_latency_initiation_tensor 32,32
+
+#execute branch insts on spec unit 1
+#<enabled>,<num_units>,<max_latency>,<ID_OC_SPEC>,<OC_EX_SPEC>,<NAME>
+-specialized_unit_1 1,4,4,4,4,BRA
+-trace_opcode_latency_initiation_spec_op_1 4,4
+
+#TEX unit, make fixed latency for all tex insts
+-specialized_unit_2 1,4,200,4,4,TEX
+-trace_opcode_latency_initiation_spec_op_2 200,4
+
+#tensor unit
+-specialized_unit_3 1,4,32,4,4,TENSOR
+-trace_opcode_latency_initiation_spec_op_3 32,32
+
+#UDP unit, for turing and above
+#for more info about UDP, see https://www.hotchips.org/hc31/HC31_2.12_NVIDIA_final.pdf
+-specialized_unit_4 1,4,4,4,4,UDP
+-trace_opcode_latency_initiation_spec_op_4 4,1

accelsim/gtx1080/gpgpusim.trace.config

@@ -1,23 +0,0 @@
--trace_opcode_latency_initiation_int 2,2
--trace_opcode_latency_initiation_sp 2,1
--trace_opcode_latency_initiation_dp 64,64
--trace_opcode_latency_initiation_sfu 21,8
--trace_opcode_latency_initiation_tensor 32,32
-
-#execute branch insts on spec unit 1
-#<enabled>,<num_units>,<max_latency>,<ID_OC_SPEC>,<OC_EX_SPEC>,<NAME>
--specialized_unit_1 1,4,4,4,4,BRA
--trace_opcode_latency_initiation_spec_op_1 4,4
-
-#TEX unit, make fixed latency for all tex insts
--specialized_unit_2 1,4,200,4,4,TEX
--trace_opcode_latency_initiation_spec_op_2 200,4
-
-#tensor unit
--specialized_unit_3 1,4,32,4,4,TENSOR
--trace_opcode_latency_initiation_spec_op_3 32,32
-
-#UDP unit, for turing and above
-#for more info about UDP, see https://www.hotchips.org/hc31/HC31_2.12_NVIDIA_final.pdf
--specialized_unit_4 1,4,4,4,4,UDP
--trace_opcode_latency_initiation_spec_op_4 4,1

accelsim/sim/Cargo.toml

@@ -5,7 +5,7 @@ edition = "2021"
 publish = false
 
 [features]
-default = []
+default = ["upstream"]
 upstream = []
 
 [package.metadata.cargo-feature-combinations]
@@ -16,13 +16,16 @@ need_cuda = true
 need_accelsim_build_dependencies = true
 
 [dependencies]
-color-eyre = "0"
+tokio = { version = "1", features = ["full"] }
 async-process = "1"
+futures = "0.3"
+
+color-eyre = "0"
 log = "0"
+
 serde_json = "1"
 env_logger = "0"
 clap = { version = "4", features = [ "derive" ] }
-tokio = { version = "1", features = ["full"] }
 
 accelsim = { path = "../" }
 utils = { path = "../../utils" }

accelsim/sim/build.rs

@@ -83,33 +83,32 @@ fn main() -> eyre::Result<()> {
     println!("cargo:rerun-if-env-changed=BUILD");
     println!("cargo:rerun-if-env-changed=build.rs");
 
+    let mut implementations = vec![accelsim::locate(false)?];
     #[cfg(feature = "upstream")]
-    let use_upstream = true;
-    #[cfg(not(feature = "upstream"))]
-    let use_upstream = false;
+    implementations.push(accelsim::locate(true)?);
 
-    let accel_path = accelsim::locate(use_upstream)?;
+    for accel_path in implementations {
+        let force = accelsim::build::is_force();
+        println!("cargo:warning=force={}", &force);
 
-    let force = accelsim::build::is_force();
-    println!("cargo:warning=force={}", &force);
+        // this does not work well, because accelsim builds in-tree, hence every
+        // build modifies ../accel-sim-framework-dev/ ... /build which triggers a rebuild
+        // Workaround: for now, just use the FORCE / BUILD flag with "yes"
+        // println!("cargo:rerun-if-changed=../accel-sim-framework-dev/");
 
-    // this does not work well, because accelsim builds in-tree, hence every
-    // build modifies ../accel-sim-framework-dev/ ... /build which triggers a rebuild
-    // Workaround: for now, just use the FORCE / BUILD flag with "yes"
-    // println!("cargo:rerun-if-changed=../accel-sim-framework-dev/");
+        if force {
+            println!("cargo:rerun-if-changed={}", accel_path.display());
+        }
 
-    if force {
-        println!("cargo:rerun-if-changed={}", accel_path.display());
-    }
-
-    println!(
-        "cargo:warning=using accelsim source at {}",
-        &accel_path.display()
-    );
+        println!(
+            "cargo:warning=using accelsim source at {}",
+            &accel_path.display()
+        );
 
-    let cuda_path = utils::find_cuda().ok_or(eyre::eyre!("CUDA not found"))?;
-    println!("cargo:warning=using cuda at {}", &cuda_path.display());
+        let cuda_path = utils::find_cuda().ok_or(eyre::eyre!("CUDA not found"))?;
+        println!("cargo:warning=using cuda at {}", &cuda_path.display());
 
-    build_accelsim(&accel_path, &cuda_path, force)?;
+        build_accelsim(&accel_path, &cuda_path, force)?;
+    }
     Ok(())
 }

accelsim/sim/src/lib.rs

@@ -28,6 +28,7 @@ pub fn render_sim_script(
     profile: &str,
     config: &SimConfig,
     setup_env_path: &Path,
+    extra_sim_args: Vec<String>,
 ) -> eyre::Result<String> {
     let mut sim_sh = vec![];
     sim_sh.push("#!/usr/bin/env bash".to_string());
@@ -53,7 +54,7 @@ pub fn render_sim_script(
         )
     })?;
 
-    let mut trace_cmd: Vec<String> = vec![
+    let mut sim_cmd: Vec<String> = vec![
         accelsim_bin.to_string_lossy().to_string(),
         "-trace".to_string(),
         kernelslist.to_string_lossy().to_string(),
@@ -64,24 +65,42 @@ pub fn render_sim_script(
     let trace_config = config.trace_config().as_deref().map(Path::canonicalize);
     match trace_config {
         Some(Ok(config)) if config.is_file() => {
-            trace_cmd.extend(["-config".to_string(), config.to_string_lossy().to_string()]);
+            sim_cmd.extend(["-config".to_string(), config.to_string_lossy().to_string()]);
         }
         _ => {}
     }
-    sim_sh.push(trace_cmd.join(" "));
+
+    // extra simulatin arguments have highest precedence
+    sim_cmd.extend(extra_sim_args);
+
+    sim_sh.push(sim_cmd.join(" "));
     Ok(sim_sh.join("\n"))
 }
 
-pub async fn simulate_trace(
+pub async fn simulate_trace<A>(
     traces_dir: impl AsRef<Path>,
     kernelslist: impl AsRef<Path>,
     config: &SimConfig,
     timeout: Option<Duration>,
-) -> eyre::Result<(std::process::Output, Duration)> {
+    extra_sim_args: A,
+    stream_output: bool,
+    use_upstream: Option<bool>,
+) -> eyre::Result<(std::process::Output, Duration)>
+where
+    A: IntoIterator,
+    <A as IntoIterator>::Item: Into<String>,
+{
     #[cfg(feature = "upstream")]
-    let use_upstream = true;
+    let use_upstream_default = true;
     #[cfg(not(feature = "upstream"))]
-    let use_upstream = false;
+    let use_upstream_default = false;
+
+    let use_upstream = use_upstream.unwrap_or(use_upstream_default);
+    #[cfg(not(feature = "upstream"))]
+    if use_upstream {
+        eyre::bail!("accelsim-sim was not compiled with upstream accelsim");
+    }
+    log::debug!("upstream = {}", use_upstream);
 
     let accelsim_path = accelsim::locate(use_upstream)?;
     let profile = accelsim::profile();
@@ -91,6 +110,8 @@ pub async fn simulate_trace(
         .canonicalize()
         .wrap_err_with(|| format!("{} does not exist", accelsim_bin.display()))?;
 
+    log::debug!("using accelsim binary at {}", accelsim_bin.display());
+
     let sim_root = accelsim_path.join("gpu-simulator/");
     let setup_env_path = sim_root.join("setup_environment.sh");
     let setup_env_path = setup_env_path
@@ -114,13 +135,15 @@ pub async fn simulate_trace(
         eyre::bail!("config dir {} is not a directory", config_dir.display());
     }
 
+    let extra_sim_args: Vec<String> = extra_sim_args.into_iter().map(Into::into).collect();
     let tmp_sim_sh = render_sim_script(
         &accelsim_bin,
         &kernelslist,
         &config_dir,
         profile,
         config,
         &setup_env_path,
+        extra_sim_args,
     )?;
     log::debug!("{}", &tmp_sim_sh);
 
@@ -143,10 +166,33 @@ pub async fn simulate_trace(
     cmd.env("CUDA_INSTALL_PATH", &*cuda_path.to_string_lossy());
     log::debug!("command: {:?}", &cmd);
 
+    let get_cmd_output = async {
+        if stream_output {
+            use futures::{AsyncBufReadExt, StreamExt};
+            let mut stdout: Vec<u8> = Vec::new();
+            let mut child = cmd.stdout(async_process::Stdio::piped()).spawn()?;
+
+            let mut line_reader = futures::io::BufReader::new(child.stdout.take().unwrap()).lines();
+            while let Some(line) = line_reader.next().await {
+                let line = line?;
+                println!("{}", line);
+                stdout.extend(line.into_bytes());
+                stdout.write(b"\n")?;
+            }
+            Ok(std::process::Output {
+                status: child.status().await?,
+                stdout,
+                stderr: Vec::new(),
+            })
+        } else {
+            cmd.output().await
+        }
+    };
+
     let start = Instant::now();
     let result = match timeout {
-        Some(timeout) => tokio::time::timeout(timeout, cmd.output()).await,
-        None => Ok(cmd.output().await),
+        Some(timeout) => tokio::time::timeout(timeout, get_cmd_output).await,
+        None => Ok(get_cmd_output.await),
     };
     let result = result??;
     let dur = start.elapsed();

accelsim/sim/src/main.rs

@@ -7,8 +7,6 @@ use color_eyre::eyre::{self, WrapErr};
 use std::io::Write;
 use std::time::Instant;
 
-// todo add an output dir
-
 #[tokio::main]
 async fn main() -> eyre::Result<()> {
     env_logger::init();
@@ -35,11 +33,21 @@ async fn main() -> eyre::Result<()> {
         .canonicalize()
         .wrap_err_with(|| format!("kernelslist at {} does not exist", kernelslist.display()))?;
 
+    println!(
+        "simulating {} [upstream={:?}]",
+        kernelslist.display(),
+        options.use_upstream
+    );
+
+    let extra_args: &[String] = &[];
     let (output, _dur) = sim::simulate_trace(
         &traces_dir,
         &kernelslist,
         &options.sim_config,
         options.timeout,
+        extra_args,
+        options.stream_output.unwrap_or(true),
+        options.use_upstream,
     )
     .await?;
 
@@ -62,23 +70,25 @@ async fn main() -> eyre::Result<()> {
     let mut log_file = utils::fs::open_writable(&log_file_path)?;
     log_file.write_all(stdout.as_bytes())?;
 
-    let stats_file_path = options
-        .stats_file
-        .unwrap_or(log_file_path.with_extension("json"));
+    println!("wrote log to {}", log_file_path.display());
 
-    let log_reader = std::io::Cursor::new(stdout);
-    let parse_options = accelsim::parser::Options::default();
-    let stats = accelsim::parser::parse_stats(log_reader, &parse_options)?;
+    // let log_reader = std::io::Cursor::new(stdout);
+    // let parse_options = accelsim::parser::Options::default();
+    // let stats = accelsim::parser::parse_stats(log_reader, &parse_options)?;
 
-    println!("{:#?}", &stats);
+    // println!("{:#?}", &stats);
 
-    let converted: Result<stats::Stats, _> = stats.clone().try_into();
-    println!("{:#?}", &converted);
+    // let converted: Result<stats::Stats, _> = stats.clone().try_into();
+    // println!("{:#?}", &converted);
     // for stat in stats.iter() {
     //     println!("{}", &stat);
     // }
 
-    let flat_stats: Vec<_> = stats.into_iter().collect();
-    serde_json::to_writer_pretty(utils::fs::open_writable(stats_file_path)?, &flat_stats)?;
+    // let stats_file_path = options
+    //     .stats_file
+    //     .unwrap_or(log_file_path.with_extension("json"));
+
+    // let flat_stats: Vec<_> = stats.into_iter().collect();
+    // serde_json::to_writer_pretty(utils::fs::open_writable(stats_file_path)?, &flat_stats)?;
     Ok(())
 }

accelsim/src/cache.h

@@ -1778,8 +1778,8 @@ class cache_config {
 
 protected:
   void exit_parse_error() {
-    printf("GPGPU-Sim uArch: cache configuration parsing error (%s)\n",
-           m_config_string);
+    fprintf(stderr, "GPGPU-Sim uArch: cache configuration parsing error (%s)\n",
+            m_config_string);
     abort();
   }
 

accelsim/src/options.rs

@@ -32,6 +32,16 @@ pub struct Options {
         value_parser = parse_duration_string,
     )]
     pub timeout: Option<Duration>,
+
+    #[clap(long = "progress", aliases = ["follow"], help = "stream simulation output")]
+    pub stream_output: Option<bool>,
+
+    #[clap(
+        long = "upstream",
+        action = clap::ArgAction::SetTrue,
+        help = "use upstream accelsim implementation (unmodified)"
+    )]
+    pub use_upstream: Option<bool>,
 }
 
 #[derive(Parser, Debug, Default)]

accelsim/src/stats.rs

@@ -153,7 +153,7 @@ impl TryFrom<Stats> for stats::Stats {
         .map(|(kind, stat)| (kind, stats.get(&key!(stat)).copied().unwrap_or(0.0) as u64))
         .collect();
 
-        dbg!(&stats);
+        // dbg!(&stats);
 
         // todo
         let instructions = stats::InstructionCounts::default();