RTL Simulation Enhancements

.gitignore

@@ -6,6 +6,7 @@ target
 *.stamp
 *.vcd
 *.swp
+*.swo
 *.log
 *#
 *~

bin/enable_printfs

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+#=============================================================================
+# this enables the PRINTF_COND around points of interest
+#=============================================================================
+
+import sys
+import re
+from textwrap import dedent as dd
+
+#=============================================================================
+# subroutines
+#=============================================================================
+def print_usage():
+  print(dd("""
+    enable_printfs <in_file> <out_file> [pattern1] [pattern2] ...
+    -------------------------------------------------------------
+    Selectively enables verbose printf() in your verilog. Right
+    now, you either get no verbose logging, or 100% verbose logging, but many
+    times, you might want to just enable logging in a specific module. THIS
+    SCRIPT IS A TEMPORARY HACK: post-processes the verilog file to remove the
+    `ifdef PRINTF_COND wrappers around the specific printf()s you care about.
+    You must manually specify python-compatible regex patterns on the cmdline
+    to match() the strings in the printf you care about. Multiple patterns are
+    treated as logical OR, not AND.
+  """))
+  exit(1)
+
+#=============================================================================
+if __name__ == "__main__":
+  if len(sys.argv) < 3:
+    print_usage()
+
+  # parse inputs
+  in_file = sys.argv[1]
+  out_file = sys.argv[2]
+  trigger = re.compile(".*`ifdef PRINTF_COND.*")
+  targets = []
+  for idx in range(3, len(sys.argv)):
+    if len(sys.argv[idx]) > 0:
+      targets.append(re.compile(sys.argv[idx]))
+
+  # read input lines
+  in_lines = []
+  with open(in_file, "r") as f:
+    in_lines = f.readlines()
+
+  # enable PRINTF_COND lines around points of interest
+  enables = 0
+  out_lines = []
+  idx = 0
+  idx_end = len(in_lines)
+  while(idx < idx_end):
+    if trigger.match(in_lines[idx]):
+      matches_target = False
+      for target in targets:
+        if target.match(in_lines[idx+4]):
+          matches_target = True
+          break
+      if matches_target:
+        enables += 1
+        out_lines += in_lines[idx+3:idx+6]
+      else:
+        out_lines += in_lines[idx:idx+9]
+      idx += 9
+    else:
+      out_lines.append(in_lines[idx])
+      idx += 1
+
+  # write to output file
+  with open(out_file, "w") as f:
+    f.writelines(out_lines)
+
+  print("INFO: enabled {} printfs in {}".format(enables, out_file))
+

bin/numa_prefix

@@ -0,0 +1,49 @@
+#!/usr/bin/env perl
+
+#============================================================================
+# really simple script, which just prints out the numactl cmd to 
+# prefix before your actual command. it determines this based on free memory
+# size attached to every node.
+#============================================================================
+
+use strict;
+use warnings;
+
+my $path = `which numactl`;
+if(length($path) > 0) {
+  my ($head_line, @rest) = map {chomp; $_} `numactl -H`;
+
+  if($head_line =~ /available: (\d+) nodes/) {
+    my $node_count     = $1;
+    my $best_node_id   = undef
+    my $best_cpus      = undef;
+    my $best_free_size = undef;
+
+    foreach my $num (1..$node_count) {
+      my $cpus_line     = shift(@rest);
+      my $mem_size_line = shift(@rest);
+      my $mem_free_line = shift(@rest);
+
+      if($cpus_line =~ /node (\d+) cpus: (\d.*\d)$/) {
+        my ($node_id, $cpus) = ($1, $2);
+        $cpus =~ s/\s+/,/g;
+
+        if($mem_free_line =~ /node $node_id free: (\d+) \S+$/) {
+          my $free_size = $1;
+          if(!defined($best_free_size) || ($free_size > $best_free_size)) {
+            $best_node_id   = $node_id;
+            $best_cpus      = $cpus;
+            $best_free_size = $free_size;
+          }
+        } else {
+          die("malformed mem-free line: $mem_free_line\n");
+        }
+      } else {
+        die("malformed cpus line: $cpus_line\n");
+      }
+    }
+    print("numactl -m $best_node_id -C $best_cpus --");
+  } else {
+    die("malformed head line: $head_line\n");
+  }
+}

common.mk

@@ -7,11 +7,20 @@ SHELL=/bin/bash
 # extra make variables/rules from subprojects
 #
 # EXTRA_GENERATOR_REQS - requirements needed for the main generator
-# EXTRA_SIM_FLAGS - runtime simulation flags
-# EXTRA_SIM_CC_FLAGS - cc flags for simulators
-# EXTRA_SIM_SOURCES - simulation sources needed for simulator
-# EXTRA_SIM_REQS - requirements to build the simulator
-#########################################################################################
+# EXTRA_SIM_FLAGS      - runtime simulation flags
+# EXTRA_SIM_CFLAGS     - CFLAGS for building simulators
+# EXTRA_SIM_CXXFLAGS   - CXXFLAGS for building simulators
+# EXTRA_SIM_LDFLAGS    - LDFLAGS for building simulators
+# EXTRA_SIM_SOURCES    - simulation sources needed for simulator
+# EXTRA_SIM_REQS       - requirements to build the simulator
+#########################################################################################
+EXTRA_SIM_FLAGS    ?=
+EXTRA_SIM_CXXFLAGS ?=
+EXTRA_SIM_CFLAGS   ?=
+EXTRA_SIM_LDFLAGS  ?=
+EXTRA_SIM_SOURCES  ?=
+EXTRA_SIM_REQS     ?=
+
 include $(base_dir)/generators/ariane/ariane.mk
 include $(base_dir)/generators/tracegen/tracegen.mk
 include $(base_dir)/generators/nvdla/nvdla.mk

docs/Simulation/Software-RTL-Simulation.rst

@@ -138,3 +138,16 @@ An open-source vcd-capable waveform viewer is `GTKWave <http://gtkwave.sourcefor
 For a VCS simulation, this will generate a vpd file (this is a proprietary waveform representation format used by Synopsys) that can be loaded to vpd-supported waveform viewers.
 If you have Synopsys licenses, we recommend using the DVE waveform viewer.
 
+.. _sw-sim-verilator-opts:
+
+Additional Verilator Options
+-------------------------------
+When building the verilator simulator there are several additional options:
+
+.. code-block:: shell
+
+   make VERILATOR_THREADS=8 ENABLE_PRINTF_PATTERN='<python-regex>'
+
+The ``VERILATOR_THREADS=<num>`` option enables the compiled verilator simulator to use ``<num>`` parallel threads. on a multi-socket machine, you will want to make sure all threads are on the same socket by using ``numactl``. You can also just use the ``numa_prefix`` wrapper, which is a simple wrapper around ``numactl`` that runs your verilated simulator like this: ``$(numa_prefix) ./simulator-<name> <simulator-args>``.
+
+The ``ENABLE_PRINTF_PATTERN`` option selectively enables hardware printf's based on a text match of the format string. For example, if you wanted to enable the rocket-core's instruction log, and nothing else, you could run ``make VERILATOR_THREADS=8 ENABLE_PRINTF_PATTERN='.*DASM\(%x\)'``. Don't forget to start the line with a ``.*``, since the pattern matches start at the beginning of the line.

generators/utilities/src/main/resources/csrc/emulator.cc

@@ -1,11 +1,15 @@
 // See LICENSE.SiFive for license details.
 // See LICENSE.Berkeley for license details.
 
-#include "verilated.h"
 #if VM_TRACE
 #include <memory>
+#if CY_FST_TRACE
+#include "verilated_fst_c.h"
+#else
+#include "verilated.h"
 #include "verilated_vcd_c.h"
-#endif
+#endif // CY_FST_TRACE
+#endif // VM_TRACE
 #include <fesvr/dtm.h>
 #include <fesvr/tsi.h>
 #include "remote_bitbang.h"
@@ -16,6 +20,8 @@
 #include <stdlib.h>
 #include <unistd.h>
 #include <getopt.h>
+// needed for s_vpi_vlog_info, which is needed for multithreading
+#include <vpi_user.h>
 
 // For option parsing, which is split across this file, Verilog, and
 // FESVR's HTIF, a few external files must be pulled in. The list of
@@ -51,9 +57,16 @@ double sc_time_stamp()
   return trace_count;
 }
 
-extern "C" int vpi_get_vlog_info(void* arg)
+// need to pull htif_argc/htif_argv out here so the thread that calls tick()
+// for the HTIF device can initialize properly with the cmdline args. this
+// was pulled out here for multithreading to work
+static int htif_argc;
+static char **htif_argv = NULL;
+extern "C" int vpi_get_vlog_info(s_vpi_vlog_info *vlog_info_s)
 {
-  return 0;
+  vlog_info_s->argc = htif_argc;
+  vlog_info_s->argv = htif_argv;
+  return 1;
 }
 
 static void usage(const char * program_name)
@@ -119,10 +132,10 @@ int main(int argc, char** argv)
   // Port numbers are 16 bit unsigned integers. 
   uint16_t rbb_port = 0;
 #if VM_TRACE
+  const char* vcdfile_name = NULL; 
   FILE * vcdfile = NULL;
   uint64_t start = 0;
 #endif
-  char ** htif_argv = NULL;
   int verilog_plusargs_legal = 1;
 
   dramsim = 0;
@@ -162,6 +175,7 @@ int main(int argc, char** argv)
       case 'D': dramsim = 1;                break;
 #if VM_TRACE
       case 'v': {
+        vcdfile_name = optarg;
         vcdfile = strcmp(optarg, "-") == 0 ? stdout : fopen(optarg, "w");
         if (!vcdfile) {
           std::cerr << "Unable to open " << optarg << " for VCD write\n";
@@ -250,7 +264,7 @@ int main(int argc, char** argv)
     usage(argv[0]);
     return 1;
   }
-  int htif_argc = 1 + argc - optind;
+  htif_argc = 1 + argc - optind;
   htif_argv = (char **) malloc((htif_argc) * sizeof (char *));
   htif_argv[0] = argv[0];
   for (int i = 1; optind < argc;) htif_argv[i++] = argv[optind++];
@@ -267,17 +281,21 @@ int main(int argc, char** argv)
 
 #if VM_TRACE
   Verilated::traceEverOn(true); // Verilator must compute traced signals
+#if CY_FST_TRACE
+  std::unique_ptr<VerilatedFstC> tfp(new VerilatedFstC);
+  if (vcdfile_name) {
+    tile->trace(tfp.get(), 99);  // Trace 99 levels of hierarchy
+    tfp->open(vcdfile_name);
+  }
+#else
   std::unique_ptr<VerilatedVcdFILE> vcdfd(new VerilatedVcdFILE(vcdfile));
   std::unique_ptr<VerilatedVcdC> tfp(new VerilatedVcdC(vcdfd.get()));
   if (vcdfile) {
     tile->trace(tfp.get(), 99);  // Trace 99 levels of hierarchy
     tfp->open("");
   }
-#endif
-
-  jtag = new remote_bitbang_t(rbb_port);
-  dtm = new dtm_t(htif_argc, htif_argv);
-  tsi = new tsi_t(htif_argc, htif_argv);
+#endif // CY_FST_TRACE
+#endif // VM_TRACE
 
   signal(SIGTERM, handle_sigterm);
 
@@ -307,8 +325,7 @@ int main(int argc, char** argv)
   tile->reset = 0;
   done_reset = true;
 
-  while (!dtm->done() && !jtag->done() && !tsi->done() &&
-         !tile->io_success && trace_count < max_cycles) {
+  do {
     tile->clock = 0;
     tile->eval();
 #if VM_TRACE
@@ -325,6 +342,13 @@ int main(int argc, char** argv)
 #endif
     trace_count++;
   }
+  // for verilator multithreading. need to do 1 loop before checking if
+  // tsi exists, since tsi is created by verilated thread on the first 
+  // serial_tick. 
+  while ((!dtm || !dtm->done()) && 
+         (!jtag || !jtag->done()) && 
+         (!tsi || !tsi->done()) &&
+         !tile->io_success && trace_count < max_cycles);
 
 #if VM_TRACE
   if (tfp)
@@ -333,17 +357,17 @@ int main(int argc, char** argv)
     fclose(vcdfile);
 #endif
 
-  if (dtm->exit_code())
+  if (dtm && dtm->exit_code())
   {
     fprintf(stderr, "*** FAILED *** via dtm (code = %d, seed %d) after %ld cycles\n", dtm->exit_code(), random_seed, trace_count);
     ret = dtm->exit_code();
   }
-  else if (tsi->exit_code())
+  else if (tsi && tsi->exit_code())
   {
     fprintf(stderr, "*** FAILED *** (code = %d, seed %d) after %ld cycles\n", tsi->exit_code(), random_seed, trace_count);
     ret = tsi->exit_code();
   }
-  else if (jtag->exit_code())
+  else if (jtag && jtag->exit_code())
   {
     fprintf(stderr, "*** FAILED *** via jtag (code = %d, seed %d) after %ld cycles\n", jtag->exit_code(), random_seed, trace_count);
     ret = jtag->exit_code();

scripts/build-toolchains.sh

@@ -6,8 +6,8 @@
 set -e
 set -o pipefail
 
-RDIR=$(pwd)
-CHIPYARD_DIR="${CHIPYARD_DIR:-$(git rev-parse --show-toplevel)}"
+DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+CHIPYARD_DIR="$(dirname "$DIR")"
 
 usage() {
     echo "usage: ${0} [OPTIONS] [riscv-tools | esp-tools | ec2fast]"
@@ -134,18 +134,16 @@ SRCDIR="$(pwd)/toolchains" module_all qemu --prefix="${RISCV}" --target-list=ris
 git submodule update --init $CHIPYARD_DIR/tools/dromajo/dromajo-src
 make -C $CHIPYARD_DIR/tools/dromajo/dromajo-src/src
 
-cd "$RDIR"
-
 # create specific env.sh
-{
-    echo "# auto-generated by build-toolchains.sh"
-    echo "export CHIPYARD_TOOLCHAIN_SOURCED=1"
-    echo "export RISCV=$(printf '%q' "$RISCV")"
-    echo "export PATH=\${RISCV}/bin:\${PATH}"
-    echo "export LD_LIBRARY_PATH=\${RISCV}/lib\${LD_LIBRARY_PATH:+":\${LD_LIBRARY_PATH}"}"
-} > env-$TOOLCHAIN.sh
+cat > "$CHIPYARD_DIR/env-$TOOLCHAIN.sh" <<EOF
+# auto-generated by build-toolchains.sh
+export CHIPYARD_TOOLCHAIN_SOURCED=1
+export RISCV=$(printf '%q' "$RISCV")
+export PATH=\${RISCV}/bin:\${PATH}
+export LD_LIBRARY_PATH=\${RISCV}/lib\${LD_LIBRARY_PATH:+":\${LD_LIBRARY_PATH}"}
+EOF
 
 # create general env.sh
 echo "# line auto-generated by build-toolchains.sh" >> env.sh
-echo "source \$( realpath \$(dirname "\${BASH_SOURCE[0]:-\${\(%\):-%x}}") )/env-$TOOLCHAIN.sh" >> env.sh
+echo "source $(printf '%q' "$CHIPYARD_DIR/env-$TOOLCHAIN.sh")" >> env.sh
 echo "Toolchain Build Complete!"

scripts/build-util.sh

@@ -15,7 +15,7 @@ case ${ncpu} in
 esac
 
 # Allow user to override MAKE
-[ -n "${MAKE}" ] || MAKE=$(command -v gnumake || command -v gmake || command -v make)
+[ -n "${MAKE:+x}" ] || MAKE=$(command -v gnumake || command -v gmake || command -v make)
 readonly MAKE