diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index eaa9321b4d..94a3f19cc2 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -42,7 +42,7 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} - name: "Build verilator emulator" run: | - nix build '.#t1.${{ matrix.config }}.ip.difftest' -L --no-link --cores 64 + nix build '.#t1.${{ matrix.config }}.ip.verilator-emu' -L --no-link --cores 64 - name: "Build all testcases" run: | # Build testcases with vlen 1024 and vlen 4096 @@ -81,7 +81,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} - name: "Build verilator emulator with trace" - run: nix build '.#t1.${{ matrix.config }}.ip.difftest-trace' -L --no-link --cores 64 + run: nix build '.#t1.${{ matrix.config }}.ip.verilator-emu-trace' -L --no-link --cores 64 test-emit: if: '! github.event.pull_request.draft' diff --git a/difftest/Cargo.toml b/difftest/Cargo.toml index 00abdf691b..b4488928c7 100644 --- a/difftest/Cargo.toml +++ b/difftest/Cargo.toml @@ -6,6 +6,7 @@ members = [ "offline", "online_dpi", "online_drive", + "online_vcs", ] exclude = [ "spike_interfaces" diff --git a/difftest/default.nix b/difftest/default.nix index 8b3626cbaa..efcd77d71e 100644 --- a/difftest/default.nix +++ b/difftest/default.nix @@ -26,6 +26,7 @@ let ./offline ./online_dpi ./online_drive + ./online_vcs ./test_common ./Cargo.lock ./Cargo.toml @@ -43,6 +44,7 @@ let ]; buildFeatures = lib.optionals verilated.enable-trace [ "trace" ]; + buildAndTestSubdir = "./online_drive"; env = { VERILATED_INC_DIR = "${verilated}/include"; diff --git a/difftest/online_vcs/Cargo.toml b/difftest/online_vcs/Cargo.toml new file mode 100644 index 0000000000..ff32311b7e --- /dev/null +++ b/difftest/online_vcs/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "online_vcs" +edition = "2021" +version.workspace = true + +[dependencies] +online_dpi = { path = "../online_dpi", features = ["svvpi"] } + +[features] +trace = ["online_dpi/trace"] diff --git a/difftest/online_vcs/build.rs b/difftest/online_vcs/build.rs new file mode 100644 index 0000000000..1144bac367 --- /dev/null +++ b/difftest/online_vcs/build.rs @@ -0,0 +1,14 @@ +use std::path::Path; +use std::env; + +fn main() { + let vcs_lib_dir = env::var("VCS_LIB_DIR").unwrap(); + println!("cargo:rustc-link-search=native={}", Path::new(&vcs_lib_dir).display()); + let vcs_compiled_lib_dir = env::var("VCS_COMPILED_LIB_DIR").unwrap(); + println!("cargo:rustc-env=LD_LIBRARY_PATH={}", Path::new(&vcs_lib_dir).display()); + println!("cargo:rustc-link-search=native={}", Path::new(&vcs_compiled_lib_dir).display()); + println!("cargo::rustc-link-lib=TestBench"); + println!("cargo::rustc-link-lib=vcsnew64"); + println!("cargo::rustc-link-lib=vcs_tls"); + println!("cargo::rustc-link-lib=vfs"); +} diff --git a/difftest/online_vcs/src/main.rs b/difftest/online_vcs/src/main.rs new file mode 100644 index 0000000000..00c7fbde88 --- /dev/null +++ b/difftest/online_vcs/src/main.rs @@ -0,0 +1,30 @@ +// force link with online_dpi +extern crate online_dpi; + +use std::{ + ffi::{c_char, c_int, c_void, CString}, + ptr, +}; + +fn main() { + let c_args: Vec = std::env::args().map(|arg| CString::new(arg).unwrap()).collect(); + + let mut c_args_ptr: Vec<*const c_char> = c_args.iter().map(|arg| arg.as_ptr()).collect(); + c_args_ptr.push(ptr::null()); + + let argc = c_args.len() as c_int; + let argv = c_args_ptr.as_ptr() as *mut *mut c_char; + + unsafe { + vcs_main(argc, argv); + VcsInit(); + VcsSimUntil(1<<31); + } +} + +extern "C" { + fn vcs_main(argc: c_int, argv: *mut *mut c_char) -> c_int; + fn VcsInit() -> c_void; + fn VcsSimUntil(c: c_int) -> c_void; +} + diff --git a/difftest/vcs-emu.nix b/difftest/vcs-emu.nix new file mode 100644 index 0000000000..4da9c46490 --- /dev/null +++ b/difftest/vcs-emu.nix @@ -0,0 +1,69 @@ +{ lib +, callPackage +, elaborateConfig + +, rustPlatform + +, rust-analyzer +, rust-bindgen + +, vcStaticInstallPath +, vcs-lib + +, cmake +, clang-tools +}: + +let + spike_interfaces = callPackage ./spike_interfaces { }; + + self = rustPlatform.buildRustPackage { + name = "t1-vcs-emu"; + src = with lib.fileset; toSource { + root = ./.; + fileset = unions [ + ./spike_rs + ./offline + ./online_dpi + ./online_drive + ./online_vcs + ./test_common + ./Cargo.lock + ./Cargo.toml + ]; + }; + + buildInputs = [ + spike_interfaces + vcs-lib + ]; + + buildFeatures = lib.optionals vcs-lib.enable-trace [ "trace" ]; + buildAndTestSubdir = "./online_vcs"; + + env = { + VCS_LIB_DIR = "${vcStaticInstallPath}/vcs-mx/linux64/lib"; + VCS_COMPILED_LIB_DIR = "${vcs-lib}/lib"; + DESIGN_VLEN = elaborateConfig.parameter.vLen; + DESIGN_DLEN = elaborateConfig.parameter.dLen; + }; + + cargoLock = { + lockFile = ./Cargo.lock; + }; + + dontUseCmakeConfigure = true; + + passthru = { + devShell = self.overrideAttrs (old: { + nativeBuildInputs = old.nativeBuildInputs ++ [ + rust-analyzer + rust-bindgen + clang-tools + ]; + }); + inherit spike_interfaces; + }; + }; +in +self diff --git a/ipemu/src/TestBench.scala b/ipemu/src/TestBench.scala index 3e23afc8ce..9d24b3f8c5 100644 --- a/ipemu/src/TestBench.scala +++ b/ipemu/src/TestBench.scala @@ -37,11 +37,6 @@ class TestBench(generator: SerializableModuleGenerator[T1, T1Parameter]) extends | export "DPI-C" function dump_wave; | function dump_wave(input string file); |`ifdef VCS - | $$fsdbDumpfile(file); - | $$fsdbDumpvars("+all"); - | $$fsdbDumpSVA(); - | $$fsdbDumpvars(0); - | $$fsdbDumpon; |`endif |`ifdef VERILATOR | $$dumpfile(file); diff --git a/nix/overlay.nix b/nix/overlay.nix index 4dae99039a..c6027ba316 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -23,6 +23,11 @@ rec { circt-full = final.callPackage ./pkgs/circt-full.nix { }; rvv-codegen = final.callPackage ./pkgs/rvv-codegen.nix { }; add-determinism = final.callPackage ./pkgs/add-determinism { }; # faster strip-undetereminism + # Using VCS need to set VC_STATIC_HOME and SNPSLMD_LICENSE_FILE to impure env, and add sandbox dir to VC_STATIC_HOME + vcs-fhs-env = final.callPackage ./pkgs/vcs-fhs-env.nix { + vcStaticInstallPath = builtins.getEnv "VC_STATIC_HOME"; + snpsLicenseFile = builtins.getEnv "SNPSLMD_LICENSE_FILE"; + }; mill = let jre = final.jdk21; in (prev.mill.override { inherit jre; }).overrideAttrs (_: { diff --git a/nix/pkgs/vcs-fhs-env.nix b/nix/pkgs/vcs-fhs-env.nix new file mode 100644 index 0000000000..37baa934d9 --- /dev/null +++ b/nix/pkgs/vcs-fhs-env.nix @@ -0,0 +1,87 @@ +{ buildFHSEnv + # e.g. /opt/synopsys/vc_static/V-2023.12 +, vcStaticInstallPath + # e.g. port@addr +, snpsLicenseFile +}: +buildFHSEnv { + name = "vcs-fhs-env"; + profile = '' + export TCL_TZ=UTC + export VC_STATIC_HOME=${vcStaticInstallPath} + export VCS_HOME=${vcStaticInstallPath}/vcs-mx + export VCS_TARGET_ARCH=amd64 + export VCS_ARCH_OVERRIDE=linux + export VERDI_HOME=${vcStaticInstallPath}/verdi + export NOVAS_HOME=${vcStaticInstallPath}/verdi + export SPYGLASS_HOME=${vcStaticInstallPath}/SG_COMPAT/SPYGLASS_HOME + export SNPS_VERDI_CBUG_LCA=1 + export SNPSLMD_LICENSE_FILE=${snpsLicenseFile} + + export PATH=${vcStaticInstallPath}/bin:$PATH + export PATH=${vcStaticInstallPath}/verdi/bin:$PATH + export PATH=${vcStaticInstallPath}/vcs-mx/bin:$PATH + export PATH=${vcStaticInstallPath}/SG_COMPAT/SPYGLASS_HOME/bin:$PATH + + export LD_LIBRARY_PATH=/usr/lib64/ + export LD_LIBRARY_PATH=${vcStaticInstallPath}/verdi/share/PLI/lib/LINUX64:$LD_LIBRARY_PATH + export LD_LIBRARY_PATH=${vcStaticInstallPath}/verdi/share/NPI/lib/LINUX64:$LD_LIBRARY_PATH + ''; + targetPkgs = (ps: with ps; [ + libGL + util-linux + libxcrypt-legacy + coreutils-full + ncurses5 + gmp5 + bzip2 + glib + bc + time + elfutils + ncurses5 + e2fsprogs + cyrus_sasl + expat + sqlite + (nssmdns.overrideAttrs rec { + version = "0.14.1"; + src = fetchFromGitHub { + owner = "avahi"; + repo = "nss-mdns"; + rev = "v${version}"; + hash = "sha256-7RqV0hyfcZ168QfeHVtCJpyP4pI6cMeekJ2hDHNurdA="; + }; + }) + (libkrb5.overrideAttrs rec { + version = "1.18.2"; + src = fetchurl { + url = "https://kerberos.org/dist/krb5/${lib.versions.majorMinor version}/krb5-${version}.tar.gz"; + hash = "sha256-xuTJ7BqYFBw/XWbd8aE1VJBQyfq06aRiDumyIIWHOuA="; + }; + sourceRoot = "krb5-${version}/src"; + }) + (gnugrep.overrideAttrs rec { + version = "3.1"; + doCheck = false; + src = fetchurl { + url = "mirror://gnu/grep/grep-${version}.tar.xz"; + hash = "sha256-22JcerO7PudXs5JqXPqNnhw5ka0kcHqD3eil7yv3oH4="; + }; + }) + keyutils + graphite2 + libpulseaudio + gcc + gnumake + xorg.libX11 + xorg.libXft + xorg.libXScrnSaver + xorg.libXext + xorg.libxcb + xorg.libXau + xorg.libXrender + xorg.libXcomposite + xorg.libXi + ]); +} diff --git a/nix/t1/default.nix b/nix/t1/default.nix index 1e12d07d5a..e67333b59c 100644 --- a/nix/t1/default.nix +++ b/nix/t1/default.nix @@ -62,27 +62,63 @@ lib.makeScope newScope ip = rec { recurseForDerivations = true; - elaborate = innerSelf.callPackage ./elaborate.nix { target = "ip"; /* use-binder = true; */ }; + # T1 RTL. + elaborate = innerSelf.callPackage ./elaborate.nix { target = "ip"; }; mlirbc = innerSelf.callPackage ./mlirbc.nix { inherit elaborate; }; - rtl = innerSelf.callPackage ./rtl.nix { inherit mlirbc; }; - + rtl = innerSelf.callPackage ./rtl.nix { + inherit mlirbc; + mfcArgs = lib.escapeShellArgs [ + "-O=release" + "--disable-all-randomization" + "--split-verilog" + "--preserve-values=all" + "--strip-debug-info" + "--strip-fir-debug-info" + "--verification-flavor=sva" + "--lowering-options=verifLabels,omitVersionComment,emittedLineLength=240,locationInfoStyle=none" + ]; + }; omreader = self.omreader-unwrapped.mkWrapper { inherit mlirbc; }; - om = innerSelf.callPackage ./om.nix { inherit mlirbc; }; - emu-elaborate = innerSelf.callPackage ./elaborate.nix { target = "ipemu"; /* use-binder = true; */ }; + emu-elaborate = innerSelf.callPackage ./elaborate.nix { target = "ipemu"; }; emu-mlirbc = innerSelf.callPackage ./mlirbc.nix { elaborate = emu-elaborate; }; - emu-omreader = self.omreader-unwrapped.mkWrapper { mlirbc = emu-mlirbc; }; - emu-rtl = innerSelf.callPackage ./rtl.nix { mlirbc = emu-mlirbc; }; - - verilated = innerSelf.callPackage ./verilated.nix { rtl = emu-rtl; }; - verilated-trace = innerSelf.callPackage ./verilated.nix { rtl = emu-rtl; enable-trace = true; }; - - emu = innerSelf.callPackage ./ipemu.nix { rtl = ip.emu-rtl; stdenv = moldStdenv; }; - emu-trace = innerSelf.callPackage ./ipemu.nix { rtl = emu-rtl; stdenv = moldStdenv; do-trace = true; }; - difftest = innerSelf.callPackage ../../difftest/default.nix { inherit verilated; }; - difftest-trace = innerSelf.callPackage ../../difftest/default.nix { verilated = verilated-trace; }; + # T1 Verilator Emulator + verilator-emu-omreader = self.omreader-unwrapped.mkWrapper { mlirbc = emu-mlirbc; }; + verilator-emu-rtl = innerSelf.callPackage ./rtl.nix { + mlirbc = emu-mlirbc; + mfcArgs = lib.escapeShellArgs [ + "-O=release" + "--split-verilog" + "--preserve-values=all" + "--verification-flavor=if-else-fatal" + "--lowering-options=verifLabels,omitVersionComment" + "--strip-debug-info" + ]; + }; + verilator-emu-rtl-verilated = innerSelf.callPackage ./verilated.nix { rtl = verilator-emu-rtl; stdenv = moldStdenv; }; + verilator-emu-rtl-verilated-trace = innerSelf.callPackage ./verilated.nix { rtl = verilator-emu-rtl; stdenv = moldStdenv; enable-trace = true; }; + verilator-emu = innerSelf.callPackage ../../difftest/default.nix { verilated = verilator-emu-rtl-verilated; }; + verilator-emu-trace = innerSelf.callPackage ../../difftest/default.nix { verilated = verilator-emu-rtl-verilated-trace; }; + + # T1 VCS Emulator + vcs-emu-omreader = self.omreader-unwrapped.mkWrapper { mlirbc = emu-mlirbc; }; + vcs-emu-rtl = innerSelf.callPackage ./rtl.nix { + mlirbc = emu-mlirbc; + mfcArgs = lib.escapeShellArgs [ + "-O=release" + "--split-verilog" + "--preserve-values=all" + "--verification-flavor=sva" + "--lowering-options=verifLabels,omitVersionComment" + "--strip-debug-info" + ]; + }; + vcs-emu-compiled = innerSelf.callPackage ./vcs.nix { rtl = vcs-emu-rtl; }; + vcs-emu-compiled-trace = innerSelf.callPackage ./vcs.nix { rtl = vcs-emu-rtl; enable-trace = true; }; + vcs-emu = innerSelf.callPackage ../../difftest/vcs-emu.nix { vcs-lib = vcs-emu-compiled; vcStaticInstallPath = builtins.getEnv "VC_STATIC_HOME"; }; + vcs-emu-trace = innerSelf.callPackage ../../difftest/vcs-emu.nix { vcs-lib = vcs-emu-compiled-trace; vcStaticInstallPath = builtins.getEnv "VC_STATIC_HOME"; }; }; subsystem = rec { diff --git a/nix/t1/elaborate.nix b/nix/t1/elaborate.nix index 691a701cd3..a531ac1dff 100644 --- a/nix/t1/elaborate.nix +++ b/nix/t1/elaborate.nix @@ -12,7 +12,7 @@ }: assert lib.assertMsg - (lib.elem target [ "ip" "ipemu" "subsystem" "subsystememu" ]) + (lib.elem target [ "ip" "ipemu" ]) "Unknown elaborate target ${target}"; let diff --git a/nix/t1/ipemu.nix b/nix/t1/ipemu.nix deleted file mode 100644 index 2002af62f1..0000000000 --- a/nix/t1/ipemu.nix +++ /dev/null @@ -1,53 +0,0 @@ -{ stdenv -, lib - - # emulator deps -, cmake -, libargs -, spdlog -, fmt -, libspike -, dramsim3 -, nlohmann_json -, ninja -, verilator -, zlib - -, rtl - -, do-trace ? false -}: - -stdenv.mkDerivation { - name = "t1-${rtl.elaborateConfig}-ipemu" + lib.optionalString do-trace "-trace"; - - src = ../../ipemu/csrc; - - # CMakeLists.txt will read the environment - env.VERILATE_SRC_DIR = toString rtl; - - cmakeFlags = lib.optionals do-trace [ - "-DVERILATE_TRACE=ON" - ]; - - nativeBuildInputs = [ - cmake - verilator - ninja - ]; - - buildInputs = [ - zlib - libargs - spdlog - fmt - libspike - dramsim3 - nlohmann_json - ]; - - meta = { - mainProgram = "emulator"; - description = "IP emulator for config ${rtl.elaborateConfig}"; - }; -} diff --git a/nix/t1/rtl.nix b/nix/t1/rtl.nix index 9f3beed965..5b9d43369d 100644 --- a/nix/t1/rtl.nix +++ b/nix/t1/rtl.nix @@ -1,19 +1,12 @@ { stdenvNoCC , lib - +, mfcArgs , circt , mlirbc }: let - mfcArgs = lib.escapeShellArgs [ - "-O=debug" - "--split-verilog" - "--preserve-values=named" - "--lowering-options=verifLabels,omitVersionComment" - "--strip-debug-info" - ]; - fixupFilelist = lib.elem mlirbc.elaborateTarget [ "ipemu" "subsystememu" ]; + fixupFilelist = lib.elem mlirbc.elaborateTarget [ "ipemu" ]; in stdenvNoCC.mkDerivation { name = "t1-${mlirbc.elaborateConfig}-${mlirbc.elaborateTarget}-rtl"; diff --git a/nix/t1/vcs.nix b/nix/t1/vcs.nix new file mode 100644 index 0000000000..06fb282105 --- /dev/null +++ b/nix/t1/vcs.nix @@ -0,0 +1,55 @@ +{ lib +, stdenv +, configName +, rtl +, vcs-fhs-env +, enable-trace ? false +}: +stdenv.mkDerivation { + name = "${configName}-vcs"; + + # require license + __noChroot = true; + + src = rtl; + + nativeBuildInputs = [ vcs-fhs-env ]; + + buildPhase = '' + runHook preBuild + + echo "[nix] running VCS" + ${vcs-fhs-env}/bin/vcs-fhs-env vcs \ + -sverilog \ + -full64 \ + -slave \ + -e vcs_main \ + -file filelist.f \ + -timescale=1ns/1ps \ + ${lib.optionalString enable-trace "--trace-fst"} \ + ${lib.optionalString enable-trace "-P $VERDI_HOME/share/PLI/VCS/LINUX64/novas.tab $VERDI_HOME/share/PLI/VCS/LINUX64/pli.a"} \ + ${lib.optionalString enable-trace "-debug_access+pp+dmptf+thread"} \ + ${lib.optionalString enable-trace "-kdb=common_elab,hgldd_all"} \ + -o libTestBench.so + + runHook postBuild + ''; + + passthru = { + inherit enable-trace; + }; + + installPhase = '' + runHook preInstall + + mkdir -p $out/ $out/lib + cp libTestBench.so $out/lib + cp *.so $out/lib + + # backup daidir + mkdir -p $out/share/daidir + cp -r libTestBench.so.daidir/* $out/share/daidir + + runHook postInstall + ''; +} diff --git a/tests/default.nix b/tests/default.nix index d84a1effed..009a9be32a 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -4,8 +4,8 @@ , newScope , rv32-stdenv , runCommand -, difftest -, difftest-trace +, verilator-emu +, verilator-emu-trace }: let @@ -20,7 +20,7 @@ let scope = lib.recurseIntoAttrs (lib.makeScope newScope (casesSelf: { recurseForDerivations = true; - inherit difftest difftest-trace; + inherit verilator-emu verilator-emu-trace; makeEmuResult = casesSelf.callPackage ./make-emu-result.nix { }; diff --git a/tests/make-emu-result.nix b/tests/make-emu-result.nix index f31553f4fd..129de6c0fa 100644 --- a/tests/make-emu-result.nix +++ b/tests/make-emu-result.nix @@ -3,8 +3,8 @@ , stdenvNoCC , jq , zstd -, difftest -, difftest-trace +, verilator-emu +, verilator-emu-trace , elaborateConfigJson }: @@ -19,7 +19,7 @@ let dontUnpack = true; - difftestDriver = "${difftest}/bin/online_drive"; + difftestDriver = "${verilator-emu}/bin/online_drive"; difftestArgs = [ "--elf-file" "${testCase}/bin/${testCase.pname}.elf" @@ -76,7 +76,7 @@ let ''; passthru.with-trace = self.overrideAttrs (old: { - difftestDriver = "${difftest-trace}/bin/online_drive"; + difftestDriver = "${verilator-emu-trace}/bin/online_drive"; difftestArgs = old.difftestArgs ++ [ "--wave-path" "${placeholder "out"}/wave.fst" ]; postCheck = '' if [ ! -r "$out/wave.fst" ]; then @@ -89,7 +89,7 @@ let passthru.with-offline = self.overrideAttrs (old: { preInstall = '' set +e - "${difftest}/bin/offline" \ + "${verilator-emu}/bin/offline" \ --elf-file ${testCase}/bin/${testCase.pname}.elf \ --log-file $out/rtl-event.jsonl \ --log-level ERROR &> $out/offline-check-journal