[build system] build test case when TEST_CASE_DIR is unset

.github/passed/v1024l8b2-test/debug/passed.json

@@ -144,16 +144,16 @@
   "vluxseg8ei16.v-codegen": 10673,
   "vluxseg8ei32.v-codegen": 5350,
   "vluxseg8ei8.v-codegen": 19493,
-  "vmacc.vv-codegen": 43678,
-  "vmacc.vx-codegen": 119289,
+  "vmacc.vv-codegen": 43680,
+  "vmacc.vx-codegen": 116642,
   "vmadc.vi-codegen": 52084,
   "vmadc.vim-codegen": 50942,
   "vmadc.vv-codegen": 16267,
   "vmadc.vvm-codegen": 18182,
   "vmadc.vx-codegen": 67405,
   "vmadc.vxm-codegen": 73757,
-  "vmadd.vv-codegen": 43678,
-  "vmadd.vx-codegen": 119289,
+  "vmadd.vv-codegen": 43680,
+  "vmadd.vx-codegen": 116642,
   "vmand.mm-codegen": 15637,
   "vmandn.mm-codegen": 15637,
   "vmax.vv-codegen": 38466,
@@ -200,14 +200,14 @@
   "vmsne.vv-codegen": 39939,
   "vmsne.vx-codegen": 144294,
   "vmsof.m-codegen": 1007,
-  "vmul.vv-codegen": 38550,
-  "vmul.vx-codegen": 159646,
-  "vmulh.vv-codegen": 38550,
-  "vmulh.vx-codegen": 159646,
-  "vmulhsu.vv-codegen": 38550,
-  "vmulhsu.vx-codegen": 159646,
-  "vmulhu.vv-codegen": 38550,
-  "vmulhu.vx-codegen": 159646,
+  "vmul.vv-codegen": 38466,
+  "vmul.vx-codegen": 155966,
+  "vmulh.vv-codegen": 38466,
+  "vmulh.vx-codegen": 155966,
+  "vmulhsu.vv-codegen": 38466,
+  "vmulhsu.vx-codegen": 155966,
+  "vmulhu.vv-codegen": 38466,
+  "vmulhu.vx-codegen": 155966,
   "vmv.s.x-codegen": 759,
   "vmv.v.i-codegen": 25764,
   "vmv.v.v-codegen": 11858,
@@ -225,10 +225,10 @@
   "vnclipu.wi-codegen": 61631,
   "vnclipu.wv-codegen": 18337,
   "vnclipu.wx-codegen": 84824,
-  "vnmsac.vv-codegen": 43678,
-  "vnmsac.vx-codegen": 119289,
-  "vnmsub.vv-codegen": 43678,
-  "vnmsub.vx-codegen": 119289,
+  "vnmsac.vv-codegen": 43680,
+  "vnmsac.vx-codegen": 116642,
+  "vnmsub.vv-codegen": 43680,
+  "vnmsub.vx-codegen": 116642,
   "vnsra.wi-codegen": 61631,
   "vnsra.wv-codegen": 18337,
   "vnsra.wx-codegen": 84824,
@@ -286,8 +286,8 @@
   "vsll.vv-codegen": 38550,
   "vsll.vx-codegen": 118978,
   "vsm.v-codegen": 276,
-  "vsmul.vv-codegen": 38550,
-  "vsmul.vx-codegen": 118978,
+  "vsmul.vv-codegen": 38466,
+  "vsmul.vx-codegen": 116398,
   "vsoxei16.v-codegen": 16092,
   "vsoxei32.v-codegen": 11996,
   "vsoxei8.v-codegen": 21779,
@@ -407,19 +407,19 @@
   "vwaddu.vx-codegen": 89464,
   "vwaddu.wv-codegen": 22975,
   "vwaddu.wx-codegen": 108533,
-  "vwmacc.vv-codegen": 28732,
-  "vwmacc.vx-codegen": 107197,
-  "vwmaccsu.vv-codegen": 28732,
-  "vwmaccsu.vx-codegen": 107197,
-  "vwmaccu.vv-codegen": 28732,
-  "vwmaccu.vx-codegen": 107197,
-  "vwmaccus.vx-codegen": 107197,
-  "vwmul.vv-codegen": 20180,
-  "vwmul.vx-codegen": 127420,
-  "vwmulsu.vv-codegen": 20180,
-  "vwmulsu.vx-codegen": 127420,
-  "vwmulu.vv-codegen": 20180,
-  "vwmulu.vx-codegen": 127420,
+  "vwmacc.vv-codegen": 28702,
+  "vwmacc.vx-codegen": 106982,
+  "vwmaccsu.vv-codegen": 28702,
+  "vwmaccsu.vx-codegen": 106982,
+  "vwmaccu.vv-codegen": 28702,
+  "vwmaccu.vx-codegen": 106982,
+  "vwmaccus.vx-codegen": 106982,
+  "vwmul.vv-codegen": 20212,
+  "vwmul.vx-codegen": 127468,
+  "vwmulsu.vv-codegen": 20212,
+  "vwmulsu.vx-codegen": 127468,
+  "vwmulu.vv-codegen": 20212,
+  "vwmulu.vx-codegen": 127468,
   "vwredsum.vs-codegen": 19461,
   "vwredsumu.vs-codegen": 19461,
   "vwsub.vv-codegen": 20212,

build.sc

@@ -355,18 +355,21 @@ trait Emulator
   }
 }
 
-def testsOutDir = sys.env.get("TEST_CASE_DIR").map(os.Path(_)).getOrElse {
-  val tmpDir = os.temp.dir(dir = os.pwd / "out", prefix = "TEST_CASE_DIR", deleteOnExit = false)
-  os.makeDir(tmpDir / "configs")
-  tmpDir
+def testsCaseDir = sys.env.get("TEST_CASE_DIR").map(os.Path(_)).getOrElse {
+  val testsCaseDir = os.pwd / "out" / "testCaseDir.dest"
+  if (!os.exists(testsCaseDir)) {
+    os.makeDir(testsCaseDir)
+    os.symlink(testsCaseDir / "configs", os.pwd / "tests" / "configs")
+  }
+  testsCaseDir
 }
 def testConfigs = os
-  .walk(testsOutDir / "configs")
+  .walk(testsCaseDir / "configs")
   .filter(_.ext == "json")
   .filter(p => !ujson.read(os.read(p)).obj("fp").bool)
   .map(_.baseName)
 def fpTestConfigs = os
-  .walk(testsOutDir / "configs")
+  .walk(testsCaseDir / "configs")
   .filter(_.ext == "json")
   .filter(p => ujson.read(os.read(p)).obj("fp").bool)
   .map(_.baseName)
@@ -400,8 +403,22 @@ trait RunVerilatorEmulator
   def configDir:  T[os.Path] = T(os.pwd / "run")
   def configFile: T[os.Path] = T(configDir() / s"$config.json")
   def runConfig:  T[ujson.Value.Value] = T(ujson.read(os.read(configFile())))
-  def testConfig: T[ujson.Value.Value] = T(ujson.read(os.read(testsOutDir / "configs" / s"$testTask.json")))
-  def binPath: T[os.Path] = T(testsOutDir / os.RelPath(testConfig().obj("elf").obj("path").str))
+  def testConfig: T[ujson.Value.Value] = T(ujson.read(os.read(testsCaseDir / "configs" / s"$testTask.json")))
+  def binPath: T[os.Path] = T {
+    if (testConfig().obj.get("elf").isDefined) {
+      testsCaseDir / os.RelPath(testConfig().obj("elf").obj("path").str)
+    } else {
+      import scala.util.chaining._
+      val testType = testConfig().obj("type").str
+      val testName = testConfig().obj("name").str
+      os.proc("mill", "-i", "show", s"$testType[$testName].elf")
+        .call(os.pwd / "tests")
+        .out
+        .text()
+        .pipe(ujson.read(_).str.split(":")(3))
+        .pipe(os.Path(_))
+    }
+  }
 
   def run = T {
     def wave: String = runConfig().obj.get("wave").map(_.str).getOrElse((T.dest / "wave").toString)
@@ -677,29 +694,32 @@ trait SubsystemEmulator
     PathRef(buildDir().path / "emulator")
   }
 
-  def defaultCommandName = "elf"
+  override def defaultCommandName() = "elf"
 }
 
 object runSubsystemEmu extends mill.Cross[RunSubsystemEmu](
-  os.walk(os.pwd / "tests" / "configs").filter(_.ext == "json").map(_.baseName).toSeq
+  os.walk(testsCaseDir).filter(_.ext == "json").map(_.baseName).toSeq
 )
 trait RunSubsystemEmu extends Cross.Module[String] with TaskModule {
   override def defaultCommandName() = "run"
 
   val test: String = crossValue
-  def testConfigDir = T(os.pwd / "tests" / "configs" / s"$test.json")
-  def testConfig = T(ujson.read(os.read(testConfigDir())))
+  def testConfig = T(ujson.read(os.read(testsCaseDir / "configs" / s"$test.json")))
   def testName = T(testConfig().apply("name").str)
   def testType = T(testConfig().apply("type").str)
 
   import scala.util.chaining._
   def testElf = T (
-    os.proc(Seq("mill", "-i", "show", s"${testType()}[${testName()}].bin"))
-      .call(os.pwd / "tests")
-      .out
-      .text
-      .pipe(ujson.read(_).str.split(":")(3))
-      .pipe(os.Path(_))
+    if (testConfig().obj.get("elf").isDefined) {
+      testsCaseDir / os.RelPath(testConfig().obj("elf").obj("path").str)
+    } else {
+      os.proc(Seq("mill", "-i", "show", s"${testType()}[${testName()}].bin"))
+        .call(os.pwd / "tests")
+        .out
+        .text
+        .pipe(ujson.read(_).str.split(":")(3))
+        .pipe(os.Path(_))
+    }
   )
 
   def run = T {

flake.nix

@@ -87,11 +87,9 @@
                 CODEGEN_CFG_PATH = "${pkgs.rvv-codegen}/configs";
               };
             };
-            # This devShell is used for running testcase
+            # This devShell is used only for running testcase
             testcase = mkLLVMShell {
-              # TODO: Currently, the emulator needs all the dependencies to run a test case ,
-              # but most of them are used to get version information, so they should be cleaned up one day.
-              buildInputs = commonDeps ++ chiselDeps ++ testcaseDeps ++ emulatorDeps ++ [ pkgs.metals ];
+              buildInputs = commonDeps ++ chiselDeps ++ emulatorDeps;
 
               env = {
                 TEST_CASE_DIR = "${pkgs.rvv-testcase}";

tests/build.sc

@@ -174,36 +174,3 @@ trait AsmCase
   val config: String = crossValue
   override def moduleName = "asm"
 }
-
-object caseBuild
-  extends Cross[CaseBuilder](
-  os.walk(os.pwd / "configs")
-    .filter(_.ext == "json")
-    .map(f => s"${ujson.read(os.read(f)).obj("name").str}-${ujson.read(os.read(f)).obj("type").str}")
-)
-
-trait CaseBuilder
-  extends Cross.Module[String] {
-  val task: String = crossValue
-  def run = T {
-    // prepare
-    val outputDir = os.pwd / os.up / "tests-out"
-    os.remove.all(os.pwd / "out")
-    os.makeDir.all(outputDir)
-    val IndexedSeq(name, module) = task.split("-").toSeq
-    os.makeDir.all(outputDir / "cases" / module)
-    os.makeDir.all(outputDir / "configs")
-
-    // build elf
-    val rawElfPath = os.proc("mill", "--no-server", "show", s"$module[$name].elf").call(os.pwd).out.text
-    val elfPath = os.Path(ujson.read(rawElfPath).str.split(":")(3))
-
-    // write elf path into test config
-    val origConfig = ujson.read(os.read(os.pwd / "configs" / s"$task.json"))
-    origConfig("elf") = ujson.Obj("path" -> s"cases/$module/${elfPath.last}")
-
-    // install, override if file exists
-    os.move.into(elfPath, outputDir / "cases" / module, true)
-    os.write.over(outputDir / "configs" / s"$task.json", ujson.write(origConfig))
-  }
-}

tests/readme.md

@@ -24,14 +24,33 @@ You can run `nix develop .#testcase-bootstrap` to have this env set up for you.
 ## How to resolve all the tests
 
 ```bash
-mill resolve _[_].elf
+mill -i resolve _[_].elf
 
 # Get asm test only
-mill resolve asm[_].elf
+mill -i resolve asm[_].elf
 ```
 
 ## How to run the test
 
+```bash
+nix develop .#testcase
+
+# List all runnable tests
+mill -i resolve verilatorEmulator[v1024l8b2-test,_,debug].run
+
+# Run a specific test
+mill -i verilatorEmulator[v1024l8b2-test,matmul-mlir,debug].run
+```
+
+## How to build single test and run it
+
+```bash
+unset TEST_CASE_DIR
+mill -i verilatorEmulator[$emulator,$YOUR_TEST,$debug].run
+```
+
+## How to manually rebuild all the tests
+
 Test cases are run by an emulator specified in the `build.sc` file which located at the project root.
 `mill` will create a bunch of `verilatorEmulator[__]` objects by reading test file in env `TEST_CASE_DIR`.
 The `TEST_CASE_DIR` must point to a directory with the following layout:
@@ -72,26 +91,3 @@ you can type `mill -i verilatorEmulator[v1024l8b2-test,matmul-mlir,debug].run` i
 
 To reduce all the tedious setup steps, you can use the provided `.#testcase` shell:
 
-```bash
-nix develop .#testcase
-mill -i verilatorEmulator[v1024l8b2-test,matmul-mlir,debug].run
-```
-
-## How to build single test and run it
-
-Suppose you want to build and run the mlir/hello.mlir test:
-
-```bash
-# build
-pushd tests
-nix develop .#testcase-bootstrap
-mill --no-server caseBuild[hello-mlir].run
-exit
-popd
-
-# run
-nix develop .#testcase
-export TEST_CASE_DIR=$PWD/tests-out
-mill --no-server verilatorEmulator[v1024l8b2-test,hello-mlir,debug].run
-```
-