From 8d81c6db9a9e6f2605f122c25c0dd8c018a4a840 Mon Sep 17 00:00:00 2001
From: Avimitin <dev@avimit.in>
Date: Thu, 18 Jul 2024 14:21:41 +0800
Subject: [PATCH] [ci] save offline check result in nix

Signed-off-by: Avimitin <dev@avimit.in>
---
 .github/workflows/pr.yml  |  3 ++-
 script/ci/src/Main.scala  | 49 +++++++++++++++++++++++----------------
 tests/default.nix         |  4 +++-
 tests/make-emu-result.nix | 12 ++++++++++
 4 files changed, 46 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index fc0238b1cd..9282e7a9fc 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -183,7 +183,8 @@ jobs:
           ref: ${{ github.head_ref }}
       - name: "Print step summary"
         run: |
-          nix run ".#ci-helper" -- postCI --cycle-update-file-path ./cycle-update.md
+          nix run ".#ci-helper" -- postCI --failed-tests-file-path ./failed-tests.md --cycle-update-file-path ./cycle-update.md
+          cat ./failed-tests.md >> $GITHUB_STEP_SUMMARY
           cat ./cycle-update.md >> $GITHUB_STEP_SUMMARY
       - name: "Commit cycle updates"
         run: |
diff --git a/script/ci/src/Main.scala b/script/ci/src/Main.scala
index f822838479..bc067deedb 100644
--- a/script/ci/src/Main.scala
+++ b/script/ci/src/Main.scala
@@ -162,7 +162,11 @@ object Main:
 
         val testResultPath =
           try
-            os.Path(nixResolvePath(s".#t1.$config.cases.$caseName.emu-result"))
+            os.Path(
+              nixResolvePath(
+                s".#t1.$config.cases.$caseName.emu-result.with-offline"
+              )
+            )
           catch
             case _ =>
               Logger.error(s"Emulation for config $config, case $caseName fail")
@@ -179,24 +183,10 @@ object Main:
               Logger.fatal("Got error from emulation, exiting CI")
 
         Logger.info("Checking RTL event with offline difftest")
-        val handle = os
-          .proc(
-            "nix",
-            "run",
-            ".#t1-helper",
-            "--",
-            "difftest",
-            "--config",
-            config,
-            "--case-attr",
-            caseName,
-            "--log-level",
-            "ERROR"
-          )
-          .call(stdout = os.Inherit, stderr = os.Inherit, check = false)
-        val testSuccess = handle.exitCode == 0
+        val testSuccess =
+          os.read(testResultPath / "offline-check-status").trim() == "0"
         if !testSuccess then
-          Logger.error(s"$caseName run failed")
+          Logger.error(s"Offline check for $caseName ($config) failed")
           allFailedTest :+ s"t1.$config.cases.$caseName"
         else allFailedTest
     end findFailedTests
@@ -252,11 +242,21 @@ object Main:
   //   * collect and report cycle update
   @main
   def postCI(
+      @arg(
+        name = "failed-tests-file-path",
+        doc = "specify the failed tests markdown file output path"
+      ) failedTestsFilePath: String,
       @arg(
         name = "cycle-update-file-path",
         doc = "specify the cycle update markdown file output path"
       ) cycleUpdateFilePath: String
   ) =
+    val failedTestsFile = os.Path(failedTestsFilePath, os.pwd)
+    os.write.over(failedTestsFile, "## Failed Tests\n")
+
+    val cycleUpdateRecordFile = os.Path(cycleUpdateFilePath, os.pwd)
+    os.write.over(cycleUpdateRecordFile, "## Cycle Update\n")
+
     os.walk(os.pwd / ".github" / "cases")
       .filter(_.last == "default.json")
       .foreach: file =>
@@ -267,9 +267,18 @@ object Main:
         val emuResultPath =
           os.Path(nixResolvePath(s".#t1.$config.cases._allEmuResult"))
 
+        Logger.info("Collecting failed tests")
+        os.walk(emuResultPath)
+          .filter(path => path.last == "offline-check-status")
+          .filter(path => os.read(path).trim() != "0")
+          .map(path => {
+            val caseName = path.segments.toSeq.reverse.drop(1).head
+            val journal = os.read(path / os.up / "offline-check-journal")
+            os.write.append(failedTestsFile, s"* ${config} - ${caseName}\n")
+            os.write.append(failedTestsFile, s"```text\n${journal}\n```\n")
+          })
+
         Logger.info("Collecting cycle update info")
-        val cycleUpdateRecordFile = os.Path(cycleUpdateFilePath, os.pwd)
-        os.write.over(cycleUpdateRecordFile, "## Cycle Update\n")
         val perfCycleRegex = raw"total_cycles:\s(\d+)".r
         val allCycleUpdates = os
           .walk(emuResultPath)
diff --git a/tests/default.nix b/tests/default.nix
index 4a7831c083..d84a1effed 100644
--- a/tests/default.nix
+++ b/tests/default.nix
@@ -80,7 +80,9 @@ let
         (caseDrv: ''
           _caseOutDir=$out/${caseDrv.pname}
           mkdir -p "$_caseOutDir"
-          cp ${caseDrv.emu-result}/perf.txt "$_caseOutDir"/
+          cp ${caseDrv.emu-result.with-offline}/perf.txt "$_caseOutDir"/
+          cp ${caseDrv.emu-result.with-offline}/offline-check-status "$_caseOutDir"/
+          cp ${caseDrv.emu-result.with-offline}/offline-check-journal "$_caseOutDir"/
         '')
         allCases);
     in
diff --git a/tests/make-emu-result.nix b/tests/make-emu-result.nix
index cab6cb92a2..dbe2a25baa 100644
--- a/tests/make-emu-result.nix
+++ b/tests/make-emu-result.nix
@@ -86,6 +86,18 @@ let
         fi
       '';
     });
+
+    passthru.with-offline = self.overrideAttrs (old: {
+      preInstall = ''
+        set +e
+        "${difftest}/bin/offline" \
+          --elf-file ${testCase}/bin/${testCase.pname}.elf \
+          --log-file $out/rtl-event.jsonl \
+          --log-level ERROR 2>&1 > $out/offline-check-journal
+        printf "$?" > $out/offline-check-status
+        set -e
+      '';
+    });
   };
 in
 self