[script] migrate all post CI action to Scala script

- Use nix derivation to cache and capture emulator output
- Use built-in S3 nix cache to replace GitHub Artifacts
- Produce GitHub step summary in Scala script

Signed-off-by: Avimitin <[email protected]>

.github/workflows/pr.yml

@@ -157,26 +157,13 @@ jobs:
       fail-fast: false
       matrix: ${{ fromJSON(needs.gen-matrix.outputs.ci-tests) }}
     runs-on: [self-hosted, linux, nixos]
-    outputs:
-      result: ${{ steps.ci-run.outputs.result }}
     steps:
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: "Run testcases"
-        id: ci-run
         run: |
-          nix develop -c t1-helper runTests --jobs "${{ matrix.jobs }}" \
-            --resultDir test-results-$(head -c 10 /dev/urandom | base32)
-
-      - uses: actions/upload-artifact@v4
-        if: ${{ !cancelled() }}
-        with:
-          name: test-reports-${{ matrix.id }}
-          path: |
-            test-results-*/failed-tests.md
-            test-results-*/cycle-updates.md
-            test-results-*/*_cycle.json
+          nix develop -c t1-helper runTests --jobs "${{ matrix.jobs }}"
 
   report:
     name: "Report CI result"
@@ -191,21 +178,14 @@ jobs:
         with:
           fetch-depth: 0
           ref: ${{ github.head_ref }}
-      - uses: actions/download-artifact@v4
-        with:
-          pattern: test-reports-*
-          merge-multiple: true
       - name: "Print step summary"
         run: |
-          echo -e "\n## Failed tests\n" >> $GITHUB_STEP_SUMMARY
-          shopt -s nullglob
-          cat test-results-*/failed-tests.md >> $GITHUB_STEP_SUMMARY
-          echo -e "\n## Cycle updates\n" >> $GITHUB_STEP_SUMMARY
-          shopt -s nullglob
-          cat test-results-*/cycle-updates.md >> $GITHUB_STEP_SUMMARY
+          nix develop -c t1-helper postCI --failed-test-file-path ./failed-test.md --cycle-update-file-path ./cycle-update.md
+          cat ./failed-test.md >> $GITHUB_STEP_SUMMARY
+          echo >> $GITHUB_STEP_SUMMARY
+          cat ./cycled-update.md >> $GITHUB_STEP_SUMMARY
       - name: "Commit cycle updates"
         run: |
-          nix develop -c t1-helper mergeCycleData
           git config user.name github-actions
           git config user.email [email protected]
           changed_cases=$(git diff --name-only '.github/cases/**/default.json')
@@ -218,17 +198,3 @@ jobs:
           else
             echo "No cycle change detect"
           fi
-      - uses: geekyeggo/delete-artifact@v5
-        with:
-          # test-reports has been used, it can be deleted
-          name: test-reports-*
-
-  clean-after-cancelled:
-    name: "Clean test reports [ON CANCELLED]"
-    if: ${{ cancelled() }}
-    runs-on: [self-hosted, linux, nixos]
-    needs: [run-testcases]
-    steps:
-      - uses: geekyeggo/delete-artifact@v5
-        with:
-          name: test-reports-*

script/src/Main.scala

@@ -494,56 +494,6 @@ object Main:
     println(toMatrixJson(scheduleTasks(testPlans, runnersAmount)))
   }
 
-  def writeCycleUpdates(
-      testName: String,
-      testRunDir: os.Path,
-      resultDir: os.Path
-  ): Unit =
-    val isEmulatorTask = raw"([^,]+),([^,]+)".r
-    testName match
-      case isEmulatorTask(e, t) =>
-        val passedFile = os.pwd / os.RelPath(s".github/cases/$e/default.json")
-        val original = ujson.read(os.read(passedFile))
-
-        val perfCycleRegex = raw"total_cycles:\s(\d+)".r
-        val newCycleCount = os.read
-          .lines(testRunDir / os.RelPath(s"$e/$t/perf.txt"))
-          .apply(0) match
-          case perfCycleRegex(cycle) => cycle.toInt
-          case _ =>
-            throw new Exception("perf.txt file is not format as expected")
-
-        val oldCycleCount = original.obj.get(t).map(_.num.toInt).getOrElse(-1)
-        val cycleUpdateFile = resultDir / "cycle-updates.md"
-        Logger.info(f"job '$testName' cycle $oldCycleCount -> $newCycleCount")
-        oldCycleCount match
-          case -1 =>
-            os.write.append(
-              cycleUpdateFile,
-              s"* 🆕 $testName: NaN -> $newCycleCount\n"
-            )
-          case _ =>
-            if oldCycleCount > newCycleCount then
-              os.write.append(
-                cycleUpdateFile,
-                s"* 🚀 $testName: $oldCycleCount -> $newCycleCount\n"
-              )
-            else if oldCycleCount < newCycleCount then
-              os.write.append(
-                cycleUpdateFile,
-                s"* 🐢 $testName: $oldCycleCount -> $newCycleCount\n"
-              )
-
-            val newCycleFile = resultDir / s"${e}_cycle.json"
-            val newCycleRecord =
-              if os.exists(newCycleFile) then ujson.read(os.read(newCycleFile))
-              else ujson.Obj()
-
-            newCycleRecord(t) = newCycleCount
-            os.write.over(newCycleFile, ujson.write(newCycleRecord, indent = 2))
-      case _ => throw new Exception(f"unknown job format '$testName'")
-  end writeCycleUpdates
-
   // Run jobs and give a brief result report
   // - Log of tailed tests will be tailed and copied into $resultDir/failed-logs/$testName.log
   // - List of failed tests will be written into $resultDir/failed-tests.md
@@ -556,23 +506,18 @@ object Main:
   @main
   def runTests(
       jobs: String,
-      resultDir: Option[os.Path],
       dontBail: Flag = Flag(false)
   ): Unit =
     if jobs == "" then
       Logger.info("No test found, exiting")
       return
 
-    var actualResultDir = resultDir.getOrElse(os.pwd / "test-results")
-    val testRunDir = os.pwd / "testrun"
-    os.makeDir.all(actualResultDir / "failed-logs")
-
     val allJobs = jobs.split(";")
     def findFailedTests() = allJobs.zipWithIndex.foldLeft(Seq[String]()):
       (allFailedTest, currentTest) =>
         val (testName, index) = currentTest
         val Array(config, caseName) = testName.split(",")
-        println()
+        println("\n")
         Logger.info(
           s"${BOLD}[${index + 1}/${allJobs.length}]${RESET} Running test case $caseName with config $config"
         )
@@ -635,38 +580,74 @@ object Main:
         )
   end runTests
 
+  // PostCI do the below four things:
+  //   * read default.json at .github/cases/$config/default.json
+  //   * generate case information for each entry in default.json (cycle, run success)
+  //   * collect and report failed tests
+  //   * collect and report cycle update
   @main
-  def mergeCycleData(filePat: String = "default.json") =
-    Logger.info("Updating cycle data")
-    val original = os
-      .walk(os.pwd / ".github" / "cases")
-      .filter(_.last == filePat)
-      .map: path =>
-        val config = path.segments.toSeq.reverse(1)
-        (config, ujson.read(os.read(path)))
-      .toMap
-    os.walk(os.pwd)
-      .filter(_.last.endsWith("_cycle.json"))
-      .map: path =>
-        val config = path.last.split("_")(0)
-        Logger.trace(s"Reading new cycle data from $path")
-        (config, ujson.read(os.read(path)))
-      .foreach:
-        case (name, latest) =>
-          val old = original.apply(name)
-          latest.obj.foreach:
-            case (k, v) => old.update(k, v)
-
-    original.foreach:
-      case (name, data) =>
-        val config = name.split(",")(0)
-        os.write.over(
-          os.pwd / ".github" / "cases" / config / filePat,
-          ujson.write(data, indent = 2)
-        )
+  def postCI(
+    @arg(
+      name = "failed-test-file-path",
+      doc = "specify the failed test markdown file output path"
+    ) failedTestsFilePath: String,
+    @arg(
+      name = "cycle-update-file-path",
+      doc = "specify the cycle update markdown file output path"
+    ) cycleUpdateFilePath: String,
+  ) =
+    case class CaseStatus(config: String, caseName: String, isFailed: Boolean, oldCycle: Int, newCycle: Int)
+
+    def collectCaseStatus(config: String, caseName: String, cycle: Int): CaseStatus =
+      val emuResultPath = os.Path(nixResolvePath(s".#t1.$config.cases.$caseName.emu-result"))
+      val testFail = os.read(emuResultPath / "emu-success") == "0"
+
+      val perfCycleRegex = raw"total_cycles:\s(\d+)".r
+      val newCycle = os.read
+        .lines(emuResultPath / "perf.txt")
+        .apply(0) match
+        case perfCycleRegex(cycle) => cycle.toInt
+        case _ =>
+          throw new Exception("perf.txt file is not format as expected")
+      CaseStatus(
+        config = config,
+        caseName = caseName,
+        isFailed = testFail,
+        oldCycle = cycle,
+        newCycle = newCycle,
+      )
+    end collectCaseStatus
 
-    Logger.info("Cycle data updated")
-  end mergeCycleData
+    val allCycleRecords =
+      os.walk(os.pwd / ".github" / "cases").filter(_.last == "default.json")
+    allCycleRecords.foreach: file =>
+      val config = file.segments.toSeq.reverse.apply(1)
+      var cycleRecord = ujson.read(os.read(file))
+      val allCaseStatus = cycleRecord.obj.map(rec => rec match {
+        case(caseName, cycle) => collectCaseStatus(config, caseName, cycle.num.toInt)
+      })
+
+      val failedCases = allCaseStatus.filter(c => c.isFailed).map(c => s"* `.#t1.${c.config}.cases.${c.caseName}`")
+      val failedTestsRecordFile = os.Path(failedTestsFilePath, os.pwd)
+      os.write.over(failedTestsRecordFile, "## Failed tests\n")
+      os.write.append(failedTestsRecordFile, failedCases)
+
+      val cycleUpdateRecordFile = os.Path(cycleUpdateFilePath, os.pwd)
+      os.write.over(cycleUpdateRecordFile, "## Cycle Update\n")
+      val allCycleUpdates = allCaseStatus.filter(c => c.oldCycle != c.newCycle).map:
+        caseStatus => caseStatus match
+          case CaseStatus(_, caseName, _, oldCycle, newCycle) =>
+            cycleRecord(caseName) = newCycle
+            if oldCycle == -1 then
+              s"* 🆕 ${caseName}: NaN -> ${newCycle}"
+            else if oldCycle > newCycle then
+              s"* 🚀 $caseName: $oldCycle -> $newCycle"
+            else
+              s"* 🐢 $caseName: $oldCycle -> $newCycle"
+      os.write.append(cycleUpdateRecordFile, allCycleUpdates.mkString("\n"))
+
+      os.write.over(file, ujson.write(cycleRecord, indent = 2))
+  end postCI
 
   @main
   def generateTestPlan() =