[script] migrate all post CI action to Scala script

- Use nix derivation to cache and capture emulator output - Use built-in S3 nix cache to replace GitHub Artifacts - Produce GitHub step summary in Scala script Signed-off-by: Avimitin <[email protected]>
chipsalliance · Jun 17, 2024 · f810d39 · f810d39
1 parent 6165ba7
commit f810d39
Show file tree

Hide file tree

Showing 3 changed files with 126 additions and 147 deletions.
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -157,26 +157,13 @@ jobs:
       fail-fast: false
       matrix: ${{ fromJSON(needs.gen-matrix.outputs.ci-tests) }}
     runs-on: [self-hosted, linux, nixos]
-    outputs:
-      result: ${{ steps.ci-run.outputs.result }}
     steps:
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: "Run testcases"
-        id: ci-run
         run: |
-          nix develop -c t1-helper runTests --jobs "${{ matrix.jobs }}" \
-            --resultDir test-results-$(head -c 10 /dev/urandom | base32)
-
-      - uses: actions/upload-artifact@v4
-        if: ${{ !cancelled() }}
-        with:
-          name: test-reports-${{ matrix.id }}
-          path: |
-            test-results-*/failed-tests.md
-            test-results-*/cycle-updates.md
-            test-results-*/*_cycle.json
+          nix develop -c t1-helper runTests --jobs "${{ matrix.jobs }}"
 
   report:
     name: "Report CI result"
@@ -191,21 +178,14 @@ jobs:
         with:
           fetch-depth: 0
           ref: ${{ github.head_ref }}
-      - uses: actions/download-artifact@v4
-        with:
-          pattern: test-reports-*
-          merge-multiple: true
       - name: "Print step summary"
         run: |
-          echo -e "\n## Failed tests\n" >> $GITHUB_STEP_SUMMARY
-          shopt -s nullglob
-          cat test-results-*/failed-tests.md >> $GITHUB_STEP_SUMMARY
-          echo -e "\n## Cycle updates\n" >> $GITHUB_STEP_SUMMARY
-          shopt -s nullglob
-          cat test-results-*/cycle-updates.md >> $GITHUB_STEP_SUMMARY
+          nix develop -c t1-helper postCI --failed-test-file-path ./failed-test.md --cycle-update-file-path ./cycle-update.md
+          cat ./failed-test.md >> $GITHUB_STEP_SUMMARY
+          echo >> $GITHUB_STEP_SUMMARY
+          cat ./cycle-update.md >> $GITHUB_STEP_SUMMARY
       - name: "Commit cycle updates"
         run: |
-          nix develop -c t1-helper mergeCycleData
           git config user.name github-actions
           git config user.email [email protected]
           changed_cases=$(git diff --name-only '.github/cases/**/default.json')
@@ -218,17 +198,3 @@ jobs:
           else
             echo "No cycle change detect"
           fi
-      - uses: geekyeggo/delete-artifact@v5
-        with:
-          # test-reports has been used, it can be deleted
-          name: test-reports-*
-
-  clean-after-cancelled:
-    name: "Clean test reports [ON CANCELLED]"
-    if: ${{ cancelled() }}
-    runs-on: [self-hosted, linux, nixos]
-    needs: [run-testcases]
-    steps:
-      - uses: geekyeggo/delete-artifact@v5
-        with:
-          name: test-reports-*
diff --git a/script/src/Main.scala b/script/src/Main.scala
@@ -494,56 +494,6 @@ object Main:
     println(toMatrixJson(scheduleTasks(testPlans, runnersAmount)))
   }
 
-  def writeCycleUpdates(
-      testName: String,
-      testRunDir: os.Path,
-      resultDir: os.Path
-  ): Unit =
-    val isEmulatorTask = raw"([^,]+),([^,]+)".r
-    testName match
-      case isEmulatorTask(e, t) =>
-        val passedFile = os.pwd / os.RelPath(s".github/cases/$e/default.json")
-        val original = ujson.read(os.read(passedFile))
-
-        val perfCycleRegex = raw"total_cycles:\s(\d+)".r
-        val newCycleCount = os.read
-          .lines(testRunDir / os.RelPath(s"$e/$t/perf.txt"))
-          .apply(0) match
-          case perfCycleRegex(cycle) => cycle.toInt
-          case _ =>
-            throw new Exception("perf.txt file is not format as expected")
-
-        val oldCycleCount = original.obj.get(t).map(_.num.toInt).getOrElse(-1)
-        val cycleUpdateFile = resultDir / "cycle-updates.md"
-        Logger.info(f"job '$testName' cycle $oldCycleCount -> $newCycleCount")
-        oldCycleCount match
-          case -1 =>
-            os.write.append(
-              cycleUpdateFile,
-              s"* 🆕 $testName: NaN -> $newCycleCount\n"
-            )
-          case _ =>
-            if oldCycleCount > newCycleCount then
-              os.write.append(
-                cycleUpdateFile,
-                s"* 🚀 $testName: $oldCycleCount -> $newCycleCount\n"
-              )
-            else if oldCycleCount < newCycleCount then
-              os.write.append(
-                cycleUpdateFile,
-                s"* 🐢 $testName: $oldCycleCount -> $newCycleCount\n"
-              )
-
-            val newCycleFile = resultDir / s"${e}_cycle.json"
-            val newCycleRecord =
-              if os.exists(newCycleFile) then ujson.read(os.read(newCycleFile))
-              else ujson.Obj()
-
-            newCycleRecord(t) = newCycleCount
-            os.write.over(newCycleFile, ujson.write(newCycleRecord, indent = 2))
-      case _ => throw new Exception(f"unknown job format '$testName'")
-  end writeCycleUpdates
-
   // Run jobs and give a brief result report
   // - Log of tailed tests will be tailed and copied into $resultDir/failed-logs/$testName.log
   // - List of failed tests will be written into $resultDir/failed-tests.md
@@ -556,60 +506,57 @@ object Main:
   @main
   def runTests(
       jobs: String,
-      resultDir: Option[os.Path],
       dontBail: Flag = Flag(false)
   ): Unit =
     if jobs == "" then
       Logger.info("No test found, exiting")
       return
 
-    var actualResultDir = resultDir.getOrElse(os.pwd / "test-results")
-    val testRunDir = os.pwd / "testrun"
-    os.makeDir.all(actualResultDir / "failed-logs")
-
     val allJobs = jobs.split(";")
     def findFailedTests() = allJobs.zipWithIndex.foldLeft(Seq[String]()):
       (allFailedTest, currentTest) =>
         val (testName, index) = currentTest
         val Array(config, caseName) = testName.split(",")
-        println()
+        println("\n")
         Logger.info(
           s"${BOLD}[${index + 1}/${allJobs.length}]${RESET} Running test case $caseName with config $config"
         )
 
-        val testResultPath = os.Path(nixResolvePath(s".#t1.$config.cases.$caseName.emu-result"))
-        val testSuccess = os.read(testResultPath / "emu-success").trim().toInt == 1
+        val testResultPath =
+          os.Path(nixResolvePath(s".#t1.$config.cases.$caseName.emu-result"))
+        val testSuccess =
+          os.read(testResultPath / "emu-success").trim().toInt == 1
 
-        if ! testSuccess then
+        if !testSuccess then
           Logger.error(s"Test case $testName failed")
           val err = os.read(testResultPath / "emu.log")
           Logger.error(s"Detail error: $err")
 
         Logger.info("Running difftest")
-        val diffTestSuccess = try
-          difftest(
-            config = config,
-            caseAttr = caseName,
-            logLevel = "ERROR"
-          )
-          true
-        catch
-          err =>
-            Logger.error(s"difftest run failed: $err")
-            false
+        val diffTestSuccess =
+          try
+            difftest(
+              config = config,
+              caseAttr = caseName,
+              logLevel = "ERROR"
+            )
+            true
+          catch
+            err =>
+              Logger.error(s"difftest run failed: $err")
+              false
 
         if diffTestSuccess != testSuccess then
-          Logger.fatal("Got different online and offline difftest result, please check this test manually. CI aborted.")
+          Logger.fatal(
+            "Got different online and offline difftest result, please check this test manually. CI aborted."
+          )
 
-        if ! testSuccess then
-          allFailedTest :+ s"t1.$config.cases.$caseName"
-        else
-          allFailedTest
+        if !testSuccess then allFailedTest :+ s"t1.$config.cases.$caseName"
+        else allFailedTest
     end findFailedTests
 
     val failedTests = findFailedTests()
-    if failedTests.isEmpty then
-      Logger.info(s"All tests passed")
+    if failedTests.isEmpty then Logger.info(s"All tests passed")
     else
       val listOfFailJobs =
         failedTests.map(job => s"* $job").appended("").mkString("\n")
@@ -635,38 +582,89 @@ object Main:
         )
   end runTests
 
+  // PostCI do the below four things:
+  //   * read default.json at .github/cases/$config/default.json
+  //   * generate case information for each entry in default.json (cycle, run success)
+  //   * collect and report failed tests
+  //   * collect and report cycle update
   @main
-  def mergeCycleData(filePat: String = "default.json") =
-    Logger.info("Updating cycle data")
-    val original = os
-      .walk(os.pwd / ".github" / "cases")
-      .filter(_.last == filePat)
-      .map: path =>
-        val config = path.segments.toSeq.reverse(1)
-        (config, ujson.read(os.read(path)))
-      .toMap
-    os.walk(os.pwd)
-      .filter(_.last.endsWith("_cycle.json"))
-      .map: path =>
-        val config = path.last.split("_")(0)
-        Logger.trace(s"Reading new cycle data from $path")
-        (config, ujson.read(os.read(path)))
-      .foreach:
-        case (name, latest) =>
-          val old = original.apply(name)
-          latest.obj.foreach:
-            case (k, v) => old.update(k, v)
-
-    original.foreach:
-      case (name, data) =>
-        val config = name.split(",")(0)
-        os.write.over(
-          os.pwd / ".github" / "cases" / config / filePat,
-          ujson.write(data, indent = 2)
-        )
+  def postCI(
+      @arg(
+        name = "failed-test-file-path",
+        doc = "specify the failed test markdown file output path"
+      ) failedTestsFilePath: String,
+      @arg(
+        name = "cycle-update-file-path",
+        doc = "specify the cycle update markdown file output path"
+      ) cycleUpdateFilePath: String
+  ) =
+    case class CaseStatus(
+        caseName: String,
+        isFailed: Boolean,
+        oldCycle: Int,
+        newCycle: Int
+    )
+
+    def collectCaseStatus(
+        emuResultPath: os.Path,
+        caseName: String,
+        cycle: Int
+    ): CaseStatus =
+      val testFail = os.read(emuResultPath / caseName / "emu-success") == "0"
+
+      val perfCycleRegex = raw"total_cycles:\s(\d+)".r
+      val newCycle = os.read
+        .lines(emuResultPath / caseName / "perf.txt")
+        .apply(0) match
+        case perfCycleRegex(cycle) => cycle.toInt
+        case _ =>
+          throw new Exception("perf.txt file is not format as expected")
+      CaseStatus(
+        caseName = caseName,
+        isFailed = testFail,
+        oldCycle = cycle,
+        newCycle = newCycle
+      )
+    end collectCaseStatus
+
+    val allCycleRecords =
+      os.walk(os.pwd / ".github" / "cases").filter(_.last == "default.json")
+    allCycleRecords.foreach: file =>
+      val config = file.segments.toSeq.reverse.apply(1)
+      var cycleRecord = ujson.read(os.read(file))
+
+      val allEmuResultPath = nixResolvePath(s".#t1.$config.cases._allEmuResult")
+
+      val allCaseStatus = cycleRecord.obj.map(rec =>
+        rec match {
+          case (caseName, cycle) =>
+            collectCaseStatus(os.Path(allEmuResultPath), caseName, cycle.num.toInt)
+        }
+      )
 
-    Logger.info("Cycle data updated")
-  end mergeCycleData
+      val failedCases = allCaseStatus
+        .filter(c => c.isFailed)
+        .map(c => s"* `.#t1.${config}.cases.${c.caseName}`")
+      val failedTestsRecordFile = os.Path(failedTestsFilePath, os.pwd)
+      os.write.over(failedTestsRecordFile, "## Failed tests\n")
+      os.write.append(failedTestsRecordFile, failedCases)
+
+      val cycleUpdateRecordFile = os.Path(cycleUpdateFilePath, os.pwd)
+      os.write.over(cycleUpdateRecordFile, "## Cycle Update\n")
+      val allCycleUpdates = allCaseStatus
+        .filter(c => c.oldCycle != c.newCycle)
+        .map: caseStatus =>
+          caseStatus match
+            case CaseStatus(caseName, _, oldCycle, newCycle) =>
+              cycleRecord(caseName) = newCycle
+              if oldCycle == -1 then s"* 🆕 ${caseName}: NaN -> ${newCycle}"
+              else if oldCycle > newCycle then
+                s"* 🚀 $caseName: $oldCycle -> $newCycle"
+              else s"* 🐢 $caseName: $oldCycle -> $newCycle"
+      os.write.append(cycleUpdateRecordFile, allCycleUpdates.mkString("\n"))
+
+      os.write.over(file, ujson.write(cycleRecord, indent = 2))
+  end postCI
 
   @main
   def generateTestPlan() =

diff --git a/tests/default.nix b/tests/default.nix
@@ -61,6 +61,21 @@ let
     inherit (scope) mlir intrinsic asm perf codegen rvv_bench;
   };
 
+  _allEmuResult =
+    let
+      allCases = lib.filter lib.isDerivation
+        (lib.concatLists (map lib.attrValues (lib.attrValues scopeStripped)));
+      script = ''
+        mkdir -p $out
+      '' + (lib.concatMapStringsSep "\n"
+        (caseDrv: ''
+          mkdir -p $out/${caseDrv.pname}
+          cp ${caseDrv.emu-result}/* $out/${caseDrv.pname}/
+        '')
+        allCases);
+    in
+    runCommand "catch-all-emu-result" { } script;
+
   all =
     let
       allCases = lib.filter lib.isDerivation
@@ -77,4 +92,4 @@ let
     in
     runCommand "build-all-testcases" { } script;
 in
-lib.recurseIntoAttrs (scopeStripped // { inherit all; })
+lib.recurseIntoAttrs (scopeStripped // { inherit all _allEmuResult; })