diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index c51e3044ff..b4f995cb2e 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -157,26 +157,13 @@ jobs: fail-fast: false matrix: ${{ fromJSON(needs.gen-matrix.outputs.ci-tests) }} runs-on: [self-hosted, linux, nixos] - outputs: - result: ${{ steps.ci-run.outputs.result }} steps: - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} - name: "Run testcases" - id: ci-run run: | - nix develop -c t1-helper runTests --jobs "${{ matrix.jobs }}" \ - --resultDir test-results-$(head -c 10 /dev/urandom | base32) - - - uses: actions/upload-artifact@v4 - if: ${{ !cancelled() }} - with: - name: test-reports-${{ matrix.id }} - path: | - test-results-*/failed-tests.md - test-results-*/cycle-updates.md - test-results-*/*_cycle.json + nix develop -c t1-helper runTests --jobs "${{ matrix.jobs }}" report: name: "Report CI result" @@ -191,21 +178,14 @@ jobs: with: fetch-depth: 0 ref: ${{ github.head_ref }} - - uses: actions/download-artifact@v4 - with: - pattern: test-reports-* - merge-multiple: true - name: "Print step summary" run: | - echo -e "\n## Failed tests\n" >> $GITHUB_STEP_SUMMARY - shopt -s nullglob - cat test-results-*/failed-tests.md >> $GITHUB_STEP_SUMMARY - echo -e "\n## Cycle updates\n" >> $GITHUB_STEP_SUMMARY - shopt -s nullglob - cat test-results-*/cycle-updates.md >> $GITHUB_STEP_SUMMARY + nix develop -c t1-helper postCI --failed-test-file-path ./failed-test.md --cycle-update-file-path ./cycle-update.md + cat ./failed-test.md >> $GITHUB_STEP_SUMMARY + echo >> $GITHUB_STEP_SUMMARY + cat ./cycled-update.md >> $GITHUB_STEP_SUMMARY - name: "Commit cycle updates" run: | - nix develop -c t1-helper mergeCycleData git config user.name github-actions git config user.email github-actions@github.com changed_cases=$(git diff --name-only '.github/cases/**/default.json') @@ -218,17 +198,3 @@ jobs: else echo "No cycle change detect" fi - - uses: geekyeggo/delete-artifact@v5 - with: - # test-reports has been used, it can be deleted - name: test-reports-* - - clean-after-cancelled: - name: "Clean test reports [ON CANCELLED]" - if: ${{ cancelled() }} - runs-on: [self-hosted, linux, nixos] - needs: [run-testcases] - steps: - - uses: geekyeggo/delete-artifact@v5 - with: - name: test-reports-* diff --git a/script/src/Main.scala b/script/src/Main.scala index b7afe8254c..f745cfe2ab 100644 --- a/script/src/Main.scala +++ b/script/src/Main.scala @@ -494,56 +494,6 @@ object Main: println(toMatrixJson(scheduleTasks(testPlans, runnersAmount))) } - def writeCycleUpdates( - testName: String, - testRunDir: os.Path, - resultDir: os.Path - ): Unit = - val isEmulatorTask = raw"([^,]+),([^,]+)".r - testName match - case isEmulatorTask(e, t) => - val passedFile = os.pwd / os.RelPath(s".github/cases/$e/default.json") - val original = ujson.read(os.read(passedFile)) - - val perfCycleRegex = raw"total_cycles:\s(\d+)".r - val newCycleCount = os.read - .lines(testRunDir / os.RelPath(s"$e/$t/perf.txt")) - .apply(0) match - case perfCycleRegex(cycle) => cycle.toInt - case _ => - throw new Exception("perf.txt file is not format as expected") - - val oldCycleCount = original.obj.get(t).map(_.num.toInt).getOrElse(-1) - val cycleUpdateFile = resultDir / "cycle-updates.md" - Logger.info(f"job '$testName' cycle $oldCycleCount -> $newCycleCount") - oldCycleCount match - case -1 => - os.write.append( - cycleUpdateFile, - s"* 🆕 $testName: NaN -> $newCycleCount\n" - ) - case _ => - if oldCycleCount > newCycleCount then - os.write.append( - cycleUpdateFile, - s"* 🚀 $testName: $oldCycleCount -> $newCycleCount\n" - ) - else if oldCycleCount < newCycleCount then - os.write.append( - cycleUpdateFile, - s"* 🐢 $testName: $oldCycleCount -> $newCycleCount\n" - ) - - val newCycleFile = resultDir / s"${e}_cycle.json" - val newCycleRecord = - if os.exists(newCycleFile) then ujson.read(os.read(newCycleFile)) - else ujson.Obj() - - newCycleRecord(t) = newCycleCount - os.write.over(newCycleFile, ujson.write(newCycleRecord, indent = 2)) - case _ => throw new Exception(f"unknown job format '$testName'") - end writeCycleUpdates - // Run jobs and give a brief result report // - Log of tailed tests will be tailed and copied into $resultDir/failed-logs/$testName.log // - List of failed tests will be written into $resultDir/failed-tests.md @@ -556,17 +506,12 @@ object Main: @main def runTests( jobs: String, - resultDir: Option[os.Path], dontBail: Flag = Flag(false) ): Unit = if jobs == "" then Logger.info("No test found, exiting") return - var actualResultDir = resultDir.getOrElse(os.pwd / "test-results") - val testRunDir = os.pwd / "testrun" - os.makeDir.all(actualResultDir / "failed-logs") - val allJobs = jobs.split(";") def findFailedTests() = allJobs.zipWithIndex.foldLeft(Seq[String]()): (allFailedTest, currentTest) => @@ -635,38 +580,74 @@ object Main: ) end runTests + // PostCI do the below four things: + // * read default.json at .github/cases/$config/default.json + // * generate case information for each entry in default.json (cycle, run success) + // * collect and report failed tests + // * collect and report cycle update @main - def mergeCycleData(filePat: String = "default.json") = - Logger.info("Updating cycle data") - val original = os - .walk(os.pwd / ".github" / "cases") - .filter(_.last == filePat) - .map: path => - val config = path.segments.toSeq.reverse(1) - (config, ujson.read(os.read(path))) - .toMap - os.walk(os.pwd) - .filter(_.last.endsWith("_cycle.json")) - .map: path => - val config = path.last.split("_")(0) - Logger.trace(s"Reading new cycle data from $path") - (config, ujson.read(os.read(path))) - .foreach: - case (name, latest) => - val old = original.apply(name) - latest.obj.foreach: - case (k, v) => old.update(k, v) - - original.foreach: - case (name, data) => - val config = name.split(",")(0) - os.write.over( - os.pwd / ".github" / "cases" / config / filePat, - ujson.write(data, indent = 2) - ) + def postCI( + @arg( + name = "failed-test-file-path", + doc = "specify the failed test markdown file output path" + ) failedTestsFilePath: String, + @arg( + name = "cycle-update-file-path", + doc = "specify the cycle update markdown file output path" + ) cycleUpdateFilePath: String, + ) = + case class CaseStatus(config: String, caseName: String, isFailed: Boolean, oldCycle: Int, newCycle: Int) + + def collectCaseStatus(config: String, caseName: String, cycle: Int): CaseStatus = + val emuResultPath = os.Path(nixResolvePath(s".#t1.$config.cases.$caseName.emu-result")) + val testFail = os.read(emuResultPath / "emu-success") == "0" + + val perfCycleRegex = raw"total_cycles:\s(\d+)".r + val newCycle = os.read + .lines(emuResultPath / "perf.txt") + .apply(0) match + case perfCycleRegex(cycle) => cycle.toInt + case _ => + throw new Exception("perf.txt file is not format as expected") + CaseStatus( + config = config, + caseName = caseName, + isFailed = testFail, + oldCycle = cycle, + newCycle = newCycle, + ) + end collectCaseStatus - Logger.info("Cycle data updated") - end mergeCycleData + val allCycleRecords = + os.walk(os.pwd / ".github" / "cases").filter(_.last == "default.json") + allCycleRecords.foreach: file => + val config = file.segments.toSeq.reverse.apply(1) + var cycleRecord = ujson.read(os.read(file)) + val allCaseStatus = cycleRecord.obj.map(rec => rec match { + case(caseName, cycle) => collectCaseStatus(config, caseName, cycle.num.toInt) + }) + + val failedCases = allCaseStatus.filter(c => c.isFailed).map(c => s"* `.#t1.${c.config}.cases.${c.caseName}`") + val failedTestsRecordFile = os.Path(failedTestsFilePath, os.pwd) + os.write.over(failedTestsRecordFile, "## Failed tests\n") + os.write.append(failedTestsRecordFile, failedCases) + + val cycleUpdateRecordFile = os.Path(cycleUpdateFilePath, os.pwd) + os.write.over(cycleUpdateRecordFile, "## Cycle Update\n") + val allCycleUpdates = allCaseStatus.filter(c => c.oldCycle != c.newCycle).map: + caseStatus => caseStatus match + case CaseStatus(_, caseName, _, oldCycle, newCycle) => + cycleRecord(caseName) = newCycle + if oldCycle == -1 then + s"* 🆕 ${caseName}: NaN -> ${newCycle}" + else if oldCycle > newCycle then + s"* 🚀 $caseName: $oldCycle -> $newCycle" + else + s"* 🐢 $caseName: $oldCycle -> $newCycle" + os.write.append(cycleUpdateRecordFile, allCycleUpdates.mkString("\n")) + + os.write.over(file, ujson.write(cycleRecord, indent = 2)) + end postCI @main def generateTestPlan() =