From 8be78ef85d73505feaac7fcbe6e486112bf532f9 Mon Sep 17 00:00:00 2001 From: Avimitin Date: Wed, 19 Jun 2024 13:15:35 +0800 Subject: [PATCH] [ci] stop using path resolve for all testcase Signed-off-by: Avimitin --- script/src/Main.scala | 155 +++++++++++++++++--------------------- tests/default.nix | 11 ++- tests/make-emu-result.nix | 2 +- 3 files changed, 78 insertions(+), 90 deletions(-) diff --git a/script/src/Main.scala b/script/src/Main.scala index 8cf2b5894..24897cd4d 100644 --- a/script/src/Main.scala +++ b/script/src/Main.scala @@ -598,73 +598,54 @@ object Main: doc = "specify the cycle update markdown file output path" ) cycleUpdateFilePath: String ) = - case class CaseStatus( - caseName: String, - isFailed: Boolean, - oldCycle: Int, - newCycle: Int - ) - - def collectCaseStatus( - config: String, - caseName: String, - cycle: Int - ): CaseStatus = - val emuResultPath = os.Path(nixResolvePath(s".#t1.$config.cases.$caseName.emu-result")) - val testFail = os.read(emuResultPath / "emu-success") == "0" - - val perfCycleRegex = raw"total_cycles:\s(\d+)".r - val newCycle = os.read - .lines(emuResultPath / "perf.txt") - .apply(0) match - case perfCycleRegex(cycle) => cycle.toInt - case _ => - throw new Exception("perf.txt file is not format as expected") - CaseStatus( - caseName = caseName, - isFailed = testFail, - oldCycle = cycle, - newCycle = newCycle - ) - end collectCaseStatus - - val allCycleRecords = - os.walk(os.pwd / ".github" / "cases").filter(_.last == "default.json") - allCycleRecords.foreach: file => - val config = file.segments.toSeq.reverse.apply(1) - var cycleRecord = ujson.read(os.read(file)) - - nixResolvePath(s".#t1.$config.cases._allEmuResult") - - val allCaseStatus = cycleRecord.obj.map(rec => - rec match { - case (caseName, cycle) => - collectCaseStatus(config, caseName, cycle.num.toInt) - } - ) - - val failedCases = allCaseStatus - .filter(c => c.isFailed) - .map(c => s"* `.#t1.${config}.cases.${c.caseName}`") - val failedTestsRecordFile = os.Path(failedTestsFilePath, os.pwd) - os.write.over(failedTestsRecordFile, "## Failed tests\n") - os.write.append(failedTestsRecordFile, failedCases) - - val cycleUpdateRecordFile = os.Path(cycleUpdateFilePath, os.pwd) - os.write.over(cycleUpdateRecordFile, "## Cycle Update\n") - val allCycleUpdates = allCaseStatus - .filter(c => c.oldCycle != c.newCycle) - .map: caseStatus => - caseStatus match - case CaseStatus(caseName, _, oldCycle, newCycle) => + os.walk(os.pwd / ".github" / "cases") + .filter(_.last == "default.json") + .foreach: file => + val config = file.segments.toSeq.reverse.apply(1) + var cycleRecord = ujson.read(os.read(file)) + + Logger.info("Fetching CI results") + val emuResultPath = + os.Path(nixResolvePath(s".#t1.$config.cases._allEmuResult")) + + Logger.info("Collecting failed cases") + val failedCases = os + .walk(emuResultPath) + .filter(path => path.last == "emu-success") + .filter(path => os.read(path) == "0") + .map(path => path.segments.toSeq.reverse.drop(1).head) + .map(caseName => s"* `.#t1.${config}.cases.${caseName}`") + val failedTestsRecordFile = os.Path(failedTestsFilePath, os.pwd) + os.write.over(failedTestsRecordFile, "## Failed tests\n") + os.write.append(failedTestsRecordFile, failedCases) + + Logger.info("Collecting cycle update info") + val cycleUpdateRecordFile = os.Path(cycleUpdateFilePath, os.pwd) + os.write.over(cycleUpdateRecordFile, "## Cycle Update\n") + val perfCycleRegex = raw"total_cycles:\s(\d+)".r + val allCycleUpdates = os + .walk(emuResultPath) + .filter(path => path.last == "perf.txt") + .map(path => { + val cycle = os.read.lines(path).head match + case perfCycleRegex(cycle) => cycle.toInt + case _ => + throw new Exception("perf.txt file is not format as expected") + val caseName = path.segments.toSeq.reverse.drop(1).head + (caseName, cycle, cycleRecord.obj(caseName).num.toInt) + }) + .filter((_, newCycle, oldCycle) => newCycle != oldCycle) + .map: + case (caseName, newCycle, oldCycle) => cycleRecord(caseName) = newCycle if oldCycle == -1 then s"* 🆕 ${caseName}: NaN -> ${newCycle}" else if oldCycle > newCycle then s"* 🚀 $caseName: $oldCycle -> $newCycle" else s"* 🐢 $caseName: $oldCycle -> $newCycle" - os.write.append(cycleUpdateRecordFile, allCycleUpdates.mkString("\n")) - os.write.over(file, ujson.write(cycleRecord, indent = 2)) + os.write.append(cycleUpdateRecordFile, allCycleUpdates.mkString("\n")) + + os.write.over(file, ujson.write(cycleRecord, indent = 2)) end postCI @main @@ -745,21 +726,21 @@ object Main: @main def difftest( - @arg( - name = "config", - short = 'c', - doc = "specify the elaborate config for running test case" - ) config: String, - @arg( - name = "case-attr", - short = 'C', - doc = "Specify test case attribute to run diff test" - ) caseAttr: String, - @arg( - name = "log-level", - short = 'L', - doc = "Specify log level to run diff test" - ) logLevel: String = "ERROR" + @arg( + name = "config", + short = 'c', + doc = "specify the elaborate config for running test case" + ) config: String, + @arg( + name = "case-attr", + short = 'C', + doc = "Specify test case attribute to run diff test" + ) caseAttr: String, + @arg( + name = "log-level", + short = 'L', + doc = "Specify log level to run diff test" + ) logLevel: String = "ERROR" ): Unit = val difftest = nixResolvePath(".#t1-simulator") @@ -778,15 +759,19 @@ object Main: val eventLog = nixResolvePath(s"${fullCaseAttr}.emu-result") Logger.trace("Running zstd to get event log") - os.proc(Seq( - "zstd", - "--decompress", - "-f", - s"${eventLog}/rtl-event.log.zstd", - "-o", - s"${config}-${caseAttr}.event.log" - )).call(stdout = os.Inherit, stderr = os.Inherit) - Logger.info(s"Starting t1-simulator with DLEN ${dLen}, VLEN ${vLen} for ${fullCaseAttr}") + os.proc( + Seq( + "zstd", + "--decompress", + "-f", + s"${eventLog}/rtl-event.log.zstd", + "-o", + s"${config}-${caseAttr}.event.log" + ) + ).call(stdout = os.Inherit, stderr = os.Inherit) + Logger.info( + s"Starting t1-simulator with DLEN ${dLen}, VLEN ${vLen} for ${fullCaseAttr}" + ) os.proc( Seq( s"${difftest}/bin/t1-simulator", diff --git a/tests/default.nix b/tests/default.nix index 19154e5b6..4afd3d38b 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -62,7 +62,7 @@ let inherit (scope) mlir intrinsic asm perf codegen rvv_bench; }; - # This derivation is for internal use only. + # This derivation is for internal CI use only. # We have a large test suite used in CI, but resolving each test individually is too slow for production. # This "fake" derivation serves as a workaround, making all tests dependencies of this single derivation. # This allows Nix to resolve the path only once, while still pulling all tests into the local Nix store. @@ -74,14 +74,17 @@ let allCases = lib.filter (val: lib.isDerivation val && lib.hasAttr val.pname testPlan) (lib.concatLists (map lib.attrValues (lib.attrValues scopeStripped))); script = '' - echo "fake-derivation" > $out + mkdir -p $out '' + (lib.concatMapStringsSep "\n" (caseDrv: '' - echo ${caseDrv.emu-result} > /dev/null + _caseOutDir=$out/${caseDrv.pname} + mkdir -p "$_caseOutDir" + cp ${caseDrv.emu-result}/perf.txt "$_caseOutDir"/ + cp ${caseDrv.emu-result}/emu-success "$_caseOutDir"/ '') allCases); in - runCommand "catch-all-emu-result" { } script; + runCommand "catch-${configName}-all-emu-result-for-ci" { } script; all = let diff --git a/tests/make-emu-result.nix b/tests/make-emu-result.nix index 5aa2070a3..d6a0d8b2b 100644 --- a/tests/make-emu-result.nix +++ b/tests/make-emu-result.nix @@ -9,7 +9,7 @@ # makeEmuResult arg testCase: -runCommand "get-emu-result" { nativeBuildInputs = [ zstd ]; } '' +runCommand "get-${testCase.pname}-emu-result" { nativeBuildInputs = [ zstd ]; } '' echo "[NIX] Running test case ${testCase.pname}" mkdir -p "$out"