From b79954e729aeeb954d6253c3c7b7c50241476fee Mon Sep 17 00:00:00 2001 From: Asuna Date: Mon, 24 Jun 2024 11:53:56 +0200 Subject: [PATCH] [ci] add tests for omreader --- .github/workflows/pr.yml | 5 +- script/emu/src/Main.scala | 344 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 348 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 04dbe91eb..fc2afc196 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -163,7 +163,10 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} - name: "Run testcases" run: | - nix run ".#ci-helper" -- runTests --jobs "${{ matrix.jobs }}" + nix develop -c t1-helper runTests --jobs "${{ matrix.jobs }}" + - name: "Run OM tests" + run: | + nix develop -c t1-helper runOMTests --jobs "${{ matrix.jobs }}" report: name: "Report CI result" diff --git a/script/emu/src/Main.scala b/script/emu/src/Main.scala index 4b18eb360..92088c27c 100644 --- a/script/emu/src/Main.scala +++ b/script/emu/src/Main.scala @@ -407,6 +407,350 @@ object Main: ).call(stdout = os.Inherit, stderr = os.Inherit, stdin = os.Inherit) Logger.info(s"RTLs store in $finalOutLink") + // + // CI + // + // The below script will try to read all the tests in ../../.github/cases/**/default.json, + // arranging them together by their required cycle, and using GitHub "matrix" feature to manage + // and separate those test job to multiple machines. + // + // We define that, the term "bucket" refers to a list of test job, concating by ';' into text. + // "Bucket" will be recorded in "matrix" payload field "jobs". Each machine will run a "bucket" of tests. + // + // Function `generateMatrix` will be used to produce necessary information to feed the GitHub matrix. + // Function `runTests` will parse the GitHub matrix, run a "bucket" of tests and generate GitHub CI report. + // + // The final "matrix" will have json data like: { include: [ { jobs: "taskA;taskB", id: 1 }, { jobs: "taskC;taskD", id: 2 } ] }. + // + + case class Bucket(buffer: Seq[String] = Seq(), totalCycle: Int = 0): + def cons(data: (String, Int)): Bucket = + val (testName, cycle) = data + Bucket(buffer ++ Seq(testName), totalCycle + cycle) + + def mkString(sep: String = ";") = buffer.mkString(sep) + + // Read test case and their cycle data from the given paths. + // Test cases will be grouped into a single bucket, and then partitioned into given `bucketSize` of sub-bucket. + // Each sub-bucket will have similar weight, so that the time cost will be similar between each runners. + // + // For example: + // + // [ {A: 100}, {B: 180}, {C: 300}, {D:200} ] => [[A,C], [B,D]] + // + // @param allTasksFile List of the default.json file path + // @param bucketSize Specify the size of the output Seq + def scheduleTasks(allTasksFile: Seq[os.Path], bucketSize: Int): Seq[String] = + // Produce a list of ("config,testName", cycle) pair + val allCycleData = allTasksFile.flatMap: file => + Logger.trace(s"Generate tests from file: $file") + val config = file.segments.toSeq.reverse.apply(1) + ujson + .read(os.read(file)) + .obj + .map { case (caseName, cycle) => + (s"$config,$caseName", cycle.num.toInt) + } + .toSeq + + val (unProcessedData, normalData) = + allCycleData.partition: + case (_, cycle) => cycle <= 0 + + // Initialize a list of buckets + val cargoInit = + (0 until math.min(bucketSize, allCycleData.length)).map(_ => Bucket()) + // Group tests that have cycle data into subset by their cycle size + val cargoStaged = normalData + .sortBy(_._2)(Ordering[Int].reverse) + .foldLeft(cargoInit): (cargo, elem) => + val smallest = cargo.minBy(_.totalCycle) + cargo.updated(cargo.indexOf(smallest), smallest.cons(elem)) + + // For unprocessed data, just split them into subset that have equal size + if unProcessedData.nonEmpty then + val chunkSize = unProcessedData.length.toDouble / bucketSize.toDouble + val cargoFinal = unProcessedData + .grouped(math.ceil(chunkSize).toInt) + .zipWithIndex + .foldLeft(cargoStaged): (cargo, chunkWithIndex) => + val (chunk, idx) = chunkWithIndex + val newBucket = chunk.foldLeft(cargoStaged.apply(idx)): + (bucket, data) => bucket.cons(data) + cargo.updated(idx, newBucket) + + cargoFinal.map(_.buffer.mkString(";")).toSeq + else cargoStaged.map(_.buffer.mkString(";")).toSeq + end scheduleTasks + + // Turn Seq( "A;B", "C;D" ) to GitHub Action matrix style json: { "include": [ { "jobs": "A;B", id: 1 }, { "jobs": "C;D", id: 2 } ] } + // + // @param buckets Seq of String that is already packed into bucket using the `buckets` function + // @param outputFile Path to the output json file + def toMatrixJson(buckets: Seq[String]) = + ujson.Obj("include" -> buckets.zipWithIndex.map: (bucket, i) => + ujson.Obj( + "jobs" -> ujson.Str(bucket), + "id" -> ujson.Num(i + 1) + )) + + // Read default tests information from '.github/cases/default.txt' file, and use that information to generate GitHub CI matrix. + // The result will be printed to stdout, and should be pipe into $GITHUB_OUTPUT + @main + def generateCiMatrix( + runnersAmount: Int, + testPlanFile: String = "default.json" + ) = { + val testPlans = + os.walk(os.pwd / ".github" / "cases").filter(_.last == testPlanFile) + println(toMatrixJson(scheduleTasks(testPlans, runnersAmount))) + } + + // Run jobs and give a brief result report + // - Log of tailed tests will be tailed and copied into $resultDir/failed-logs/$testName.log + // - List of failed tests will be written into $resultDir/failed-tests.md + // - Report of cycle updates will be written into $resultDir/cycle-updates.md + // - New cycle file will be written into $resultDir/$config_$runConfig_cycle.json + // + // @param: jobs A semicolon-separated list of job names of the form $config,$caseName,$runConfig + // @param: resultDir output directory of the test results, default to ./test-results + // @param: dontBail don't throw exception when test fail. Useful for postpr. + @main + def runTests( + jobs: String, + dontBail: Flag = Flag(false) + ): Unit = + if jobs == "" then + Logger.info("No test found, exiting") + return + + val allJobs = jobs.split(";") + def findFailedTests() = allJobs.zipWithIndex.foldLeft(Seq[String]()): + (allFailedTest, currentTest) => + val (testName, index) = currentTest + val Array(config, caseName) = testName.split(",") + println("\n") + Logger.info( + s"${BOLD}[${index + 1}/${allJobs.length}]${RESET} Running test case $caseName with config $config" + ) + + val testResultPath = + os.Path(nixResolvePath(s".#t1.$config.cases.$caseName.emu-result")) + val testSuccess = + os.read(testResultPath / "emu-success").trim().toInt == 1 + + if !testSuccess then + Logger.error(s"Test case $testName failed") + val err = os.read(testResultPath / "emu.log") + Logger.error(s"Detail error: $err") + + Logger.info("Running difftest") + val diffTestSuccess = + try + difftest( + config = config, + caseAttr = caseName, + logLevel = "ERROR" + ) + true + catch + err => + Logger.error(s"difftest run failed: $err") + false + + if diffTestSuccess != testSuccess then + Logger.fatal( + "Got different online and offline difftest result, please check this test manually. CI aborted." + ) + + if !testSuccess then allFailedTest :+ s"t1.$config.cases.$caseName" + else allFailedTest + end findFailedTests + + val failedTests = findFailedTests() + if failedTests.isEmpty then Logger.info(s"All tests passed") + else + val listOfFailJobs = + failedTests.map(job => s"* $job").appended("").mkString("\n") + val failedJobsWithError = failedTests + .map(testName => + val testResult = os.Path(nixResolvePath(s".#$testName.emu-result")) + val emuLog = os.read(testResult / "emu.log") + if emuLog.nonEmpty then + s"* $testName\n >>> ERROR SUMMARY <<<\n${emuLog}" + else + s"* $testName\n >>> OTHER ERROR <<<\n${os.read(testResult / "emu.journal")}" + ) + .appended("") + .mkString("\n") + + if dontBail.value then + Logger.error( + s"${BOLD}${failedTests.length} tests failed${RESET}:\n${failedJobsWithError}" + ) + else + Logger.fatal( + s"${BOLD}${failedTests.length} tests failed${RESET}:\n${failedJobsWithError}" + ) + end runTests + + @main + def runOMTests( + jobs: String, + ): Unit = + val configs = jobs.split(";").map(_.split(",")(0)) + configs.distinct.foreach: config => + Seq("omreader", "emu-omreader").foreach: target => + val command = Seq( + "nix", + "run", + s".#t1.$config.ip.$target", + "--", + "run", + "--dump-methods" + ) + println("\n") + Logger.info(s"Running OM test with command $BOLD'${command.mkString(" ")}'$RESET") + val outputs = os.proc(command).call().out.trim() + Logger.trace(s"Outputs:\n${outputs}") + + Seq("vlen =", "dlen =").foreach: keyword => + if outputs.contains(keyword) then + Logger.info(s"Keyword $BOLD'$keyword'$RESET found - ${GREEN}Pass!$RESET") + else + Logger.fatal(s"Keyword $BOLD'$keyword'$RESET not found - ${RED}Fail!$RESET") + end runOMTests + + // PostCI do the below four things: + // * read default.json at .github/cases/$config/default.json + // * generate case information for each entry in default.json (cycle, run success) + // * collect and report failed tests + // * collect and report cycle update + @main + def postCI( + @arg( + name = "failed-test-file-path", + doc = "specify the failed test markdown file output path" + ) failedTestsFilePath: String, + @arg( + name = "cycle-update-file-path", + doc = "specify the cycle update markdown file output path" + ) cycleUpdateFilePath: String + ) = + os.walk(os.pwd / ".github" / "cases") + .filter(_.last == "default.json") + .foreach: file => + val config = file.segments.toSeq.reverse.apply(1) + var cycleRecord = ujson.read(os.read(file)) + + Logger.info("Fetching CI results") + val emuResultPath = + os.Path(nixResolvePath(s".#t1.$config.cases._allEmuResult")) + + Logger.info("Collecting failed cases") + val failedCases = os + .walk(emuResultPath) + .filter(path => path.last == "emu-success") + .filter(path => os.read(path) == "0") + .map(path => path.segments.toSeq.reverse.drop(1).head) + .map(caseName => s"* `.#t1.${config}.cases.${caseName}`") + val failedTestsRecordFile = os.Path(failedTestsFilePath, os.pwd) + os.write.over(failedTestsRecordFile, "## Failed tests\n") + os.write.append(failedTestsRecordFile, failedCases) + + Logger.info("Collecting cycle update info") + val cycleUpdateRecordFile = os.Path(cycleUpdateFilePath, os.pwd) + os.write.over(cycleUpdateRecordFile, "## Cycle Update\n") + val perfCycleRegex = raw"total_cycles:\s(\d+)".r + val allCycleUpdates = os + .walk(emuResultPath) + .filter(path => path.last == "perf.txt") + .map(path => { + val cycle = os.read.lines(path).head match + case perfCycleRegex(cycle) => cycle.toInt + case _ => + throw new Exception("perf.txt file is not format as expected") + val caseName = path.segments.toSeq.reverse.drop(1).head + (caseName, cycle, cycleRecord.obj(caseName).num.toInt) + }) + .filter((_, newCycle, oldCycle) => newCycle != oldCycle) + .map: + case (caseName, newCycle, oldCycle) => + cycleRecord(caseName) = newCycle + if oldCycle == -1 then s"* 🆕 ${caseName}: NaN -> ${newCycle}" + else if oldCycle > newCycle then + s"* 🚀 $caseName: $oldCycle -> $newCycle" + else s"* 🐢 $caseName: $oldCycle -> $newCycle" + + os.write.append(cycleUpdateRecordFile, allCycleUpdates.mkString("\n")) + + os.write.over(file, ujson.write(cycleRecord, indent = 2)) + end postCI + + @main + def generateTestPlan() = + val allCases = + os.walk(os.pwd / ".github" / "cases").filter(_.last == "default.json") + val testPlans = allCases.map: caseFilePath => + caseFilePath.segments.dropWhile(_ != "cases").drop(1).next + + println(ujson.write(Map("config" -> testPlans))) + end generateTestPlan + + @main + def generateRegressionTestPlan(runnersAmount: Int): Unit = + // Find emulator configs + val emulatorConfigs: Seq[String] = + os.walk(os.pwd / ".github" / "cases") + .filter: path => + path.last == "default.json" + .map: path => + // We have a list of pwd/.github/cases//default.json string, + // but all we need is the name. + path.segments.toSeq.reverse.drop(1).head + + import scala.util.chaining._ + val testPlans: Seq[String] = emulatorConfigs.flatMap: configName => + val allCasesPath = nixResolvePath(s".#t1.$configName.cases.all") + os.walk(os.Path(allCasesPath) / "configs") + .filter: path => + path.ext == "json" + .map: path => + // configs/ directory have a list of .json files, we need those + val testName = path.segments.toSeq.last.stripSuffix(".json") + s"$configName,$testName" + + def getTestPlan(filePat: String): Seq[String] = + os.walk(os.pwd / ".github" / "cases") + .filter: path => + path.last == filePat + .flatMap: path => + val config = path.segments.toSeq.reverse.drop(1).head + os.read(path) + .pipe(raw => ujson.read(raw)) + .pipe(json => json.obj.keys.map(testName => s"$config,$testName")) + + val currentTestPlan = getTestPlan("default.json") + val perfCases = getTestPlan("perf.json") + + // We don't have much information for this tests, so randomly split them into same size buckets + // Merge Seq( "A", "B", "C", "D" ) into Seq( "A;B", "C;D" ) + def buckets(alltests: Seq[String], bucketSize: Int): Seq[String] = + scala.util.Random + .shuffle(alltests) + .grouped( + math.ceil(alltests.size.toDouble / bucketSize).toInt + ) + .toSeq + .map(_.mkString(";")) + + val finalTestPlan = + (testPlans.toSet -- currentTestPlan.toSet -- perfCases.toSet).toSeq + buckets(finalTestPlan, runnersAmount) + .pipe(toMatrixJson) + .pipe(println) + end generateRegressionTestPlan + @main def difftest( @arg(