From 776b77c9f1b4595d3660041a4bc4853a4c08b8e8 Mon Sep 17 00:00:00 2001 From: Jake Howlett <107993905+jhowlett-scottlogic@users.noreply.github.com> Date: Thu, 20 Oct 2022 13:33:54 +0100 Subject: [PATCH] docs: Fail if more scenarios fail (#106) Added new steps into the test.yml workflow. 1. upload results as an artifact 2. downloads most recent push to master's test results and compares them. --- .github/workflows/test.yml | 21 ++++++++++++++-- CICD/compareTestResults.js | 49 ++++++++++++++++++++++++++++++++++++++ README.md | 15 +++++------- package.json | 2 +- src/index.js | 8 +++++-- src/log.js | 7 +++--- src/testGenerators.js | 24 +++++++++++++++---- 7 files changed, 103 insertions(+), 23 deletions(-) create mode 100644 CICD/compareTestResults.js mode change 100755 => 100644 src/index.js diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 372d85a..12356f0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,9 +28,26 @@ jobs: npm install npm install --global - - name: test-generators + - name: Test generators run: npm run test:generators continue-on-error: true + - name: Upload test results + uses: actions/upload-artifact@v3 + with: + name: test-results + path: test-results.json - + - name: Download previous test results + id: download-artifact + uses: dawidd6/action-download-artifact@v2 + with: + workflow: test.yml + branch: master + name: test-results + path: CICD_${{github.event.pull_request.head.sha}} + workflow_conclusion: completed + + + - name: Compare test results + run: node CICD/compareTestResults test-results.json CICD_${{github.event.pull_request.head.sha}}/test-results.json diff --git a/CICD/compareTestResults.js b/CICD/compareTestResults.js new file mode 100644 index 0000000..a65ccc1 --- /dev/null +++ b/CICD/compareTestResults.js @@ -0,0 +1,49 @@ +const fs = require("fs"); + +const log = require("../src/log.js"); + +log.setLogLevel(log.logLevels.verbose); + +// Extract cl arguments +const clArgs = process.argv.slice(2); + +if (clArgs.length !== 2) { + log.error("Incorrect number of arguments"); + process.exit(1); +} + +const resultType = ["New", "Previous"]; +let results = []; + +for (let ii = 0; ii < 2; ii++) { + try { + results[ii] = JSON.parse(fs.readFileSync(clArgs[ii], "utf-8")); + log.verbose( + `${log.underline}${resultType[ii]} test results${log.resetStyling}` + ); + log.verbose(results[ii]); + } catch (ee) { + log.error(clArgs[ii] + " : " + ee.message); + process.exit(1); + } +} + +Object.entries(results[1]).forEach(([language, oldResult]) => { + let newResult; + if ((newResult = results[0][language]) != undefined) { + let limit = newResult.scenarios > oldResult.scenarios; + if (limit < 0) limit = 0; + if (newResult.failed - oldResult.failed > limit) { + log.error( + `${language} ${log.redBackground}${log.blackForeground} FAILED ${log.resetStyling}` + ); + log.verbose("There are more newly failing tests than added test."); + log.verbose( + "This is an indication that an existing test is now failing." + ); + process.exit(1); + } + } +}); + +process.exit(0); diff --git a/README.md b/README.md index a1d55fe..abac559 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ The CLI tool provided by this repository is the primary interface for the Forge: ``` % openapi-forge help forge -Usage: openapi-generator forge [options] +Usage: openapi-forge forge [options] Forge the API client from an OpenAPI specification. This command takes an OpenAPI schema, and uses the given generator to create a client library. @@ -132,13 +132,10 @@ Usage: openapi-forge test-generators [options] Test language specific generators. Options: - -g, --generators Narrow down the generators to test. Each letter is a generator, combine letters to test multiple generators, options are: - c (CSharp), t (TypeScript) (default: "ct") - -c, --csharp Sets the location of the CSharp generator. Default is a directory called 'openapi-forge-csharp' in the same location as - openapi-forge (default: "../../openapi-forge-csharp") - -t, --typescript Sets the location of the TypeScript generator. Default is a directory called 'openapi-forge-typescript' in the same - location as openapi-forge (default: "../../openapi-forge-typescript") - -l, --logLevel Sets the logging level, options are: quiet ('quiet', 'q' or '0'), standard (default) ('standard', 's' or '1'), verbose - ('verbose', 'v' or '2') (default: "1") + -g, --generators Narrow down the generators to test. Each letter is a generator, combine letters to test multiple generators, options are: c (CSharp), t (TypeScript) (default: "ct") + -c, --csharp Sets the location of the CSharp generator. Default is a directory named 'openapi-forge-csharp' in the same location as openapi-forge (default: "./openapi-forge-csharp") + -t, --typescript Sets the location of the TypeScript generator. Default is a directory named 'openapi-forge-typescript' in the same location as openapi-forge (default: "./openapi-forge-typescript") + -l, --logLevel Sets the logging level, options are: quiet ('quiet', 'q' or '0'), standard (default) ('standard', 's' or '1'), verbose ('verbose', 'v' or '2') (default: "1") + -o, --outputFile [file] Writes the testing results to a JSON file, defaults to "test-results.json" -h, --help display help for command ``` diff --git a/package.json b/package.json index 99690b5..380f6f0 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "Effortlessly create OpenAPI clients from the fiery furnace of our forge", "main": "src/generate.js", "scripts": { - "test:generators": "openapi-forge test-generators -l v", + "test:generators": "openapi-forge test-generators -l v -o", "format:check": "prettier --check .", "format:write": "prettier --write .", "lint:check": "eslint .", diff --git a/src/index.js b/src/index.js old mode 100755 new mode 100644 index b9b99d4..db3ce99 --- a/src/index.js +++ b/src/index.js @@ -5,7 +5,7 @@ const generate = require("./generate"); const testGenerators = require("./testGenerators"); const program = new Command(); -program.name("openapi-generator"); +program.name("openapi-forge"); program .command("forge") @@ -60,8 +60,12 @@ program "Sets the logging level, options are: quiet ('quiet', 'q' or '0'), standard (default) ('standard', 's' or '1'), verbose ('verbose', 'v' or '2')", "1" ) + .option( + "-o, --outputFile [file]", + `Writes the testing results to a JSON file, defaults to "${testGenerators.defaultResultFile}"` + ) .action(async (options) => { - testGenerators(options); + testGenerators.testGenerators(options); }); program.parse(); diff --git a/src/log.js b/src/log.js index 1ab8ceb..c3e9e28 100644 --- a/src/log.js +++ b/src/log.js @@ -24,13 +24,13 @@ function getLogLevel() { } function setLogLevel(level) { - if (level === "0" || level === "q" || level === "quiet") { + if ([logLevels.quiet, "0", "q", "quiet"].includes(level)) { logLevel = logLevels.quiet; setSilentShell(); - } else if (level === "1" || level === "s" || level === "standard") { + } else if ([logLevels.standard, "1", "s", "standard"].includes(level)) { logLevel = logLevels.standard; setSilentShell(); - } else if (level === "2" || level === "v" || level === "verbose") { + } else if ([logLevels.verbose, "2", "v", "verbose"].includes(level)) { logLevel = logLevels.verbose; } return; @@ -157,7 +157,6 @@ module.exports = { underline, resetStyling, divider, - logLevel, logLevels, shellOptions, getLogLevel, diff --git a/src/testGenerators.js b/src/testGenerators.js index 057c8f1..8293593 100644 --- a/src/testGenerators.js +++ b/src/testGenerators.js @@ -7,6 +7,8 @@ const shell = require("shelljs"); const log = require("./log"); const generatorResolver = require("./generatorResolver"); +const defaultResultFile = "test-results.json"; + const typescriptData = { languageString: "TypeScript", languageLetter: "t", @@ -65,15 +67,12 @@ function checkTestResultForErrors(result) { if (result.failed !== 0) { return 1; } - if (result.undefined !== 0) { + if (result.undef !== 0) { return 1; } if (result.skipped !== 0) { return 1; } - if (result.passed !== result.scenarios) { - return 1; - } return 0; } @@ -81,6 +80,8 @@ async function testGenerators(options) { let resultArray = {}; let exitCode = 0; + const curDir = process.cwd(); + log.setLogLevel(options.logLevel); const typescript = options.generators.includes(typescriptData.languageLetter); @@ -168,7 +169,20 @@ async function testGenerators(options) { if (Object.keys(resultArray).length) { if (!log.isQuiet()) console.table(resultArray); } + + if (options.outputFile) { + if (typeof options.outputFile === "boolean") + options.outputFile = defaultResultFile; + fs.writeFileSync( + path.join(curDir, options.outputFile), + JSON.stringify(resultArray) + ); + } + process.exit(exitCode); } -module.exports = testGenerators; +module.exports = { + defaultResultFile, + testGenerators, +};