docs: Fail if more scenarios fail (#106)

Added new steps into the test.yml workflow.
1. upload results as an artifact
2. downloads most recent push to master's test results and compares them.

.github/workflows/test.yml

@@ -28,9 +28,26 @@ jobs:
           npm install
           npm install --global
 
-      - name: test-generators
+      - name: Test generators
         run: npm run test:generators
         continue-on-error: true
 
+      - name: Upload test results
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-results
+          path: test-results.json
 
-
+      - name: Download previous test results
+        id: download-artifact
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          workflow: test.yml
+          branch: master
+          name: test-results
+          path: CICD_${{github.event.pull_request.head.sha}}
+          workflow_conclusion: completed
+
+
+      - name: Compare test results
+        run: node CICD/compareTestResults test-results.json CICD_${{github.event.pull_request.head.sha}}/test-results.json 

CICD/compareTestResults.js

@@ -0,0 +1,49 @@
+const fs = require("fs");
+
+const log = require("../src/log.js");
+
+log.setLogLevel(log.logLevels.verbose);
+
+// Extract cl arguments
+const clArgs = process.argv.slice(2);
+
+if (clArgs.length !== 2) {
+  log.error("Incorrect number of arguments");
+  process.exit(1);
+}
+
+const resultType = ["New", "Previous"];
+let results = [];
+
+for (let ii = 0; ii < 2; ii++) {
+  try {
+    results[ii] = JSON.parse(fs.readFileSync(clArgs[ii], "utf-8"));
+    log.verbose(
+      `${log.underline}${resultType[ii]} test results${log.resetStyling}`
+    );
+    log.verbose(results[ii]);
+  } catch (ee) {
+    log.error(clArgs[ii] + " : " + ee.message);
+    process.exit(1);
+  }
+}
+
+Object.entries(results[1]).forEach(([language, oldResult]) => {
+  let newResult;
+  if ((newResult = results[0][language]) != undefined) {
+    let limit = newResult.scenarios > oldResult.scenarios;
+    if (limit < 0) limit = 0;
+    if (newResult.failed - oldResult.failed > limit) {
+      log.error(
+        `${language} ${log.redBackground}${log.blackForeground} FAILED ${log.resetStyling}`
+      );
+      log.verbose("There are more newly failing tests than added test.");
+      log.verbose(
+        "This is an indication that an existing test is now failing."
+      );
+      process.exit(1);
+    }
+  }
+});
+
+process.exit(0);

README.md

@@ -88,7 +88,7 @@ The CLI tool provided by this repository is the primary interface for the Forge:
 
 ```
 % openapi-forge help forge
-Usage: openapi-generator forge [options] <schema> <generator>
+Usage: openapi-forge forge [options] <schema> <generator>
 
 Forge the API client from an OpenAPI specification. This command takes an
 OpenAPI schema, and uses the given generator to create a client library.
@@ -132,13 +132,10 @@ Usage: openapi-forge test-generators [options]
 Test language specific generators.
 
 Options:
-  -g, --generators <gens>            Narrow down the generators to test. Each letter is a generator, combine letters to test multiple generators, options are:
-                                     c (CSharp), t (TypeScript) (default: "ct")
-  -c, --csharp <csharpPath>          Sets the location of the CSharp generator. Default is a directory called 'openapi-forge-csharp' in the same location as
-                                     openapi-forge (default: "../../openapi-forge-csharp")
-  -t, --typescript <typescriptPath>  Sets the location of the TypeScript generator. Default is a directory called 'openapi-forge-typescript' in the same
-                                     location as openapi-forge (default: "../../openapi-forge-typescript")
-  -l, --logLevel <level>             Sets the logging level, options are: quiet ('quiet', 'q' or '0'), standard (default) ('standard', 's' or '1'), verbose
-                                     ('verbose', 'v' or '2') (default: "1")
+  -g, --generators <gens>            Narrow down the generators to test. Each letter is a generator, combine letters to test multiple generators, options are: c (CSharp), t (TypeScript) (default: "ct")
+  -c, --csharp <csharpPath>          Sets the location of the CSharp generator. Default is a directory named 'openapi-forge-csharp' in the same location as openapi-forge (default: "./openapi-forge-csharp")
+  -t, --typescript <typescriptPath>  Sets the location of the TypeScript generator. Default is a directory named 'openapi-forge-typescript' in the same location as openapi-forge (default: "./openapi-forge-typescript")
+  -l, --logLevel <level>             Sets the logging level, options are: quiet ('quiet', 'q' or '0'), standard (default) ('standard', 's' or '1'), verbose ('verbose', 'v' or '2') (default: "1")
+  -o, --outputFile [file]            Writes the testing results to a JSON file, defaults to "test-results.json"
   -h, --help                         display help for command
 ```

package.json

@@ -4,7 +4,7 @@
   "description": "Effortlessly create OpenAPI clients from the fiery furnace of our forge",
   "main": "src/generate.js",
   "scripts": {
-    "test:generators": "openapi-forge test-generators -l v",
+    "test:generators": "openapi-forge test-generators -l v -o",
     "format:check": "prettier --check .",
     "format:write": "prettier --write .",
     "lint:check": "eslint .",

src/index.js

@@ -5,7 +5,7 @@ const generate = require("./generate");
 const testGenerators = require("./testGenerators");
 const program = new Command();
 
-program.name("openapi-generator");
+program.name("openapi-forge");
 
 program
   .command("forge")
@@ -60,8 +60,12 @@ program
     "Sets the logging level, options are: quiet ('quiet', 'q' or '0'), standard (default) ('standard', 's' or '1'), verbose ('verbose', 'v' or '2')",
     "1"
   )
+  .option(
+    "-o, --outputFile [file]",
+    `Writes the testing results to a JSON file, defaults to "${testGenerators.defaultResultFile}"`
+  )
   .action(async (options) => {
-    testGenerators(options);
+    testGenerators.testGenerators(options);
   });
 
 program.parse();

src/log.js

@@ -24,13 +24,13 @@ function getLogLevel() {
 }
 
 function setLogLevel(level) {
-  if (level === "0" || level === "q" || level === "quiet") {
+  if ([logLevels.quiet, "0", "q", "quiet"].includes(level)) {
     logLevel = logLevels.quiet;
     setSilentShell();
-  } else if (level === "1" || level === "s" || level === "standard") {
+  } else if ([logLevels.standard, "1", "s", "standard"].includes(level)) {
     logLevel = logLevels.standard;
     setSilentShell();
-  } else if (level === "2" || level === "v" || level === "verbose") {
+  } else if ([logLevels.verbose, "2", "v", "verbose"].includes(level)) {
     logLevel = logLevels.verbose;
   }
   return;
@@ -157,7 +157,6 @@ module.exports = {
   underline,
   resetStyling,
   divider,
-  logLevel,
   logLevels,
   shellOptions,
   getLogLevel,

src/testGenerators.js

@@ -7,6 +7,8 @@ const shell = require("shelljs");
 const log = require("./log");
 const generatorResolver = require("./generatorResolver");
 
+const defaultResultFile = "test-results.json";
+
 const typescriptData = {
   languageString: "TypeScript",
   languageLetter: "t",
@@ -65,22 +67,21 @@ function checkTestResultForErrors(result) {
   if (result.failed !== 0) {
     return 1;
   }
-  if (result.undefined !== 0) {
+  if (result.undef !== 0) {
     return 1;
   }
   if (result.skipped !== 0) {
     return 1;
   }
-  if (result.passed !== result.scenarios) {
-    return 1;
-  }
   return 0;
 }
 
 async function testGenerators(options) {
   let resultArray = {};
   let exitCode = 0;
 
+  const curDir = process.cwd();
+
   log.setLogLevel(options.logLevel);
 
   const typescript = options.generators.includes(typescriptData.languageLetter);
@@ -168,7 +169,20 @@ async function testGenerators(options) {
   if (Object.keys(resultArray).length) {
     if (!log.isQuiet()) console.table(resultArray);
   }
+
+  if (options.outputFile) {
+    if (typeof options.outputFile === "boolean")
+      options.outputFile = defaultResultFile;
+    fs.writeFileSync(
+      path.join(curDir, options.outputFile),
+      JSON.stringify(resultArray)
+    );
+  }
+
   process.exit(exitCode);
 }
 
-module.exports = testGenerators;
+module.exports = {
+  defaultResultFile,
+  testGenerators,
+};