Improve dev constraint script (#690)

* add option to specify test content be made from template or use all invalid

* improve scaffolding for negative test cases

* introduce re-run script

* update contributing docs for rerun info

* improve scaffolding

* adjust constraint script to match feedback

* Update dev-constraint.js

.gitignore

@@ -42,3 +42,5 @@ src/validations/test/rules/rev5/sar-result.html
 
 /reports
 /sarif
+@rerun.txt
+NUL

cucumber.json

@@ -1,12 +1,29 @@
 {
   "default": {
-    "requireModule": ["ts-node/register"],
-    "import": ["features/**/*.ts"],
+    "import": [
+      "import { register } from 'ts-node'",
+      "register({ esm: true, experimentalSpecifierResolution: 'node' })",
+      "features/**/*.ts"
+    ],
     "format": [
       ["junit", "reports/junit-constraints.xml"],
-      ["html", "reports/constraints.html"]
+      ["html", "reports/constraints.html"],
+      ["rerun","@rerun.txt"]
     ],
     "retry": 2,
     "retryTagFilter": "@flaky"
+  },
+  "rerun": {
+    "import": [
+      "import { register } from 'ts-node'",
+      "register({ esm: true, experimentalSpecifierResolution: 'node' })",
+      "features/**/*.ts"
+    ],
+    "format": [
+      ["junit", "reports/junit-constraints-rerun.xml"],
+      ["html", "reports/constraints-rerun.html"]
+    ],
+    "retry": 0,
+    "paths": ["@rerun.txt"]
   }
-}
+}

features/fedramp_extensions.feature

@@ -95,6 +95,8 @@ Examples:
   | interconnection-security-PASS.yaml |
   | missing-response-components-FAIL.yaml |
   | missing-response-components-PASS.yaml |
+  | missing-response-components-test-FAIL.yaml |
+  | missing-response-components-test-PASS.yaml |
   | privilege-level-FAIL.yaml |
   | privilege-level-PASS.yaml |
   | resource-has-base64-or-rlink-FAIL.yaml |
@@ -167,7 +169,7 @@ Examples:
   | information-type-system |
   | interconnection-direction |
   | interconnection-security |
-  | missing-response-components |
+  | missing-response-components-test |
   | privilege-level |
   | prop-response-point-has-cardinality-one |
   | resource-has-base64-or-rlink |

features/steps/fedramp_extensions_steps.ts

@@ -27,6 +27,7 @@ let currentTestCase: {
   pipelines: [];
   expectations: [{ "constraint-id": string; result: string }];
 };
+let currentTestCaseFileName:string;
 let processedContentPath: string;
 let ignoreDocument: string = "oscal-external-constraints.xml";
 let metaschemaDocuments: string[] = [];
@@ -157,13 +158,20 @@ When("I process the constraint unit test {string}", async function (testFile) {
     "unit-tests"
   );
   const filePath = join(constraintTestDir, testFile);
+  currentTestCaseFileName = testFile;
   const fileContents = readFileSync(filePath, "utf8");
   currentTestCase = load(fileContents) as any;
 });
 
 Then("the constraint unit test should pass", async function () {
   const result = await processTestCase(currentTestCase);
-  expect(result.status).to.equal("pass", result.errorMessage);
+  const testType = currentTestCaseFileName.includes("FAIL") ? "Negative" : "Positive";
+
+  const errorMessage = result.errorMessage 
+    ? `${testType} test failed: ${result.errorMessage}`
+    : `${testType} test failed without a specific error message`;
+
+  expect(result.status).to.equal("pass", errorMessage);
 });
 
 async function processTestCase({ "test-case": testCase }: any) {
@@ -217,9 +225,13 @@ async function processTestCase({ "test-case": testCase }: any) {
       console.log("Using cached validation result from "+cacheKey);
       sarifResponse = validationCache.get(cacheKey)!;
     }else{
+      let args = [];
+      if(currentTestCaseFileName.includes("FAIL")){
+        args.push("--disable-schema-validation")
+      }
     sarifResponse = await validateWithSarif([
       processedContentPath,
-      "--sarif-include-pass",
+      ...args,
       ...metaschemaDocuments.flatMap((x) => ["-c", x]),
     ]);
     validationCache.set(cacheKey,sarifResponse);

package.json

@@ -7,7 +7,8 @@
     "build:validation-ui": "cd src/web && npm install && npm run build && cd ../..",
     "federalist": "make init-repo && npm run build:validation-ui && npm run link:validation-ui",
     "link:validation-ui": "ln -sf ./src/web/dist _site",
-    "test": "cross-env NODE_OPTIONS=\"--loader ts-node/esm --no-warnings --experimental-specifier-resolution=node\" cucumber-js",
+    "test": "cross-env-shell NODE_OPTIONS=\"--loader ts-node/esm --no-warnings --experimental-specifier-resolution=node\" cucumber-js 2>/dev/null 2>NUL",
+    "test:failed": "cross-env NODE_OPTIONS=\"--loader ts-node/esm --no-warnings --experimental-specifier-resolution=node\" cucumber-js -p rerun",
     "test:constraints": "cross-env NODE_OPTIONS=\"--loader ts-node/esm --no-warnings --experimental-specifier-resolution=node\" cucumber-js --tags @constraints",
     "test:coverage": "cross-env NODE_OPTIONS=\"--loader ts-node/esm --no-warnings --experimental-specifier-resolution=node\" cucumber-js --tags @full-coverage",
     "mq": "node ./src/scripts/dev-metaschema-eval.js",
@@ -22,6 +23,7 @@
     "jsdom": "^25.0.0",
     "oscal": "^1.4.7",
     "ts-node": "^10.9.2",
+    "xml-formatter": "^3.6.3",
     "xml2js": "^0.6.2"
   },
   "devDependencies": {