diff --git a/.gitignore b/.gitignore index bf1e4bf2a..8b00e3dc1 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,5 @@ src/validations/test/rules/rev5/poam-result.html src/validations/test/rules/rev5/sar-result.html /reports /sarif +@rerun.txt +NUL diff --git a/cucumber.json b/cucumber.json index df6655347..9122b9c93 100644 --- a/cucumber.json +++ b/cucumber.json @@ -1,12 +1,29 @@ { "default": { - "requireModule": ["ts-node/register"], - "import": ["features/**/*.ts"], + "import": [ + "import { register } from 'ts-node'", + "register({ esm: true, experimentalSpecifierResolution: 'node' })", + "features/**/*.ts" + ], "format": [ ["junit", "reports/junit-constraints.xml"], - ["html", "reports/constraints.html"] + ["html", "reports/constraints.html"], + ["rerun","@rerun.txt"] ], "retry": 2, "retryTagFilter": "@flaky" + }, + "rerun": { + "import": [ + "import { register } from 'ts-node'", + "register({ esm: true, experimentalSpecifierResolution: 'node' })", + "features/**/*.ts" + ], + "format": [ + ["junit", "reports/junit-constraints-rerun.xml"], + ["html", "reports/constraints-rerun.html"] + ], + "retry": 0, + "paths": ["@rerun.txt"] } -} +} \ No newline at end of file diff --git a/features/fedramp_extensions.feature b/features/fedramp_extensions.feature index a729209f9..d4e58de82 100644 --- a/features/fedramp_extensions.feature +++ b/features/fedramp_extensions.feature @@ -95,6 +95,8 @@ Examples: | interconnection-security-PASS.yaml | | missing-response-components-FAIL.yaml | | missing-response-components-PASS.yaml | + | missing-response-components-test-FAIL.yaml | + | missing-response-components-test-PASS.yaml | | privilege-level-FAIL.yaml | | privilege-level-PASS.yaml | | resource-has-base64-or-rlink-FAIL.yaml | @@ -167,7 +169,7 @@ Examples: | information-type-system | | interconnection-direction | | interconnection-security | - | missing-response-components | + | missing-response-components-test | | privilege-level | | prop-response-point-has-cardinality-one | | resource-has-base64-or-rlink | diff --git a/features/steps/fedramp_extensions_steps.ts b/features/steps/fedramp_extensions_steps.ts index 0d58130e7..b2ebc683e 100644 --- a/features/steps/fedramp_extensions_steps.ts +++ b/features/steps/fedramp_extensions_steps.ts @@ -27,6 +27,7 @@ let currentTestCase: { pipelines: []; expectations: [{ "constraint-id": string; result: string }]; }; +let currentTestCaseFileName:string; let processedContentPath: string; let ignoreDocument: string = "oscal-external-constraints.xml"; let metaschemaDocuments: string[] = []; @@ -157,13 +158,20 @@ When("I process the constraint unit test {string}", async function (testFile) { "unit-tests" ); const filePath = join(constraintTestDir, testFile); + currentTestCaseFileName = testFile; const fileContents = readFileSync(filePath, "utf8"); currentTestCase = load(fileContents) as any; }); Then("the constraint unit test should pass", async function () { const result = await processTestCase(currentTestCase); - expect(result.status).to.equal("pass", result.errorMessage); + const testType = currentTestCaseFileName.includes("FAIL") ? "Negative" : "Positive"; + + const errorMessage = result.errorMessage + ? `${testType} test failed: ${result.errorMessage}` + : `${testType} test failed without a specific error message`; + + expect(result.status).to.equal("pass", errorMessage); }); async function processTestCase({ "test-case": testCase }: any) { @@ -217,9 +225,13 @@ async function processTestCase({ "test-case": testCase }: any) { console.log("Using cached validation result from "+cacheKey); sarifResponse = validationCache.get(cacheKey)!; }else{ + let args = []; + if(currentTestCaseFileName.includes("FAIL")){ + args.push("--disable-schema-validation") + } sarifResponse = await validateWithSarif([ processedContentPath, - "--sarif-include-pass", + ...args, ...metaschemaDocuments.flatMap((x) => ["-c", x]), ]); validationCache.set(cacheKey,sarifResponse); diff --git a/package-lock.json b/package-lock.json index bf9c0e920..f16784200 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "jsdom": "^25.0.0", "oscal": "^1.4.7", "ts-node": "^10.9.2", + "xml-formatter": "^3.6.3", "xml2js": "^0.6.2" }, "devDependencies": { @@ -3774,6 +3775,18 @@ } } }, + "node_modules/xml-formatter": { + "version": "3.6.3", + "resolved": "https://registry.npmjs.org/xml-formatter/-/xml-formatter-3.6.3.tgz", + "integrity": "sha512-++x1TlRO1FRlQ82AZ4WnoCSufaI/PT/sycn4K8nRl4gnrNC1uYY2VV/67aALZ2m0Q4Q/BLj/L69K360Itw9NNg==", + "license": "MIT", + "dependencies": { + "xml-parser-xo": "^4.1.2" + }, + "engines": { + "node": ">= 16" + } + }, "node_modules/xml-name-validator": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-5.0.0.tgz", @@ -3783,6 +3796,15 @@ "node": ">=18" } }, + "node_modules/xml-parser-xo": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/xml-parser-xo/-/xml-parser-xo-4.1.2.tgz", + "integrity": "sha512-Z/DRB0ZAKj5vAQg++XsfQQKfT73Vfj5n5lKIVXobBDQEva6NHWUTxOA6OohJmEcpoy8AEqBmSGkXXAnFwt5qAA==", + "license": "MIT", + "engines": { + "node": ">= 16" + } + }, "node_modules/xml2js": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz", diff --git a/package.json b/package.json index cae542c27..971d727ed 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,8 @@ "build:validation-ui": "cd src/web && npm install && npm run build && cd ../..", "federalist": "make init-repo && npm run build:validation-ui && npm run link:validation-ui", "link:validation-ui": "ln -sf ./src/web/dist _site", - "test": "cross-env NODE_OPTIONS=\"--loader ts-node/esm --no-warnings --experimental-specifier-resolution=node\" cucumber-js", + "test": "cross-env-shell NODE_OPTIONS=\"--loader ts-node/esm --no-warnings --experimental-specifier-resolution=node\" cucumber-js 2>/dev/null 2>NUL", + "test:failed": "cross-env NODE_OPTIONS=\"--loader ts-node/esm --no-warnings --experimental-specifier-resolution=node\" cucumber-js -p rerun", "test:constraints": "cross-env NODE_OPTIONS=\"--loader ts-node/esm --no-warnings --experimental-specifier-resolution=node\" cucumber-js --tags @constraints", "test:coverage": "cross-env NODE_OPTIONS=\"--loader ts-node/esm --no-warnings --experimental-specifier-resolution=node\" cucumber-js --tags @full-coverage", "mq": "node ./src/scripts/dev-metaschema-eval.js", @@ -22,6 +23,7 @@ "jsdom": "^25.0.0", "oscal": "^1.4.7", "ts-node": "^10.9.2", + "xml-formatter": "^3.6.3", "xml2js": "^0.6.2" }, "devDependencies": { diff --git a/src/scripts/dev-constraint.js b/src/scripts/dev-constraint.js index c35749678..f665681b9 100644 --- a/src/scripts/dev-constraint.js +++ b/src/scripts/dev-constraint.js @@ -2,8 +2,10 @@ import fs from 'fs'; import path from 'path'; import xml2js from 'xml2js'; import yaml from 'js-yaml'; +import {JSDOM} from "jsdom" import { execSync } from 'child_process'; import inquirer from 'inquirer'; +import xmlFormatter from 'xml-formatter'; const prompt = inquirer.createPromptModule(); @@ -59,18 +61,41 @@ function extractConstraints(xmlObject) { async function getAllConstraints() { const files = fs.readdirSync(constraintsDir).filter(file => file.endsWith('.xml') && file !== ignoreDocument); let allConstraints = []; + let allContext = {}; for (const file of files) { const filePath = path.join(constraintsDir, file); - const xmlObject = await parseXml(filePath); - const constraints = extractConstraints(xmlObject); - allConstraints = [...allConstraints, ...constraints]; + const xmlContent = fs.readFileSync(filePath, 'utf8'); + const dom = new JSDOM(xmlContent, { contentType: "text/xml" }); + const document = dom.window.document; + + // Select all elements with an 'id' attribute + const constraintElements = document.querySelectorAll('[id]'); + + constraintElements.forEach(constraintElement => { + const id = constraintElement.getAttribute('id'); + + // Find the parent 'context' element + let contextElement = constraintElement.closest('context'); + + if (contextElement) { + // Find the 'metapath' element within the context + const metapathElement = contextElement.querySelector('metapath'); + const context = metapathElement ? metapathElement.getAttribute('target') : ''; + + allConstraints.push(id); + allContext[id] = context; + + console.log(`Constraint ${id} context: ${context}`); // Debug log + } else { + console.log(`Warning: No context found for constraint ${id}`); + } + }); } - return [...new Set(allConstraints)].sort(); + return { constraints: [...new Set(allConstraints)].sort(), allContext }; } - function analyzeTestFiles() { const testFiles = fs.readdirSync(testDir).filter(file => file.endsWith('.yaml') || file.endsWith('.yml')); const testResults = {}; @@ -103,7 +128,9 @@ function analyzeTestFiles() { return testResults; } -async function scaffoldTest(constraintId) { + + +async function scaffoldTest(constraintId,context) { const { confirm } = await prompt([ { type: 'confirm', @@ -112,26 +139,151 @@ async function scaffoldTest(constraintId) { default: true } ]); + + if (!confirm) { + console.log(`Skipping test scaffolding for ${constraintId}`); + return; + } + const { model } = await prompt([ { type: 'string', name: 'model', - message: `what is the constraint targeting?`, + message: `What is the constraint targeting?`, default: "ssp" } ]); + console.log(`Context for ${constraintId}:\n${context}`); - if (!confirm) { - console.log(`Skipping test scaffolding for ${constraintId}`); - return; + const { useTemplate } = await prompt([ + { + type: 'list', + name: 'useTemplate', + message: `Choose the content for the negative test:`, + choices: [ + { name: `Create new ${constraintId}-INVALID.xml`, value: 'new' }, + { name: 'Select an existing content file to copy', value: 'select' } + ] + } + ]); + + let invalidContent; + if (useTemplate === 'new') { + const templatePath = path.join(__dirname, '..', '..', 'src', 'validations', 'constraints', 'content', `${model}-all-VALID.xml`); + const newInvalidPath = path.join(__dirname, '..', '..', 'src', 'validations', 'constraints', 'content', `${model}-${constraintId}-INVALID.xml`); + + try { + // Read the template XML + const templateXml = fs.readFileSync(templatePath, 'utf8'); + const dom = new JSDOM(templateXml, { contentType: "text/xml" }); + const document = dom.window.document; + + console.log(`Context for ${constraintId}: ${context}`); // Debug log + + if (!context || typeof context !== 'string' || context.trim() === '') { + throw new Error('Invalid or empty context'); + } + + // Prepare the XPath + const contextParts = context.split('/').filter(part => part !== ''); + let xpathExpression = '//' + contextParts[contextParts.length - 1]; + + console.log(`Attempting to evaluate XPath: ${xpathExpression}`); + + // Use XPath to select the nodes specified by the context + const xpathResult = document.evaluate( + xpathExpression, + document, + null, + dom.window.XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, + null + ); + + if (xpathResult.snapshotLength > 0) { + // Create a new document + const newDoc = document.implementation.createDocument(null, null, null); + + // Function to recursively clone nodes and their ancestors while preserving namespaces + function cloneWithAncestors(node, newParent) { + if (node.parentNode && node.parentNode.nodeType === dom.window.Node.ELEMENT_NODE) { + // Clone the parent node, ensuring we carry over the namespace + const parentClone = newDoc.createElementNS( + node.parentNode.namespaceURI, + node.parentNode.nodeName + ); + + // Clone the attributes (except schema declaration) + Array.from(node.parentNode.attributes).forEach(attr => { + if (!attr.name.includes('schemaLocation')) { + parentClone.setAttributeNS(attr.namespaceURI, attr.name, attr.value); + } + }); + + // Recursively clone its ancestors + cloneWithAncestors(node.parentNode, parentClone); + parentClone.appendChild(newParent); + } else { + newDoc.appendChild(newParent); + } + } + + // Clone only the first matching node and its ancestors + const relevantNode = xpathResult.snapshotItem(0); + const relevantClone = newDoc.importNode(relevantNode, true); + cloneWithAncestors(relevantNode, relevantClone); + + // Serialize the new document + const serializer = new dom.window.XMLSerializer(); + let filteredXml = serializer.serializeToString(newDoc); + + // Format the XML with indentation + filteredXml = xmlFormatter(filteredXml, { + indentation: ' ', // Two spaces for indentation + collapseContent: true, + lineSeparator: '\n' + }); + // Write the new invalid XML file + fs.writeFileSync(newInvalidPath, filteredXml, 'utf8'); + console.log(`Created new ${model}-${constraintId}-INVALID.xml file`); + invalidContent = `../content/${model}-${constraintId}-INVALID.xml`; + } else { + throw new Error('Could not find the specified context in the template.'); + } + } catch (error) { + console.log(`Warning: ${error.message}. Using the full template.`); + console.log(`Error details:`, error); + fs.copyFileSync(templatePath, newInvalidPath); + invalidContent = `../content/${model}-${constraintId}-INVALID.xml`; + } + } else { + const contentDir = path.join(__dirname, '..', '..', 'src', 'validations', 'constraints', 'content'); + const contentFiles = fs.readdirSync(contentDir).filter(file => file.endsWith('.xml')); + const { selectedContent } = await prompt([ + { + type: 'list', + name: 'selectedContent', + message: 'Select an existing content file to copy:', + choices: contentFiles + } + ]); + + // Create a new invalid XML file based on the selected content + const selectedContentPath = path.join(contentDir, selectedContent); + const newInvalidPath = path.join(contentDir, `${model}-${constraintId}-INVALID.xml`); + + // Copy the selected content to the new file + fs.copyFileSync(selectedContentPath, newInvalidPath); + console.log(`Created new ${model}-${constraintId}-INVALID.xml file based on ${selectedContent}`); + + invalidContent = `../content/${model}-${constraintId}-INVALID.xml`; } const positivetestCase = { 'test-case': { name: `Positive Test for ${constraintId}`, description: `This test case validates the behavior of constraint ${constraintId}`, - content:"../content/"+ model+'-all-VALID.xml', + content: `../content/${model}-all-VALID.xml`, expectations: [ { 'constraint-id': constraintId, @@ -144,7 +296,7 @@ async function scaffoldTest(constraintId) { 'test-case': { name: `Negative Test for ${constraintId}`, description: `This test case validates the behavior of constraint ${constraintId}`, - content:"../content/"+ model+"-all-INVALID.xml", + content: invalidContent, expectations: [ { 'constraint-id': constraintId, @@ -159,11 +311,13 @@ async function scaffoldTest(constraintId) { const fileNamePASS = `${constraintId.toLowerCase()}-PASS.yaml`; const fileNameFAIL = `${constraintId.toLowerCase()}-FAIL.yaml`; const positiveFilePath = path.join(testDir, fileNamePASS); - const negativefilePath = path.join(testDir,fileNameFAIL) + const negativefilePath = path.join(testDir, fileNameFAIL); fs.writeFileSync(positiveFilePath, positiveYamlContent, 'utf8'); fs.writeFileSync(negativefilePath, negativeYamlContent, 'utf8'); console.log(`Scaffolded test for ${constraintId} at ${positiveFilePath}`); console.log(`Scaffolded test for ${constraintId} at ${negativefilePath}`); + + return true; } async function selectConstraints(allConstraints) { @@ -262,9 +416,8 @@ async function runCucumberTest(constraintId, testFiles) { async function main() { - const allConstraints = await getAllConstraints(); + const {constraints:allConstraints,allContext} = await getAllConstraints(); console.log(`Found ${allConstraints.length} constraints.`); - const selectedConstraints = await selectConstraints(allConstraints); console.log(`Selected ${selectedConstraints.length} constraints for analysis.`); @@ -276,7 +429,9 @@ async function main() { if (!testCoverage) { console.log(`${constraintId}: No tests found`); - const scaffold = await scaffoldTest(constraintId); + var context = allContext[constraintId] + console.log(`${context}: constraint context`); + const scaffold = await scaffoldTest(constraintId,context); if (scaffold) { const passed = await runCucumberTest(constraintId, { pass_file: `${constraintId}-PASS.yaml`, fail_file: `${constraintId}-FAIL.yaml` }); console.log(`${constraintId}: Test ${passed ? 'passed' : 'failed'}`); diff --git a/src/validations/CONTRIBUTING.md b/src/validations/CONTRIBUTING.md index 6c1c35863..a59bb6548 100644 --- a/src/validations/CONTRIBUTING.md +++ b/src/validations/CONTRIBUTING.md @@ -109,6 +109,19 @@ docker-compose run \ /root/vendor/xspec/bin/xspec.sh -s -j ./test/test_all.xspec ``` + +### Rerunning Failed Tests + +After running the full test suite, you may want to rerun only the failed tests for quicker debugging. To do this, you can use the following npm script: + +```sh +npm run test:failed +``` + +This command will rerun only the tests that failed in the previous execution. It's particularly useful when you're fixing issues and want to verify that your changes have resolved the failures without running the entire test suite again. + +Note: Make sure you've run the full test suite at least once before using this command, as it relies on the `@rerun.txt` file generated during the initial run. + ## Adding tests to the harness To add new tests, add an import to the `./test/test_all.xpec` file. For example: