diff --git a/.github/workflows/python-push.yml b/.github/workflows/python-push.yml index 9deb0d1e0..8cef045ae 100644 --- a/.github/workflows/python-push.yml +++ b/.github/workflows/python-push.yml @@ -9,6 +9,8 @@ on: branches: - main +permissions: {} + jobs: set-versions: runs-on: ubuntu-latest @@ -110,11 +112,16 @@ jobs: url: https://pypi.org/p/compliance-trestle if: github.ref == 'refs/heads/main' && github.repository == 'oscal-compass/compliance-trestle' steps: + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.PRIVATE_KEY }} - uses: actions/checkout@v4 with: submodules: true fetch-depth: 0 - token: ${{ secrets.ADMIN_PAT }} + token: ${{ steps.app-token.outputs.token }} - name: Set up Python ${{ needs.set-versions.outputs.max }} uses: actions/setup-python@v5 with: @@ -126,9 +133,9 @@ jobs: # This action uses Python Semantic Release v8 - name: Python Semantic Release id: release - uses: python-semantic-release/python-semantic-release@v9.8.0 + uses: python-semantic-release/python-semantic-release@v9.8.8 with: - github_token: ${{ secrets.ADMIN_PAT }} + github_token: ${{ steps.app-token.outputs.token }} - name: Publish package distributions to PyPI uses: pypa/gh-action-pypi-publish@release/v1 @@ -137,10 +144,10 @@ jobs: if: steps.release.outputs.released == 'true' - name: Publish package distributions to GitHub Releases - uses: python-semantic-release/upload-to-gh-release@v9.8.0 + uses: python-semantic-release/upload-to-gh-release@v9.8.9 if: steps.release.outputs.released == 'true' with: - github_token: ${{ secrets.ADMIN_PAT }} + github_token: ${{ steps.app-token.outputs.token }} deploy-docs: runs-on: ubuntu-latest @@ -151,11 +158,16 @@ jobs: # Temporary hack: allow develop as well as master to deploy docs. if: github.ref == 'refs/heads/main' steps: + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.PRIVATE_KEY }} - uses: actions/checkout@v4 with: submodules: true fetch-depth: 0 - token: ${{ secrets.ADMIN_PAT }} + token: ${{ steps.app-token.outputs.token }} - name: Set up Python ${{ needs.set-versions.outputs.max }} uses: actions/setup-python@v5 # This is deliberately not using a custom credential as it relies on native github actions token to have push rights. @@ -178,16 +190,31 @@ jobs: cancel-in-progress: true if: github.ref == 'refs/heads/main' steps: + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.PRIVATE_KEY }} - uses: actions/checkout@v4 with: submodules: true ref: main fetch-depth: 0 - token: ${{ secrets.ADMIN_PAT }} + token: ${{ steps.app-token.outputs.token }} + - name: Get GitHub App User ID + id: get-user-id + run: echo "user-id=$(gh api "/users/${{ env.SLUG }}[bot]" --jq .id)" >> "$GITHUB_OUTPUT" + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + SLUG: ${{ steps.app-token.outputs.app-slug }} - name: Configure Git run: | - git config user.name "Vikas Agarwal" - git config user.email "<>" + git config --global user.name '${{ env.SLUG }}[bot]' + git config --global user.email '${{ env.ID }}+${{ env.SLUG }}[bot]@users.noreply.github.com' + env: + SLUG: ${{ steps.app-token.outputs.app-slug }} + ID: ${{ steps.get-user-id.outputs.user-id }} + # https://docs.github.com/en/actions/security-for-github-actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable - name: Merge Main to Develop run: | git checkout develop diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 3d8ef5178..6eac49c70 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -151,7 +151,7 @@ jobs: - name: Upload artifact if: steps.core-version.outputs.core == 'true' - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: coverage path: coverage.xml @@ -184,7 +184,7 @@ jobs: run: | make develop - name: Get coverage - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: coverage - name: SonarCloud Scan diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 000000000..88c2c299a --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,38 @@ +name: Stale Issues and PRs +on: + schedule: + - cron: '17 1 * * *' + workflow_dispatch: + +permissions: + contents: read + +jobs: + stale: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + env: + STALE_WARNING_DAYS: 90 + STALE_CLOSURE_DAYS: 30 + steps: + - uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9 + with: + stale-issue-label: stale + exempt-issue-labels: good-first-issue, help-wanted, exempt-from-stale + stale-issue-message: > + This issue has been automatically marked as stale because it has not had activity within ${{ env.STALE_WARNING_DAYS }} days. + It will be automatically closed if no further activity occurs within ${{ env.STALE_CLOSURE_DAYS }} days. + close-issue-message: > + This issue has been automatically closed due to inactivity. + days-before-issue-stale: ${{ env.STALE_WARNING_DAYS }} + days-before-issue-close: ${{ env.STALE_CLOSURE_DAYS }} + stale-pr-label: stale + stale-pr-message: > + This pull request has been automatically marked as stale because it has not had activity within ${{ env.STALE_WARNING_DAYS }} days. + It will be automatically closed if no further activity occurs within ${{ env.STALE_CLOSURE_DAYS }} days. + close-pr-message: > + This pull request has been automatically closed due to inactivity. Please reopen if this PR is still being worked on. + days-before-pr-stale: ${{ env.STALE_WARNING_DAYS }} + days-before-pr-close: ${{ env.STALE_CLOSURE_DAYS }} \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9973e7249..326339266 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,7 @@ + repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 # Use the ref you want to point at + rev: v5.0.0 # Use the ref you want to point at hooks: - id: check-merge-conflict - id: check-yaml @@ -13,13 +14,13 @@ repos: - id: yapf args: [--in-place, --parallel, --recursive, --style, .yapf-config] files: "^(trestle|tests|scripts)" - stages: [commit] + stages: [pre-commit] additional_dependencies: [toml] - repo: https://github.com/PyCQA/flake8 - rev: 5.0.4 + rev: 7.1.1 hooks: - id: flake8 - args: [--extend-ignore, "P1,C812,C813,C814,C815,C816,W503,W605,B017,B028", "--illegal-import-packages=filecmp"] + args: [--extend-ignore, "P1,C812,C813,C814,C815,C816,W503,W605,B017,B028"] additional_dependencies: [ flake8-2020, @@ -37,14 +38,13 @@ repos: flake8-quotes, flake8-string-format, flake8-use-fstring, - flake8-illegal-import, pep8-naming, ] files: "^(tests|scripts)" exclude: "(oscal/|third_party)" - stages: [commit] + stages: [pre-commit] - id: flake8 - args: [--extend-ignore, "P1,C812,C813,C814,C815,C816,W503,W605,B017,B028", "--illegal-import-packages=filecmp"] + args: [--extend-ignore, "P1,C812,C813,C814,C815,C816,W503,W605,B017,B028"] additional_dependencies: [ flake8-2020, @@ -62,16 +62,15 @@ repos: flake8-quotes, flake8-string-format, flake8-use-fstring, - flake8-illegal-import, pep8-naming, flake8-bandit, dlint ] files: "^(trestle)" exclude: "(oscal/)" - stages: [commit] + stages: [pre-commit] - repo: https://github.com/executablebooks/mdformat - rev: 0.7.16 + rev: 0.7.17 hooks: - id: mdformat exclude: "CHANGELOG.md|docs/mkdocs_code_of_conduct.md|docs/maintainers.md|docs/api_reference|tests/data/author|docs/contributing/mkdocs_contributing.md|tests/data/jinja_markdown_include|tests/data/jinja_cmd/number_captions_data.md|tests/data/jinja_cmd/number_captions_expected_output.md" @@ -80,3 +79,4 @@ repos: - mdformat-config - mdformat-frontmatter - mdformat-gfm + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1484f000e..9c1734b44 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,6 +32,17 @@ review to indicate acceptance. A change requires LGTMs from one of the maintaine For a list of the maintainers, see the [maintainers](https://oscal-compass.github.io/compliance-trestle/maintainers/) page. +### Trestle updating, testing and release logistics + +Contributors should make a working copy (branch or fork) from the develop branch of `trestle`. +Contributors should update the working copy with changes, then create a pull request to merge into the develop branch. +Contributors must include test cases to meet at least the minimum code coverage requirements. +Upon approval from reviewer(s), the working copy is squashed and merged into the develop branch. +Upon a cadence established by the maintainers, the develop branch is merged into the main branch and a new release is uniquely numbered and pushed to [pypi](https://pypi.org/project/compliance-trestle/). + +`trestle` employs `semantic release` to automatically control release numbering. +Code deliveries should be tagged with prefix `fix:` for changes that are bug fixes or `feat:` for changes that are new features. See [allowed_tags](https://python-semantic-release.readthedocs.io/en/latest/commit-parsing.html#:~:text=The%20default%20configuration%20options%20for%20semantic_release.commit_parser.AngularCommitParser%20are%3A) for a list of supported tags. + ### Trestle merging and release workflow `trestle` is operating on a simple, yet opinionated, method for continuous integration. It's designed to give developers a coherent understanding of the objectives of other past developers. @@ -86,7 +97,7 @@ Software License 2.0. Using the SPDX format is the simplest approach. e.g. ```text -# Copyright (c) 2020 IBM Corp. All rights reserved. +# Copyright (c) 2024 The OSCAL Compass Authors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -164,7 +175,7 @@ Test discovery should be automatic when you select a .py file for editing. After Sometimes the discovery fails - and you may need to resort to uninstalling the python extension and reinstalling it - perhaps also shutting down code and restarting. This is a lightweight operation and seems to be safe and usually fixes any problems. -Test disovery will fail or stop if any of the tests have errors in them - so be sure to monitor the Problems panel at the bottom for problems in the code. +Test discovery will fail or stop if any of the tests have errors in them - so be sure to monitor the Problems panel at the bottom for problems in the code. Note that there are many panels available in Output - so be sure to check `Python Test Log` for errors and output from the tests. @@ -179,8 +190,22 @@ Trestle relies on reference data from two NIST repositories for testing: Both of these repositories are submodules in the trestle project. In order to develop / test trestle the submodules must be checked out with `git submodule update --init` or `make submodules`. +### Code testing + +Tests must exist for at least 96% of trestle Python code. To determine the code coverage locally during development: + +```bash +make test-cov +``` + +A PR without sufficient test coverage will fail the trestle CI process and will not be approved or merged. + +Additional code scrutiny is applied in the trestle CI pipeline by [SonarCloud](https://sonarcloud.io/project/overview?id=compliance-trestle). Any failures must be addressed before code merging. + ### Code style and formating +Python code should generally follow [PEP 8](https://peps.python.org/pep-0008/). + `trestle` uses [yapf](https://github.com/google/yapf) for code formatting and [flake8](https://flake8.pycqa.org/en/latest/) for code styling. It also uses [pre-commit](https://pre-commit.com/) hooks that are integrated into the development process and the CI. When you run `make develop` you are ensuring that the pre-commit hooks are installed and updated to their latest versions for this repository. This ensures that all delivered code has been properly formatted and passes the linter rules. See the [pre-commit configuration file](https://github.com/oscal-compass/compliance-trestle/blob/develop/.pre-commit-config.yaml) for details on `yapf` and `flake8` configurations. @@ -248,4 +273,4 @@ ______________________________________________________________________ ##### Overview of process to take OSCAL models and upgrade trestle Python code - +![](images/trestle-OSCAL-upgrade.png) diff --git a/Makefile b/Makefile index 0482664ff..362e6e68d 100644 --- a/Makefile +++ b/Makefile @@ -86,7 +86,7 @@ docs-automation:: python ./scripts/website_automation.py docs-validate:: docs-automation - mkdocs build -c -s + mkdocs build -v -c -s rm -rf site docs-serve: docs-automation @@ -117,4 +117,4 @@ pylint-test: pylint tests --rcfile=.pylintrc_tests check-for-changes: - python scripts/have_files_changed.py -u \ No newline at end of file + python scripts/have_files_changed.py -u diff --git a/README.md b/README.md index 2407cf760..e9b28a9e6 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ ![[Code Coverage](https://sonarcloud.io/dashboard?id=compliance-trestle)](https://sonarcloud.io/api/project_badges/measure?project=compliance-trestle&metric=coverage) ![[Quality gate](https://sonarcloud.io/dashboard?id=compliance-trestle)](https://sonarcloud.io/api/project_badges/measure?project=compliance-trestle&metric=alert_status) ![[Pypi](https://pypi.org/project/compliance-trestle/)](https://img.shields.io/pypi/dm/compliance-trestle) -![GitHub Actions status](https://img.shields.io/github/workflow/status/oscal-compass/compliance-trestle/Trestle%20PR%20pipeline?event=push) +![GitHub Actions status](https://github.com/oscal-compass/compliance-trestle/actions/workflows/python-test.yml/badge.svg?branch=develop) Trestle is an ensemble of tools that enable the creation, validation, and governance of documentation artifacts for compliance needs. It leverages NIST's [OSCAL](https://pages.nist.gov/OSCAL/) as a standard data format for interchange between tools and people, and provides an opinionated approach to OSCAL adoption. @@ -112,7 +112,7 @@ If you would like to see the detailed LICENSE click [here](LICENSE). Consult [contributors](https://github.com/oscal-compass/compliance-trestle/graphs/contributors) for a list of authors and [maintainers](MAINTAINERS.md) for the core team. ```text -# Copyright (c) 2020 IBM Corp. All rights reserved. +# Copyright (c) 2024 The OSCAL Compass Authors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -125,5 +125,17 @@ Consult [contributors](https://github.com/oscal-compass/compliance-trestle/graph # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - ``` + +______________________________________________________________________ + +We are a Cloud Native Computing Foundation sandbox project. + + + + + + +The Linux Foundation® (TLF) has registered trademarks and uses trademarks. For a list of TLF trademarks, see [Trademark Usage](https://www.linuxfoundation.org/legal/trademark-usage)". + +*Trestle was originally created by IBM.* diff --git a/ROADMAP.md b/ROADMAP.md index b2d2a70d8..370bf52c4 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -18,6 +18,12 @@ We use 12-week iterations for high-level initiatives that must be broken down an Each roadmap item is represented as a GitHub Issue. Discussions or any feedback on current roadmap items can take place in threads on the corresponding issue. To stay up to date with roadmap items, please join the OSCAL-Compass community [meetings](https://github.com/oscal-compass/community). +**Help Us Grow `compliance-trestle`!** Community contributions are essential to the project's success. To ensure we prioritize the most active and relevant issues, we're using `actions/stale` to automatically close older ones. Your participation is greatly appreciated! + +#### Our Triaging Process + +All new issues in `compliance-trestle` are added to the [project](https://github.com/orgs/oscal-compass/projects/2) with a status of New. All issues that need to be triaged are viewable [here](https://github.com/orgs/oscal-compass/projects/2/views/8). Once there is enough information to move forward on the issue, it can be moved to the `Backlog` by applying the `Backlog` label and moving the status to `Ready`. + ### How to add an item to the roadmap? **Contributors are encouraged to get feedback early by submitting issues to new work they would like to complete and getting feedback.** diff --git a/docs/api_reference/trestle.core.jinja.base.md b/docs/api_reference/trestle.core.jinja.base.md new file mode 100644 index 000000000..b929616d6 --- /dev/null +++ b/docs/api_reference/trestle.core.jinja.base.md @@ -0,0 +1,2 @@ +::: trestle.core.jinja.base +handler: python diff --git a/docs/api_reference/trestle.core.jinja.ext.md b/docs/api_reference/trestle.core.jinja.ext.md new file mode 100644 index 000000000..5da0b570e --- /dev/null +++ b/docs/api_reference/trestle.core.jinja.ext.md @@ -0,0 +1,2 @@ +::: trestle.core.jinja.ext +handler: python diff --git a/docs/api_reference/trestle.core.jinja.filters.md b/docs/api_reference/trestle.core.jinja.filters.md new file mode 100644 index 000000000..40a4d4e5e --- /dev/null +++ b/docs/api_reference/trestle.core.jinja.filters.md @@ -0,0 +1,2 @@ +::: trestle.core.jinja.filters +handler: python diff --git a/docs/api_reference/trestle.core.jinja.md b/docs/api_reference/trestle.core.jinja.md deleted file mode 100644 index ea057f7e1..000000000 --- a/docs/api_reference/trestle.core.jinja.md +++ /dev/null @@ -1,2 +0,0 @@ -::: trestle.core.jinja -handler: python diff --git a/docs/api_reference/trestle.core.jinja.tags.md b/docs/api_reference/trestle.core.jinja.tags.md new file mode 100644 index 000000000..53e6638a0 --- /dev/null +++ b/docs/api_reference/trestle.core.jinja.tags.md @@ -0,0 +1,2 @@ +::: trestle.core.jinja.tags +handler: python diff --git a/docs/api_reference/trestle.core.plugins.md b/docs/api_reference/trestle.core.plugins.md new file mode 100644 index 000000000..54e01680b --- /dev/null +++ b/docs/api_reference/trestle.core.plugins.md @@ -0,0 +1,2 @@ +::: trestle.core.plugins +handler: python diff --git a/docs/cli.md b/docs/cli.md index f7ef55b9e..bd44ac92e 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -113,11 +113,11 @@ Users can query the contents of files using `trestle describe`, and probe the co OSCAL models are rich and contain multiple nested data structures. Given this, a mechanism is required to address _elements_ /_attributes_ within an oscal object. -This accessing method is called 'element path' and is similar to _jsonPath_. Commands provide element path by a `-e` argument where available, e.g. trestle split -f catalog.json -e 'catalog.metadata.\*'. This path is used whenever specifying an attribute or model, rather than exposing trestle's underlying object model name. Users can refer to [NIST's json outline](https://pages.nist.gov/OSCAL/reference/latest/complete/json-outline/) to understand object names in trestle. +This accessing method is called 'element path' and is similar to _jsonPath_. Commands provide element path by a `-e` argument where available, e.g. trestle split -f catalog.json -e 'catalog.metadata.\*'. This path is used whenever specifying an attribute or model, rather than exposing trestle's underlying object model name. Users can refer to [NIST's json outline](https://pages.nist.gov/OSCAL-Reference/models/latest/complete/json-outline/) to understand object names in trestle. ### Rules for element path -1. Element path is an expression of the attribute names, [in json form](https://pages.nist.gov/OSCAL/reference/latest/complete/json-outline/) , concatenated by a period (`.`). +1. Element path is an expression of the attribute names, [in json form](https://pages.nist.gov/OSCAL-Reference/models/latest/complete/json-outline/) , concatenated by a period (`.`). 1. E.g. The metadata in a catalog is referred to as `catalog.metadata` 1. Element paths are relative to the file. 1. e.g. For `metadata.json` roles would be referred to as `metadata.roles`, from the catalog file that would be `catalog.metadata.roles` diff --git a/docs/contributing/github_actions_setup.md b/docs/contributing/github_actions_setup.md index 8cf0bbb8f..b3136f69f 100644 --- a/docs/contributing/github_actions_setup.md +++ b/docs/contributing/github_actions_setup.md @@ -5,7 +5,7 @@ The variables are documented here such that trestle can be setup on a fork etc. ## Secrets -- `ADMIN_PAT`: Github PAT with sufficient write access to merge content into `develop` and commit to `gh-pages` and `main` +- `APP_ID` and `PRIVATE_KEY`: GitHub App information with sufficient write access to merge content into `develop` and commit to `gh-pages` and `main` - `SONAR_TOKEN`: Token to sonarcloud with rights to the appropriate project. diff --git a/docs/contributing/images/trestle-OSCAL-upgrade.png b/docs/contributing/images/trestle-OSCAL-upgrade.png new file mode 100644 index 000000000..63e73799f Binary files /dev/null and b/docs/contributing/images/trestle-OSCAL-upgrade.png differ diff --git a/docs/contributing/plugins.md b/docs/contributing/plugins.md index f4f7d6582..89330caf6 100644 --- a/docs/contributing/plugins.md +++ b/docs/contributing/plugins.md @@ -13,17 +13,22 @@ The plugin project should be organized as shown below. ```text compliance-trestle-fedramp ├── trestle_fedramp -│ ├── __init.py__ +│ ├── __init__.py │ ├── commands -| | ├── __init.py__ +| | ├── __init__.py | | ├── validate.py +| ├── jinja_ext +| | ├── __init__.py +| | ├── filters.py │ ├── ├── ``` Trestle uses a naming convention to discover the top-level module of the plugin projects. It expects the top-level module to be named `trestle_{plugin_name}`. This covention must be followed by plugins to be discoverable by trestle. In the above example, the top-level module is named as `trestle_fedramp` so that it can be autmatically discovered by trestle. All the python source files should be created inside this module (folder). -The top-evel module should contain a `commands` directory where all the plugin command files should be stored. Each command should have its own python file. In the above exaample, `validate.py` file conatins one command for this plugin. Other python files or folders should be created in the top-level module folder, outside the `commands` folder. This helps in keeping the commands separate and in their discovery by trestle. +To add commands to the CLI interface, the top-level module should contain a `commands` directory where all the plugin command files should be stored. Each command should have its own python file. In the above example, `validate.py` file contains one command for this plugin. Other python files or folders should be created in the top-level module folder, outside the `commands` folder. This helps in keeping the commands separate and in their discovery by trestle. + +To add jinja extensions available during `trestle author jinja`, the top-level module should contain a `jinja_ext` directory where all extension files should be stored. Each extension should have its own python file. In the above example, `filters.py` file contains a single extension class, which may define many filters or custom tags. Supporting code should be created in the top-level module folder, outside the `jinja_ext` folder. This helps in keeping the extensions separate and in their discovery by trestle. ## Command Creation @@ -61,3 +66,28 @@ There should be a command class for example, `ValidateCmd` which should either e The docstring of the command class is used as the help message for the command. Input arguments to the command should be specified in `_init_arguments` method as shown above. The acutal code of the command is contained in`_run` method. This method is called by ilcli when the command is excuted on the commandline. The command arguments can be accessed from the `args` input parameter as shown above. The command should return `0` in case of successful execution, or any number greater than 0 in case of failure. Please see `trestle.core.commands.common.return_codes.CmdReturnCodes` class for specific return codes in case of failure. The command class should conatin the `name` field which should be set to the desired command name. In the above example, the command is called `fedramp-validate`. This name is automatically added to the list of sub-command names of trestle during the plugin discovery process. This command can then be invoked as `trestle {name}` from the commandline e.g., `trestle fedramp-validate`. Any input parameters to the command can also be passed on the commandline after the command name. + +## Jinja Extension Creation + +The plugin extension should be created as shown in the below code snippet. + +```python +from jinja2 import Environment +from trestle.core.jinja.base import TrestleJinjaExtension + +def _mark_tktk(value: str) -> str: + """Mark a value with TKTK to easily find it for future revision.""" + return f'TKTK {value} TKTK' + + +class Filters(TrestleJinjaExtension): + def __init__(self, environment: Environment) -> None: + super(Filters, self).__init__(environment) + + environment.filters['tktk'] = _mark_tktk + +``` + +There should be an extension class, for example `Filters` that must extend from `TrestleJinjaExtension` or `jinja2.ext.Extention`. The `__init__` method must call init for its superclass. Beyond that, any behavior for standard [jinja2 custom extensions](https://jinja.palletsprojects.com/en/3.1.x/extensions/#module-jinja2.ext) is supported. + +Examples for implementing extensions can be found at `trestle/core/jinja/tags.py` and `trestle/core/jinja/filters.py` diff --git a/docs/contributing/trestle_oscal_object_model.md b/docs/contributing/trestle_oscal_object_model.md index 71083d6d2..a77b47a4f 100644 --- a/docs/contributing/trestle_oscal_object_model.md +++ b/docs/contributing/trestle_oscal_object_model.md @@ -9,7 +9,7 @@ This functionality, which is built on [pydantic](https://pydantic-docs.helpmanua ## Mapping and variance with OSCAL names. -The underlying object model that trestle relies on is the json schema published by NIST [here](https://github.com/usnistgov/OSCAL/tree/main/json/schema). In understanding these models the [model reference page](https://pages.nist.gov/OSCAL/reference/1.0.0/) is an indispensable source. +The underlying object model that trestle relies on is the json schema published by NIST [here](https://github.com/usnistgov/OSCAL/releases/latest). In understanding these models the [model reference page](https://pages.nist.gov/OSCAL-Reference/models/) is an indispensable source. When generating the python data class based models we have tried to be as faithful as we can to the naming convention provided by OSCAL. This is the hierarchy of rules that we have used: diff --git a/docs/index.md b/docs/index.md index bee864d33..3759cecaf 100644 --- a/docs/index.md +++ b/docs/index.md @@ -6,7 +6,7 @@ ![[Code Coverage](https://sonarcloud.io/dashboard?id=compliance-trestle)](https://sonarcloud.io/api/project_badges/measure?project=compliance-trestle&metric=coverage) ![[Quality gate](https://sonarcloud.io/dashboard?id=compliance-trestle)](https://sonarcloud.io/api/project_badges/measure?project=compliance-trestle&metric=alert_status) ![[Pypi](https://pypi.org/project/compliance-trestle/)](https://img.shields.io/pypi/dm/compliance-trestle) -![GitHub Actions status](https://img.shields.io/github/workflow/status/oscal-compass/compliance-trestle/Trestle%20PR%20pipeline?event=push) +![GitHub Actions status](https://github.com/oscal-compass/compliance-trestle/actions/workflows/python-test.yml/badge.svg?branch=develop) Trestle is an ensemble of tools that enable the creation, validation, and governance of documentation artifacts for compliance needs. It leverages NIST's [OSCAL](https://pages.nist.gov/OSCAL/documentation/) as a standard data format for interchange between tools and people, and provides an opinionated approach to OSCAL adoption. @@ -65,7 +65,7 @@ natively supports only `json` and `yaml` formats at this time. Future roadmap anticipates that support for xml [import](https://github.com/oscal-compass/compliance-trestle/issues/177) and [upstream references](https://github.com/oscal-compass/compliance-trestle/issues/178) will be enabled. However, it is expected that full support will remain only for `json` and `yaml`. -Users needing to import XML OSCAL artifacts are recommended to look at NIST's XML to json conversion page [here](https://github.com/usnistgov/OSCAL/tree/master/json#oscal-xml-to-json-converters). +Users needing to import XML OSCAL artifacts are recommended to look at NIST's OSCAL converters page [here](https://github.com/usnistgov/OSCAL/blob/main/build/README.md#converters). ## Python codebase, easy installation via pip @@ -77,15 +77,15 @@ Compliance trestle is currently stable and is based on NIST OSCAL version 1.0.4, ## Contributing to Trestle -Our project welcomes external contributions. Please consult [contributing](contributing/mkdocs_contributing/) to get started. +Our project welcomes external contributions. Please consult [contributing](contributing/mkdocs_contributing.md) to get started. ## License & Authors -If you would like to see the detailed LICENSE click [here](license/). -Consult [contributors](https://github.com/oscal-compass/compliance-trestle/graphs/contributors) for a list of authors and [maintainers](maintainers/) for the core team. +If you would like to see the detailed LICENSE click [here](license.md). +Consult [contributors](https://github.com/oscal-compass/compliance-trestle/graphs/contributors) for a list of authors and [maintainers](maintainers.md) for the core team. ```text -# Copyright (c) 2020 IBM Corp. All rights reserved. +# Copyright (c) 2024 The OSCAL Compass Authors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -98,5 +98,17 @@ Consult [contributors](https://github.com/oscal-compass/compliance-trestle/graph # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - ``` + +______________________________________________________________________ + +We are a Cloud Native Computing Foundation sandbox project. + + + + + + +The Linux Foundation® (TLF) has registered trademarks and uses trademarks. For a list of TLF trademarks, see [Trademark Usage](https://www.linuxfoundation.org/legal/trademark-usage)". + +*Trestle was originally created by IBM.* diff --git a/docs/plugins/compliance-trestle-fedramp.md b/docs/plugins/compliance-trestle-fedramp.md index 8b5069bb9..843039eef 100644 --- a/docs/plugins/compliance-trestle-fedramp.md +++ b/docs/plugins/compliance-trestle-fedramp.md @@ -4,7 +4,7 @@ This plugin provides functionality for validating an SSP for FedRAMP compliance. ## `trestle fedramp-validate` -This command allows users to validate existing OSCAL SSP file (in JSON or YAML format) for FedRAMP compliance. For example, `trestle fedramp-validate -f /local_dir/ssp.json -o report/` will validate `ssp.json` file for fedramp complaince and store the validation reports in `report` folder. +This command allows users to validate existing OSCAL SSP file (in JSON or YAML format) for FedRAMP compliance. For example, `trestle fedramp-validate -f /local_dir/ssp.json -o report/` will validate `ssp.json` file for fedramp compliance and store the validation reports in `report` folder. The following options are supported: @@ -12,3 +12,15 @@ The following options are supported: - `-o or --output`: specifies the name of the output directory where the validation reports will be stored. It may be an absolute or relative path. The output directory should already exist. This is also a required option. The validation reports are created in XML and HTML format and provide details on which part of the SSP are not complaint as per FedRAMP specification. + +## `trestle fedramp-transform` + +This command allows users to extract information from an OSCAL SSP and transform it into a Word document based on the FedRAMP SSP Appendix A Template. The templates for the High, Moderate, and Low baseline security control requirements were retrieved from this [location](https://www.fedramp.gov/documents-templates/) and are bundled with the application. The `Control Summary Information` tables are populated for each control based on the OSCAL SSP. + +For example, `trestle fedramp-transform -n ssp-name -l 'high' -o my_ssp.docx` will transform the OSCAL SSP file `ssp-name` into a Word document `my_ssp.docx` based on the SSP Appendix A - High FedRAMP Security Controls template. + +The following options are supported: + +- `-n or --ssp-name`: The name of the OSCAL SSP imported into trestle workspace. This is a required option. +- `-l or --level`: The baseline level corresponding to the template. This is high, moderate, low. This is a required option. +- `-o or --output-file`: The output location for the populated Word document. This is also a required option. diff --git a/docs/trestle_author_jinja.md b/docs/trestle_author_jinja.md index 45001f513..9f9df1349 100644 --- a/docs/trestle_author_jinja.md +++ b/docs/trestle_author_jinja.md @@ -113,7 +113,7 @@ Trestle provides custom jinja tags for use specifically with markdown: `mdsectio 1. `{% md_clean_include 'path_to_file.md' heading_level=2 %}` 1. The heading level argument adjusts to (based on the number of hashes) the most significant heading in the document, if headings exist. -`mdsection_include` is similar to the native `md_clean_include` except that.: +`mdsection_include` is similar to `md_clean_include` except that: 1. `mdsection_include` requires an second positional argument which is the title of a heading, from a markdown file, which you want the content from. @@ -129,6 +129,23 @@ Trestle provides custom jinja tags for use specifically with markdown: `mdsectio 1. `format` where a python [datetime strftime format string](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes) is provided to format the output. E.g. `{% md_datestamp format='%B %d, %Y' %}` results in `December 28, 2021` being inserted. 1. `newline` is a boolean to control the addition of a double newline after the inserted date string. For example `{% md_datestamp newline=false %}` inserts a date in the default format, without additional newlines. +## Custom Jinja filters. + +Trestle provides custom jinja filters for help processing SSP data. + +- `as_list` will return the passed value, or an empty list if `None` is passed in. + - Example: `{% for party in ssp.metadata.parties | as_list %}` +- `get_default` operates the same as the built-in Jinja `default` filter, with the optional second parameter set to `True` + - Example: `{{ control_interface.get_prop(user, 'user-property') | get_default('[Property]') }}` +- `first_or_none` will return the first element of a list, or `None` if the list is empty or `None` itself. + - Example: `{% set address = party.addresses | first_or_none %}` +- `get_party` will return the `Party` found in `ssp.metadata.parties` for a given `uuid` + - Example: `{% set organization = party.member_of_organizations | first_or_none | get_party(ssp) %}` +- `parties_for_role` will yield individual `Party` entries when given a list of `ResponsibleParty` and a `role-id` + - Example: `{% for party in ssp.metadata.responsible_parties | parties_for_role("prepared-by", ssp) %}` +- `diagram_href` will return the `Link.href` where `Link.rel == 'diagram'` when given a `Diagram` object + - Example: `![{{diagram.caption}}]({{ diagram | diagram_href }})` + ## Generate controls as individual markdown pages. Trestle's Jinja functionality allows its users to generate individual markdown pages for each control from a resolved profile catalog. Such functionality can be used later on to pack individual pages into docs of various formats. @@ -144,17 +161,17 @@ To achieve that, we can create a simple Jinja template that would be used to gen {{ control_writer.write_control_with_sections( control, profile, - group_title, - ['statement', 'objective', 'expected_evidence', 'implementation_guidance', 'table_of_parameters'], + group_title, + ['statement', 'objective', 'expected_evidence', 'implementation_guidance', 'table_of_parameters'], { 'statement':'Control Statement', - 'objective':'Control Objective', - 'expected_evidence':'Expected Evidence', - 'implementation_guidance':'Implementation Guidance', + 'objective':'Control Objective', + 'expected_evidence':'Expected Evidence', + 'implementation_guidance':'Implementation Guidance', 'table_of_parameters':'Control Parameters' } - ) - + ) + | safe }} ``` diff --git a/docs/tutorials/continuous-compliance/continuous-compliance.md b/docs/tutorials/continuous-compliance/continuous-compliance.md index 0bd7e0d3e..a8f5b0c8c 100644 --- a/docs/tutorials/continuous-compliance/continuous-compliance.md +++ b/docs/tutorials/continuous-compliance/continuous-compliance.md @@ -17,7 +17,7 @@ Moreover, assuring continuous compliance across multiple cloud vendors can compl Common sense dictates that standardization would simplify matters. The National Institute of Standards and Technologies (NIST) is developing the Open Security Controls Assessment Language ([OSCAL](https://pages.nist.gov/OSCAL)). -The compliance-[trestle](https://oscal-compass.github.io/compliance-trestle/) open source github project is an effort to employ [OSCAL](https://pages.nist.gov/OSCAL) for compliance standardization and automation. Of great utility is the [trestle](https://oscal-compass.github.io/compliance-trestle/) oscal module that facilitates transformation of data to/from Python object representations in accordance with the [OSCAL](https://pages.nist.gov/OSCAL) schemas. +The compliance-[trestle](../../index.md) open source github project is an effort to employ [OSCAL](https://pages.nist.gov/OSCAL) for compliance standardization and automation. Of great utility is the [trestle](../../index.md) oscal module that facilitates transformation of data to/from Python object representations in accordance with the [OSCAL](https://pages.nist.gov/OSCAL) schemas. #### Simple Continuous Compliance Architecture @@ -25,7 +25,7 @@ The compliance-[trestle](https://oscal-compass.github.io/compliance-trestle/) op Cloud Services can often be configured to monitor (and sometimes enforce) policies. Examples include OpenShift Compliance Operator and Tanium. However, the compliance reporting “raw” data produced is unique to each. -Two steps are needed to ascertain your compliance posture. Step 1 is to transform available compliance “raw” data into standardized form ([OSCAL](https://pages.nist.gov/OSCAL)). Step 2 is to examine the [OSCAL](https://pages.nist.gov/OSCAL) data and assemble a compliance posture for the controls and components of interest. And [trestle](https://oscal-compass.github.io/compliance-trestle/) is the go-to solution. +Two steps are needed to ascertain your compliance posture. Step 1 is to transform available compliance “raw” data into standardized form ([OSCAL](https://pages.nist.gov/OSCAL)). Step 2 is to examine the [OSCAL](https://pages.nist.gov/OSCAL) data and assemble a compliance posture for the controls and components of interest. And [trestle](../../index.md) is the go-to solution. #### Step 1 – Transformation @@ -33,14 +33,14 @@ The bad news is that a transformer to [OSCAL](https://pages.nist.gov/OSCAL) is n However, there is plenty of good news: -- a transformer for your Cloud Service type may already exist, such as: [Tanium to OSCAL](https://github.com/oscal-compass/compliance-trestle/blob/main/trestle/tasks/tanium-result-to-oscal-ar.py), [OpenShift Compliance Operator to OSCAL](https://github.com/oscal-compass/compliance-trestle/blob/main/trestle/tasks/xccdf_result_to_oscal_ar.py) +- a transformer for your Cloud Service type may already exist, such as: [Tanium to OSCAL](https://github.com/oscal-compass/compliance-trestle/blob/main/trestle/tasks/tanium_result_to_oscal_ar.py), [OpenShift Compliance Operator to OSCAL](https://github.com/oscal-compass/compliance-trestle/blob/main/trestle/tasks/xccdf_result_to_oscal_ar.py) - once a transformer for a Cloud Service type has been written, it can be open-sourced/re-used -- writing a transformer is fairly easy: just a few lines of Python code using [trestle](https://oscal-compass.github.io/compliance-trestle/) as a foundation +- writing a transformer is fairly easy: just a few lines of Python code using [trestle](../../index.md) as a foundation In the case of Tanium, the [OSCAL](https://pages.nist.gov/OSCAL) compliance data document is a *System Assessment Results* fragment with *Findings* and *Observations*, while in the case of OpenShift Compliance Operator there are *Observations* only. -Tutorials are available to show you: how to [run a transformer](https://oscal-compass.github.io/compliance-trestle/tutorials/task.tanuim-to-oscal/transformation/), how to [write a transformer](https://oscal-compass.github.io/compliance-trestle/tutorials/task.transformer-construction/transformer-construction/). +Tutorials are available to show you: how to [run a transformer](../task.tanium-result-to-oscal-ar/transformation.md), how to [write a transformer](../task.transformer-construction/transformer-construction.md). #### Step 2 – Reporting -Coming soon is a [trestle](https://oscal-compass.github.io/compliance-trestle/) tool to assemble the [OSCAL](https://pages.nist.gov/OSCAL) fragments documents together using [OSCAL](https://pages.nist.gov/OSCAL) compliance configuration data (*System Assessment Plan* and *System Security Plan*) into a complete *System Assessment Results*. +Coming soon is a [trestle](../../index.md) tool to assemble the [OSCAL](https://pages.nist.gov/OSCAL) fragments documents together using [OSCAL](https://pages.nist.gov/OSCAL) compliance configuration data (*System Assessment Plan* and *System Security Plan*) into a complete *System Assessment Results*. diff --git a/docs/tutorials/ssp_profile_catalog_authoring/ssp_profile_catalog_authoring.md b/docs/tutorials/ssp_profile_catalog_authoring/ssp_profile_catalog_authoring.md index 9a1d2acbc..0286b5d58 100644 --- a/docs/tutorials/ssp_profile_catalog_authoring/ssp_profile_catalog_authoring.md +++ b/docs/tutorials/ssp_profile_catalog_authoring/ssp_profile_catalog_authoring.md @@ -793,8 +793,7 @@ The markdown header lists all the rules that apply to this control, along with t The values for rule parameters are specified using the normal `SetParameter` mechanism in the ControlImplementation, but it's important to note that there are two different types of `SetParameter`: Those that apply to the normal parameters of the control, and those that apply strictly to the rules. -Note that markdown for a control is only created if there are rules associated with the control, and within the markdown the only parts written out that -prompt for responses are parts that have rules assigned. Thus the output markdown directory may be highly pruned of both controls and groups of controls if only some controls have rules associated. +Note that markdown is created for all of the controls in the `Component` control implementation. However, when processing control parts, the part is only written out if there are associated rules or pre-filled implementation descriptions. In addition, the rules should be regarded as read-only from the editing perspective, and you cannot change the rules associated with a control or its parts. But you may edit the rule parameter values as described in the comments of the markdown file under `x-trestle-comp-def-rules-param-vals`. You may also edit the prose and implementation status associated with a statement part at the bottom of the markdown file. diff --git a/docs/tutorials/task.ocp4-cis-profile-to-oscal-catalog/transformation.md b/docs/tutorials/task.ocp4-cis-profile-to-oscal-catalog/transformation.md index 5b36df015..00a669a2e 100644 --- a/docs/tutorials/task.ocp4-cis-profile-to-oscal-catalog/transformation.md +++ b/docs/tutorials/task.ocp4-cis-profile-to-oscal-catalog/transformation.md @@ -1,6 +1,6 @@ # Tutorial: Setup for and use of ComplianceAsCode profile to OSCAL Catalog transformer -Here are step by step instructions for setup and transformation of [ComplianceAsCode](https://github.com/ComplianceAsCode/content) profile data files into [NIST](https://www.nist.gov/) standard [OSCAL](https://pages.nist.gov/OSCAL/) [Catalog](https://pages.nist.gov/OSCAL/reference/latest/catalog/json-outline/) using the [compliance-trestle](https://oscal-compass.github.io/compliance-trestle/) tool. +Here are step by step instructions for setup and transformation of [ComplianceAsCode](https://github.com/ComplianceAsCode/content) profile data files into [NIST](https://www.nist.gov/) standard [OSCAL](https://pages.nist.gov/OSCAL/) [Catalog](https://pages.nist.gov/OSCAL-Reference/models/latest/catalog/json-outline/) using the [compliance-trestle](../../index.md) tool. ## *Objective* @@ -12,7 +12,7 @@ The second is a one-command transformation from `.profile` to `OSCAL.json`. ## *Step 1: Install trestle in a Python virtual environment* -Follow the instructions [here](https://oscal-compass.github.io/compliance-trestle/python_trestle_setup/) to install trestle in a virtual environment. +Follow the instructions [here](../../python_trestle_setup.md) to install trestle in a virtual environment. ## *Step 2: Transform profile data (CIS benchmarks)* diff --git a/docs/tutorials/task.ocp4-cis-profile-to-oscal-cd/transformation.md b/docs/tutorials/task.ocp4-cis-profile-to-oscal-cd/transformation.md index 23184aea1..58dcef079 100644 --- a/docs/tutorials/task.ocp4-cis-profile-to-oscal-cd/transformation.md +++ b/docs/tutorials/task.ocp4-cis-profile-to-oscal-cd/transformation.md @@ -1,6 +1,6 @@ # Tutorial: Setup for and use of ComplianceAsCode profile to OSCAL Component Definition transformer -Here are step by step instructions for setup and transformation of [ComplianceAsCode](https://github.com/ComplianceAsCode/content) profile data files into [NIST](https://www.nist.gov/) standard [OSCAL](https://pages.nist.gov/OSCAL/) [Component Definition](https://pages.nist.gov/OSCAL/reference/latest/component-definition/json-outline/) using the [compliance-trestle](https://oscal-compass.github.io/compliance-trestle/) tool. +Here are step by step instructions for setup and transformation of [ComplianceAsCode](https://github.com/ComplianceAsCode/content) profile data files into [NIST](https://www.nist.gov/) standard [OSCAL](https://pages.nist.gov/OSCAL/) [Component Definition](https://pages.nist.gov/OSCAL-Reference/models/v1.1.2/complete/json-reference/#/component-definition) using the [compliance-trestle](https://oscal-compass.github.io/compliance-trestle/) tool. ## *Objective* diff --git a/docs/tutorials/task.transformer-construction/transformer-construction.md b/docs/tutorials/task.transformer-construction/transformer-construction.md index b6924437c..e5573cd5e 100644 --- a/docs/tutorials/task.transformer-construction/transformer-construction.md +++ b/docs/tutorials/task.transformer-construction/transformer-construction.md @@ -18,7 +18,7 @@ The objective here is to transform your compliance data into valid OSCAL, in par Examples of existing transformers included with trestle are for the OpenShift Compliance Operator [OSCO](https://github.com/oscal-compass/compliance-trestle/blob/develop/trestle/tasks/xccdf_result_to_oscal_ar.py) and -[Tanium](https://github.com/oscal-compass/compliance-trestle/blob/develop/trestle/tasks/tanium-result-to-oscal-ar.py). +[Tanium](https://github.com/oscal-compass/compliance-trestle/blob/develop/trestle/tasks/tanium_result_to_oscal_ar.py). ## *Overview* @@ -407,7 +407,7 @@ There are 2 transformers in trestle. The [xccdf-result-to-oscal-ar](https://github.com/oscal-compass/compliance-trestle/blob/develop/trestle/tasks/xccdf_result_to_oscal_ar.py) transformer emits OSCAL Observations, the simplest partial result. -The [tanium-result-to-oscal-ar](https://github.com/oscal-compass/compliance-trestle/blob/develop/trestle/tasks/tanium-result-to-oscal-ar.py) +The [tanium-result-to-oscal-ar](https://github.com/oscal-compass/compliance-trestle/blob/develop/trestle/tasks/tanium_result_to_oscal_ar.py) transformer emits OSCAL Findings, a more complex partial result. Table of approximate lines of code. diff --git a/images/trestle-OSCAL-upgrade.png b/images/trestle-OSCAL-upgrade.png index eae3bb6ef..63e73799f 100644 Binary files a/images/trestle-OSCAL-upgrade.png and b/images/trestle-OSCAL-upgrade.png differ diff --git a/mkdocs.yml b/mkdocs.yml index 26b7a9821..0cbad372e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -3,6 +3,13 @@ extra: analytics: property: G-XT3KGMHSY8 provider: google + consent: + title: Cookie consent + description: >- + We use cookies to recognize your repeated visits and preferences, as well as + to measure the effectiveness of our documentation and whether users find what + they're searching for. With your consent, you're helping us to make our documentation + better. markdown_extensions: - admonition - markdown_include.include @@ -44,6 +51,7 @@ nav: - Trestle's object model: contributing/trestle_oscal_object_model.md - Developer Certificate of Originality: contributing/DCO.md - Trestle plugin mechanism: contributing/plugins.md + - GitHub actions setup: contributing/github_actions_setup.md - Known limitations: errors.md - Demos: demonstrations-content.md - Plugins: @@ -124,7 +132,11 @@ nav: - duplicates_validator: api_reference/trestle.core.duplicates_validator.md - generators: api_reference/trestle.core.generators.md - generic_oscal: api_reference/trestle.core.generic_oscal.md - - jinja: api_reference/trestle.core.jinja.md + - jinja: + - base: api_reference/trestle.core.jinja.base.md + - ext: api_reference/trestle.core.jinja.ext.md + - filters: api_reference/trestle.core.jinja.filters.md + - tags: api_reference/trestle.core.jinja.tags.md - links_validator: api_reference/trestle.core.links_validator.md - markdown: - base_markdown_node: api_reference/trestle.core.markdown.base_markdown_node.md @@ -144,6 +156,7 @@ nav: - object_factory: api_reference/trestle.core.object_factory.md - parser: api_reference/trestle.core.parser.md - pipeline: api_reference/trestle.core.pipeline.md + - plugins: api_reference/trestle.core.plugins.md - profile_resolver: api_reference/trestle.core.profile_resolver.md - refs_validator: api_reference/trestle.core.refs_validator.md - remote: @@ -192,11 +205,25 @@ nav: - transformer_helper: api_reference/trestle.transforms.transformer_helper.md - transformer_singleton: api_reference/trestle.transforms.transformer_singleton.md plugins: +# warning don't use `macros` - search +- autorefs +- htmlproofer: + enabled: true + validate_rendered_template: false + validate_external_urls: true + raise_error_after_finish: false + raise_error_excludes: + # This is to remove some false positives for links which work. + # Anchors are validated again by core mkdocs + 404: ['#trestle.*'] - mkdocstrings: + default_handler: python handlers: python: options: + show_inheritance_diagram: true + inherited_members: false group_by_category: true show_category_heading: true show_if_no_docstring: true @@ -206,8 +233,6 @@ plugins: - '!^__json' - '!^__config__' new_path_syntax: true - watch: - - trestle repo_name: oscal-compass/compliance-trestle repo_url: https://github.com/oscal-compass/compliance-trestle site_description: Documentation for compliance-trestle package. @@ -222,3 +247,8 @@ theme: accent: purple primary: teal scheme: slate +validation: + omitted_files: warn + absolute_links: warn # Or 'relative_to_docs' - new in MkDocs 1.6 + unrecognized_links: warn + anchors: warn # New in MkDocs 1.6 diff --git a/scripts/flatten_schema.py b/scripts/flatten_schema.py index 0e1df3a29..cf155cf9a 100644 --- a/scripts/flatten_schema.py +++ b/scripts/flatten_schema.py @@ -56,7 +56,7 @@ def _replace_ref(self, d, ref_set): def _replace_refs(self, obj, ref_set): """Given an object recurse into it replacing any found ref: defs with what is in def_list.""" - if type(obj) == dict: + if isinstance(obj, dict): # first check if it is a simple $ref line and replace it directly if len(obj.items()) == 1 and obj.get(self._ref_str, None) is not None: return self._replace_ref(obj, ref_set) @@ -71,9 +71,9 @@ def _replace_refs(self, obj, ref_set): if changed: dirty = True return new_dict, ref_set, dirty - elif type(obj) == str: + elif isinstance(obj, str): return obj, ref_set, False - elif type(obj) == list: + elif isinstance(obj, list): n_list = len(obj) changed = False dirty = False @@ -82,7 +82,7 @@ def _replace_refs(self, obj, ref_set): if changed: dirty = True return obj, ref_set, dirty - elif type(obj) == tuple: + elif isinstance(obj, tuple): new_val, ref_set, changed = self._replace_refs(obj[1], ref_set) return (obj[0], new_val), ref_set, changed if hasattr(obj, '__iter__'): diff --git a/scripts/oscal_normalize.py b/scripts/oscal_normalize.py index 73a37b295..5b2027936 100644 --- a/scripts/oscal_normalize.py +++ b/scripts/oscal_normalize.py @@ -235,7 +235,7 @@ def add_ref_pattern(self, p, line): new_refs = p.findall(line) if new_refs: for r in new_refs: - if type(r) == tuple: + if isinstance(r, tuple): for s in r: self.add_ref_if_good(s) else: diff --git a/scripts/schema_integrity.py b/scripts/schema_integrity.py index fc004e443..cc1ed60f6 100644 --- a/scripts/schema_integrity.py +++ b/scripts/schema_integrity.py @@ -77,11 +77,11 @@ def recursive_ref(self, ref_key: str, dict_of_interest: Dict[str, Any]) -> List[ for key, value in dict_of_interest.items(): if key == ref_key: returner.append(value) - elif type(value) == dict: + elif isinstance(value, dict): returner = returner + self.recursive_ref(ref_key, value) - elif type(value) == list: + elif isinstance(value, list): for item in value: - if type(item) == dict: + if isinstance(item, dict): returner = returner + self.recursive_ref(ref_key, item) elif key == ref_key: returner.append(value) diff --git a/setup.cfg b/setup.cfg index 116f70029..c1b26c678 100644 --- a/setup.cfg +++ b/setup.cfg @@ -79,8 +79,9 @@ dev = types-requests types-setuptools # # Docs website - mkdocs==1.5.0 - mkdocstrings[python-legacy]==0.19.0 + mkdocs>=1.6.0 + mkdocstrings[python]>=0.25.2 + mkdocs-htmlproofer-plugin mkdocs-material markdown-include pymdown-extensions diff --git a/tests/conftest.py b/tests/conftest.py index d4053b29d..21704a0f8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -32,8 +32,10 @@ from trestle.cli import Trestle from trestle.common.err import TrestleError from trestle.oscal import catalog as cat +from trestle.oscal.common import Party from trestle.oscal.component import ComponentDefinition, DefinedComponent from trestle.oscal.profile import Profile +from trestle.oscal.ssp import SystemSecurityPlan TEST_CONFIG: dict = {} @@ -190,6 +192,20 @@ def sample_catalog_subgroups(): return catalog_obj +@pytest.fixture(scope='function') +def sample_party() -> Party: + """Return a valid Party object.""" + return gens.generate_sample_model(Party, True, 3) + + +@pytest.fixture(scope='function') +def sample_system_security_plan(sample_party: Party) -> SystemSecurityPlan: + """Return a valid SSP object with some contents.""" + ssp: SystemSecurityPlan = gens.generate_sample_model(SystemSecurityPlan, True, 2) + ssp.metadata.parties = [gens.generate_sample_model(Party, True, 3), sample_party] + return ssp + + @pytest.fixture(scope='function') def sample_component_definition(): """Return a valid ComponentDefinition object with some contents.""" diff --git a/tests/data/csv/rule-name-overlap.csv b/tests/data/csv/rule-name-overlap.csv new file mode 100644 index 000000000..d1570e0af --- /dev/null +++ b/tests/data/csv/rule-name-overlap.csv @@ -0,0 +1,7 @@ +$$Component_Title,$$Component_Description,$$Component_Type,$$Rule_Id,$$Rule_Description,Check_Id,Check_Description,$$Namespace,Target_Component,$$Control_Id_List,$$Profile_Source,$$Profile_Description +OSCO,OSCO,validation,RULE-1.1,RULE1.1,Check.1,Check.1,http://oscal-compass/compliance-trestle/schemas/oscal/cd,Target-A,,, +OSCO,OSCO,validation,RULE-1.1,RULE1.1,Check.2,Check.2,http://oscal-compass/compliance-trestle/schemas/oscal/cd,Target-A,,, +OSCO,OSCO,validation,RULE-1.1,RULE1.1,Check.3,Check.3,http://oscal-compass/compliance-trestle/schemas/oscal/cd,Target-A,,, +OSCO,OSCO,validation,RULE-1.1,RULE1.1,Check.1,Check.1,http://oscal-compass/compliance-trestle/schemas/oscal/cd,Target-B,,, +OSCO,OSCO,validation,RULE-1.1,RULE1.1,Check.3,Check.2,http://oscal-compass/compliance-trestle/schemas/oscal/cd,Target-B,,, +OSCO,OSCO,validation,RULE-1.1,RULE1.1,Check.2,Check.3,http://oscal-compass/compliance-trestle/schemas/oscal/cd,Target-B,,, diff --git a/tests/data/json/comp_def_c.json b/tests/data/json/comp_def_c.json new file mode 100644 index 000000000..f56d434d1 --- /dev/null +++ b/tests/data/json/comp_def_c.json @@ -0,0 +1,121 @@ +{ + "component-definition": { + "uuid": "2652b814-2a6b-4b6d-a0ae-8bc7a007209f", + "metadata": { + "title": "comp def c", + "last-modified": "2021-07-19T14:03:03+00:00", + "version": "0.21.0", + "oscal-version": "1.0.2", + "roles": [ + { + "id": "prepared-by", + "title": "Indicates the organization that created this content." + }, + { + "id": "prepared-for", + "title": "Indicates the organization for which this content was created.." + }, + { + "id": "content-approver", + "title": "Indicates the organization responsible for all content represented in the \"document\"." + } + ], + "parties": [ + { + "uuid": "ce1f379a-fcdd-485a-a7b7-6f02c0763dd2", + "type": "organization", + "name": "ACME", + "remarks": "ACME company" + }, + { + "uuid": "481856b6-16e4-4993-a3ed-2fb242ce235b", + "type": "organization", + "name": "Customer", + "remarks": "Customer for the Component Definition" + }, + { + "uuid": "2dc8b17f-daca-44a1-8a1d-c290120ea5e2", + "type": "organization", + "name": "ISV", + "remarks": "ISV for the Component Definition" + } + ], + "responsible-parties": [ + { + "role-id": "prepared-by", + "party-uuids": [ + "ce1f379a-fcdd-485a-a7b7-6f02c0763dd2" + ] + }, + { + "role-id": "prepared-for", + "party-uuids": [ + "481856b6-16e4-4993-a3ed-2fb242ce235b", + "2dc8b17f-daca-44a1-8a1d-c290120ea5e2" + ] + }, + { + "role-id": "content-approver", + "party-uuids": [ + "ce1f379a-fcdd-485a-a7b7-6f02c0763dd2" + ] + } + ] + }, + "components": [ + { + "uuid": "8220b305-0271-45f9-8a21-40ab6f197f70", + "type": "Service", + "title": "comp_cc", + "description": "comp cc", + "control-implementations": [ + { + "uuid": "76e89b67-3d6b-463d-90df-ec56a46c6069", + "source": "trestle://profiles/comp_prof_aa/profile.json", + "description": "trestle comp prof cc", + "implemented-requirements": [ + { + "uuid": "ca5ea4c5-ba51-4b1d-932a-5606891b7500", + "control-id": "ac-1", + "description": "imp req prose for ac-1 from comp cc", + "responsible-roles": [ + { + "role-id": "prepared-by", + "party-uuids": [ + "ce1f379a-fcdd-485a-a7b7-6f02c0763dd2" + ] + }, + { + "role-id": "prepared-for", + "party-uuids": [ + "481856b6-16e4-4993-a3ed-2fb242ce235b", + "2dc8b17f-daca-44a1-8a1d-c290120ea5e2" + ] + }, + { + "role-id": "content-approver", + "party-uuids": [ + "ce1f379a-fcdd-485a-a7b7-6f02c0763dd2" + ] + } + ], + "statements": [ + { + "statement-id": "ac-1_smt.a", + "uuid": "2652b814-2a6b-4b6d-a0ae-8bc7a0072200", + "description": "statement prose for part a. from comp cc" + } + ] + }, + { + "uuid": "ca5ea4c5-ba51-4b1d-932a-5606891b7599", + "control-id": "ac-3", + "description": "imp req prose for ac-3 from comp cc" + } + ] + } + ] + } + ] + } +} \ No newline at end of file diff --git a/tests/data/tasks/csv-to-oscal-cd/test-csv-to-oscal-cd-rule-name-overlap.config b/tests/data/tasks/csv-to-oscal-cd/test-csv-to-oscal-cd-rule-name-overlap.config new file mode 100644 index 000000000..b3604de03 --- /dev/null +++ b/tests/data/tasks/csv-to-oscal-cd/test-csv-to-oscal-cd-rule-name-overlap.config @@ -0,0 +1,7 @@ +[task.csv-to-oscal-cd] + +title = Component definition for validation with rule overlaps +version = V1.1 +csv-file = tests/data/csv/rule-name-overlap.csv +output-dir = tmp +output-overwrite = true diff --git a/tests/trestle/core/base_model_test.py b/tests/trestle/core/base_model_test.py index bc1ff567c..61581bb5c 100644 --- a/tests/trestle/core/base_model_test.py +++ b/tests/trestle/core/base_model_test.py @@ -86,7 +86,7 @@ def test_is_oscal_base() -> None: """Test that the typing information is as expected.""" catalog = simple_catalog() - assert (isinstance(catalog, ospydantic.OscalBaseModel)) + assert isinstance(catalog, ospydantic.OscalBaseModel) def test_no_timezone_exception() -> None: @@ -104,7 +104,7 @@ def test_with_timezone() -> None: popo_json = json.loads(jsoned_catalog) time = popo_json['metadata']['last-modified'] - assert (type(time) == str) + assert isinstance(time, str) assert ('Z' in time or '+' in time or '-' in time) diff --git a/tests/trestle/core/commands/author/component_test.py b/tests/trestle/core/commands/author/component_test.py index 508b14f71..0160371c4 100644 --- a/tests/trestle/core/commands/author/component_test.py +++ b/tests/trestle/core/commands/author/component_test.py @@ -202,3 +202,64 @@ def test_component_generate_more_than_one_param(tmp_trestle_dir: pathlib.Path, m # now assemble component generated assemble_cmd = f'trestle author component-assemble -m {md_path} -n {comp_name} -o {assem_name}' test_utils.execute_command_and_assert(assemble_cmd, CmdReturnCodes.SUCCESS.value, monkeypatch) + + +def test_component_workflow_no_rules(tmp_trestle_dir: pathlib.Path, monkeypatch: MonkeyPatch) -> None: + """Test component generate and assemble with no rules set.""" + comp_name = test_utils.setup_component_generate(tmp_trestle_dir, 'comp_def_c') + ac1_path = tmp_trestle_dir / 'md_comp/comp_cc/comp_prof_aa/ac/ac-1.md' + + orig_component, _ = model_utils.ModelUtils.load_model_for_class( + tmp_trestle_dir, comp_name, comp.ComponentDefinition + ) + + generate_cmd = f'trestle author component-generate -n {comp_name} -o {md_path}' + test_utils.execute_command_and_assert(generate_cmd, CmdReturnCodes.SUCCESS.value, monkeypatch) + + # Check that the example md file looks correct + _, tree = MarkdownProcessor().process_markdown(ac1_path) + + imp_req_md = """## What is the solution and how is it implemented? + + + + + +imp req prose for ac-1 from comp cc + +### Implementation Status: planned + +______________________________________________________________________ +""" # noqa E501 + + node = tree.get_node_for_key('## What is the solution and how is it implemented?') + assert node.content.raw_text == imp_req_md + + part_a_md = """## Implementation for part a. + +statement prose for part a. from comp cc + +### Implementation Status: planned + +______________________________________________________________________""" + + node = tree.get_node_for_key('## Implementation for part a.') + assert node.content.raw_text == part_a_md + + test_utils.substitute_text_in_file( + ac1_path, '### Implementation Status: planned', f'### Implementation Status: {const.STATUS_IMPLEMENTED}' + ) + # Check that the changes make it into the JSON + assem_name = 'assem_comp' + assemble_cmd = f'trestle author component-assemble -m {md_path} -n {comp_name} -o {assem_name}' + test_utils.execute_command_and_assert(assemble_cmd, CmdReturnCodes.SUCCESS.value, monkeypatch) + assem_component, _ = model_utils.ModelUtils.load_model_for_class( + tmp_trestle_dir, assem_name, comp.ComponentDefinition + ) + + # Check the ac-1 implementation status and that the model changed + assert not model_utils.ModelUtils.models_are_equivalent(orig_component, assem_component) # type: ignore + imp_reqs = assem_component.components[0].control_implementations[0].implemented_requirements # type: ignore + imp_req = next((i_req for i_req in imp_reqs if i_req.control_id == 'ac-1'), None) + assert imp_req.description == 'imp req prose for ac-1 from comp cc' + assert ControlInterface.get_status_from_props(imp_req).state == const.STATUS_IMPLEMENTED # type: ignore diff --git a/tests/trestle/core/generator_test.py b/tests/trestle/core/generator_test.py index 5ebe2f4e8..e50b08540 100644 --- a/tests/trestle/core/generator_test.py +++ b/tests/trestle/core/generator_test.py @@ -154,7 +154,7 @@ def test_gen_control() -> None: def test_ensure_optional_exists() -> None: """Explicit test to ensure that optional variables are populated.""" my_catalog = gens.generate_sample_model(catalog.Catalog, include_optional=True, depth=-1) - assert type(my_catalog.controls[0]) == catalog.Control + assert isinstance(my_catalog.controls[0], catalog.Control) def test_gen_party() -> None: diff --git a/tests/trestle/core/jinja/filters_test.py b/tests/trestle/core/jinja/filters_test.py new file mode 100644 index 000000000..24848bf24 --- /dev/null +++ b/tests/trestle/core/jinja/filters_test.py @@ -0,0 +1,69 @@ +# -*- mode:python; coding:utf-8 -*- + +# Copyright (c) 2024 The OSCAL Compass Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for trestle custom jinja filters functionality.""" + +from typing import Any, List, Optional + +import pytest + +from trestle.core.jinja.filters import (diagram_href, first_or_none, get_party, parties_for_role) +from trestle.oscal.common import Link, Party, ResponsibleParty +from trestle.oscal.ssp import Diagram, SystemSecurityPlan + + +@pytest.mark.parametrize( + 'links,expected', + [ + ( + [ + Link(rel='other', href='./path/to/local/thing'), + Link(rel='diagram', href='https://host.name/path/to/diagram.png') + ], + 'https://host.name/path/to/diagram.png' + ), ([Link(rel='other', href='./path/to/local/file')], ''), ([], ''), (None, '') + ] +) +def test_diagram_href(links: Optional[List[Link]], expected: str) -> None: + """Test retrieving the link href for rel='diagram'.""" + diagram = Diagram(uuid='26c1c7df-fb67-45ba-b60f-35d8b5c1d1dc', links=links) + assert diagram_href(diagram) == expected + + +@pytest.mark.parametrize('actual,expected', [[['ok'], 'ok'], ([], None), (None, None)]) +def test_first_or_none(actual: Optional[List[Any]], expected: Optional[Any]) -> None: + """Test behavior of retrieving the first element or None for empty or missing list.""" + assert first_or_none(actual) == expected + + +def test_get_party(sample_system_security_plan: SystemSecurityPlan, sample_party: Party) -> None: + """Test behavior of retrieving a ssp.metadata.parties entry by UUID.""" + assert get_party(sample_party.uuid, ssp=sample_system_security_plan) == sample_party + + +def test_parties_for_role(sample_system_security_plan: SystemSecurityPlan, sample_party: Party) -> None: + """Test behavior of retrieving all parties for a given role-id.""" + sample_system_security_plan.metadata.responsible_parties = [ + ResponsibleParty(role_id='pytest-tester', party_uuids=[sample_party.uuid]) + ] + result = list( + parties_for_role( + sample_system_security_plan.metadata.responsible_parties, + role_id='pytest-tester', + ssp=sample_system_security_plan + ) + ) + assert len(result) == 1 + assert result[0] == sample_party diff --git a/tests/trestle/core/jinja_test.py b/tests/trestle/core/jinja/tags_test.py similarity index 98% rename from tests/trestle/core/jinja_test.py rename to tests/trestle/core/jinja/tags_test.py index 09a888ed4..0885ad047 100644 --- a/tests/trestle/core/jinja_test.py +++ b/tests/trestle/core/jinja/tags_test.py @@ -22,7 +22,7 @@ import pytest -import trestle.core.jinja as tres_jinja +import trestle.core.jinja.tags as tres_jinja from trestle.core.markdown import markdown_const JINJA_MD = 'jinja_markdown_include' diff --git a/tests/trestle/core/remote/cache_test.py b/tests/trestle/core/remote/cache_test.py index 51dbfe99a..1c7fdac39 100644 --- a/tests/trestle/core/remote/cache_test.py +++ b/tests/trestle/core/remote/cache_test.py @@ -276,13 +276,13 @@ def test_fetcher_factory(tmp_trestle_dir: pathlib.Path, monkeypatch: MonkeyPatch as_file_uri('user/oscal_file.json'), as_file_uri('../user/oscal_file.json')]: fetcher = cache.FetcherFactory.get_fetcher(tmp_trestle_dir, uri) - assert type(fetcher) == cache.LocalFetcher + assert isinstance(fetcher, cache.LocalFetcher) # paths with drive letter for uri in ['C:\\Users\\user\\this.json', 'C:/Users/user/this.json', 'C:file.json']: if file_utils.is_windows(): fetcher = cache.FetcherFactory.get_fetcher(tmp_trestle_dir, uri) - assert type(fetcher) == cache.LocalFetcher + assert isinstance(fetcher, cache.LocalFetcher) else: with pytest.raises(TrestleError): cache.FetcherFactory.get_fetcher(tmp_trestle_dir, uri) @@ -291,15 +291,15 @@ def test_fetcher_factory(tmp_trestle_dir: pathlib.Path, monkeypatch: MonkeyPatch monkeypatch.setenv('myusername', 'user123') monkeypatch.setenv('mypassword', 'somep4ss') fetcher = cache.FetcherFactory.get_fetcher(tmp_trestle_dir, https_uri) - assert type(fetcher) == cache.HTTPSFetcher + assert isinstance(fetcher, cache.HTTPSFetcher) sftp_uri = 'sftp://user@hostname:/path/to/file.json' fetcher = cache.FetcherFactory.get_fetcher(tmp_trestle_dir, sftp_uri) - assert type(fetcher) == cache.SFTPFetcher + assert isinstance(fetcher, cache.SFTPFetcher) sftp_uri = 'sftp://user@hostname:2000/path/to/file.json' fetcher = cache.FetcherFactory.get_fetcher(tmp_trestle_dir, sftp_uri) - assert type(fetcher) == cache.SFTPFetcher + assert isinstance(fetcher, cache.SFTPFetcher) def test_fetcher_expiration(tmp_trestle_dir: pathlib.Path) -> None: diff --git a/tests/trestle/tasks/csv_to_oscal_cd_test.py b/tests/trestle/tasks/csv_to_oscal_cd_test.py index 3fd7d2800..88af85754 100644 --- a/tests/trestle/tasks/csv_to_oscal_cd_test.py +++ b/tests/trestle/tasks/csv_to_oscal_cd_test.py @@ -908,10 +908,10 @@ def test_execute_correct_rule_key(tmp_path: pathlib.Path) -> None: retval = tgt.execute() assert retval == TaskOutcome.SUCCESS # insure expected key exists - expected_key = (component_title, component_type, rule_id) + expected_key = (component_title, component_type, rule_id, None, None) assert expected_key in tgt._csv_mgr.get_rule_keys() # insure unexpected key does not exist - unexpected_key = (component_description, component_type, rule_id) + unexpected_key = (component_description, component_type, rule_id, None, None) assert unexpected_key not in tgt._csv_mgr.get_rule_keys() @@ -1345,6 +1345,32 @@ def test_execute_with_ignored_risk_properties(tmp_path: pathlib.Path) -> None: assert prop.name != 'Risk_Adjustment' +def test_execute_rule_name_overlap(tmp_path: pathlib.Path) -> None: + """Test execute rule name overlap.""" + _, section = _get_config_section_init(tmp_path, 'test-csv-to-oscal-cd-rule-name-overlap.config') + rows = _get_rows(section['csv-file']) + with mock.patch('trestle.tasks.csv_to_oscal_cd.csv.reader') as mock_csv_reader: + mock_csv_reader.return_value = rows + tgt = csv_to_oscal_cd.CsvToOscalComponentDefinition(section) + retval = tgt.execute() + assert retval == TaskOutcome.SUCCESS + # read component-definition + fp = pathlib.Path(tmp_path) / 'component-definition.json' + cd = ComponentDefinition.oscal_read(fp) + # spot check + component = cd.components[0] + assert component.type.lower() == 'validation' + assert len(component.props) == 20 + assert component.props[0].name == 'Rule_Id' + assert component.props[0].value == 'RULE-1.1' + assert component.props[3].name == 'Target_Component' + assert component.props[3].value == 'Target-A' + assert component.props[12].name == 'Rule_Id' + assert component.props[12].value == 'RULE-1.1' + assert component.props[15].name == 'Target_Component' + assert component.props[15].value == 'Target-B' + + def test_execute_add_user_property(tmp_path: pathlib.Path) -> None: """Test execute add user property.""" _, section = _get_config_section_init(tmp_path, 'test-csv-to-oscal-cd-bp.config') diff --git a/trestle/cli.py b/trestle/cli.py index badb563f1..918b0e711 100644 --- a/trestle/cli.py +++ b/trestle/cli.py @@ -14,11 +14,8 @@ # limitations under the License. """Starting point for the Trestle CLI.""" -import importlib -import inspect import logging import pathlib -import pkgutil from trestle.common import const, log from trestle.core.commands.assemble import AssembleCmd @@ -38,6 +35,7 @@ from trestle.core.commands.task import TaskCmd from trestle.core.commands.validate import ValidateCmd from trestle.core.commands.version import VersionCmd +from trestle.core.plugins import discovered_plugins logger = logging.getLogger('trestle') @@ -63,29 +61,14 @@ class Trestle(CommandBase): VersionCmd ] - discovered_plugins = { - name: importlib.import_module(name) - for finder, - name, - ispkg in pkgutil.iter_modules() - if name.startswith('trestle_') - } - - logger.debug(discovered_plugins) # This block is uncovered as trestle cannot find plugins in it's unit tests - it is the base module. - for plugin, value in discovered_plugins.items(): # pragma: nocover - for _, module, _ in pkgutil.iter_modules([pathlib.Path(value.__path__[0], 'commands')]): - logger.debug(module) - command_module = importlib.import_module(f'{plugin}.commands.{module}') - clsmembers = inspect.getmembers(command_module, inspect.isclass) - logger.debug(clsmembers) - for _, cmd_cls in clsmembers: - # add commands (derived from CommandPlusDocs or CommandBase) to subcommands list - if issubclass(cmd_cls, CommandBase): - # don't add CommandPlusDocs or CommandBase - if cmd_cls is not CommandPlusDocs and cmd_cls is not CommandBase: - subcommands.append(cmd_cls) - logger.info(f'{cmd_cls} added to subcommands from plugin {plugin}') + for plugin, cmd_cls in discovered_plugins('commands'): # pragma: nocover + # add commands (derived from CommandPlusDocs or CommandBase) to subcommands list + if issubclass(cmd_cls, CommandBase): + # don't add CommandPlusDocs or CommandBase + if cmd_cls is not CommandPlusDocs and cmd_cls is not CommandBase: + subcommands.append(cmd_cls) + logger.info(f'{cmd_cls} added to subcommands from plugin {plugin}') def _init_arguments(self) -> None: self.add_argument('-v', '--verbose', help=const.DISPLAY_VERBOSE_OUTPUT, action='count', default=0) diff --git a/trestle/common/model_utils.py b/trestle/common/model_utils.py index d24991abd..47e901b7c 100644 --- a/trestle/common/model_utils.py +++ b/trestle/common/model_utils.py @@ -68,7 +68,7 @@ def load_distributed( Return a tuple of Model Type (e.g. class 'trestle.oscal.catalog.Catalog'), Model Alias (e.g. 'catalog.metadata') and Instance of the Model. If the model is decomposed/split/distributed, the instance of the model contains - the decomposed models loaded recursively. + the decomposed models loaded recursively. Note: This does not validate the model. You must either validate the model separately or use the load_validate @@ -171,7 +171,8 @@ def load_model_for_class( If you need to load an existing model but its content type may not be known, use this method. But the file content type should be specified if it is somehow known. - Note: This does not validate the model. If you want to validate the model use the load_validate utilities. + Note: + This does not validate the model. If you want to validate the model use the load_validate utilities. """ root_model_path = ModelUtils._root_path_for_top_level_model( trestle_root, model_name, model_class diff --git a/trestle/core/catalog/catalog_writer.py b/trestle/core/catalog/catalog_writer.py index a255dcd12..13cb1e584 100644 --- a/trestle/core/catalog/catalog_writer.py +++ b/trestle/core/catalog/catalog_writer.py @@ -355,31 +355,24 @@ def _update_values(set_param: comp.SetParameter, control_param_dict) -> None: ci_set_params = ControlInterface.get_set_params_from_item(control_imp) for imp_req in as_list(control_imp.implemented_requirements): control_part_id_map = part_id_map.get(imp_req.control_id, {}) - control_rules, statement_rules, _ = ControlInterface.get_rule_list_for_imp_req(imp_req) - if control_rules or statement_rules: - if control_rules: - status = ControlInterface.get_status_from_props(imp_req) - comp_info = ComponentImpInfo(imp_req.description, control_rules, [], status) - self._catalog_interface.add_comp_info(imp_req.control_id, context.comp_name, '', comp_info) - set_params = copy.deepcopy(ci_set_params) - set_params.update(ControlInterface.get_set_params_from_item(imp_req)) - for set_param in set_params.values(): - self._catalog_interface.add_comp_set_param(imp_req.control_id, context.comp_name, set_param) - for statement in as_list(imp_req.statements): - rule_list, _ = ControlInterface.get_rule_list_for_item(statement) - if rule_list: - status = ControlInterface.get_status_from_props(statement) - if statement.statement_id not in control_part_id_map: - label = statement.statement_id - logger.warning( - f'No statement label found for statement id {label}. Defaulting to {label}.' - ) - else: - label = control_part_id_map[statement.statement_id] - comp_info = ComponentImpInfo(statement.description, rule_list, [], status) - self._catalog_interface.add_comp_info( - imp_req.control_id, context.comp_name, label, comp_info - ) + status = ControlInterface.get_status_from_props(imp_req) + control_rules, _ = ControlInterface.get_rule_list_for_item(imp_req) + comp_info = ComponentImpInfo(imp_req.description, control_rules, [], status) + self._catalog_interface.add_comp_info(imp_req.control_id, context.comp_name, '', comp_info) + set_params = copy.deepcopy(ci_set_params) + set_params.update(ControlInterface.get_set_params_from_item(imp_req)) + for set_param in set_params.values(): + self._catalog_interface.add_comp_set_param(imp_req.control_id, context.comp_name, set_param) + for statement in as_list(imp_req.statements): + status = ControlInterface.get_status_from_props(statement) + if statement.statement_id not in control_part_id_map: + label = statement.statement_id + logger.warning(f'No statement label found for statement id {label}. Defaulting to {label}.') + else: + label = control_part_id_map[statement.statement_id] + rule_list, _ = ControlInterface.get_rule_list_for_item(statement) + comp_info = ComponentImpInfo(statement.description, rule_list, [], status) + self._catalog_interface.add_comp_info(imp_req.control_id, context.comp_name, label, comp_info) catalog_merger = CatalogMerger(self._catalog_interface) diff --git a/trestle/core/commands/author/jinja.py b/trestle/core/commands/author/jinja.py index 56d01ca5e..55b2752a8 100644 --- a/trestle/core/commands/author/jinja.py +++ b/trestle/core/commands/author/jinja.py @@ -35,7 +35,7 @@ from trestle.core.commands.common.return_codes import CmdReturnCodes from trestle.core.control_interface import ControlInterface, ParameterRep from trestle.core.docs_control_writer import DocsControlWriter -from trestle.core.jinja import MDCleanInclude, MDDatestamp, MDSectionInclude +from trestle.core.jinja.ext import extensions from trestle.core.profile_resolver import ProfileResolver from trestle.core.ssp_io import SSPMarkdownWriter from trestle.oscal.profile import Profile @@ -191,10 +191,7 @@ def jinja_ify( """Run jinja over an input file with additional booleans.""" template_folder = pathlib.Path.cwd() jinja_env = Environment( - loader=FileSystemLoader(template_folder), - extensions=[MDSectionInclude, MDCleanInclude, MDDatestamp], - trim_blocks=True, - autoescape=True + loader=FileSystemLoader(template_folder), extensions=extensions(), trim_blocks=True, autoescape=True ) template = jinja_env.get_template(str(r_input_file)) # create boolean dict @@ -283,7 +280,7 @@ def jinja_multiple_md( jinja_env = Environment( loader=FileSystemLoader(template_folder), - extensions=[MDSectionInclude, MDCleanInclude, MDDatestamp], + extensions=extensions(), trim_blocks=True, autoescape=True ) @@ -315,7 +312,7 @@ def render_template(template: Template, lut: Dict[str, Any], template_folder: pa dict_loader = DictLoader({str(random_name): new_output}) jinja_env = Environment( loader=ChoiceLoader([dict_loader, FileSystemLoader(template_folder)]), - extensions=[MDCleanInclude, MDSectionInclude, MDDatestamp], + extensions=extensions(), autoescape=True, trim_blocks=True ) diff --git a/trestle/core/control_interface.py b/trestle/core/control_interface.py index 4e92c2bfc..1a2642d20 100644 --- a/trestle/core/control_interface.py +++ b/trestle/core/control_interface.py @@ -713,7 +713,8 @@ def _param_values_assignment_str( param_str = ControlInterface._param_selection_as_str(param, True, False) # otherwise use param aggregated values if present if not param_str: - param_str = ControlInterface._param_as_aggregated_value(param, param_dict, True, False) + if any(as_list(param_dict[prop.value].values) for prop in as_list(param.props) if prop.value in param_dict): + param_str = ControlInterface._param_as_aggregated_value(param, param_dict, True, False) # finally use label and param_id as fallbacks if not param_str: param_str = param.label if param.label else param.id diff --git a/trestle/core/control_reader.py b/trestle/core/control_reader.py index c1881c488..884910ac5 100644 --- a/trestle/core/control_reader.py +++ b/trestle/core/control_reader.py @@ -283,9 +283,6 @@ def read_implemented_requirement(control_file: pathlib.Path, imp_req.statements = [] comp_dict = md_comp_dict[comp_name] for label, comp_info in comp_dict.items(): - # only assemble responses with associated rules - if not comp_info.rules: - continue # if no label it applies to the imp_req itself rather than a statement if not label: imp_req.description = ControlReader._handle_empty_prose(comp_info.prose, control_id) diff --git a/trestle/core/control_writer.py b/trestle/core/control_writer.py index 481e23931..62703bd5f 100644 --- a/trestle/core/control_writer.py +++ b/trestle/core/control_writer.py @@ -129,7 +129,7 @@ def _insert_comp_info( level = 3 if context.purpose == ContextPurpose.COMPONENT else 4 if part_label in comp_info: info = comp_info[part_label] - if context.purpose in [ContextPurpose.COMPONENT, ContextPurpose.SSP] and not info.rules: + if context.purpose in [ContextPurpose.COMPONENT, ContextPurpose.SSP] and not self._include_component(info): return self._md_file.new_paragraph() if info.prose: @@ -266,23 +266,35 @@ def _add_implementation_response_prompts( self._insert_comp_info(part_label, dic, context) self._md_file.new_hr() + def _include_component(self, comp_info: ComponentImpInfo) -> bool: + """ + Check if a component has the required Markdown fields. + + Notes: This is a simple function to centralize logic to check + when a component meets the requirement to get written to Markdown. + """ + if comp_info.rules or comp_info.prose: + return True + return False + def _skip_part(self, context: ControlContext, part_label: str, comp_dict: CompDict) -> bool: """ Check if a part should be skipped based on rules and context. Notes: The default logic is to conditionally add control parts based - on whether the component has rules associated with that part. This can be + on whether the component has rules or existing prose associated with that part. This can be changed using the control context for SSP markdown. """ if context.purpose == ContextPurpose.SSP and context.include_all_parts: return False else: - no_applied_rules = True + skip_item = True for _, dic in comp_dict.items(): - if part_label in dic and dic[part_label].rules: - no_applied_rules = False - break - return no_applied_rules + if part_label in dic: + if self._include_component(dic[part_label]): + skip_item = False + break + return skip_item def _dump_subpart_infos(self, level: int, part: Dict[str, Any]) -> None: name = part['name'] diff --git a/trestle/core/jinja/__init__.py b/trestle/core/jinja/__init__.py new file mode 100644 index 000000000..348baa8dc --- /dev/null +++ b/trestle/core/jinja/__init__.py @@ -0,0 +1,16 @@ +# -*- mode:python; coding:utf-8 -*- + +# Copyright (c) 2024 The OSCAL Compass Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Trestle core.jinja functionality.""" diff --git a/trestle/core/jinja/base.py b/trestle/core/jinja/base.py new file mode 100644 index 000000000..5e851f149 --- /dev/null +++ b/trestle/core/jinja/base.py @@ -0,0 +1,48 @@ +# -*- mode:python; coding:utf-8 -*- + +# Copyright (c) 2024 The OSCAL Compass Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Trestle core.jinja base class.""" +from jinja2 import lexer, nodes +from jinja2.environment import Environment +from jinja2.ext import Extension + + +class TrestleJinjaExtension(Extension): + """Class to define common methods to be inherited from for use in trestle.""" + + max_tag_parse = 20 + + def __init__(self, environment: Environment) -> None: + """Ensure enviroment is set and carried into class vars.""" + super().__init__(environment) + + @staticmethod + def parse_expression(parser): + """Safely parse jinja expression.""" + # Licensed under MIT from: + # https://github.com/MoritzS/jinja2-django-tags/blob/master/jdj_tags/extensions.py#L424 + # Due to how the jinja2 parser works, it treats "foo" "bar" as a single + # string literal as it is the case in python. + # But the url tag in django supports multiple string arguments, e.g. + # "{% url 'my_view' 'arg1' 'arg2' %}". + # That's why we have to check if it's a string literal first. + token = parser.stream.current + if token.test(lexer.TOKEN_STRING): + expr = nodes.Const(token.value, lineno=token.lineno) + next(parser.stream) + else: + expr = parser.parse_expression(False) + + return expr diff --git a/trestle/core/jinja/ext.py b/trestle/core/jinja/ext.py new file mode 100644 index 000000000..09795b118 --- /dev/null +++ b/trestle/core/jinja/ext.py @@ -0,0 +1,40 @@ +# -*- mode:python; coding:utf-8 -*- + +# Copyright (c) 2024 The OSCAL Compass Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Trestle core.jinja extension loading functionality.""" +import functools +import logging +from typing import List + +from trestle.core.jinja import filters, tags +from trestle.core.jinja.base import Extension, TrestleJinjaExtension +from trestle.core.plugins import discovered_plugins + +logger = logging.getLogger(__name__) + + +@functools.cache +def extensions() -> List[Extension]: + """Return list of Jinja extensions packaged with compliance-trestle and included from plugins.""" + extensions = [tags.MDSectionInclude, tags.MDCleanInclude, tags.MDDatestamp, filters.JinjaSSPFilters] + # This block is uncovered as trestle cannot find plugins in it's unit tests - it is the base module. + for plugin, ext_cls in discovered_plugins('jinja_ext'): # pragma: nocover + # add extensions (derived from TrestleJinjaExtension) to extensions list + if issubclass(ext_cls, Extension): + # don't add Extension or TrestleJinjaExtension + if ext_cls is not TrestleJinjaExtension and ext_cls is not Extension: + extensions.append(ext_cls) + logger.info(f'{ext_cls} added to jinja extensions from plugin {plugin}') + return extensions diff --git a/trestle/core/jinja/filters.py b/trestle/core/jinja/filters.py new file mode 100644 index 000000000..574281999 --- /dev/null +++ b/trestle/core/jinja/filters.py @@ -0,0 +1,76 @@ +# -*- mode:python; coding:utf-8 -*- + +# Copyright (c) 2024 The OSCAL Compass Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Trestle utilities to customize jinja filters.""" +import logging +from typing import Any, Iterator, List, Optional + +from jinja2.environment import Environment + +from trestle.common.list_utils import as_list, get_default +from trestle.core.jinja.base import TrestleJinjaExtension +from trestle.oscal.common import Party, ResponsibleParty +from trestle.oscal.ssp import Diagram, SystemSecurityPlan + +logger = logging.getLogger(__name__) + + +def first_or_none(value: Optional[List[Any]]) -> Optional[Any]: + """Retrieve the first array entry, or None for lists that are None or empty.""" + return next(iter(as_list(value)), None) + + +def get_party(uuid: str, ssp: SystemSecurityPlan) -> Optional[Party]: + """Get the metadata.parties entry for this UUID.""" + return next((x for x in as_list(ssp.metadata.parties) if x.uuid == uuid), None) + + +def parties_for_role(responsible_parties: List[ResponsibleParty], role_id: str, + ssp: SystemSecurityPlan) -> Iterator[Party]: + """Get a list of parties from a list of responsible_parties and a given role_id.""" + logger.debug(f'Finding parties for role: {role_id}') + for responsible_party in as_list(responsible_parties): + if responsible_party.role_id == role_id: + logger.debug( + f'Found responsible party for role_id: {role_id} with {len(responsible_party.party_uuids)} parties' + ) + for uuid in responsible_party.party_uuids: + logger.debug(f'Looking for parties with uuid: {uuid}') + party = get_party(uuid, ssp) + if party: + yield party + + +def diagram_href(diagram: Optional[Diagram]) -> str: + """Retrieve the diagrams's link href.""" + if diagram: + return next((link.href for link in as_list(diagram.links) if link.rel == 'diagram'), '') + else: + return '' + + +class JinjaSSPFilters(TrestleJinjaExtension): + """Collection of useful OSCAL-specific filters.""" + + def __init__(self, environment: Environment) -> None: + """Initialize class and add filters.""" + super(JinjaSSPFilters, self).__init__(environment) + + environment.filters['as_list'] = as_list + environment.filters['get_default'] = get_default + environment.filters['first_or_none'] = first_or_none + environment.filters['get_party'] = get_party + environment.filters['parties_for_role'] = parties_for_role + environment.filters['diagram_href'] = diagram_href diff --git a/trestle/core/jinja.py b/trestle/core/jinja/tags.py similarity index 86% rename from trestle/core/jinja.py rename to trestle/core/jinja/tags.py index 3f97e219f..4ce11f725 100644 --- a/trestle/core/jinja.py +++ b/trestle/core/jinja/tags.py @@ -13,18 +13,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Trestle utilities to customize .""" +"""Trestle utilities to customize jinja tags.""" import logging from datetime import date import frontmatter -from jinja2 import lexer, nodes +from jinja2 import lexer from jinja2.environment import Environment -from jinja2.ext import Extension from jinja2.parser import Parser from trestle.common import err +from trestle.core.jinja.base import TrestleJinjaExtension from trestle.core.markdown import docs_markdown_node, markdown_const logger = logging.getLogger(__name__) @@ -43,36 +43,6 @@ def adjust_heading_level(input_md: str, expected: int) -> str: return output_md -class TrestleJinjaExtension(Extension): - """Class to define common methods to be inherited from for use in trestle.""" - - # This - max_tag_parse = 20 - - def __init__(self, environment: Environment) -> None: - """Ensure enviroment is set and carried into class vars.""" - super().__init__(environment) - - @staticmethod - def parse_expression(parser): - """Safely parse jinja expression.""" - # Licensed under MIT from: - # https://github.com/MoritzS/jinja2-django-tags/blob/master/jdj_tags/extensions.py#L424 - # Due to how the jinja2 parser works, it treats "foo" "bar" as a single - # string literal as it is the case in python. - # But the url tag in django supports multiple string arguments, e.g. - # "{% url 'my_view' 'arg1' 'arg2' %}". - # That's why we have to check if it's a string literal first. - token = parser.stream.current - if token.test(lexer.TOKEN_STRING): - expr = nodes.Const(token.value, lineno=token.lineno) - next(parser.stream) - else: - expr = parser.parse_expression(False) - - return expr - - class MDSectionInclude(TrestleJinjaExtension): """Inject the parameter of the tag as the resulting content.""" diff --git a/trestle/core/markdown/markdown_validator.py b/trestle/core/markdown/markdown_validator.py index 3cd91cea5..35b8ce30c 100644 --- a/trestle/core/markdown/markdown_validator.py +++ b/trestle/core/markdown/markdown_validator.py @@ -181,8 +181,8 @@ def compare_keys( return False for key in template.keys(): if key in candidate.keys(): - if type(template[key]) == dict: - if type(candidate[key]) == dict: + if isinstance(template[key], dict): + if isinstance(candidate[key], dict): status = cls.compare_keys(template[key], candidate[key], ignore_fields) if not status: return status diff --git a/trestle/core/plugins.py b/trestle/core/plugins.py new file mode 100644 index 000000000..80d10b512 --- /dev/null +++ b/trestle/core/plugins.py @@ -0,0 +1,47 @@ +# -*- mode:python; coding:utf-8 -*- + +# Copyright (c) 2024 The OSCAL Compass Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Plugin discovery code.""" + +import importlib +import inspect +import logging +import pathlib +import pkgutil +from typing import Any, Iterator, Tuple + +logger = logging.getLogger(__name__) + +_discovered_plugins = { + name: importlib.import_module(name) + for finder, + name, + ispkg in pkgutil.iter_modules() + if name.startswith('trestle_') +} + + +def discovered_plugins(search_module: str) -> Iterator[Tuple[str, Any]]: + """Yield discovered plugin classes within a given module name.""" + logger.debug(_discovered_plugins) + # This block is uncovered as trestle cannot find plugins in it's unit tests - it is the base module. + for plugin, value in _discovered_plugins.items(): # pragma: nocover + for _, module, _ in pkgutil.iter_modules([pathlib.Path(value.__path__[0], search_module)]): + logger.debug(module) + plugin_module = importlib.import_module(f'{plugin}.{search_module}.{module}') + clsmembers = inspect.getmembers(plugin_module, inspect.isclass) + logger.debug(clsmembers) + for _, plugin_cls in clsmembers: + yield (plugin, plugin_cls) diff --git a/trestle/tasks/cis_xlsx_to_oscal_catalog.py b/trestle/tasks/cis_xlsx_to_oscal_catalog.py index 1fd5d52d5..3eff90df5 100644 --- a/trestle/tasks/cis_xlsx_to_oscal_catalog.py +++ b/trestle/tasks/cis_xlsx_to_oscal_catalog.py @@ -41,7 +41,7 @@ logger = logging.getLogger(__name__) -timestamp = datetime.datetime.utcnow().replace(microsecond=0).replace(tzinfo=datetime.timezone.utc).isoformat() +timestamp = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() class XlsxHelper: diff --git a/trestle/tasks/csv_to_oscal_cd.py b/trestle/tasks/csv_to_oscal_cd.py index a7343e02d..c8bf32dff 100644 --- a/trestle/tasks/csv_to_oscal_cd.py +++ b/trestle/tasks/csv_to_oscal_cd.py @@ -52,6 +52,7 @@ RULE_DESCRIPTION = 'Rule_Description' PROFILE_SOURCE = 'Profile_Source' PROFILE_DESCRIPTION = 'Profile_Description' +TARGET_COMPONENT = 'Target_Component' CHECK_ID = 'Check_Id' CHECK_DESCRIPTION = 'Check_Description' FETCHER_ID = 'Fetcher_Id' @@ -145,6 +146,26 @@ def row_property_builder(row: int, name: str, value, ns: str, class_: str, remar return prop +def is_validation(component_type: str) -> bool: + """Check for validation component.""" + return component_type.lower().strip() == validation + + +def synthesize_rule_key( + component_title: str, + component_type: str, + rule_id: str, + check_id: Union[str, None], + target_component: Union[str, None] +) -> tuple: + """Synthesize rule_key.""" + if is_validation(component_type): + rval = (component_title, component_type, rule_id, check_id, target_component) + else: + rval = (component_title, component_type, rule_id, None, None) + return rval + + class CsvToOscalComponentDefinition(TaskBase): """ Task to create OSCAL ComponentDefinition json. @@ -168,7 +189,13 @@ def print_info(self) -> None: """Print the help string.""" name = self.name oscal_name = 'component_definition' - # + # help note identifiers + note01 = '1' + note02 = '2' + note03 = '3' + note04 = '4' + note05 = '5' + # help generation logger.info(f'Help information for {name} task.') logger.info('') logger.info(f'Purpose: From csv produce OSCAL {oscal_name} file.') @@ -188,23 +215,25 @@ def print_info(self) -> None: text1 = ' required columns: ' for text2 in CsvColumn.get_required_column_names(): if text2 in [f'{RULE_DESCRIPTION}', f'{PROFILE_SOURCE}', f'{PROFILE_DESCRIPTION}', f'{CONTROL_ID_LIST}']: - text2 += ' (see note 1)' + text2 += f' (see note {note01})' logger.info(text1 + '$$' + text2) text1 = ' ' text1 = ' optional columns: ' for text2 in CsvColumn.get_optional_column_names(): if text2 in [f'{ORIGINAL_RISK_RATING}', f'{ADJUSTED_RISK_RATING}', f'{RISK_ADJUSTMENT}']: - text2 += ' (see note 1)' + text2 += f' (see note {note01})' + elif text2 in [f'{TARGET_COMPONENT}']: + text2 += f' (see note {note03})' else: - text2 += ' (see note 2)' + text2 += f' (see note {note02})' logger.info(text1 + '$' + text2) text1 = ' ' for text2 in CsvColumn.get_parameter_column_names(): - text2 += ' (see notes 1, 4)' + text2 += f' (see notes {note01}, {note05})' logger.info(text1 + '$' + text2) text1 = ' ' text1 = ' comment columns: ' - text2 = 'Informational (see note 3)' + text2 = f'Informational (see note {note04})' logger.info(text1 + '#' + text2) text1 = ' output-dir = ' text2 = '(required) the path of the output directory for synthesized OSCAL .json files.' @@ -226,23 +255,26 @@ def print_info(self) -> None: text2 = '' logger.info(text1 + text2) text1 = 'Notes: ' - text2 = '[1] column is ignored for validation component type' + text2 = f'[{note01}] column is ignored for validation component type' logger.info(text1 + text2) text1 = ' ' - text2 = '[2] column is required for validation component type' + text2 = f'[{note02}] column is required for validation component type' logger.info(text1 + text2) text1 = ' ' - text2 = '[3] column name starting with # causes column to be ignored' + text2 = f'[{note03}] column is optional for validation component type' + text3 = f', but may be needed to prevent {RULE_ID} collisions' + logger.info(text1 + text2 + text3) + text1 = ' ' + text2 = f'[{note04}] column name starting with # causes column to be ignored' logger.info(text1 + text2) text1 = ' ' - text2 = '[4] additional parameters are specified by adding a common suffix per set' + text2 = f'[{note05}] additional parameters are specified by adding a common suffix per set' text3 = f', for example: {PARAMETER_ID}_1, {PARAMETER_DESCRIPTION}_1, ...{PARAMETER_ID}_2...' logger.info(text1 + text2 + text3) def configure(self) -> bool: """Configure.""" - self._timestamp = datetime.datetime.utcnow().replace(microsecond=0).replace(tzinfo=datetime.timezone.utc - ).isoformat() + self._timestamp = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() # config verbosity self._quiet = self._config.get('quiet', False) self._verbose = not self._quiet @@ -388,7 +420,7 @@ def _calculate_set_params(self, mod_rules: List) -> tuple: add_set_params = [] mod_set_params = [] for key in cd_set_params: - rule_key = (key[0], key[1], key[2]) + rule_key = synthesize_rule_key(key[0], key[1], key[2], None, None) if rule_key not in mod_rules: continue if key in csv_set_params: @@ -397,7 +429,7 @@ def _calculate_set_params(self, mod_rules: List) -> tuple: del_set_params.append(key) logger.debug(f'params del: {key}') for key in csv_set_params: - rule_key = (key[0], key[1], key[2]) + rule_key = synthesize_rule_key(key[0], key[1], key[2], None, None) if rule_key not in mod_rules: continue if key in cd_set_params: @@ -416,7 +448,7 @@ def _calculate_control_mappings(self, mod_rules: List) -> tuple: add_control_mappings = [] mod_control_mappings = [] for key in cd_controls: - rule_key = (key[0], key[1], key[2]) + rule_key = synthesize_rule_key(key[0], key[1], key[2], None, None) if rule_key not in mod_rules: continue if key in csv_controls: @@ -425,7 +457,7 @@ def _calculate_control_mappings(self, mod_rules: List) -> tuple: del_control_mappings.append(key) logger.debug(f'ctl-maps del: {key}') for key in csv_controls: - rule_key = (key[0], key[1], key[2]) + rule_key = synthesize_rule_key(key[0], key[1], key[2], None, None) if rule_key not in mod_rules: continue if key in cd_controls: @@ -580,7 +612,7 @@ def rules_add(self, add_rules: List[str]) -> None: def _is_validation(self, rule_key: tuple) -> bool: """Check for validation component.""" component_type = self._csv_mgr.get_value(rule_key, COMPONENT_TYPE) - return component_type.lower() == validation + return is_validation(component_type) def _add_rule_prop( self, control_implementation: ControlImplementation, control_mappings: List[str], rule_key: tuple @@ -770,6 +802,7 @@ def set_params_del(self, del_set_params: List[str]) -> None: source = tokens[3] description = tokens[4] param_id = tokens[5] + # ctl impl control_implementation = self._cd_mgr.find_control_implementation( component_title, component_type, source, description ) @@ -792,12 +825,13 @@ def set_params_add(self, add_set_params: List[str]) -> None: source = tokens[3] description = tokens[4] param_id = tokens[5] + # ctl impl control_implementation = self._cd_mgr.find_control_implementation( component_title, component_type, source, description ) control_implementation.set_parameters = as_list(control_implementation.set_parameters) # add - rule_key = _CsvMgr.get_rule_key(component_title, component_type, rule_id) + rule_key = synthesize_rule_key(component_title, component_type, rule_id, None, None) values = [self._csv_mgr.get_default_value_by_id(rule_key, param_id)] set_parameter = SetParameter( param_id=param_id, @@ -814,6 +848,7 @@ def set_params_mod(self, mod_set_params: List[str]) -> None: source = tokens[3] description = tokens[4] param_id = tokens[5] + # ctl impl control_implementation = self._cd_mgr.find_control_implementation( component_title, component_type, source, description ) @@ -822,7 +857,7 @@ def set_params_mod(self, mod_set_params: List[str]) -> None: for set_parameter in self._set_parameter_generator(set_parameters): if set_parameter.param_id != param_id: continue - rule_key = _CsvMgr.get_rule_key(component_title, component_type, rule_id) + rule_key = synthesize_rule_key(component_title, component_type, rule_id, None, None) values = [self._csv_mgr.get_default_value_by_id(rule_key, param_id)] replacement = SetParameter( param_id=param_id, @@ -855,6 +890,7 @@ def control_mappings_del(self, del_control_mappings: List[str]) -> None: source = tokens[3] description = tokens[4] smt_id = tokens[5] + # ctl-id control_id = derive_control_id(smt_id) control_implementation = self._cd_mgr.find_control_implementation( component_title, component_type, source, description @@ -881,13 +917,14 @@ def control_mappings_add(self, add_control_mappings: List[str]) -> None: source = tokens[3] description = tokens[4] smt_id = tokens[5] + # ctl-id control_id = derive_control_id(smt_id) control_implementation = self._cd_mgr.find_control_implementation( component_title, component_type, source, description ) implemented_requirement = self._get_implemented_requirement(control_implementation, control_id) # namespace - rule_key = (tokens[0], tokens[1], tokens[2]) + rule_key = synthesize_rule_key(tokens[0], tokens[1], tokens[2], None, None) ns = self._get_namespace(rule_key) # create rule implementation (as property) name = RULE_ID @@ -1191,7 +1228,7 @@ def accounting_rule_definitions(self, component: DefinedComponent) -> None: if component.props: for prop in component.props: if prop.name == RULE_ID: - key = (component.title, component.type, prop.value) + key = synthesize_rule_key(component.title, component.type, prop.value, None, None) value = prop.remarks self._cd_rules_map[key] = value logger.debug(f'cd: {key} {self._cd_rules_map[key]}') @@ -1381,6 +1418,7 @@ class CsvColumn(): _columns_optional = [ f'{CHECK_ID}', f'{CHECK_DESCRIPTION}', + f'{TARGET_COMPONENT}', f'{ORIGINAL_RISK_RATING}', f'{ADJUSTED_RISK_RATING}', f'{RISK_ADJUSTMENT}', @@ -1457,6 +1495,7 @@ def get_required_column_names_validation() -> List[str]: f'{PARAMETER_VALUE_ALTERNATIVES}', f'{CHECK_ID}', f'{CHECK_DESCRIPTION}', + f'{TARGET_COMPONENT}', f'{ORIGINAL_RISK_RATING}', f'{ADJUSTED_RISK_RATING}', f'{RISK_ADJUSTMENT}', @@ -1505,6 +1544,7 @@ def get_filtered_optional_column_names() -> List[str]: f'{RULE_ID}', f'{CHECK_ID}', f'{CHECK_DESCRIPTION}', + f'{TARGET_COMPONENT}', ] @staticmethod @@ -1553,7 +1593,9 @@ def __init__(self, csv_path: pathlib.Path) -> None: component_description = self.get_row_value(row, f'{COMPONENT_DESCRIPTION}') rule_id = self.get_row_value(row, f'{RULE_ID}') # rule sets - key = _CsvMgr.get_rule_key(component_title, component_type, rule_id) + check_id = self.get_row_value(row, f'{CHECK_ID}', default=None) + target_component = self.get_row_value(row, f'{TARGET_COMPONENT}', default=None) + key = synthesize_rule_key(component_title, component_type, rule_id, check_id, target_component) if key in self._csv_rules_map: text = f'row "{row_num}" contains duplicate {RULE_ID} "{rule_id}"' raise RuntimeError(text) @@ -1594,11 +1636,6 @@ def _control_mappings( key = (component_description, component_type, rule_id, source, description, control) self._csv_controls_map[key] = [row_num, row] - @staticmethod - def get_rule_key(component_title: str, component_type: str, rule_id: str) -> tuple: - """Get rule_key.""" - return (component_title, component_type, rule_id) - def get_parameter_id_column_names(self) -> List[str]: """Get parameter_id column_names.""" col_names = [] @@ -1725,9 +1762,9 @@ def get_row_number(self, rule_key: tuple) -> List: """Get row number for rule.""" return self._csv_rules_map[rule_key][0] - def get_row_value(self, row: List[str], name: str) -> str: + def get_row_value(self, row: List[str], name: str, default='') -> str: """Get value for specified name.""" - rval = '' + rval = default index = self.get_col_index(name) if index >= 0: rval = row[index] diff --git a/trestle/tasks/ocp4_cis_profile_to_oscal_catalog.py b/trestle/tasks/ocp4_cis_profile_to_oscal_catalog.py index fc2c00596..4b0f48f18 100644 --- a/trestle/tasks/ocp4_cis_profile_to_oscal_catalog.py +++ b/trestle/tasks/ocp4_cis_profile_to_oscal_catalog.py @@ -64,8 +64,7 @@ def __init__(self, config_object: Optional[configparser.SectionProxy]) -> None: config_object: Config section associated with the task. """ super().__init__(config_object) - self._timestamp = datetime.datetime.utcnow().replace(microsecond=0).replace(tzinfo=datetime.timezone.utc - ).isoformat() + self._timestamp = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() def print_info(self) -> None: """Print the help string.""" diff --git a/trestle/tasks/ocp4_cis_profile_to_oscal_cd.py b/trestle/tasks/ocp4_cis_profile_to_oscal_cd.py index f04d4eaa0..cfc1cf7bb 100644 --- a/trestle/tasks/ocp4_cis_profile_to_oscal_cd.py +++ b/trestle/tasks/ocp4_cis_profile_to_oscal_cd.py @@ -63,8 +63,7 @@ def __init__(self, config_object: Optional[configparser.SectionProxy]) -> None: config_object: Config section associated with the task. """ super().__init__(config_object) - self._timestamp = datetime.datetime.utcnow().replace(microsecond=0).replace(tzinfo=datetime.timezone.utc - ).isoformat() + self._timestamp = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() def set_timestamp(self, timestamp: str) -> None: """Set the timestamp.""" diff --git a/trestle/tasks/oscal_catalog_to_csv.py b/trestle/tasks/oscal_catalog_to_csv.py index b68ed8457..73b1d5b2e 100644 --- a/trestle/tasks/oscal_catalog_to_csv.py +++ b/trestle/tasks/oscal_catalog_to_csv.py @@ -37,7 +37,7 @@ logger = logging.getLogger(__name__) -timestamp = datetime.datetime.utcnow().replace(microsecond=0).replace(tzinfo=datetime.timezone.utc).isoformat() +timestamp = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() recurse = True diff --git a/trestle/tasks/xlsx_to_oscal_cd.py b/trestle/tasks/xlsx_to_oscal_cd.py index fb966f15e..d55acf303 100644 --- a/trestle/tasks/xlsx_to_oscal_cd.py +++ b/trestle/tasks/xlsx_to_oscal_cd.py @@ -65,8 +65,7 @@ def __init__(self, config_object: Optional[configparser.SectionProxy]) -> None: """ super().__init__(config_object) self.xlsx_helper = XlsxHelper() - self._timestamp = datetime.datetime.utcnow().replace(microsecond=0).replace(tzinfo=datetime.timezone.utc - ).isoformat() + self._timestamp = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() def set_timestamp(self, timestamp: str) -> None: """Set the timestamp.""" diff --git a/trestle/tasks/xlsx_to_oscal_profile.py b/trestle/tasks/xlsx_to_oscal_profile.py index faaa2ce74..84d587e0c 100644 --- a/trestle/tasks/xlsx_to_oscal_profile.py +++ b/trestle/tasks/xlsx_to_oscal_profile.py @@ -57,8 +57,7 @@ def __init__(self, config_object: Optional[configparser.SectionProxy]) -> None: """ super().__init__(config_object) self.xlsx_helper = XlsxHelper() - self._timestamp = datetime.datetime.utcnow().replace(microsecond=0).replace(tzinfo=datetime.timezone.utc - ).isoformat() + self._timestamp = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() def set_timestamp(self, timestamp: str) -> None: """Set the timestamp.""" diff --git a/trestle/transforms/transformer_factory.py b/trestle/transforms/transformer_factory.py index 99f23e6c2..ba84bb4f3 100644 --- a/trestle/transforms/transformer_factory.py +++ b/trestle/transforms/transformer_factory.py @@ -28,7 +28,7 @@ class TransformerBase(ABC): """Abstract base interface for all transformers.""" # the current time for consistent timestamping - _timestamp = datetime.datetime.utcnow().replace(microsecond=0).replace(tzinfo=datetime.timezone.utc).isoformat() + _timestamp = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() @staticmethod def set_timestamp(value: str) -> None: