diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..58eca65fd --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,3 @@ +# These owners will be the default owners for everything in +# the repo. +* @oscal-compass/compliance-trestle-maintainers \ No newline at end of file diff --git a/.github/workflows/docs-update.yml b/.github/workflows/docs-update.yml new file mode 100644 index 000000000..8e4d0951c --- /dev/null +++ b/.github/workflows/docs-update.yml @@ -0,0 +1,75 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions +name: Trestle Docs update +on: + push: + branches: + - develop + tags: + - v* + +jobs: + set-versions: + runs-on: ubuntu-latest + outputs: + min: ${{ steps.versions.outputs.min }} + max: ${{ steps.versions.outputs.max }} + steps: + - uses: actions/checkout@v4 + - id: versions + run: | + min_version=$(jq '.PYTHON_MIN' -r version.json) + max_version=$(jq '.PYTHON_MAX' -r version.json) + echo "min=$min_version" + echo "max=$max_version" + echo "min=$min_version" >> $GITHUB_OUTPUT + echo "max=$max_version" >> $GITHUB_OUTPUT + mike-version: + runs-on: ubuntu-latest + needs: [ set-versions] + outputs: + mver: ${{ steps.versions.mver }} + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ needs.set-versions.outputs.max }} + uses: actions/setup-python@v5 + # This is deliberately not using a custom credential as it relies on native github actions token to have push rights. + with: + python-version: ${{ needs.set-versions.outputs.max }} + - id: versions + run: | + mike_version=$(python ./scripts/check-version.py ${{ env.GITHUB_REF }}) + echo "mver=$mike_version" >> $GITHUB_OUTPUT + deploy-docs: + runs-on: ubuntu-latest + needs: [ mike-version ] + steps: + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.PRIVATE_KEY }} + - uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 0 + token: ${{ steps.app-token.outputs.token }} + - name: Set up Python ${{ needs.set-versions.outputs.max }} + uses: actions/setup-python@v5 + # This is deliberately not using a custom credential as it relies on native github actions token to have push rights. + with: + python-version: ${{ needs.set-versions.outputs.max }} + - name: Install build tools + run: | + make develop + - name: Install documenation dependencies + run: | + make docs-ubuntu-deps + - name: Create release + shell: bash + run: | + mike deploy ${{ needs.mike-version.outputs.mver }} + - name: Ensure latest is latest + shell: bash + run: | + mike set-default latest \ No newline at end of file diff --git a/.github/workflows/python-push.yml b/.github/workflows/python-push.yml index 9d5909115..2ef301f4b 100644 --- a/.github/workflows/python-push.yml +++ b/.github/workflows/python-push.yml @@ -153,41 +153,6 @@ jobs: with: github_token: ${{ steps.app-token.outputs.token }} - deploy-docs: - runs-on: ubuntu-latest - needs: [ deploy, set-versions ] - concurrency: - group: ${{ github.ref }}-${{ github.workflow }}-${{ github.job }}-docs - cancel-in-progress: true - # Temporary hack: allow develop as well as master to deploy docs. - if: github.ref == 'refs/heads/main' - steps: - - uses: actions/create-github-app-token@v1 - id: app-token - with: - app-id: ${{ secrets.APP_ID }} - private-key: ${{ secrets.PRIVATE_KEY }} - - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 0 - token: ${{ steps.app-token.outputs.token }} - - name: Set up Python ${{ needs.set-versions.outputs.max }} - uses: actions/setup-python@v5 - # This is deliberately not using a custom credential as it relies on native github actions token to have push rights. - with: - python-version: ${{ needs.set-versions.outputs.max }} - - name: Install build tools - run: | - make develop - - name: Install documenation dependencies - run: | - make docs-ubuntu-deps - - name: Create release - shell: bash - run: | - mkdocs gh-deploy - merge-main-to-develop: name: Merge main -> develop runs-on: ubuntu-latest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7b9622fde..fdd4c9759 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -28,9 +28,11 @@ pull request so it can be tracked. ### Merge approval The project maintainers use LGTM (Looks Good To Me) in comments on the code -review to indicate acceptance. A change requires LGTMs from one of the maintainers. +review to indicate acceptance. -For a list of the maintainers, see the [maintainers](https://oscal-compass.github.io/compliance-trestle/maintainers/) page. +A change requires LGTMs from at least two reviewers. One of the reviewers must be a [`CODEOWNER`](https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners). + +For a list of the maintainers (also codeowners), see the [maintainers](https://oscal-compass.github.io/compliance-trestle/maintainers/) page. ### Trestle updating, testing and release logistics diff --git a/docs/js/.keep b/docs/js/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/mkdocs.yml b/mkdocs.yml index 51c762f3e..5f808ec26 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -36,7 +36,7 @@ plugins: raise_error_excludes: # This is to remove some false positives for links which work. # Anchors are validated again by core mkdocs - 404: ['#trestle.*'] + 404: [ '#trestle.*' ] - mkdocstrings: default_handler: python handlers: @@ -61,6 +61,15 @@ plugins: - css/mkdocstrings.css htmlmin_opts: remove_comments: true +- mike: + # These fields are all optional; the defaults are as below... + alias_type: symlink + redirect_template: null + deploy_prefix: '' + canonical_version: latest + version_selector: true + css_dir: css + javascript_dir: js repo_name: oscal-compass/compliance-trestle repo_url: https://github.com/oscal-compass/compliance-trestle site_description: Documentation for compliance-trestle package. diff --git a/setup.cfg b/setup.cfg index d965db771..882f204a8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,7 @@ include_package_data = True install_requires = attrs ilcli - cryptography==42.0.4 + cryptography==43.0.3 paramiko==3.4.0 ruamel.yaml furl @@ -79,6 +79,7 @@ dev = types-requests types-setuptools # # Docs website + mike mkdocs>=1.6.0 mkdocs-awesome-pages-plugin mkdocstrings[python]>=0.25.2 diff --git a/tests/trestle/core/control_io_test.py b/tests/trestle/core/control_io_test.py index 1ecf96985..5a6744863 100644 --- a/tests/trestle/core/control_io_test.py +++ b/tests/trestle/core/control_io_test.py @@ -30,7 +30,7 @@ from trestle.common.model_utils import ModelUtils from trestle.core.catalog.catalog_interface import CatalogInterface from trestle.core.control_context import ContextPurpose, ControlContext -from trestle.core.control_interface import ControlInterface, ParameterRep +from trestle.core.control_interface import ComponentImpInfo, ControlInterface, ParameterRep from trestle.core.control_reader import ControlReader from trestle.core.control_writer import ControlWriter from trestle.core.markdown.control_markdown_node import ControlMarkdownNode, tree_context @@ -45,7 +45,7 @@ case_3 = 'indent end abrupt' case_4 = 'no items' -control_text = """--- +control_text = r"""--- x-trestle-global: sort-id: xy-09 --- @@ -131,7 +131,7 @@ def test_read_write_controls( part_b3 = common.Part(id='ac-1_smt.b.3', name='item', prose='b.3 prose', props=[prop]) prop.value = 'c' part_c = common.Part(id='ac-1_smt.c', name='item', prose='c prose', props=[prop]) - sec_1_text = """ + sec_1_text = r""" General comment on separate lines @@ -395,7 +395,47 @@ def test_write_control_header_params(overwrite_header_values, tmp_path: pathlib. assert test_utils.controls_equivalent(orig_control_read, new_control_read) -statement_text = """ +def test_merge_control_update(tmp_path: pathlib.Path, testdata_dir: pathlib.Path) -> None: + """Test merging of control header params after spec update.""" + src_control_path = pathlib.Path(testdata_dir / 'author/controls/control_with_components.md') + control_path = tmp_path / 'ac-1.md' + shutil.copyfile(src_control_path, control_path) + orig_control_read, group_title = ControlReader.read_control(control_path, False) + assert group_title == 'Access Control' + context = ControlContext.generate(ContextPurpose.COMPONENT, True, tmp_path, tmp_path, True) + # Given updated template comp_dict from the component definition json + context.comp_dict = { + 'This System': { + '': ComponentImpInfo( + prose='', rules=[], props=[], status=common.ImplementationStatus(state='planned', remarks=None) + ), + 'a.': ComponentImpInfo( + prose='Text for fancy thing component', + rules=[], + props=[], + status=common.ImplementationStatus(state='planned', remarks=None) + ), + 'c.': ComponentImpInfo( + prose='Just for the default component', + rules=[], + props=[], + status=common.ImplementationStatus(state='planned', remarks=None) + ), + 'd.': ComponentImpInfo( + prose='Example extra component', + rules=[], + props=[], + status=common.ImplementationStatus(state='planned', remarks=None) + ) + } + } + control_writer = ControlWriter() + control_writer.write_control_for_editing(context, orig_control_read, tmp_path, group_title, {}, []) + assert context.comp_dict['This System']['c.'].status.state == 'operational', 'State must be merged' + assert context.comp_dict['This System']['d.'].status.state == 'planned', 'New template state must be merged' + + +statement_text = r""" # xy-9 - \[My Group Title\] Fancy Control diff --git a/trestle/core/catalog/catalog_interface.py b/trestle/core/catalog/catalog_interface.py index 2278ddbf4..da8c5929b 100644 --- a/trestle/core/catalog/catalog_interface.py +++ b/trestle/core/catalog/catalog_interface.py @@ -904,5 +904,5 @@ def generate_control_rule_info(self, part_id_map: Dict[str, Dict[str, str]], con if len(dup_comp_uuids) > 0: # throw an exception if there are repeated component uuids for comp_uuid in dup_comp_uuids: - logger.error(f'Component uuid { comp_uuid } is duplicated') + logger.error(f'Component uuid {comp_uuid} is duplicated') raise TrestleError('Component uuids cannot be duplicated between different component definitions') diff --git a/trestle/core/control_interface.py b/trestle/core/control_interface.py index 1a2642d20..ab3c4c95e 100644 --- a/trestle/core/control_interface.py +++ b/trestle/core/control_interface.py @@ -109,8 +109,8 @@ class ControlInterface: @staticmethod def _wrap_label(label: str): - l_side = '\[' - r_side = '\]' + l_side = r'\[' + r_side = r'\]' wrapped = '' if label == '' else f'{l_side}{label}{r_side}' return wrapped @@ -591,6 +591,8 @@ def merge_dicts_deep( New items are always added from src to dest. Items present in both will be overriden dest if overwrite_header_values is True. """ + if src is None: + return for key in src.keys(): if key in dest: if depth and level == depth: diff --git a/trestle/core/control_writer.py b/trestle/core/control_writer.py index 62703bd5f..00bd5ccc3 100644 --- a/trestle/core/control_writer.py +++ b/trestle/core/control_writer.py @@ -66,7 +66,7 @@ def _add_control_statement(self, control: cat.Control, group_title: str, print_g control_title = control.title if print_group_title: - group_name = ' \[' + group_title + '\]' + group_name = r' \[' + group_title + r'\]' title = f'{control_id} -{group_name} {control_title}' @@ -516,8 +516,10 @@ def write_control_for_editing( control_file = dest_path / (control.id + const.MARKDOWN_FILE_EXT) # read the existing markdown header and content if it exists md_header, comp_dict = ControlReader.read_control_info_from_md(control_file, context) - # replace the memory comp_dict with the md one if control exists + # Merge the memory comp_dict with the md one if control exists if comp_dict: + template_comp_dict = context.comp_dict + ControlInterface.merge_dicts_deep(comp_dict, template_comp_dict, False) context.comp_dict = comp_dict header_comment_dict = {const.TRESTLE_ADD_PROPS_TAG: const.YAML_PROPS_COMMENT} diff --git a/trestle/transforms/implementations/tanium.py b/trestle/transforms/implementations/tanium.py index 33d108137..e76f21ced 100644 --- a/trestle/transforms/implementations/tanium.py +++ b/trestle/transforms/implementations/tanium.py @@ -110,7 +110,7 @@ def transform(self, blob: str) -> Results: results.__root__ = tanium_oscal_factory.results ts1 = datetime.datetime.now() self._analysis = tanium_oscal_factory.analysis - self._analysis.append(f'transform time: {ts1-ts0}') + self._analysis.append(f'transform time: {ts1 - ts0}') return results @@ -455,7 +455,7 @@ def _batch_observations(self, index: int) -> Dict[str, List[Observation]]: start = index * batch_size end = (index + 1) * batch_size end = min(end, len(self._rule_use_list)) - logger.debug(f'start: {start} end: {end-1}') + logger.debug(f'start: {start} end: {end - 1}') # process just the one chunk for i in range(start, end): rule_use = self._rule_use_list[i]