From f6af021015ed4dfd9a794e9fa52da118d0b5735a Mon Sep 17 00:00:00 2001 From: Alex Flom Date: Thu, 5 Oct 2023 07:17:35 -0600 Subject: [PATCH 01/10] feat(action): prep for multiple actions Related: PSCE-243 Signed-off-by: Alex Flom --- CONTRIBUTING.md | 6 +- actions/trestle-bot/action.yml | 91 ++++++++++++++++++++++++++ trestlebot/infra/entrypoints/create.py | 31 +++++++++ 3 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 actions/trestle-bot/action.yml create mode 100644 trestlebot/infra/entrypoints/create.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b00bc956..79ddcaf1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -67,4 +67,8 @@ cat my-token.txt | podman secret create repo-secret - ```bash podman run --entrypoint /entrypoint.sh --secret repo-secret,type=env,target=GITHUB_TOKEN --env-file=envfile -v my-trestle-space:/data -w /data localhost:5000/trestlebot:latest -``` \ No newline at end of file +``` + +### Adding a new action + +First, create an entrypoint script for the new action in the `trestlebot/infra` directory. Then add the action by creating a new directory in the `actions` directory with an `action.yml` that references your new entrypoint. \ No newline at end of file diff --git a/actions/trestle-bot/action.yml b/actions/trestle-bot/action.yml new file mode 100644 index 00000000..f94013b3 --- /dev/null +++ b/actions/trestle-bot/action.yml @@ -0,0 +1,91 @@ +name: "trestle-bot" +author: "Red Hat Product Security" +description: "A workflow automation manager for OSCAL formatted compliance content" + +inputs: + markdown_path: + description: Path relative to the repository path where the Trestle markdown files are located. See project README.md for more information. + required: true + oscal_model: + description: OSCAL Model type to assemble. Values can be catalog, profile, compdef, or ssp. + required: true + check_only: + description: "Runs tasks and exits with an error if there is a diff. Defaults to false" + required: false + default: "false" + github_token: + description: "GitHub token used to make authenticated API requests" + required: false + skip_assemble: + description: "Skip assembly task. Defaults to false" + required: false + default: "false" + skip_regenerate: + description: "Skip regenerate task. Defaults to false." + required: false + default: "false" + skip_items: + description: "Comma-separated glob patterns list of content by Trestle name to skip during task execution. For example `profile_x,profile_y*,`." + required: false + ssp_index_path: + description: Path relative to the repository path where the ssp index is located. See project README.md for information about the ssp index. + required: false + default: "ssp-index.json" + commit_message: + description: Commit message + required: false + default: "Sync automatic updates" + pull_request_title: + description: Custom pull request title + required: false + default: "Automatic updates from trestlebot" + branch: + description: Name of the Git branch to which modifications should be pushed. Required if Action is used on the `pull_request` event. + required: false + default: ${{ github.ref_name }} + target_branch: + description: Target branch (or base branch) to create a pull request against. If unset, no pull request will be created. If set, a pull request will be created using the `branch` field as the head branch. + required: false + file_pattern: + description: Comma separated file pattern list used for `git add`. For example `component-definitions/*,*json`. Defaults to (`.`) + required: false + default: '.' + repository: + description: Local file path to the git repository. Defaults to the current directory (`.`) + required: false + default: '.' + commit_user_name: + description: Name used for the commit user + required: false + default: github-actions[bot] + commit_user_email: + description: Email address used for the commit user + required: false + default: 41898282+github-actions[bot]@users.noreply.github.com + commit_author_name: + description: Name used for the commit author. Defaults to the username of whoever triggered this workflow run. + required: false + default: ${{ github.actor }} + commit_author_email: + description: Email address used for the commit author. Defaults to the email of whoever triggered this workflow run. + required: false + default: ${{ github.actor }}@users.noreply.github.com + +outputs: + changes: + description: Value is "true" if changes were committed back to the repository. + commit: + description: Full hash of the created commit. Only present if the "changes" output is "true". + pr_number: + description: Number of the submitted pull request. Only present if a pull request is submitted. + +runs: + using: "docker" + image: "Dockerfile" + entrypoint: "/entrypoint.sh" + env: + GITHUB_TOKEN: ${{ inputs.github_token }} + +branding: + icon: "check" + color: "green" diff --git a/trestlebot/infra/entrypoints/create.py b/trestlebot/infra/entrypoints/create.py new file mode 100644 index 00000000..fa2c70f3 --- /dev/null +++ b/trestlebot/infra/entrypoints/create.py @@ -0,0 +1,31 @@ +# Copyright 2023 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Create OSCAL content.""" + +import sys + + +def create_entrypoint(): + """Creates specified OSCAL formatted content.""" + pass + + +def main(): + + return create_entrypoint() + + +if __name__ == '__main__': + sys.exit(main()) \ No newline at end of file From b93830bd633b92e006c25f5e4340c6c9d303d9a9 Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Mon, 9 Oct 2023 16:27:12 -0400 Subject: [PATCH 02/10] docs: updates CONTRIBUTING.md with developer guide Signed-off-by: Jennifer Power --- CONTRIBUTING.md | 56 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 79ddcaf1..c33c0939 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,58 @@ -## Contributing +# Contributing to trestlebot + +Thank you for your interest in the trestlebot project. + +Before you start contributing, please take a moment to read through the guide below. + +> WARNING: This project is currently under initial development. APIs may be changed incompatibly from one commit to another. + +### How To Contribute + +Some initial contributions could be: + +- Improving the documentation +- Adding test coverage +- Try out issues that have the label `good first issue` +- Opening an issue for bugs or feature requests + +## Opening a Pull Request + +When submitting a pull request, please follow these guidelines: + +1. Ensure that you have an issue submitted first and reference it in your pull request. +2. Ensure that your code passes all CI tests. +3. Please keep the pull request focused on a single issue or feature, if possible. + +## License + +This project is licensed under the Apache 2.0 License - see the [LICENSE.md](LICENSE) file for details. + +## Developer Guide + +## Prerequisites + +- [Python](https://www.python.org/downloads/) - v3.8+ +- [Poetry](https://python-poetry.org/) +- [Podman](https://podman.io/docs/installation) + +### How It Works + +For workflow diagrams, see the [diagrams](./docs/diagrams/) under the `docs` folder. + +#### Components + +1. CI Provider - Runs or builds and runs trestle-bot container +2. Trestle Bot - Provides logic for managing workspace and containerized environment for use in workflows +3. Compliance-Trestle - Upstream library that provided core logic for how OSCAL content is managed + +#### Code structure + +- `cli.py and cli_base.py` - Provides top level logic for entrypoints +- `entrypoint.sh` - Bash entrypoint script that is used exclusively with the GitHub Action +- `provider.py, github.py, and gitlab.py` - Git provider abstract class and concrete implementations +- `tasks` - Pre-tasks can be configured before the main git logic is run. Any task that does workspace management should go here. +- `tasks/authored` - The `authored` package contains logic for managing authoring tasks for single instances of a top-level OSCAL model. These encapsulate logic from the `compliance-trestle` library and allows loose coupling between `tasks` and `authored` types. +- `transformers` - This contains data transformation logic; specifically for rules. ### Format and Styling From e274a8a42eca311301766016e96bca44785acd8a Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Mon, 9 Oct 2023 17:54:34 -0400 Subject: [PATCH 03/10] feat(cli): adds rules tranformation entrypoint to infra module Moves cli.py and cli_base.py to entrypoint.py and entrypoint_base.py Adds rules_transform.py to `infra/entrypoints` Update CONTRIBUTING.md, pyproject.toml, and __main__.py with new name Signed-off-by: Jennifer Power --- CONTRIBUTING.md | 2 +- pyproject.toml | 2 +- .../{test_cli.py => test_entrypoint.py} | 4 +- trestlebot/__main__.py | 4 +- trestlebot/{cli.py => entrypoint.py} | 9 +- .../{cli_base.py => entrypoint_base.py} | 0 trestlebot/infra/entrypoints/__init__.py | 18 ++++ trestlebot/infra/entrypoints/create.py | 31 ------- .../infra/entrypoints/rule_transform.py | 90 +++++++++++++++++++ 9 files changed, 121 insertions(+), 39 deletions(-) rename tests/trestlebot/{test_cli.py => test_entrypoint.py} (96%) rename trestlebot/{cli.py => entrypoint.py} (95%) rename trestlebot/{cli_base.py => entrypoint_base.py} (100%) create mode 100644 trestlebot/infra/entrypoints/__init__.py delete mode 100644 trestlebot/infra/entrypoints/create.py create mode 100644 trestlebot/infra/entrypoints/rule_transform.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c33c0939..13818460 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -47,7 +47,7 @@ For workflow diagrams, see the [diagrams](./docs/diagrams/) under the `docs` fol #### Code structure -- `cli.py and cli_base.py` - Provides top level logic for entrypoints +- `entrypoint.py and entrypoint_base.py` - Provides top level logic for entrypoints - `entrypoint.sh` - Bash entrypoint script that is used exclusively with the GitHub Action - `provider.py, github.py, and gitlab.py` - Git provider abstract class and concrete implementations - `tasks` - Pre-tasks can be configured before the main git logic is run. Any task that does workspace management should go here. diff --git a/pyproject.toml b/pyproject.toml index 34e8727e..21558769 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ readme = 'README.md' repository = 'https://github.com/RedHatProductSecurity/trestle-bot' [tool.poetry.scripts] -trestle-bot = "trestlebot.cli:main" +trestle-bot = "trestlebot.entrypoint:main" [tool.poetry.dependencies] python = '^3.8.1' diff --git a/tests/trestlebot/test_cli.py b/tests/trestlebot/test_entrypoint.py similarity index 96% rename from tests/trestlebot/test_cli.py rename to tests/trestlebot/test_entrypoint.py index 5440dc88..72b4e8a7 100644 --- a/tests/trestlebot/test_cli.py +++ b/tests/trestlebot/test_entrypoint.py @@ -22,7 +22,7 @@ import pytest -from trestlebot.cli import main as cli_main +from trestlebot.entrypoint import main as cli_main @pytest.fixture @@ -95,7 +95,7 @@ def test_with_target_branch(valid_args_dict: Dict[str, str], caplog: Any) -> Non # Patch is_github_actions since these tests will be running in # GitHub Actions - with patch("trestlebot.cli_base.is_github_actions") as mock_check, patch( + with patch("trestlebot.entrypoint_base.is_github_actions") as mock_check, patch( "sys.argv", ["trestlebot", *args_dict_to_list(args_dict)] ): mock_check.return_value = False diff --git a/trestlebot/__main__.py b/trestlebot/__main__.py index 9c84b787..d41cab89 100644 --- a/trestlebot/__main__.py +++ b/trestlebot/__main__.py @@ -14,13 +14,13 @@ # License for the specific language governing permissions and limitations # under the License. -import trestlebot.cli +import trestlebot.entrypoint def init() -> None: """Initialize trestlebot""" if __name__ == "__main__": - trestlebot.cli.main() + trestlebot.entrypoint.main() init() diff --git a/trestlebot/cli.py b/trestlebot/entrypoint.py similarity index 95% rename from trestlebot/cli.py rename to trestlebot/entrypoint.py index b5f0ffef..e849650f 100644 --- a/trestlebot/cli.py +++ b/trestlebot/entrypoint.py @@ -15,7 +15,12 @@ # under the License. -"""This module parses CLI arguments for the Trestle Bot.""" +""" +This module parses the default entrypoint for the Trestle Bot. + +This is the default entrypoint for the Trestle Bot which performs +autosync operations using compliance-trestle. +""" import argparse import logging @@ -25,7 +30,7 @@ import trestle.common.log as log from trestlebot import const -from trestlebot.cli_base import EntrypointBase, comma_sep_to_list +from trestlebot.entrypoint_base import EntrypointBase, comma_sep_to_list from trestlebot.tasks.assemble_task import AssembleTask from trestlebot.tasks.authored import types from trestlebot.tasks.base_task import TaskBase diff --git a/trestlebot/cli_base.py b/trestlebot/entrypoint_base.py similarity index 100% rename from trestlebot/cli_base.py rename to trestlebot/entrypoint_base.py diff --git a/trestlebot/infra/entrypoints/__init__.py b/trestlebot/infra/entrypoints/__init__.py new file mode 100644 index 00000000..c3788c1c --- /dev/null +++ b/trestlebot/infra/entrypoints/__init__.py @@ -0,0 +1,18 @@ +#!/usr/bin/python + +# Copyright 2023 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +A collection of entrypoints for Trestlebot. +""" diff --git a/trestlebot/infra/entrypoints/create.py b/trestlebot/infra/entrypoints/create.py deleted file mode 100644 index fa2c70f3..00000000 --- a/trestlebot/infra/entrypoints/create.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2023 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Create OSCAL content.""" - -import sys - - -def create_entrypoint(): - """Creates specified OSCAL formatted content.""" - pass - - -def main(): - - return create_entrypoint() - - -if __name__ == '__main__': - sys.exit(main()) \ No newline at end of file diff --git a/trestlebot/infra/entrypoints/rule_transform.py b/trestlebot/infra/entrypoints/rule_transform.py new file mode 100644 index 00000000..f219f8d3 --- /dev/null +++ b/trestlebot/infra/entrypoints/rule_transform.py @@ -0,0 +1,90 @@ +# Copyright 2023 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Entrypoint for component definition rules transformation.""" + +import argparse +import logging +from typing import List + +import trestle.common.log as log + +from trestlebot.entrypoint_base import EntrypointBase +from trestlebot.tasks.base_task import TaskBase +from trestlebot.tasks.rule_transform_task import RuleTransformTask +from trestlebot.transformers.validations import ValidationHandler, parameter_validation +from trestlebot.transformers.yaml_transformer import ToRulesYAMLTransformer + + +logger = logging.getLogger(__name__) + + +class RulesTransformEntrypoint(EntrypointBase): + """Entrypoint for the rules transformation operation.""" + + def __init__(self, parser: argparse.ArgumentParser) -> None: + """Initialize.""" + # Setup base arguments + super().__init__(parser) + self.setup_rules_transformation_arguments() + + def setup_rules_transformation_arguments(self) -> None: + """Setup arguments for the rule transformer entrypoint.""" + self.parser.add_argument( + "--rules-view-path", + required=True, + type=str, + help="Path to Trestle markdown files", + ) + self.parser.add_argument( + "--skip-items", + type=str, + required=False, + help="Comma-separated list of glob patterns of the chosen model type to skip when running \ + tasks", + ) + + def run(self, args: argparse.Namespace) -> None: + """Run the rule transform entrypoint.""" + + log.set_log_level_from_args(args=args) + + validation_handler: ValidationHandler = ValidationHandler(parameter_validation) + transformer: ToRulesYAMLTransformer = ToRulesYAMLTransformer(validation_handler) + + rule_transform_task: RuleTransformTask = RuleTransformTask( + args.working_dir, + args.rules_view_path, + transformer, + args.skip_items, + ) + pre_tasks: List[TaskBase] = [rule_transform_task] + + super().run_base(args, pre_tasks) + + +def main() -> None: + """Run the CLI.""" + parser = argparse.ArgumentParser( + description="Rules transformation entrypoint for trestle." + ) + rules_transform = RulesTransformEntrypoint(parser=parser) + + args = parser.parse_args() + + rules_transform.run(args) + + +if __name__ == "__main__": + main() From 8245e5a3d4e21bfcfaad9ffea11cc77785eac7fa Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Tue, 10 Oct 2023 16:30:52 -0400 Subject: [PATCH 04/10] chore: moves all entrypoints to a new directory Moves autosync entrypoint to entrypoints directory Removes infra directory Signed-off-by: Jennifer Power --- tests/testutils.py | 2 +- tests/trestlebot/__init__.py | 2 +- tests/trestlebot/entrypoints/__init__.py | 16 ++++++++++++++++ .../test_autosync.py} | 9 ++++----- trestlebot/__main__.py | 6 ++++-- trestlebot/{infra => }/entrypoints/__init__.py | 0 .../{entrypoint.py => entrypoints/autosync.py} | 11 +++-------- trestlebot/{ => entrypoints}/entrypoint_base.py | 8 ++++++++ .../{infra => }/entrypoints/rule_transform.py | 9 +++++---- trestlebot/transformers/base_transformer.py | 2 ++ 10 files changed, 44 insertions(+), 21 deletions(-) create mode 100644 tests/trestlebot/entrypoints/__init__.py rename tests/trestlebot/{test_entrypoint.py => entrypoints/test_autosync.py} (93%) rename trestlebot/{infra => }/entrypoints/__init__.py (100%) rename trestlebot/{entrypoint.py => entrypoints/autosync.py} (93%) rename trestlebot/{ => entrypoints}/entrypoint_base.py (95%) rename trestlebot/{infra => }/entrypoints/rule_transform.py (88%) diff --git a/tests/testutils.py b/tests/testutils.py index bfd2bc23..a4962a18 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -43,7 +43,7 @@ def clean(repo_path: str, repo: Optional[Repo]) -> None: - # Clean up the temporary Git repository + """Clean up the temporary Git repository.""" if repo is not None: repo.close() shutil.rmtree(repo_path) diff --git a/tests/trestlebot/__init__.py b/tests/trestlebot/__init__.py index 740d8d55..d0f44b5c 100644 --- a/tests/trestlebot/__init__.py +++ b/tests/trestlebot/__init__.py @@ -13,4 +13,4 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -"""Test package. for trestlebot top-level logic.""" +"""Test package for trestlebot top-level logic.""" diff --git a/tests/trestlebot/entrypoints/__init__.py b/tests/trestlebot/entrypoints/__init__.py new file mode 100644 index 00000000..104f95c4 --- /dev/null +++ b/tests/trestlebot/entrypoints/__init__.py @@ -0,0 +1,16 @@ +#!/usr/bin/python + +# Copyright 2023 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""Test package for entrypoints.""" diff --git a/tests/trestlebot/test_entrypoint.py b/tests/trestlebot/entrypoints/test_autosync.py similarity index 93% rename from tests/trestlebot/test_entrypoint.py rename to tests/trestlebot/entrypoints/test_autosync.py index 72b4e8a7..e6a245ab 100644 --- a/tests/trestlebot/test_entrypoint.py +++ b/tests/trestlebot/entrypoints/test_autosync.py @@ -22,7 +22,7 @@ import pytest -from trestlebot.entrypoint import main as cli_main +from trestlebot.entrypoints.autosync import main as cli_main @pytest.fixture @@ -51,7 +51,6 @@ def test_invalid_oscal_model(valid_args_dict: Dict[str, str], caplog: Any) -> No """Test invalid oscal model""" args_dict = valid_args_dict args_dict["oscal-model"] = "fake" - with patch("sys.argv", ["trestlebot", *args_dict_to_list(args_dict)]): with pytest.raises(SystemExit, match="2"): cli_main() @@ -95,9 +94,9 @@ def test_with_target_branch(valid_args_dict: Dict[str, str], caplog: Any) -> Non # Patch is_github_actions since these tests will be running in # GitHub Actions - with patch("trestlebot.entrypoint_base.is_github_actions") as mock_check, patch( - "sys.argv", ["trestlebot", *args_dict_to_list(args_dict)] - ): + with patch( + "trestlebot.entrypoints.entrypoint_base.is_github_actions" + ) as mock_check, patch("sys.argv", ["trestlebot", *args_dict_to_list(args_dict)]): mock_check.return_value = False with pytest.raises(SystemExit): diff --git a/trestlebot/__main__.py b/trestlebot/__main__.py index d41cab89..2b115f52 100644 --- a/trestlebot/__main__.py +++ b/trestlebot/__main__.py @@ -14,13 +14,15 @@ # License for the specific language governing permissions and limitations # under the License. -import trestlebot.entrypoint +# Default entrypoint for trestlebot is autosync mode + +from trestlebot.entrypoints.autosync import main as autosync_main def init() -> None: """Initialize trestlebot""" if __name__ == "__main__": - trestlebot.entrypoint.main() + autosync_main() init() diff --git a/trestlebot/infra/entrypoints/__init__.py b/trestlebot/entrypoints/__init__.py similarity index 100% rename from trestlebot/infra/entrypoints/__init__.py rename to trestlebot/entrypoints/__init__.py diff --git a/trestlebot/entrypoint.py b/trestlebot/entrypoints/autosync.py similarity index 93% rename from trestlebot/entrypoint.py rename to trestlebot/entrypoints/autosync.py index e849650f..bd4c8fba 100644 --- a/trestlebot/entrypoint.py +++ b/trestlebot/entrypoints/autosync.py @@ -30,7 +30,7 @@ import trestle.common.log as log from trestlebot import const -from trestlebot.entrypoint_base import EntrypointBase, comma_sep_to_list +from trestlebot.entrypoints.entrypoint_base import EntrypointBase, comma_sep_to_list from trestlebot.tasks.assemble_task import AssembleTask from trestlebot.tasks.authored import types from trestlebot.tasks.base_task import TaskBase @@ -83,12 +83,6 @@ def setup_autosync_arguments(self) -> None: action="store_true", help="Skip regenerate task. Defaults to false.", ) - self.parser.add_argument( - "--check-only", - required=False, - action="store_true", - help="Runs tasks and exits with an error if there is a diff", - ) self.parser.add_argument( "--ssp-index-path", required=False, @@ -112,7 +106,7 @@ def run(self, args: argparse.Namespace) -> None: if args.oscal_model not in authored_list: logger.error( f"Invalid value {args.oscal_model} for oscal model. " - f"Please use catalog, profile, compdef, or ssp." + f"Please use one of {authored_list}" ) sys.exit(const.ERROR_EXIT_CODE) @@ -158,6 +152,7 @@ def main() -> None: parser = argparse.ArgumentParser( description="Workflow automation bot for compliance-trestle" ) + auto_sync = AutoSyncEntrypoint(parser=parser) args = parser.parse_args() diff --git a/trestlebot/entrypoint_base.py b/trestlebot/entrypoints/entrypoint_base.py similarity index 95% rename from trestlebot/entrypoint_base.py rename to trestlebot/entrypoints/entrypoint_base.py index 1bf3c0bc..f63b8700 100644 --- a/trestlebot/entrypoint_base.py +++ b/trestlebot/entrypoints/entrypoint_base.py @@ -39,6 +39,8 @@ logger = logging.getLogger(__name__) +# TODO: Consider adding support for a config file or environment variables to set these values + class EntrypointBase: """Base class for all entrypoints.""" @@ -109,6 +111,12 @@ def setup_common_arguments(self) -> None: type=str, help="Email for commit author if differs from committer", ) + self.parser.add_argument( + "--check-only", + required=False, + action="store_true", + help="Runs tasks and exits with an error if there is a diff", + ) self.parser.add_argument( "--target-branch", type=str, diff --git a/trestlebot/infra/entrypoints/rule_transform.py b/trestlebot/entrypoints/rule_transform.py similarity index 88% rename from trestlebot/infra/entrypoints/rule_transform.py rename to trestlebot/entrypoints/rule_transform.py index f219f8d3..767788e2 100644 --- a/trestlebot/infra/entrypoints/rule_transform.py +++ b/trestlebot/entrypoints/rule_transform.py @@ -20,7 +20,7 @@ import trestle.common.log as log -from trestlebot.entrypoint_base import EntrypointBase +from trestlebot.entrypoints.entrypoint_base import EntrypointBase, comma_sep_to_list from trestlebot.tasks.base_task import TaskBase from trestlebot.tasks.rule_transform_task import RuleTransformTask from trestlebot.transformers.validations import ValidationHandler, parameter_validation @@ -45,13 +45,13 @@ def setup_rules_transformation_arguments(self) -> None: "--rules-view-path", required=True, type=str, - help="Path to Trestle markdown files", + help="Path to top-level rules-view directory", ) self.parser.add_argument( "--skip-items", type=str, required=False, - help="Comma-separated list of glob patterns of the chosen model type to skip when running \ + help="Comma-separated list of glob patterns for directories to skip when running \ tasks", ) @@ -60,6 +60,7 @@ def run(self, args: argparse.Namespace) -> None: log.set_log_level_from_args(args=args) + # Configure the YAML Transformer for the task validation_handler: ValidationHandler = ValidationHandler(parameter_validation) transformer: ToRulesYAMLTransformer = ToRulesYAMLTransformer(validation_handler) @@ -67,7 +68,7 @@ def run(self, args: argparse.Namespace) -> None: args.working_dir, args.rules_view_path, transformer, - args.skip_items, + comma_sep_to_list(args.skip_items), ) pre_tasks: List[TaskBase] = [rule_transform_task] diff --git a/trestlebot/transformers/base_transformer.py b/trestlebot/transformers/base_transformer.py index bac21150..6cf2de25 100644 --- a/trestlebot/transformers/base_transformer.py +++ b/trestlebot/transformers/base_transformer.py @@ -16,6 +16,8 @@ """Base transformer for rules.""" +# TODO: Evaluate if this should be contributed back to trestle + from abc import abstractmethod from typing import Any From aa81cc19271cedca2f4d1eea1009e9f4d6bf9d98 Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Tue, 10 Oct 2023 17:14:01 -0400 Subject: [PATCH 05/10] docs: updates documentation for new rules transform action and updates repo layout Signed-off-by: Jennifer Power --- CONTRIBUTING.md | 6 +-- README.md | 87 ++++-------------------------- actions/README.md | 14 +++++ actions/autosync/README.md | 88 +++++++++++++++++++++++++++++++ actions/rules-transform/README.md | 50 ++++++++++++++++++ 5 files changed, 164 insertions(+), 81 deletions(-) create mode 100644 actions/README.md create mode 100644 actions/autosync/README.md create mode 100644 actions/rules-transform/README.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 13818460..37d6aad9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -47,9 +47,9 @@ For workflow diagrams, see the [diagrams](./docs/diagrams/) under the `docs` fol #### Code structure -- `entrypoint.py and entrypoint_base.py` - Provides top level logic for entrypoints -- `entrypoint.sh` - Bash entrypoint script that is used exclusively with the GitHub Action -- `provider.py, github.py, and gitlab.py` - Git provider abstract class and concrete implementations +- `actions` - Provides specific logic for `trestle-bot` tasks that are packaged as Actions. See [README.md](./actions/README.md) for more information. +- `entrypoints` - Provides top level logic for specific tasks. These tasks are not necessarily related in any so they are not organized into a hierarchical command structure, but they do inherit logic and flag from a base class. +- `provider.py, github.py, and gitlab.py` - Git provider abstract class and concrete implementations for interacting with the API. - `tasks` - Pre-tasks can be configured before the main git logic is run. Any task that does workspace management should go here. - `tasks/authored` - The `authored` package contains logic for managing authoring tasks for single instances of a top-level OSCAL model. These encapsulate logic from the `compliance-trestle` library and allows loose coupling between `tasks` and `authored` types. - `transformers` - This contains data transformation logic; specifically for rules. diff --git a/README.md b/README.md index 8f3febb5..d16f423d 100644 --- a/README.md +++ b/README.md @@ -4,91 +4,22 @@ trestle-bot assists users in leveraging [Compliance-Trestle](https://github.com/ > WARNING: This project is currently under initial development. APIs may be changed incompatibly from one commit to another. -## Basic Configuration +## Getting Started +## GitHub Actions -```yaml +For detailed documentation on how to use each action see the README.md each each folder under [actions](./actions/). -name: Example Workflow -... +The `autosync` action will sync trestle-generated Markdown files to OSCAL JSON files in a trestle workspace. All content under the provided markdown directory when the action is run. This action supports all top-level models [supported by compliance-trestle for authoring](https://ibm.github.io/compliance-trestle/tutorials/ssp_profile_catalog_authoring/ssp_profile_catalog_authoring/). - steps: - - uses: actions/checkout@v3 - - name: Run trestlebot - id: trestlebot - uses: RedHatProductSecurity/trestle-bot@main - with: - markdown_path: "markdown/profiles" - oscal_model: "profile" -``` +The `rules-transform` actions can be used when managing [OSCAL Component Definitions](https://pages.nist.gov/OSCAL-Reference/models/v1.1.1/component-definition/json-outline/) in a trestle workspace. The action will transform rules defined in the rules YAML view to an OSCAL Component Definition JSON file. -## Inputs and Outputs +### GitLab CI -Checkout [`action.yml`](./action.yml) for a full list of supported inputs and outputs. +> Coming Soon -### Additional information on workflow inputs +## Contributing -- `markdown_path`: This is the location for Markdown generated by the `trestle author -generate` commands -- `ssp_index_path`: This is a text file that stores the component definition information by name in trestle with the ssp name. Example below - -```json - "ssp1": { - "profile": "profile1", - "component definitions": [ - "comp1", - "comp2" - ] - }, -``` - -## Action Behavior - -The purpose of this action is to sync JSON and Markdown data with `compliance-trestle` and commit changes back to the branch or submit a pull request (if desired). Below are the main use-cases/workflows available: - -- The default behavior of this action is to run a trestle `assemble` and `regenerate` tasks with the given markdown directory and model and commit the changes back to the branch the workflow ran from ( `github.ref_name` ). The branch can be changed by setting the field `branch`. If no changes exist or the changes do not exist with the file pattern set, no changes will be made and the action will exit successfully. - -```yaml - steps: - - uses: actions/checkout@v3 - - name: Run trestlebot - id: trestlebot - uses: RedHatProductSecurity/trestle-bot@main - with: - markdown_path: "markdown/profiles" - oscal_model: "profile" - branch: "another-branch" -``` - -- If the `target_branch` field is set, a pull request will be made using the `target_branch` as the base branch and `branch` as the head branch. - -```yaml - steps: - - uses: actions/checkout@v3 - - name: Run trestlebot - id: trestlebot - uses: RedHatProductSecurity/trestle-bot@main - with: - markdown_path: "markdown/profiles" - oscal_model: "profile" - branch: "autoupdate-${{ github.run_id }}" - target_branch: "main" - github_token: ${{ secret.GITHUB_TOKEN }} -``` - -- When `check_only` is set, the trestle `assemble` and `regenerate` tasks are run and the repository is checked for changes. If changes exists, the action with exit with an error. - -```yaml - steps: - - uses: actions/checkout@v3 - - name: Run trestlebot - id: trestlebot - uses: RedHatProductSecurity/trestle-bot@main - with: - markdown_path: "markdown/profiles" - oscal_model: "profile" - check_only: true -``` - -> Note: Trestle `assemble` or `regenerate` tasks may be skipped if desired using `skip_assemble: true` or `skip_regenerate: true`, respectively. +For information about contributing to trestle-bot see the [contributing guide](./CONTRIBUTING.md) See `TROUBLESHOOTING.md` for additional information. \ No newline at end of file diff --git a/actions/README.md b/actions/README.md new file mode 100644 index 00000000..cf9ed856 --- /dev/null +++ b/actions/README.md @@ -0,0 +1,14 @@ +# Actions + + +Tasks in trestle-bot that we want to expose as an Action are located here. + +# Adding a new Action + +> Note to contributors: We are trying to limit the task that we expose as actions to workspace manage operations and checks only. + +1. Add a new folder with the action name +2. Add an action.yml +3. Add an entrypoint script to handle inputs from GitHub Actions +4. Add a COPY line in the Dockerfile + diff --git a/actions/autosync/README.md b/actions/autosync/README.md new file mode 100644 index 00000000..4ebc7988 --- /dev/null +++ b/actions/autosync/README.md @@ -0,0 +1,88 @@ +# trestlebot AutoSync Action + +## Basic Configuration + + +```yaml + +name: Example Workflow +... + + steps: + - uses: actions/checkout@v3 + - name: Run trestlebot + id: trestlebot + uses: RedHatProductSecurity/trestle-bot/actions/autosync@main + with: + markdown_path: "markdown/profiles" + oscal_model: "profile" +``` + +## Inputs and Outputs + +Checkout [`action.yml`](./action.yml) for a full list of supported inputs and outputs. + +### Additional information on workflow inputs + +- `markdown_path`: This is the location for Markdown generated by the `trestle author -generate` commands +- `ssp_index_path`: This is a text file that stores the component definition information by name in trestle with the ssp name. Example below + +```json + "ssp1": { + "profile": "profile1", + "component definitions": [ + "comp1", + "comp2" + ] + }, +``` + +## Action Behavior + +The purpose of this action is to sync JSON and Markdown data with `compliance-trestle` and commit changes back to the branch or submit a pull request (if desired). Below are the main use-cases/workflows available: + +- The default behavior of this action is to run a trestle `assemble` and `regenerate` tasks with the given markdown directory and model and commit the changes back to the branch the workflow ran from ( `github.ref_name` ). The branch can be changed by setting the field `branch`. If no changes exist or the changes do not exist with the file pattern set, no changes will be made and the action will exit successfully. + +```yaml + steps: + - uses: actions/checkout@v3 + - name: Run trestlebot + id: trestlebot + uses: RedHatProductSecurity/trestle-bot/actions/autosync@main + with: + markdown_path: "markdown/profiles" + oscal_model: "profile" + branch: "another-branch" +``` + +- If the `target_branch` field is set, a pull request will be made using the `target_branch` as the base branch and `branch` as the head branch. + +```yaml + steps: + - uses: actions/checkout@v3 + - name: Run trestlebot + id: trestlebot + uses: RedHatProductSecurity/trestle-bot/actions/autosync@main + with: + markdown_path: "markdown/profiles" + oscal_model: "profile" + branch: "autoupdate-${{ github.run_id }}" + target_branch: "main" + github_token: ${{ secret.GITHUB_TOKEN }} +``` + +- When `check_only` is set, the trestle `assemble` and `regenerate` tasks are run and the repository is checked for changes. If changes exists, the action with exit with an error. + +```yaml + steps: + - uses: actions/checkout@v3 + - name: Run trestlebot + id: trestlebot + uses: RedHatProductSecurity/trestle-bot/actions/autosync@main + with: + markdown_path: "markdown/profiles" + oscal_model: "profile" + check_only: true +``` + +> Note: Trestle `assemble` or `regenerate` tasks may be skipped if desired using `skip_assemble: true` or `skip_regenerate: true`, respectively. \ No newline at end of file diff --git a/actions/rules-transform/README.md b/actions/rules-transform/README.md new file mode 100644 index 00000000..5b45a04b --- /dev/null +++ b/actions/rules-transform/README.md @@ -0,0 +1,50 @@ +# trestlebot Rules Transform Action + +## Basic Configuration + + +```yaml + +name: Example Workflow +... + + steps: + - uses: actions/checkout@v3 + - name: Run trestlebot + id: trestlebot + uses: RedHatProductSecurity/trestle-bot/actions/rules-transform@main + with: + rules_view_path: "rules/" +``` + +## Action Behavior + +The purpose of this action is to sync the rules view data in YAML to OSCAL with `compliance-trestle` and commit changes back to the branch or submit a pull request (if desired). Below are the main use-cases/workflows available: + +- The default behavior of this action is to run the rules transformation and commit the changes back to the branch the workflow ran from ( `github.ref_name` ). The branch can be changed by setting the field `branch`. If no changes exist or the changes do not exist with the file pattern set, no changes will be made and the action will exit successfully. + +```yaml + steps: + - uses: actions/checkout@v3 + - name: Run trestlebot + id: trestlebot + uses: RedHatProductSecurity/trestle-bot/actions/rules-transform@main + with: + rules_view_path: "rules/" + branch: "another-branch" +``` + +- If the `target_branch` field is set, a pull request will be made using the `target_branch` as the base branch and `branch` as the head branch. + +```yaml + steps: + - uses: actions/checkout@v3 + - name: Run trestlebot + id: trestlebot + uses: RedHatProductSecurity/trestle-bot/actions/rules-transform@main + with: + rules_view_path: "rules/" + branch: "transform-${{ github.run_id }}" + target_branch: "main" + github_token: ${{ secret.GITHUB_TOKEN }} +``` \ No newline at end of file From 3abe86b690e924b520493f31bd3d2cfc80ef6948 Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Tue, 10 Oct 2023 17:15:16 -0400 Subject: [PATCH 06/10] feat: updates Dockerfile and actions.yaml and entrypoint script to support rules-transform Signed-off-by: Jennifer Power --- Dockerfile | 6 +- actions/{trestle-bot => autosync}/action.yml | 6 +- .../autosync/auto-sync-entrypoint.sh | 0 actions/rules-transform/actions.yml | 72 +++++++++++++++++++ .../rules-transform-entrypoint.sh | 68 ++++++++++++++++++ pyproject.toml | 3 +- 6 files changed, 149 insertions(+), 6 deletions(-) rename actions/{trestle-bot => autosync}/action.yml (95%) rename entrypoint.sh => actions/autosync/auto-sync-entrypoint.sh (100%) create mode 100644 actions/rules-transform/actions.yml create mode 100644 actions/rules-transform/rules-transform-entrypoint.sh diff --git a/Dockerfile b/Dockerfile index 72588f2d..ee83cb35 100644 --- a/Dockerfile +++ b/Dockerfile @@ -64,9 +64,11 @@ RUN microdnf install -y git \ && microdnf clean all \ && rm -rf /var/lib/apt/lists/* -COPY ./entrypoint.sh / +COPY ./actions/autosync/auto-sync-entrypoint.sh / +RUN chmod +x /auto-sync-entrypoint.sh -RUN chmod +x /entrypoint.sh +COPY ./actions/rules-transform/rules-transform-entrypoint.sh / +RUN chmod +x /rules-transform-entrypoint.sh ENTRYPOINT ["python3.9", "-m" , "trestlebot"] CMD ["--help"] diff --git a/actions/trestle-bot/action.yml b/actions/autosync/action.yml similarity index 95% rename from actions/trestle-bot/action.yml rename to actions/autosync/action.yml index f94013b3..1263f6fa 100644 --- a/actions/trestle-bot/action.yml +++ b/actions/autosync/action.yml @@ -1,6 +1,6 @@ -name: "trestle-bot" +name: "trestle-bot-autosync" author: "Red Hat Product Security" -description: "A workflow automation manager for OSCAL formatted compliance content" +description: "An action to perform automatic synchronization of Trestle markdown files to OSCAL." inputs: markdown_path: @@ -82,7 +82,7 @@ outputs: runs: using: "docker" image: "Dockerfile" - entrypoint: "/entrypoint.sh" + entrypoint: "/auto-sync-entrypoint.sh" env: GITHUB_TOKEN: ${{ inputs.github_token }} diff --git a/entrypoint.sh b/actions/autosync/auto-sync-entrypoint.sh similarity index 100% rename from entrypoint.sh rename to actions/autosync/auto-sync-entrypoint.sh diff --git a/actions/rules-transform/actions.yml b/actions/rules-transform/actions.yml new file mode 100644 index 00000000..7f242b43 --- /dev/null +++ b/actions/rules-transform/actions.yml @@ -0,0 +1,72 @@ +name: "trestle-bot-rules-transform" +author: "Red Hat Product Security" +description: "A rules transform action to convert trestle rules in YAML format to OSCAL" + +inputs: + rules_view_path: + description: Path relative to the repository path where the Trestle markdown files are located. See project README.md for more information. + required: true + github_token: + description: "GitHub token used to make authenticated API requests" + required: false + skip_items: + description: "Comma-separated glob patterns list of content by Trestle name to skip during task execution. For example `profile_x,profile_y*,`." + required: false + commit_message: + description: Commit message + required: false + default: "Sync automatic updates" + pull_request_title: + description: Custom pull request title + required: false + default: "Automatic updates from trestlebot" + branch: + description: Name of the Git branch to which modifications should be pushed. Required if Action is used on the `pull_request` event. + required: false + default: ${{ github.ref_name }} + target_branch: + description: Target branch (or base branch) to create a pull request against. If unset, no pull request will be created. If set, a pull request will be created using the `branch` field as the head branch. + required: false + file_pattern: + description: Comma separated file pattern list used for `git add`. For example `component-definitions/*,*json`. Defaults to (`.`) + required: false + default: '.' + repository: + description: Local file path to the git repository. Defaults to the current directory (`.`) + required: false + default: '.' + commit_user_name: + description: Name used for the commit user + required: false + default: github-actions[bot] + commit_user_email: + description: Email address used for the commit user + required: false + default: 41898282+github-actions[bot]@users.noreply.github.com + commit_author_name: + description: Name used for the commit author. Defaults to the username of whoever triggered this workflow run. + required: false + default: ${{ github.actor }} + commit_author_email: + description: Email address used for the commit author. Defaults to the email of whoever triggered this workflow run. + required: false + default: ${{ github.actor }}@users.noreply.github.com + +outputs: + changes: + description: Value is "true" if changes were committed back to the repository. + commit: + description: Full hash of the created commit. Only present if the "changes" output is "true". + pr_number: + description: Number of the submitted pull request. Only present if a pull request is submitted. + +runs: + using: "docker" + image: "../Dockerfile" + entrypoint: "/rules-transform-entrypoint.sh" + env: + GITHUB_TOKEN: ${{ inputs.github_token }} + +branding: + icon: "check" + color: "green" diff --git a/actions/rules-transform/rules-transform-entrypoint.sh b/actions/rules-transform/rules-transform-entrypoint.sh new file mode 100644 index 00000000..2dc6274e --- /dev/null +++ b/actions/rules-transform/rules-transform-entrypoint.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +set -eu + +# Manage newest git versions (related to CVE https://github.blog/2022-04-12-git-security-vulnerability-announced/) +# +if [ -z ${GITHUB_WORKSPACE+x} ]; then + echo "Setting git safe.directory default: /github/workspace ..." + git config --global --add safe.directory /github/workspace +else + echo "Setting git safe.directory GITHUB_WORKSPACE: $GITHUB_WORKSPACE ..." + git config --global --add safe.directory "$GITHUB_WORKSPACE" +fi + +if [ -z ${INPUT_REPOSITORY+x} ]; then + echo "Skipping setting working directory as safe directory" +else + echo "Setting git safe.directory default: $INPUT_REPOSITORY..." + git config --global --add safe.directory "$INPUT_REPOSITORY" +fi + +# Initialize the command variable +command="python3.9 /trestle-bot/trestlebot/entrypoints/rules-tranform.py \ + --rules-view-path=\"${INPUT_RULES_VIEW_PATH}\" \ + --commit-message=\"${INPUT_COMMIT_MESSAGE}\" \ + --pull-request-title=\"${INPUT_PULL_REQUEST_TITLE}\" \ + --branch=\"${INPUT_BRANCH}\" \ + --file-patterns=\"${INPUT_FILE_PATTERN}\" \ + --committer-name=\"${INPUT_COMMIT_USER_NAME}\" \ + --committer-email=\"${INPUT_COMMIT_USER_EMAIL}\" \ + --author-name=\"${INPUT_COMMIT_AUTHOR_NAME}\" \ + --author-email=\"${INPUT_COMMIT_AUTHOR_EMAIL}\" \ + --working-dir=\"${INPUT_REPOSITORY}\" \ + --target-branch=\"${INPUT_TARGET_BRANCH}\" \ + --skip-items=\"${INPUT_SKIP_ITEMS}\"" + + +if [[ ${INPUT_CHECK_ONLY} == true ]]; then + command+=" --check-only" +fi + +# Only set the token value when is a target branch so pull requests can be created +if [[ -n ${INPUT_TARGET_BRANCH} ]]; then + if [[ -z ${GITHUB_TOKEN} ]]; then + echo "Set the GITHUB_TOKEN env variable." + exit 1 + fi + + command+=" --with-token - <<<\"${GITHUB_TOKEN}\"" +fi + +exec 3>&1 +output=$(eval "$command" > >(tee /dev/fd/3) 2>&1) + +commit=$(echo "$output" | grep "Commit Hash:" | sed 's/.*: //') + +if [ -n "$commit" ]; then + echo "changes=true" >> "$GITHUB_OUTPUT" + echo "commit=$commit" >> "$GITHUB_OUTPUT" +else + echo "changes=false" >> "$GITHUB_OUTPUT" +fi + +pr_number=$(echo "$output" | grep "Pull Request Number:" | sed 's/.*: //') + +if [ -n "$pr_number" ]; then + echo "pr_number=$pr_number" >> "$GITHUB_OUTPUT" +fi \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 21558769..fd932fe5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,8 @@ readme = 'README.md' repository = 'https://github.com/RedHatProductSecurity/trestle-bot' [tool.poetry.scripts] -trestle-bot = "trestlebot.entrypoint:main" +autosync = "trestlebot.entrypoints.autosync:main" +rules-transform = "trestlebot.entrypoints.rule_transform:main" [tool.poetry.dependencies] python = '^3.8.1' From 075c5279925d6ec0f55c96b3685fbf84d346052c Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Fri, 15 Sep 2023 09:29:56 -0400 Subject: [PATCH 07/10] chore: change workdir in dependency stage to remove extra files Signed-off-by: Jennifer Power --- Dockerfile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index ee83cb35..7e1958bd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,14 +44,17 @@ RUN python3.9 -m pip install --no-cache-dir --upgrade pip \ && pip install --no-cache-dir poetry=="$POETRY_VERSION" # Cache runtime deps -WORKDIR $PYSETUP_PATH -COPY ./ $PYSETUP_PATH +WORKDIR "/build" +COPY . "/build" # Install runtime deps -RUN poetry install --without tests,dev --no-root +RUN python -m venv $VENV_PATH && \ + . $VENV_PATH/bin/activate && \ + poetry install --without tests,dev --no-root # install the root project in non-editable mode -RUN poetry build -f wheel -n && \ +RUN . $VENV_PATH/bin/activate && \ + poetry build -f wheel -n && \ pip install --no-cache-dir --no-deps dist/*.whl && \ rm -rf dist *.egg-info From 1b85b715950a3e233372aed1f9de6ba65edecacd Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Wed, 11 Oct 2023 10:43:50 -0400 Subject: [PATCH 08/10] docs: adds steps for running as a container to README.md getting started Signed-off-by: Jennifer Power --- CONTRIBUTING.md | 11 +++++------ README.md | 29 +++++++++++++++++++++++++---- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 37d6aad9..8503407b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,10 +23,6 @@ When submitting a pull request, please follow these guidelines: 2. Ensure that your code passes all CI tests. 3. Please keep the pull request focused on a single issue or feature, if possible. -## License - -This project is licensed under the Apache 2.0 License - see the [LICENSE.md](LICENSE) file for details. - ## Developer Guide ## Prerequisites @@ -68,7 +64,10 @@ make test ### Run with poetry ``` -poetry run trestle-bot +poetry shell +poetry install +poetry run trestlebot-autosync +poetry run trestlebot-rules-transform ``` ### Local testing @@ -125,4 +124,4 @@ podman run --entrypoint /entrypoint.sh --secret repo-secret,type=env,target=GITH ### Adding a new action -First, create an entrypoint script for the new action in the `trestlebot/infra` directory. Then add the action by creating a new directory in the `actions` directory with an `action.yml` that references your new entrypoint. \ No newline at end of file +First, create an entrypoint script for the new action in the `trestlebot/entrypoints` directory. Then add the action by creating a new directory in the `actions` directory with an `action.yml` that references your new entrypoint. \ No newline at end of file diff --git a/README.md b/README.md index d16f423d..85271251 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ # trestle-bot -trestle-bot assists users in leveraging [Compliance-Trestle](https://github.com/IBM/compliance-trestle) in automated workflows for [OSCAL](https://github.com/usnistgov/OSCAL) formatted compliance content management. +trestle-bot assists users in leveraging [Compliance-Trestle](https://github.com/IBM/compliance-trestle) in CI/CD workflows for [OSCAL](https://github.com/usnistgov/OSCAL) formatted compliance content management. > WARNING: This project is currently under initial development. APIs may be changed incompatibly from one commit to another. ## Getting Started -## GitHub Actions +### GitHub Actions For detailed documentation on how to use each action see the README.md each each folder under [actions](./actions/). @@ -18,8 +18,29 @@ The `rules-transform` actions can be used when managing [OSCAL Component Definit > Coming Soon +### Run as a Container + +Build and run the container locally: + +```bash +podman build -f Dockerfile -t trestle-bot . +podman run -v $(pwd):/data -w /data trestle-bot +``` + +Container images are available in `quay.io`: + +```bash +podman run -v $(pwd):/data -w /data quay.io/continuouscompliance/trestle-bot: +``` + ## Contributing -For information about contributing to trestle-bot see the [contributing guide](./CONTRIBUTING.md) +For information about contributing to trestle-bot see the [CONTRIBUTING.md](./CONTRIBUTING.md) file. + +## License + +This project is licensed under the Apache 2.0 License - see the [LICENSE.md](LICENSE) file for details. + +## Troubleshooting -See `TROUBLESHOOTING.md` for additional information. \ No newline at end of file +See [TROUBLESHOOTING.md](./TROUBLESHOOTING.md) for troubleshooting tips. \ No newline at end of file From e9eba9b2efca762a16a672d5e5c8c75187de3d5c Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Wed, 11 Oct 2023 11:20:51 -0400 Subject: [PATCH 09/10] fix(actions): adds fixes to Dockerfile location and actions.yml for each each Signed-off-by: Jennifer Power --- Dockerfile | 1 + actions/autosync/action.yml | 2 +- actions/autosync/auto-sync-entrypoint.sh | 2 +- actions/rules-transform/{actions.yml => action.yml} | 2 +- actions/rules-transform/rules-transform-entrypoint.sh | 6 +----- pyproject.toml | 4 ++-- trestlebot/__main__.py | 2 +- 7 files changed, 8 insertions(+), 11 deletions(-) rename actions/rules-transform/{actions.yml => action.yml} (99%) diff --git a/Dockerfile b/Dockerfile index 7e1958bd..a19bbbd9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -67,6 +67,7 @@ RUN microdnf install -y git \ && microdnf clean all \ && rm -rf /var/lib/apt/lists/* +# Add wrappers for entrypoints that provide support the actions COPY ./actions/autosync/auto-sync-entrypoint.sh / RUN chmod +x /auto-sync-entrypoint.sh diff --git a/actions/autosync/action.yml b/actions/autosync/action.yml index 1263f6fa..3353b3c9 100644 --- a/actions/autosync/action.yml +++ b/actions/autosync/action.yml @@ -81,7 +81,7 @@ outputs: runs: using: "docker" - image: "Dockerfile" + image: "../../Dockerfile" entrypoint: "/auto-sync-entrypoint.sh" env: GITHUB_TOKEN: ${{ inputs.github_token }} diff --git a/actions/autosync/auto-sync-entrypoint.sh b/actions/autosync/auto-sync-entrypoint.sh index 732a183c..48103923 100644 --- a/actions/autosync/auto-sync-entrypoint.sh +++ b/actions/autosync/auto-sync-entrypoint.sh @@ -20,7 +20,7 @@ else fi # Initialize the command variable -command="python3.9 -m trestlebot \ +command="trestlebot-autosync \ --markdown-path=\"${INPUT_MARKDOWN_PATH}\" \ --oscal-model=\"${INPUT_OSCAL_MODEL}\" \ --ssp-index-path=\"${INPUT_SSP_INDEX_PATH}\" \ diff --git a/actions/rules-transform/actions.yml b/actions/rules-transform/action.yml similarity index 99% rename from actions/rules-transform/actions.yml rename to actions/rules-transform/action.yml index 7f242b43..9728325b 100644 --- a/actions/rules-transform/actions.yml +++ b/actions/rules-transform/action.yml @@ -62,7 +62,7 @@ outputs: runs: using: "docker" - image: "../Dockerfile" + image: "../../Dockerfile" entrypoint: "/rules-transform-entrypoint.sh" env: GITHUB_TOKEN: ${{ inputs.github_token }} diff --git a/actions/rules-transform/rules-transform-entrypoint.sh b/actions/rules-transform/rules-transform-entrypoint.sh index 2dc6274e..55444170 100644 --- a/actions/rules-transform/rules-transform-entrypoint.sh +++ b/actions/rules-transform/rules-transform-entrypoint.sh @@ -20,7 +20,7 @@ else fi # Initialize the command variable -command="python3.9 /trestle-bot/trestlebot/entrypoints/rules-tranform.py \ +command="trestlebot-rules-transform \ --rules-view-path=\"${INPUT_RULES_VIEW_PATH}\" \ --commit-message=\"${INPUT_COMMIT_MESSAGE}\" \ --pull-request-title=\"${INPUT_PULL_REQUEST_TITLE}\" \ @@ -35,10 +35,6 @@ command="python3.9 /trestle-bot/trestlebot/entrypoints/rules-tranform.py \ --skip-items=\"${INPUT_SKIP_ITEMS}\"" -if [[ ${INPUT_CHECK_ONLY} == true ]]; then - command+=" --check-only" -fi - # Only set the token value when is a target branch so pull requests can be created if [[ -n ${INPUT_TARGET_BRANCH} ]]; then if [[ -z ${GITHUB_TOKEN} ]]; then diff --git a/pyproject.toml b/pyproject.toml index fd932fe5..913aa809 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,8 +16,8 @@ readme = 'README.md' repository = 'https://github.com/RedHatProductSecurity/trestle-bot' [tool.poetry.scripts] -autosync = "trestlebot.entrypoints.autosync:main" -rules-transform = "trestlebot.entrypoints.rule_transform:main" +trestlebot-autosync = "trestlebot.entrypoints.autosync:main" +trestlebot-rules-transform = "trestlebot.entrypoints.rule_transform:main" [tool.poetry.dependencies] python = '^3.8.1' diff --git a/trestlebot/__main__.py b/trestlebot/__main__.py index 2b115f52..a8e9c8d2 100644 --- a/trestlebot/__main__.py +++ b/trestlebot/__main__.py @@ -14,7 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. -# Default entrypoint for trestlebot is autosync mode +# Default entrypoint for trestlebot is autosync mode when run with python -m trestlebot from trestlebot.entrypoints.autosync import main as autosync_main From 77b9d0118ad90ca3193b2aac02be1d54cd7062cf Mon Sep 17 00:00:00 2001 From: Jennifer Power Date: Wed, 11 Oct 2023 16:52:26 -0400 Subject: [PATCH 10/10] docs: adds PR feedback and moves "add action" steps to actions README.md Signed-off-by: Jennifer Power --- CONTRIBUTING.md | 6 +----- actions/README.md | 7 ++----- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8503407b..d1debe40 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -120,8 +120,4 @@ cat my-token.txt | podman secret create repo-secret - ```bash podman run --entrypoint /entrypoint.sh --secret repo-secret,type=env,target=GITHUB_TOKEN --env-file=envfile -v my-trestle-space:/data -w /data localhost:5000/trestlebot:latest -``` - -### Adding a new action - -First, create an entrypoint script for the new action in the `trestlebot/entrypoints` directory. Then add the action by creating a new directory in the `actions` directory with an `action.yml` that references your new entrypoint. \ No newline at end of file +``` \ No newline at end of file diff --git a/actions/README.md b/actions/README.md index cf9ed856..026d8107 100644 --- a/actions/README.md +++ b/actions/README.md @@ -1,14 +1,11 @@ # Actions -Tasks in trestle-bot that we want to expose as an Action are located here. +Tasks in trestle-bot that we want to expose as a GitHub Action are located here. # Adding a new Action > Note to contributors: We are trying to limit the task that we expose as actions to workspace manage operations and checks only. -1. Add a new folder with the action name -2. Add an action.yml -3. Add an entrypoint script to handle inputs from GitHub Actions -4. Add a COPY line in the Dockerfile +First, create an entrypoint script for the new action in the `trestlebot/entrypoints` directory. Then add the action by creating a new directory in the `actions` directory with an `action.yml` that references your new entrypoint. See the [GitHub Actions documentation](https://docs.github.com/en/actions/creating-actions/creating-a-docker-container-action) for more information.