diff --git a/CHANGES.md b/CHANGES.md index 46d0dfd0..e863571b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,7 @@ +# 0.2.0 + +- [NEW] Add Python packages fetcher and check. + # 0.1.0 - [NEW] Add compliance configuration fetcher and check. diff --git a/arboretum/__init__.py b/arboretum/__init__.py index a1adfd09..573376ee 100644 --- a/arboretum/__init__.py +++ b/arboretum/__init__.py @@ -14,4 +14,4 @@ # limitations under the License. """Arboretum - Checking your compliance & security posture, continuously.""" -__version__ = '0.1.0' +__version__ = '0.2.0' diff --git a/arboretum/common/constants.py b/arboretum/common/constants.py new file mode 100644 index 00000000..2a914400 --- /dev/null +++ b/arboretum/common/constants.py @@ -0,0 +1,18 @@ +# -*- mode:python; coding:utf-8 -*- +# Copyright (c) 2020 IBM Corp. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Common constants.""" + +# PyPI RSS feed base URL +PYPI_RSS_BASE_URL = 'https://pypi.org/rss/project' diff --git a/arboretum/technology/auditree/README.md b/arboretum/technology/auditree/README.md index acc5aa31..bed47dfb 100644 --- a/arboretum/technology/auditree/README.md +++ b/arboretum/technology/auditree/README.md @@ -1,26 +1,32 @@ # Auditree technology library -The fetchers and checks contained within this `auditree` technology folder are common tests that can be configured and -executed for the purpose of generating compliance reports and notifications using the [auditree-framework][]. -See [auditree-framework documentation](https://complianceascode.github.io/auditree-framework/) for more details. +The fetchers and checks contained within this `auditree` technology folder are +common tests that can be configured and executed for the purpose of generating +compliance reports and notifications using the [auditree-framework][]. They +validate the configuration and ensure smooth execution of an auditree instance. +See [auditree-framework documentation](https://complianceascode.github.io/auditree-framework/) +for more details. -These tests are normally executed by a CI/CD system like [Travis CI](https://travis-ci.com/) as part of another project -that uses this library package as a dependency. +These tests are normally executed by a CI/CD system like +[Travis CI](https://travis-ci.com/) as part of another project that uses this +library package as a dependency. ## Usage as a library -See [usage][usage] for specifics on including this library as a dependency and how to include the fetchers and checks -from this library in your downstream project. +See [usage][usage] for specifics on including this library as a dependency and +how to include the fetchers and checks from this library in your downstream project. ## Fetchers ### Abandoned Evidence * Class: [AbandonedEvidenceFetcher][fetch-abandoned-evidence] -* Purpose: Writes evidence that has been identified as abandoned to the evidence locker. -* Behavior: Stores abandoned evidence and abandoned evidence exceptions to the evidence locker. If the optional -`threshold` configuration setting is applied then abandoned evidence is identified as evidence that has not been -updated in over that `threshold` value otherwise the default is 30 days. TTL is set to 1 day. +* Purpose: Writes evidence that has been identified as abandoned to the evidence +locker. +* Behavior: Stores abandoned evidence and abandoned evidence exceptions to the +evidence locker. If the optional `threshold` configuration setting is applied +then abandoned evidence is identified as evidence that has not been updated in +over that `threshold` value otherwise the default is 30 days. TTL is set to 1 day. * Expected configuration elements: * org.auditree.abandoned\_evidence.threshold * Optional @@ -29,11 +35,12 @@ updated in over that `threshold` value otherwise the default is 30 days. TTL is * Use if looking to override the default of 30 days otherwise do not include. * org.auditree.abandoned\_evidence.exceptions * Optional - * Dictionary where the key/value pairs are the path to the evidence (key) and the reason for excluding it from the - abandoned evidence list (value). + * Dictionary where the key/value pairs are the path to the evidence (key) + and the reason for excluding it from the abandoned evidence list (value). * Key/Value: String/String - * Use if looking to exclude evidence files from being deemed abandoned and included as failures. All "exceptions" - will still appear on the report and will be treated as warnings rather than failures. + * Use if looking to exclude evidence files from being deemed abandoned and + included as failures. All "exceptions" will still appear on the report and + will be treated as warnings rather than failures. * Expected configuration (optional): ```json @@ -61,9 +68,12 @@ updated in over that `threshold` value otherwise the default is 30 days. TTL is ### Compliance Configuration * Class: [ComplianceConfigFetcher][fetch-compliance-config] -* Purpose: Writes the current configuration stored in the ComplianceConfig object to the evidence locker. -* Behavior: Stores the configuration used to execute the compliance fetchers to the evidence locker and sets a time to -live (TTL) to 2 hours. This fetcher will refresh the configuration evidence on every execution of the fetchers. +* Purpose: Writes the current configuration stored in the ComplianceConfig object +to the evidence locker. +* Behavior: Stores the configuration used to execute the compliance fetchers to +the evidence locker and sets a time to live (TTL) to 2 hours. This fetcher +ignores TTL and will refresh the configuration evidence on every execution of +the fetchers. * Expected configuration elements: * None * Expected configuration: @@ -76,24 +86,47 @@ live (TTL) to 2 hours. This fetcher will refresh the configuration evidence on from arboretum.technology.auditree.fetchers.fetch_compliance_config import ComplianceConfigFetcher ``` +### Python Packages + +* Class: [PythonPackageFetcher][fetch-python-packages] +* Purpose: Writes the current Python package dependency list to evidence. +* Behavior: Stores the current Python package dependency list as evidence and +the latest release information for `auditree-arboretum`, `auditree-framework` +and `auditree-harvest` are also retrieved and stored as evidence. The time to +live (TTL) is set to 1 day for all evidences. +* Expected configuration elements: + * None +* Expected configuration: + * None +* Expected credentials: + * None +* Import statement: + + ```python + from arboretum.technology.auditree.fetchers.fetch_python_packages import PythonPackageFetcher + ``` + ## Checks ### Abandoned Evidence * Class: [AbandonedEvidenceCheck][check-abandoned-evidence] -* Purpose: For every piece of evidence that has not been updated for longer than the time to live plus the specified -threshold a failure is generated and reported. -* Behavior: Performs a check that compares abandoned evidence identified on a given check execution with the last time new -abandoned evidence was found and reports on newly found abandoned evidence and possible exceptions. If no "abandoned -evidence" evidence is contained within the locker then this check traverses the evidence locker repository and -identifies evidence that has not been updated for a specific period of time and reports on abandoned evidence found for -the current check execution. The default threshold is 30 days beyond the time to live (TTL) setting. +* Purpose: For every piece of evidence that has not been updated for longer than +the time to live plus the specified threshold a failure is generated and reported. +* Behavior: Performs a check that compares abandoned evidence identified on a given +check execution with the last time new abandoned evidence was found and reports +on newly found abandoned evidence and possible exceptions. If no "abandoned evidence" +evidence is contained within the locker then this check traverses the evidence +locker repository and identifies evidence that has not been updated for a specific +period of time and reports on abandoned evidence found for the current check +execution. The default threshold is 30 days beyond the time to live (TTL) setting. * Evidence depended upon: * Abandoned evidence and exceptions * `raw/auditree/abandoned_evidence.json` * Gathered by the `auditree` provider [AbandonedEvidenceFetcher][fetch-abandoned-evidence] - * If the [AbandonedEvidenceFetcher][fetch-abandoned-evidence] is not used to store "abandoned evidence" evidence in - the locker then the tooling performs a sweep of the evidence locker metadata to assess evidence that has not been + * If the [AbandonedEvidenceFetcher][fetch-abandoned-evidence] is not used to + store "abandoned evidence" evidence in the locker then the tooling performs + a sweep of the evidence locker metadata to assess evidence that has not been updated in the timeframe specified. * Expected configuration elements: * org.auditree.abandoned\_evidence.threshold @@ -103,17 +136,19 @@ the current check execution. The default threshold is 30 days beyond the time t * Use if looking to override the default of 30 days otherwise do not include. * org.auditree.abandoned\_evidence.exceptions * Optional - * Dictionary where the key/value pairs are the path to the evidence (key) and the reason for excluding it from the - abandoned evidence list (value). + * Dictionary where the key/value pairs are the path to the evidence (key) + and the reason for excluding it from the abandoned evidence list (value). * Key/Value: String/String - * Use if looking to exclude evidence files from being deemed abandoned and included as failures. All "exceptions" - will still appear on the report and will be treated as warnings rather than failures. + * Use if looking to exclude evidence files from being deemed abandoned + and included as failures. All "exceptions" will still appear on the + report and will be treated as warnings rather than failures. * org.auditree.abandoned\_evidence.ignore\_history * Optional * Boolean * Set to `true` - * Use if collecting `raw/auditree/abandoned_evidence.json` in the evidence locker but intend to run the check - without referencing the evidence history (more rigid alerts). Otherwise do not include. + * Use if collecting `raw/auditree/abandoned_evidence.json` in the evidence + locker but intend to run the check without referencing the evidence history + (more rigid alerts). Otherwise do not include. * Expected configuration (optional): ```json @@ -141,10 +176,10 @@ the current check execution. The default threshold is 30 days beyond the time t ### Compliance Configuration * Class: [ComplianceConfigCheck][check-compliance-config] -* Purpose: Compare the configuration captured as evidence with the current configuration in the ComplianceConfig object -being used to execute the checks. -* Behavior: For every difference found between the evidence and the current configuration a failure is generated and -reported on. +* Purpose: Compare the configuration captured as evidence with the current +configuration in the ComplianceConfig object being used to execute the checks. +* Behavior: For every difference found between the evidence and the current +configuration a failure is generated and reported on. * Evidence depended upon: * Compliance tooling configuration settings * `raw/auditree/compliance_config.json` @@ -160,8 +195,39 @@ reported on. from auditree_central.provider.auditree.checks.test_compliance_config import ComplianceConfigCheck ``` +### Python Packages + +* Class: [PythonPackageCheck][check-python-packages] +* Purpose: Compare the most recent Python package evidence with the evidence +from the most recent historical version of that evidence found in the locker and +checks that the `auditree-arboretum`, `auditree-framework` and `auditree-harvest` +packages are at their most current release level. +* Behavior: For every difference found between the two versions of evidence a +warning is generated and reported on. Warnings are also generated when the +`auditree-arboretum`, `auditree-framework`, or `auditree-harvest` packages being +used are not at the current release version. +* Evidence depended upon: + * The executing environment's Python package list + * `raw/auditree/python_packages.json` + * `raw/auditree/auditree_arboretum_releases.xml` + * `raw/auditree/auditree_framework_releases.xml` + * `raw/auditree/auditree_harvest_releases.xml` + * Gathered by the `technology.auditree` [PythonPackageFetcher][fetch-python-packages] +* Expected configuration elements: + * None +* Expected configuration (optional): + * None + +* Import statement: + + ```python + from arboretum.technology.auditree.checks.test_python_packages import PythonPackageCheck + ``` + [usage]: https://github.com/ComplianceAsCode/auditree-arboretum#usage [fetch-abandoned-evidence]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/technology/auditree/fetchers/fetch_abandoned_evidence.py [fetch-compliance-config]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/technology/auditree/fetchers/fetch_compliance_config.py +[fetch-python-packages]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/technology/auditree/fetchers/fetch_python_packages.py [check-abandoned-evidence]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/technology/auditree/checks/test_abandoned_evidence.py [check-compliance-config]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/technology/auditree/checks/test_compliance_config.py +[check-python-packages]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/technology/auditree/checks/test_python_packages.py diff --git a/arboretum/technology/auditree/checks/test_python_packages.py b/arboretum/technology/auditree/checks/test_python_packages.py new file mode 100644 index 00000000..05a27185 --- /dev/null +++ b/arboretum/technology/auditree/checks/test_python_packages.py @@ -0,0 +1,162 @@ +# -*- mode:python; coding:utf-8 -*- +# Copyright (c) 2020 IBM Corp. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Execution environment Python packages checks.""" + +import json +from datetime import datetime, timedelta + +from compliance.check import ComplianceCheck +from compliance.evidence import DAY, ReportEvidence, with_raw_evidences + +from ..evidences.python_package_release import PackageReleaseEvidence + + +class PythonPackageCheck(ComplianceCheck): + """Compare the software versions used in Auditree execution.""" + + @property + def title(self): + """ + Return the title of the checks. + + :returns: the title of the checks + """ + return 'Python Packages' + + @classmethod + def setUpClass(cls): + """Initialize the check object with configuration settings.""" + cls.config.add_evidences( + [ + ReportEvidence( + 'python_packages.md', + 'auditree', + DAY, + 'Execution environment Python packages report.' + ) + ] + ) + + return cls + + @with_raw_evidences('auditree/python_packages.json') + def test_python_package_deltas(self, today): + """Check the last two Python environments against each other.""" + yesterday = None + yesterday_dt = datetime.utcnow() - timedelta(days=1) + try: + yesterday = self.locker.get_evidence( + 'raw/auditree/python_packages.json', + ignore_ttl=True, + evidence_dt=yesterday_dt + ) + except ValueError: + self.add_warnings( + 'Python Package Deltas', + ( + 'No evidence found on or prior ' + f'to {yesterday_dt.strftime("%b %d, %Y")}' + ) + ) + return + today_pkgs = json.loads(today.content) + yesterday_pkgs = json.loads(yesterday.content) + for pkg, ver in today_pkgs.items(): + if pkg not in yesterday_pkgs.keys(): + self.add_warnings('New Packages', f'{pkg} version {ver}') + elif ver != yesterday_pkgs[pkg]: + self.add_warnings( + 'Package Version Changes', + ( + f'{pkg} previous version {yesterday_pkgs[pkg]}, ' + f'current version {ver}' + ) + ) + for pkg in set(yesterday_pkgs.keys()) - set(today_pkgs.keys()): + self.add_warnings( + 'Removed Packages', f'{pkg} version {yesterday_pkgs[pkg]}' + ) + + @with_raw_evidences( + 'auditree/python_packages.json', + 'auditree/auditree_arboretum_releases.xml' + ) + def test_auditree_arboretum_version(self, packages, releases): + """Check auditree-arboretum version matches latest release.""" + self._test_versions(packages, releases, 'auditree-arboretum') + + @with_raw_evidences( + 'auditree/python_packages.json', + 'auditree/auditree_framework_releases.xml' + ) + def test_auditree_framework_version(self, packages, releases): + """Check auditree-framework version matches latest release.""" + self._test_versions(packages, releases, 'auditree-framework') + + def test_auditree_harvest_version(self, packages, releases): + """Check auditree-harvest version matches latest release.""" + self._test_versions(packages, releases, 'auditree-harvest') + + def _test_versions(self, packages, releases, package): + latest = PackageReleaseEvidence.from_evidence(releases).latest_release + version_used = json.loads(packages.content).get(package) + if version_used != latest: + self.add_warnings( + 'Latest Version Violation', + ( + f'{package} latest version {latest}, ' + f'version used {version_used}' + ) + ) + + def get_reports(self): + """ + Provide the check report name. + + :returns: the report(s) generated for this check + """ + return ['auditree/python_packages.md'] + + def msg_python_package_deltas(self): + """ + Python Packages deltas check notifier. + + :returns: notification dictionary + """ + return {'subtitle': 'Python package deltas', 'body': None} + + def msg_auditree_arboretum_version(self): + """ + Python Packages auditree-arboretum check notifier. + + :returns: notification dictionary + """ + return {'subtitle': 'auditree-arboretum version', 'body': None} + + def msg_auditree_framework_version(self): + """ + Python Packages auditree-framework check notifier. + + :returns: notification dictionary + """ + return {'subtitle': 'auditree-framework version', 'body': None} + + def msg_auditree_harvest_version(self): + """ + Python Packages auditree-harvest check notifier. + + :returns: notification dictionary + """ + return {'subtitle': 'auditree-harvest version', 'body': None} diff --git a/arboretum/technology/auditree/evidences/python_package_release.py b/arboretum/technology/auditree/evidences/python_package_release.py new file mode 100644 index 00000000..13af6318 --- /dev/null +++ b/arboretum/technology/auditree/evidences/python_package_release.py @@ -0,0 +1,38 @@ +# -*- mode:python; coding:utf-8 -*- +# Copyright (c) 2020 IBM Corp. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Python PyPI release evidence.""" + +from xml.etree.ElementTree import fromstring + +from compliance.evidence import RawEvidence + + +class PackageReleaseEvidence(RawEvidence): + """Python package release raw evidence class.""" + + @property + def latest_release(self): + """ + Get the latest release from the evidence. + + Gets the latest release version for the Python package represented + by the evidence. + """ + if not self.content: + return + if not hasattr(self, '_latest_release'): + root = fromstring(self.content) + self._latest_release = root[0].find('item').find('title').text + return self._latest_release diff --git a/arboretum/technology/auditree/fetchers/fetch_python_packages.py b/arboretum/technology/auditree/fetchers/fetch_python_packages.py new file mode 100644 index 00000000..e6dd3d1a --- /dev/null +++ b/arboretum/technology/auditree/fetchers/fetch_python_packages.py @@ -0,0 +1,91 @@ +# -*- mode:python; coding:utf-8 -*- +# Copyright (c) 2020 IBM Corp. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Execution environment Python packages fetchers.""" + +import json + +from arboretum.common.constants import PYPI_RSS_BASE_URL +from arboretum.technology.auditree.evidences.python_package_release import ( + PackageReleaseEvidence +) + +from compliance.evidence import DAY, RawEvidence, store_raw_evidence +from compliance.fetch import ComplianceFetcher + +from pkg_resources import working_set + + +class PythonPackageFetcher(ComplianceFetcher): + """Fetch the current environment's Python package list.""" + + @classmethod + def setUpClass(cls): + """Initialize the fetcher object with configuration settings.""" + cls.config.add_evidences( + [ + RawEvidence( + 'python_packages.json', + 'auditree', + DAY, + 'Python Package List' + ), + PackageReleaseEvidence( + 'auditree_arboretum_releases.xml', + 'auditree', + DAY, + 'Auditree Arboretum PyPI releases' + ), + PackageReleaseEvidence( + 'auditree_framework_releases.xml', + 'auditree', + DAY, + 'Auditree Framework PyPI releases' + ) + ] + ) + headers = { + 'Content-Type': 'application/xml', 'Accept': 'application/xml' + } + cls.session(PYPI_RSS_BASE_URL, **headers) + return cls + + @store_raw_evidence('auditree/python_packages.json') + def fetch_python_package_list(self): + """Fetch the Python packages in the current virtual environment.""" + packages = {} + for dist in working_set: + packages[dist.project_name] = dist.version + + return json.dumps(packages) + + @store_raw_evidence('auditree/auditree_arboretum_releases.xml') + def fetch_auditree_arboretum_releases(self): + """Fetch the auditree-arboretum package releases.""" + return self._fetch_pypi_releases('auditree-arboretum') + + @store_raw_evidence('auditree/auditree_framework_releases.xml') + def fetch_auditree_framework_releases(self): + """Fetch the auditree-framework package releases.""" + return self._fetch_pypi_releases('auditree-framework') + + @store_raw_evidence('auditree/auditree_harvest_releases.xml') + def fetch_auditree_harvest_releases(self): + """Fetch the auditree-harvest package releases.""" + return self._fetch_pypi_releases('auditree-harvest') + + def _fetch_pypi_releases(self, package_name): + resp = self.session().get(f'{package_name}/releases.xml') + resp.raise_for_status() + return resp.text diff --git a/arboretum/technology/auditree/templates/reports/auditree/abandoned_evidence.md.tmpl b/arboretum/technology/auditree/templates/reports/auditree/abandoned_evidence.md.tmpl index bb178564..b0c8aa6c 100644 --- a/arboretum/technology/auditree/templates/reports/auditree/abandoned_evidence.md.tmpl +++ b/arboretum/technology/auditree/templates/reports/auditree/abandoned_evidence.md.tmpl @@ -17,23 +17,32 @@ limitations under the License. # {{ test.title }} Report {{ now.strftime('%Y-%m-%d') }} -This report displays any evidence in the evidence locker that has not been updated in over {{ test.formatted_threshold }}. +This report displays any evidence in the evidence locker that has not been updated +in over {{ test.formatted_threshold }}. {%- if test.title == 'Latest Abandoned Evidence' -%} -This report display only changes to abandoned evidence since the last time new abandoned evidence was discovered. In other -words, when abandoned evidence is **initially** discovered it is displayed here. On subsequent executions of the abandoned -evidence check, the results are suppressed if the abandoned evidence is the same as the previous execution of the check to -reduce the "noise" in this report. +This report display only changes to abandoned evidence since the last time new +abandoned evidence was discovered. In other words, when abandoned evidence is +**initially** discovered it is displayed here. On subsequent executions of the +abandoned evidence check, the results are suppressed if the abandoned evidence +is the same as the previous execution of the check to reduce the "noise" in this +report. {% endif -%} -## Remediation +
+Remediation... -The purpose of this report is to highlight that evidence has not been refreshed in quite some time within the evidence locker. -It _could_ mean that a fetcher or check is malfunctioning or it _could_ mean that the evidence being flagged as abandoned is no -longer in use and should be considered for removal from the evidence locker. +The purpose of this report is to highlight that evidence has not been refreshed +in quite some time within the evidence locker. It _could_ mean that a fetcher +or check is malfunctioning or it _could_ mean that the evidence being flagged as +abandoned is no longer in use and should be considered for removal from the evidence +locker. Evidence can be safely removed from the evidence locker via +[auditree-prune](https://github.com/ComplianceAsCode/auditree-prune). -It is also possible to add abandoned evidence to an exceptions list in your configuration of this check/report thereby reducing -the severity of the notification. Doing this allows you to keep abandoned evidence around for a period of time while deferring -the removal of that abandoned evidence to later point in time. +It is also possible to add abandoned evidence to an exceptions list in your +configuration of this check/report thereby reducing the severity of the notification. +Doing this allows you to keep abandoned evidence around for a period of time while +deferring the removal of that abandoned evidence to later point in time. +
{% if test.total_issues_count(results) == 0 %} **No abandoned evidence to report** @@ -42,10 +51,13 @@ the removal of that abandoned evidence to later point in time. {% for section_heading, failures in all_failures.items() -%} ## {{ section_heading }} -The following evidence have not been updated in over {{ test.formatted_threshold }} and have been identified as abandoned. -Consider validating why this evidence was marked as abandoned. If it was due to a failing fetcher and check then resolve -that failure. If it was due to the evidence no longer being useful then consider [pruning the evidence from the evidence -locker](https://github.com/ComplianceAsCode/auditree-prune) or adding the evidence to the abandoned evidence exceptions list. +The following evidence have not been updated in over {{ test.formatted_threshold }} +and have been identified as abandoned. +Consider validating why this evidence was marked as abandoned. If it was due to a +failing fetcher and check then resolve that failure. If it was due to the evidence +no longer being useful then consider [pruning the evidence from the evidence +locker](https://github.com/ComplianceAsCode/auditree-prune) or adding the evidence +to the abandoned evidence exceptions list. {% for failure in failures %} - [`{{ failure['ae_path'] }}`]({{ test.config.get('locker.repo_url') }}/blob/master/{{ failure['ae_path'] }}) - **Last updated:** {{ failure['last_update'] }} @@ -57,8 +69,9 @@ locker](https://github.com/ComplianceAsCode/auditree-prune) or adding the eviden {% for section_heading, warnings in all_warnings.items() %} ## {{ section_heading }} -The following evidence have not been updated in over {{ test.formatted_threshold }} and have been identified as abandoned -evidence exceptions. Have a look at the reason for an evidence being included on the exceptions list and consider whether +The following evidence have not been updated in over {{ test.formatted_threshold }} +and have been identified as abandoned evidence exceptions. Have a look at the +reason for an evidence being included on the exceptions list and consider whether it makes sense to [prune that evidence from the evidence locker](https://github.com/ComplianceAsCode/auditree-prune). {% for warning in warnings %} diff --git a/arboretum/technology/auditree/templates/reports/auditree/compliance_config.md.tmpl b/arboretum/technology/auditree/templates/reports/auditree/compliance_config.md.tmpl index 779f96d2..bd575612 100644 --- a/arboretum/technology/auditree/templates/reports/auditree/compliance_config.md.tmpl +++ b/arboretum/technology/auditree/templates/reports/auditree/compliance_config.md.tmpl @@ -18,13 +18,19 @@ limitations under the License. # {{ test.title }} Report {{ now.strftime('%Y-%m-%d') }} This report displays differences between the configuration used by the fetcher -execution and the configuration used by the check execution. Fetchers and checks -are executed as separate processes but the expectation is that both processes -are executed using the same configuration settings as defined by a project's -configuration JSON file. If discrepancies are found and reported here then a -serious problem occurred during execution. +execution and the configuration used by the check execution. -## Remediation +
+More details... + +Fetchers and checks are executed as separate processes but the expectation is +that both processes are executed using the same configuration settings as defined +by a project's configuration JSON file. If discrepancies are found and reported +here then a serious problem occurred during execution. +
+ +
+Remediation... All discrepancies reported here need to be fully investigated and accounted for. Since the official execution of fetchers and checks is handled by a CI/CD tool @@ -32,6 +38,7 @@ like Travis, the execution environment should be a closed "eco-system" and there should be no reason for a difference in execution configuration between the fetchers and the checks. In the event of a problem reported here, check your CI/CD build logs for more details. +
{% if test.total_issues_count(results) == 0 %} **No execution configuration discrepancies to report** diff --git a/arboretum/technology/auditree/templates/reports/auditree/python_packages.md.tmpl b/arboretum/technology/auditree/templates/reports/auditree/python_packages.md.tmpl new file mode 100644 index 00000000..d3d29235 --- /dev/null +++ b/arboretum/technology/auditree/templates/reports/auditree/python_packages.md.tmpl @@ -0,0 +1,71 @@ +{#- -*- mode:jinja2; coding: utf-8 -*- -#} +{# +Copyright (c) 2020 IBM Corp. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +#} + +# {{ test.title }} Report {{ now.strftime('%Y-%m-%d') }} + +This report displays differences in the compliance automation execution Python +package environment since the previous execution. + +
+More details... + +Any **package version changes** +are largely informational. They _can_ be used to debug why things worked +previously but aren't working today. The compliance automation framework execution +environment depends on other of Python libraries. These libraries can have new +releases and it's not outside the realm of possibility that a new release of a +dependency could cause a problem. This report provides information that helps to +guard against that. This report also checks whether the versions of the +[auditree-arboretum](https://github.com/ComplianceAsCode/auditree-arboretum), +the [auditree-framework](https://github.com/ComplianceAsCode/auditree-framework) +and the [auditree-harvest](https://github.com/ComplianceAsCode/auditree-harvest) +packages are the most recent versions available, as is the expected behavior. +
+ +
+Remediation... + +Package version change warnings are informational but can be used in part +to debug why things worked previously but currently don't. However, if the +`auditree-arboretum`, `auditree-framework`, or the `auditree-harvest` packages +are flagged as a **latest version violation** then that needs to be explained. +It is expected that the most recent versions of each of those packages are used +during fetcher, check and report execution. +
+ +{% if test.total_issues_count(results) == 0 -%} +**No python package version differences to report** +{% else -%} +{% if test.warnings_for_check_count(results) > 0 -%} +{% for warning, instances in all_warnings.items() %} + +## {{ warning }} +{% if warning == 'Latest Version Violation'-%} +Checks that ensure that the `auditree-arboretum`, `auditree-framework` and +`auditree-harvest` packages are up to date have uncovered issues. These warnings +should be investigated because the expectation is that these packages should be +current and up to date. +{% else -%} +These findings are largely informational. They can be used to debug issues with +the execution environment. +{% endif -%} +{% for instance in instances %} +* {{ instance }} +{%- endfor -%} +{%- endfor -%} +{%- endif -%} +{%- endif -%} diff --git a/controls.json b/controls.json index 4c7faf97..8fd1188b 100644 --- a/controls.json +++ b/controls.json @@ -8,5 +8,10 @@ "auditree_evidence": { "auditree_control": ["tech.auditree"] } + }, + "arboretum.technology.auditree.checks.test_python_packages.PythonPackageCheck": { + "auditree_evidence": { + "auditree_control": ["tech.auditree"] + } } } diff --git a/test/fixtures/pypi_release_info.xml b/test/fixtures/pypi_release_info.xml new file mode 100644 index 00000000..980d0314 --- /dev/null +++ b/test/fixtures/pypi_release_info.xml @@ -0,0 +1,45 @@ + + + + + PyPI recent updates for auditree-framework + https://pypi.org/project/auditree-framework/ + Recent updates to the Python Package Index for auditree-framework + en + + 1.0.2 + https://pypi.org/project/auditree-framework/1.0.2/ + Tool to run compliance control checks as unit tests + al.finkelstein@ibm.com + Tue, 28 Jul 2020 20:48:48 GMT + + + 1.0.1 + https://pypi.org/project/auditree-framework/1.0.1/ + Tool to run compliance control checks as unit tests + al.finkelstein@ibm.com + Mon, 27 Jul 2020 17:30:15 GMT + + + 1.0.0 + https://pypi.org/project/auditree-framework/1.0.0/ + Tool to run compliance control checks as unit tests + al.finkelstein@ibm.com + Tue, 21 Jul 2020 15:38:18 GMT + + + diff --git a/test/test_evidences/__init__.py b/test/test_evidences/__init__.py new file mode 100644 index 00000000..1ec168b2 --- /dev/null +++ b/test/test_evidences/__init__.py @@ -0,0 +1,15 @@ +# -*- mode:python; coding:utf-8 -*- +# Copyright (c) 2020 IBM Corp. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Arboretum evidence helpers unit tests.""" diff --git a/test/test_evidences/test_python_package_release.py b/test/test_evidences/test_python_package_release.py new file mode 100644 index 00000000..e7f83377 --- /dev/null +++ b/test/test_evidences/test_python_package_release.py @@ -0,0 +1,47 @@ +# -*- mode:python; coding:utf-8 -*- +# Copyright (c) 2020 IBM Corp. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Python package release evidence unit tests.""" + +import unittest + +from arboretum.technology.auditree.evidences.python_package_release import ( + PackageReleaseEvidence +) + + +class PythonPackageReleaseTest(unittest.TestCase): + """PythonPackageRelease unit tests.""" + + def test_latest_release_success(self): + """Ensure that latest available release is returned.""" + evidence = PackageReleaseEvidence('foo.xml', 'bar') + evidence.set_content( + open('./test/fixtures/pypi_release_info.xml').read() + ) + self.assertEqual(evidence.latest_release, '1.0.2') + + def test_latest_release_none(self): + """Ensure latest release is None if no content.""" + evidence = PackageReleaseEvidence('foo.xml', 'bar') + self.assertIsNone(evidence.latest_release) + + def test_latest_release_already_set(self): + """Ensure that latest release only retrieved if not yet set.""" + evidence = PackageReleaseEvidence('foo.xml', 'bar') + evidence._latest_release = 'foo.bar.baz' + evidence.set_content( + open('./test/fixtures/pypi_release_info.xml').read() + ) + self.assertEqual(evidence.latest_release, 'foo.bar.baz')