Add auditree evidence locker large file check (#64)

CHANGES.md

@@ -1,3 +1,7 @@
+# [0.14.0](https://github.com/ComplianceAsCode/auditree-arboretum/releases/tag/v0.14.0)
+
+- [ADDED] Auditree evidence locker large file check has been added.
+
 # [0.13.0](https://github.com/ComplianceAsCode/auditree-arboretum/releases/tag/v0.13.0)
 
 - [ADDED] Auditree empty evidence check has been added.

arboretum/__init__.py

@@ -14,4 +14,4 @@
 # limitations under the License.
 """Arboretum - Checking your compliance & security posture, continuously."""
 
-__version__ = '0.13.0'
+__version__ = '0.14.0'

arboretum/auditree/README.md

@@ -476,6 +476,43 @@ can vary across evidences.  But the default `is_empty` criteria is as follows:
    from arboretum.auditree.checks.test_empty_evidence import EmptyEvidenceCheck
    ```
 
+### Evidence Locker Large Files
+
+* Class: [LargeFilesCheck][check-large-files]
+* Purpose: Remote hosting services have file size constraints that if violated
+will cause a remote push of the locker to be rejected.  This check identifies
+"large" sized files so they can be dealt with prior to reaching the remote
+hosting service file size constraint.
+* Behavior: Performs a check that flags all files in the evidence locker
+(evidence and non-evidence) that exceed a large file threshold setting as
+failures and flags all files within 20% of the threshold as warnings.  The
+large file threshold defaults to 50MB.
+* Evidence depended upon:
+   * This check does not depend on any evidence specifically.  It acts on **all**
+   files contained within an evidence locker.
+* Configuration elements:
+   * `locker.large_file_threshold`
+      * Optional
+      * File size threshold in bytes as an integer
+      * Use if looking to override the default of 50MB.  Otherwise do not include.
+      * NOTE: This is the same [evidence locker][] configuration setting is used by
+      the framework to produce large file execution log INFO messages.
+* Example (optional) configuration:
+
+   ```json
+   {
+     "locker": {
+       "large_file_threshold": 50000000
+     }
+   }
+   ```
+
+* Import statement:
+
+   ```python
+   from arboretum.auditree.checks.test_locker_large_files import LargeFilesCheck
+   ```
+
 ### Compliance Configuration
 
 * Class: [ComplianceConfigCheck][check-compliance-config]
@@ -734,6 +771,7 @@ getting new commits.  This check validates that.
 [auditree-framework]: https://github.com/ComplianceAsCode/auditree-framework
 [auditree-framework documentation]: https://complianceascode.github.io/auditree-framework/
 [usage]: https://github.com/ComplianceAsCode/auditree-arboretum#usage
+[evidence locker]: https://complianceascode.github.io/auditree-framework/design-principles.html#evidence-locker
 [fetch-abandoned-evidence]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/auditree/fetchers/fetch_abandoned_evidence.py
 [fetch-compliance-config]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/auditree/fetchers/fetch_compliance_config.py
 [fetch-python-packages]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/auditree/fetchers/fetch_python_packages.py
@@ -743,6 +781,7 @@ getting new commits.  This check validates that.
 [fetch-branch-protection]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/auditree/fetchers/github/fetch_branch_protection.py
 [check-abandoned-evidence]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/auditree/checks/test_abandoned_evidence.py
 [check-empty-evidence]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/auditree/checks/test_empty_evidence.py
+[check-large-files]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/auditree/checks/test_locker_large_files.py
 [check-compliance-config]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/auditree/checks/test_compliance_config.py
 [check-python-packages]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/auditree/checks/test_python_packages.py
 [check-locker-integrity]: https://github.com/ComplianceAsCode/auditree-arboretum/blob/main/arboretum/auditree/checks/test_locker_repo_integrity.py

arboretum/auditree/checks/test_locker_large_files.py

@@ -0,0 +1,90 @@
+# -*- mode:python; coding:utf-8 -*-
+# Copyright (c) 2021 IBM Corp. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Evidence locker large files check."""
+from compliance.check import ComplianceCheck
+from compliance.evidence import DAY, ReportEvidence
+from compliance.locker import LF_DEFAULT, MB
+
+
+class LargeFilesCheck(ComplianceCheck):
+    """
+    Evidence locker large files check.
+
+    This check finds all large FILES in the evidence locker.
+    """
+
+    @property
+    def title(self):
+        """
+        Return the title of the checks.
+
+        :returns: the title of the checks
+        """
+        return 'Large Evidence Locker Files'
+
+    @classmethod
+    def setUpClass(cls):
+        """Initialize the check object with configuration settings."""
+        cls.config.add_evidences(
+            [
+                ReportEvidence(
+                    'locker_large_files.md',
+                    'auditree',
+                    DAY,
+                    'Evidence locker large files report.'
+                )
+            ]
+        )
+        return cls
+
+    def test_large_files(self):
+        """Check for large files."""
+        lf_size = self.config.get('locker.large_file_threshold', LF_DEFAULT)
+        warn_size = .8 * lf_size
+        for f_path, f_size in self.locker.get_large_files(warn_size).items():
+            if f_size > lf_size:
+                # Fail if file size is over the large file threshold
+                self.add_failures(
+                    'Large Files:', f'`{f_path}` - {_size_to_str(f_size)}'
+                )
+            elif f_size > warn_size:
+                # Warn if file size is within 20% of the large file threshold
+                self.add_warnings(
+                    'Large Files (almost):',
+                    f'`{f_path}` - {_size_to_str(f_size)}'
+                )
+
+    def get_reports(self):
+        """
+        Provide the check report name.
+
+        :returns: the report(s) generated for this check.
+        """
+        return ['auditree/locker_large_files.md']
+
+    def get_notification_message(self):
+        """
+        Large files check notifier.
+
+        :returns: notification dictionary.
+        """
+        return {'body': None}
+
+
+def _size_to_str(size):
+    formatted_size = f'{size/MB:.1f} MB'
+    if formatted_size == '0.0 MB':
+        formatted_size = f'{str(size)} Bytes'
+    return formatted_size

devel.json

@@ -3,14 +3,29 @@
     "enabled": false
   },
   "locker": {
-    "repo_url": "https://github.com/my-org/my-evidence-locker-repo"
+    "repo_url": "https://github.com/my-org/my-evidence-locker-repo",
+    "large_file_threshold": 50000000
   },
   "notify": {
     "slack": {}
   },
   "org": {
     "name": "MY-ORG",
     "auditree": {
+      "abandoned_evidence": {
+         "threshold": 1234567,
+         "exceptions": {
+           "raw/foo/evidence_bar.json": "This is a good reason",
+           "raw/foo/evidence_baz.json": "This is also a good reason"
+         },
+         "ignore_history": true
+      },
+      "empty_evidence": {
+        "exceptions": [
+          "raw/foo/evidence_bar.json",
+          "raw/foo/evidence_baz.json"
+        ]
+      },
       "repo_integrity": {
         "branches": {
           "https://github.com/ComplianceAsCode/auditree-arboretum": ["main"]