feat: add store_payload_and_result opt

README.md

@@ -6,13 +6,6 @@ This extension will capture and retain a comprehensive record of all changes wit
 
 Read the [documentation](https://datashades.github.io/ckanext-event-audit/) for a full user guide.
 
-TODO:
-- [ ] add config option to exclude result and payload fields from being stored
-- [ ] allow to restrict a list of available repos (security concern)
-- [X] disable the admin interface by default (security concern)
-- [X] update `remove_events` method to allow removing events by date range
-- [X] add a cli command to remove events by date range
-
 ## Quick start
 
 1. Install the extension from `PyPI`:

ckanext/event_audit/config.py

@@ -19,6 +19,8 @@
 CONF_DATABASE_TRACK_ENABLED = "ckanext.event_audit.track.model"
 CONF_API_TRACK_ENABLED = "ckanext.event_audit.track.api"
 
+CONF_STORE_PAYLOAD_AND_RESULT = "ckanext.event_audit.store_payload_and_result"
+
 CONF_BATCH_SIZE = "ckanext.event_audit.batch.size"
 CONF_BATCH_TIMEOUT = "ckanext.event_audit.batch.timeout"
 
@@ -70,6 +72,12 @@ def is_api_log_enabled() -> bool:
     return tk.config[CONF_API_TRACK_ENABLED]
 
 
+def should_store_payload_and_result() -> bool:
+    """Check if the payload and result should be stored in the event. Works
+    only for in-built listeners."""
+    return tk.config[CONF_STORE_PAYLOAD_AND_RESULT]
+
+
 def get_batch_size() -> int:
     return tk.config[CONF_BATCH_SIZE]
 

ckanext/event_audit/config_declaration.yaml

@@ -67,6 +67,12 @@ groups:
         editable: true
         type: bool
 
+      - key: ckanext.event_audit.store_payload_and_result
+        description: Store the payload and result of the event
+        default: false
+        editable: true
+        type: bool
+
       - key: ckanext.event_audit.batch.size
         description: The number of events to batch before sending to the repository
         default: 50

ckanext/event_audit/listeners/api.py

@@ -25,6 +25,8 @@ def action_succeeded_subscriber(
         return
 
     thread_mode_enabled = config.is_threaded_mode_enabled()
+    should_store_complex_data = config.should_store_payload_and_result()
+    result = result if isinstance(result, dict) else {"result": result}
 
     event = repo.build_event(
         types.EventData(
@@ -35,8 +37,8 @@ def action_succeeded_subscriber(
                 else ""
             ),
             action=action_name,
-            payload=data_dict,
-            result=result if isinstance(result, dict) else {"result": result},
+            payload=data_dict if should_store_complex_data else {},
+            result=result if should_store_complex_data else {},
         )
     )
 

ckanext/event_audit/listeners/database.py

@@ -60,6 +60,7 @@ def after_commit(session: SQLAlchemySession):
         return
 
     thread_mode_enabled = config.is_threaded_mode_enabled()
+    should_store_complex_data = config.should_store_payload_and_result()
 
     for action, instances in session._audit_cache.items():  # type: ignore
         for instance in instances:
@@ -72,7 +73,11 @@ def after_commit(session: SQLAlchemySession):
                     action=action,
                     action_object=instance.__class__.__name__,
                     action_object_id=inspect(instance).identity[0],
-                    payload=_filter_payload(instance.__dict__),
+                    payload=(
+                        _filter_payload(instance.__dict__)
+                        if should_store_complex_data
+                        else {}
+                    ),
                 )
             )
 

ckanext/event_audit/tests/listeners/test_api.py

@@ -57,3 +57,22 @@ def _check_events(self, repo: repositories.AbstractRepository):
         events = repo.filter_events(types.Filters())
 
         assert len(events) == 1
+
+    @pytest.mark.usefixtures("with_plugins", "clean_db")
+    @pytest.mark.ckan_config(config.CONF_ACTIVE_REPO, "postgres")
+    def test_payload_and_result_arent_stored_by_default(self, repo: repositories.AbstractRepository):
+        call_action("status_show", {})
+        events = repo.filter_events(types.Filters())
+
+        assert events[0].payload == {}
+        assert events[0].result == {}
+
+
+    @pytest.mark.usefixtures("with_plugins", "clean_db")
+    @pytest.mark.ckan_config(config.CONF_ACTIVE_REPO, "postgres")
+    @pytest.mark.ckan_config(config.CONF_STORE_PAYLOAD_AND_RESULT, True)
+    def test_store_payload_and_result(self, repo: repositories.AbstractRepository):
+        call_action("status_show", {})
+        events = repo.filter_events(types.Filters())
+
+        assert events[0].result["site_title"] == "CKAN"

ckanext/event_audit/tests/listeners/test_model.py

@@ -19,11 +19,13 @@
 class TestModelListener:
     @pytest.mark.usefixtures("with_plugins", "clean_redis", "clean_db")
     @pytest.mark.ckan_config(config.CONF_ACTIVE_REPO, "redis")
+    @pytest.mark.ckan_config(config.CONF_STORE_PAYLOAD_AND_RESULT, True)
     def test_redis(self, user: dict[str, Any], repo: repositories.AbstractRepository):
         self._check_events(user, repo)
 
     @pytest.mark.usefixtures("with_plugins", "clean_db")
     @pytest.mark.ckan_config(config.CONF_ACTIVE_REPO, "cloudwatch")
+    @pytest.mark.ckan_config(config.CONF_STORE_PAYLOAD_AND_RESULT, True)
     def test_cloudwatch(
         self,
         user: dict[str, Any],
@@ -108,6 +110,7 @@ def test_cloudwatch(
 
     @pytest.mark.usefixtures("with_plugins", "clean_db")
     @pytest.mark.ckan_config(config.CONF_ACTIVE_REPO, "postgres")
+    @pytest.mark.ckan_config(config.CONF_STORE_PAYLOAD_AND_RESULT, True)
     def test_postgres(
         self, user: dict[str, Any], repo: repositories.AbstractRepository
     ):
@@ -144,3 +147,26 @@ def _check_events(
         assert events[dashboard].action_object == "Dashboard"
         assert events[dashboard].action_object_id == user["id"]
         assert events[dashboard].payload["user_id"] == user["id"]
+
+    @pytest.mark.usefixtures("with_plugins", "clean_db")
+    @pytest.mark.ckan_config(config.CONF_ACTIVE_REPO, "postgres")
+    def test_payload_and_result_arent_stored_by_default(
+        self, user: dict[str, Any], repo: repositories.AbstractRepository
+    ):
+        events = repo.filter_events(types.Filters())
+
+        user_idx = 1
+
+        assert events[user_idx].payload == {}
+
+    @pytest.mark.usefixtures("with_plugins", "clean_db")
+    @pytest.mark.ckan_config(config.CONF_ACTIVE_REPO, "postgres")
+    @pytest.mark.ckan_config(config.CONF_STORE_PAYLOAD_AND_RESULT, True)
+    def test_store_payload_and_result(
+        self, user: dict[str, Any], repo: repositories.AbstractRepository
+    ):
+        events = repo.filter_events(types.Filters())
+
+        user_idx = 1
+
+        assert events[user_idx].payload["name"] == user["name"]

docs/configure/tracking.md

@@ -26,8 +26,17 @@ ckanext.event_audit.track.model = false
 
 We can ignore specific models from being tracked by setting the `ckanext.event_audit.ignore.models` configuration option. See the [ignore](ignore.md) section for more details.
 
-## Custom trackers
+## Storing payload and result data
+
+Storing `payload` and `result` data for in-built trackers is disabled by default, as it might be too expensive to store all the data. You can enable it by setting the following configuration options:
 
-You can create and write an event anywhere in your codebase.
+```ini
+ckanext.event_audit.store_payload_and_result = true
+```
+
+???+ Warning
+    Enabling this option might have a significant impact on the storage size. Use it with caution.
+
+## Custom trackers
 
-TODO: add link to usage docs
+You can create and write an event anywhere in your codebase. See the [usage](../usage.md) section for more details.

site/configure/tracking/index.html

@@ -599,6 +599,15 @@
     </span>
   </a>
 
+</li>
+
+        <li class="md-nav__item">
+  <a href="#storing-payload-and-result-data" class="md-nav__link">
+    <span class="md-ellipsis">
+      Storing payload and result data
+    </span>
+  </a>
+
 </li>
 
         <li class="md-nav__item">
@@ -1030,6 +1039,15 @@
     </span>
   </a>
 
+</li>
+
+        <li class="md-nav__item">
+  <a href="#storing-payload-and-result-data" class="md-nav__link">
+    <span class="md-ellipsis">
+      Storing payload and result data
+    </span>
+  </a>
+
 </li>
 
         <li class="md-nav__item">
@@ -1073,9 +1091,16 @@ <h2 id="database-tracker">Database tracker</h2>
 <div class="highlight"><pre><span></span><code><span class="na">ckanext.event_audit.track.model</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">false</span>
 </code></pre></div>
 <p>We can ignore specific models from being tracked by setting the <code>ckanext.event_audit.ignore.models</code> configuration option. See the <a href="../ignore/">ignore</a> section for more details.</p>
+<h2 id="storing-payload-and-result-data">Storing payload and result data</h2>
+<p>Storing <code>payload</code> and <code>result</code> data for in-built trackers is disabled by default, as it might be too expensive to store all the data. You can enable it by setting the following configuration options:</p>
+<div class="highlight"><pre><span></span><code><span class="na">ckanext.event_audit.store_payload_and_result</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">true</span>
+</code></pre></div>
+<details class="warning" open="open">
+<summary>Warning</summary>
+<p>Enabling this option might have a significant impact on the storage size. Use it with caution.</p>
+</details>
 <h2 id="custom-trackers">Custom trackers</h2>
-<p>You can create and write an event anywhere in your codebase.</p>
-<p>TODO: add link to usage docs</p>
+<p>You can create and write an event anywhere in your codebase. See the <a href="../../usage/">usage</a> section for more details.</p>