feat: finish config and dashboard implementation

.github/workflows/test.yml

@@ -2,17 +2,19 @@ name: Tests
 on: [push, pull_request]
 jobs:
   test:
+    strategy:
+      matrix:
+        ckan-version: ["2.11", "2.10"]
+      fail-fast: false
+
     runs-on: ubuntu-latest
     container:
-      # The CKAN version tag of the Solr and Postgres containers should match
-      # the one of the container the tests run on.
-      # You can switch this base image with a custom image tailored to your project
-      image: ckan/ckan-dev:2.11
+      image: ckan/ckan-dev:${{ matrix.ckan-version }}
     services:
       solr:
-        image: ckan/ckan-solr:2.11-solr9
+        image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr9
       postgres:
-        image: ckan/ckan-postgres-dev:2.11
+        image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }}
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
@@ -33,9 +35,7 @@ jobs:
     - name: Install requirements
       # Install any extra requirements your extension has here (dev requirements, other extensions etc)
       run: |
-        pip install -r requirements.txt
-        pip install -r dev-requirements.txt
-        pip install -e .
+        pip install -e '.[dev]'
     - name: Setup extension
       # Extra initialization steps
       run: |

ckanext/event_audit/collection.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+from dominate import tags
+
+import ckan.plugins.toolkit as tk
+
+from ckanext.ap_main.collection.base import (
+    ApCollection,
+    ApColumns,
+    ApHtmxTableSerializer,
+)
+from ckanext.collection.shared import data
+
+from ckanext.event_audit import types, utils
+
+
+def event_dictizer(serializer: ApHtmxTableSerializer, row: types.Event):
+    """Return a dictionary representation of an event."""
+    data = row.model_dump()
+
+    data["bulk-action"] = data["id"]
+
+    return data
+
+
+class EventAuditData(data.Data):
+    def compute_data(self):
+        """Return a list of events.
+
+        TODO: Implement proper filtering. Right now we're fetching all the
+        events from the repository, which is not efficient (not even close).
+        """
+        repo = utils.get_active_repo()
+
+        return repo.filter_events(types.Filters())
+
+
+class EventAuditListCollection(ApCollection):
+    SerializerFactory = ApHtmxTableSerializer.with_attributes(
+        row_dictizer=event_dictizer
+    )
+
+    DataFactory = EventAuditData
+
+    ColumnsFactory = ApColumns.with_attributes(
+        names=[
+            "bulk-action",
+            "category",
+            "action",
+            "actor",
+            "action_object",
+            "action_object_id",
+            "target_type",
+            "target_id",
+            "timestamp",
+            # "result",
+            # "payload",
+            # "row_actions",
+        ],
+        width={"bulk-action": "3%", "result": "15%", "payload": "15%"},
+        sortable={"timestamp"},
+        searchable={"category", "action", "actor"},
+        labels={
+            "bulk-action": tk.literal(
+                tags.input_(
+                    type="checkbox",
+                    name="bulk_check",
+                    id="bulk_check",
+                    data_module="ap-bulk-check",
+                    data_module_selector='input[name="entity_id"]',
+                )
+            ),
+            "category": "Category",
+            "action": "Action",
+            "actor": "User",
+            "action_object": "Action Object",
+            "action_object_id": "Action Object ID",
+            "target_type": "Target Type",
+            "target_id": "Target ID",
+            "timestamp": "Timestamp",
+            # "result": "Result",
+            # "payload": "Payload",
+            # "row_actions": "Actions",
+        },
+        serializers={
+            "timestamp": [("date", {})],
+            "actor": [("user_link", {})],
+            "result": [("json_display", {})],
+            "payload": [("json_display", {})],
+        },
+    )

ckanext/event_audit/config.py

@@ -12,6 +12,7 @@
 
 CONF_IGNORED_CATEGORIES = "ckanext.event_audit.ignore.categories"
 CONF_IGNORED_ACTIONS = "ckanext.event_audit.ignore.actions"
+CONF_IGNORED_MODELS = "ckanext.event_audit.ignore.models"
 
 CONF_DATABASE_TRACK_ENABLED = "ckanext.event_audit.track.model"
 CONF_API_TRACK_ENABLED = "ckanext.event_audit.track.api"
@@ -45,6 +46,11 @@ def get_ignored_actions() -> list[str]:
     return tk.config[CONF_IGNORED_ACTIONS]
 
 
+def get_ignored_models() -> list[str]:
+    """A list of database models to ignore when logging events."""
+    return tk.config[CONF_IGNORED_MODELS]
+
+
 def is_database_log_enabled() -> bool:
     """Returns True if database logging is enabled."""
     return tk.config[CONF_DATABASE_TRACK_ENABLED]

ckanext/event_audit/config_declaration.yaml

@@ -30,6 +30,7 @@ groups:
         type: list
         example: api view
         editable: true
+        default: ''
 
       - key: ckanext.event_audit.ignore.actions
         description: |
@@ -38,6 +39,16 @@ groups:
         type: list
         example: package_create
         editable: true
+        default: editable_config_list editable_config_change get_site_user ckanext_pages_list user_show
+
+      - key: ckanext.event_audit.ignore.models
+        description: |
+          A list of models to exclude from event logging, applicable only to
+          built-in tracking methods (API, database).
+        type: list
+        example: User Package Resource
+        editable: true
+        default: Option
 
       - key: ckanext.event_audit.track.model
         description: Enable logging of database events

ckanext/event_audit/config_schema.yaml

@@ -30,15 +30,23 @@ fields:
     help_text: |
           A space separated list of categories to exclude from event logging, applicable only to
           built-in tracking methods (API, database).
-    form_snippet: textarea.html
+    form_snippet: tom_tags.html
     required: false
 
   - field_name: ckanext.event_audit.ignore.actions
     label: Ignored Actions
     help_text: |
           A space separated list of actions to exclude from event logging, applicable only to
           built-in tracking methods (API, database).
-    form_snippet: textarea.html
+    form_snippet: tom_tags.html
+    required: false
+
+  - field_name: ckanext.event_audit.ignore.models
+    label: Ignored Models
+    help_text: |
+          A space separated list of database models to exclude from event logging,
+          applicable only to built-in tracking methods (API, database).
+    form_snippet: tom_tags.html
     required: false
 
   - field_name: ckanext.event_audit.track.model

ckanext/event_audit/helpers.py

@@ -4,5 +4,6 @@
 
 from ckanext.event_audit import utils
 
+
 def event_audit_active_repo_choices(field: dict[str, Any]) -> list[dict[str, str]]:
     return [{"value": opt, "label": opt} for opt in utils.get_available_repos()]

ckanext/event_audit/listeners/api.py

@@ -19,7 +19,17 @@ def action_succeeded_subscriber(
     if not config.is_api_log_enabled():
         return
 
+    # if not utils.is_cloudwatch_conn_tested():
+    #     utils.test_cloudwatch_connection2()
+
+    # if not utils.is_cloudwatch_available():
+    #     return
+
     repo = utils.get_active_repo()
+
+    if repo._connection is False:
+        return
+
     thread_mode_enabled = config.is_threaded_mode_enabled()
 
     event = repo.build_event(

ckanext/event_audit/logic/validators.py

@@ -2,8 +2,6 @@
 
 from typing import Any
 
-from botocore.exceptions import NoCredentialsError, PartialCredentialsError, ClientError
-
 import ckan.plugins.toolkit as tk
 from ckan.types import Context
 
@@ -15,15 +13,3 @@ def audit_repo_exists(value: Any, context: Context) -> Any:
         raise tk.Invalid(f"Repository `{value}` is not registered")
 
     return value
-
-
-# def audit_cloudwatch_credentials_validator(value: Any, context: Context) -> Any:
-#     if value != "cloudwatch":
-#         return value
-
-#     try:
-#         utils.test_cloudwatch_connection()
-#     except ValueError as e:
-#         raise tk.Invalid(str(e))
-
-#     return value

ckanext/event_audit/plugin.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import os
+import yaml
 import queue
 import threading
 from datetime import datetime, timedelta
@@ -10,6 +12,8 @@
 from ckan import plugins as p
 from ckan.common import CKANConfig
 from ckan.types import SignalMapping
+from ckan.config.declaration import Declaration, Key
+from ckan.logic import clear_validators_cache
 
 from ckanext.event_audit import config, listeners, types, utils
 from ckanext.event_audit.interfaces import IEventAudit
@@ -56,7 +60,6 @@ def _is_time_to_push(self, last_push: datetime) -> bool:
         )
 
 
-@tk.blanket.config_declarations
 @tk.blanket.validators
 @tk.blanket.cli
 @tk.blanket.blueprints
@@ -66,6 +69,7 @@ class EventAuditPlugin(p.SingletonPlugin):
     p.implements(p.IConfigurer)
     p.implements(p.ISignal)
     p.implements(IEventAudit, inherit=True)
+    p.implements(p.IConfigDeclaration)
 
     event_queue = queue.Queue()
 
@@ -76,8 +80,12 @@ def update_config(self, config_: CKANConfig):
     # IConfigurable
 
     def configure(self, config_: CKANConfig) -> None:
-        if utils.get_active_repo().get_name() == "cloudwatch" and not config_.get(
-            "testing"
+        repo = utils.get_active_repo()
+
+        if (
+            not config_.get("testing")
+            and repo.get_name() == "cloudwatch"
+            and repo._connection is None
         ):
             utils.test_cloudwatch_connection()
 
@@ -100,6 +108,9 @@ def get_signal_subscriptions(self) -> SignalMapping:
             tk.signals.ckanext.signal("ap_main:collect_config_schemas"): [
                 self.collect_config_schemas_subs
             ],
+            tk.signals.ckanext.signal("collection:register_collections"): [
+                self.get_collection_factories,
+            ],
         }
 
     @staticmethod
@@ -112,13 +123,24 @@ def collect_config_sections_subs(sender: None):
                     "blueprint": "event_audit.config",
                     "info": "Event Audit",
                 },
+                {
+                    "name": "Events dashboard",
+                    "blueprint": "event_audit.dashboard",
+                    "info": "A list of all events",
+                },
             ],
         }
 
     @staticmethod
     def collect_config_schemas_subs(sender: None):
         return ["ckanext.event_audit:config_schema.yaml"]
 
+    @staticmethod
+    def get_collection_factories(sender: None):
+        from ckanext.event_audit.collection import EventAuditListCollection
+
+        return {"event-audit-list": EventAuditListCollection}
+
     # IEventAudit
 
     def skip_event(self, event: types.Event) -> bool:
@@ -128,4 +150,19 @@ def skip_event(self, event: types.Event) -> bool:
         if event.category in config.get_ignored_categories():
             return True
 
+        if event.action_object in config.get_ignored_models():
+            return True
+
         return False
+
+    # IConfigDeclaration
+
+    def declare_config_options(self, declaration: Declaration, key: Key):
+        # this call allows using custom validators in config declarations
+        # we need it for CKAN 2.10, as this PR wasn't backported
+        # https://github.com/ckan/ckan/pull/7614
+        clear_validators_cache()
+        here = os.path.dirname(__file__)
+
+        with open(os.path.join(here, "config_declaration.yaml"), "rb") as src:
+            declaration.load_dict(yaml.safe_load(src))

ckanext/event_audit/repositories/base.py

@@ -7,6 +7,15 @@
 
 
 class AbstractRepository(ABC):
+    _connection = None
+
+    def __new__(cls, *args: Any, **kwargs: Any):
+        """Singleton pattern implementation."""
+        if not hasattr(cls, "_instance"):
+            cls._instance = super().__new__(cls)
+
+        return cls._instance
+
     @classmethod
     @abstractmethod
     def get_name(cls) -> str:

ckanext/event_audit/repositories/cloudwatch.py

@@ -31,6 +31,10 @@ def __init__(
         log_group: str = "/ckan/event-audit",
         log_stream: str = "event-audit-stream",
     ):
+        # TODO: check conn?
+        if self._connection is not None:
+            return
+
         if not credentials:
             credentials = config.get_cloudwatch_credentials()
 
@@ -47,11 +51,11 @@ def __init__(
 
         try:
             self._create_log_group_if_not_exists()
-        except (NoCredentialsError, PartialCredentialsError):
+        except (NoCredentialsError, PartialCredentialsError) as e:
             raise ValueError(
                 "AWS credentials are not configured. "
                 "Please, check the extension configuration."
-            )
+            ) from e
 
     @classmethod
     def get_name(cls) -> str:
@@ -188,4 +192,4 @@ def remove_all_events(self) -> types.Result:
         except self.client.exceptions.ResourceNotFoundException as err:
             return types.Result(status=False, message=str(err))
 
-        return types.Result(status=True)
+        return types.Result(status=True, message="All events removed successfully")