diff --git a/ckanext/__init__.py b/ckanext/__init__.py index ed48ed0..6d83202 100644 --- a/ckanext/__init__.py +++ b/ckanext/__init__.py @@ -1,9 +1,9 @@ -# encoding: utf-8 - # this is a namespace package try: import pkg_resources + pkg_resources.declare_namespace(__name__) except ImportError: import pkgutil + __path__ = pkgutil.extend_path(__path__, __name__) diff --git a/ckanext/event_audit/config.py b/ckanext/event_audit/config.py index 4b922a3..1e48a77 100644 --- a/ckanext/event_audit/config.py +++ b/ckanext/event_audit/config.py @@ -1,6 +1,5 @@ import ckan.plugins.toolkit as tk - CONF_ACTIVE_REPO = "ckanext.event_audit.active_repo" diff --git a/ckanext/event_audit/migration/event_audit/README b/ckanext/event_audit/migration/event_audit/README new file mode 100644 index 0000000..98e4f9c --- /dev/null +++ b/ckanext/event_audit/migration/event_audit/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/ckanext/event_audit/migration/event_audit/alembic.ini b/ckanext/event_audit/migration/event_audit/alembic.ini new file mode 100644 index 0000000..9315037 --- /dev/null +++ b/ckanext/event_audit/migration/event_audit/alembic.ini @@ -0,0 +1,74 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = %(here)s + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# timezone to use when rendering the date +# within the migration file as well as the filename. +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to /home/berry/projects/master/ckanext-event-audit/ckanext/event_audit/migration/event_audit/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat /home/berry/projects/master/ckanext-event-audit/ckanext/event_audit/migration/event_audit/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = driver://user:pass@localhost/dbname + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/ckanext/event_audit/migration/event_audit/env.py b/ckanext/event_audit/migration/event_audit/env.py new file mode 100644 index 0000000..64132ed --- /dev/null +++ b/ckanext/event_audit/migration/event_audit/env.py @@ -0,0 +1,89 @@ +import os +from logging.config import fileConfig + +from alembic import context +from ckan.model.meta import metadata +from sqlalchemy import engine_from_config, pool + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + +name = os.path.basename(os.path.dirname(__file__)) + + +def include_object(object, object_name, type_, reflected, compare_to): + if type_ == "table": + return object_name.startswith(name) + return True + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + version_table=f"{name}_alembic_version", + include_object=include_object, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + version_table=f"{name}_alembic_version", + include_object=include_object, + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/ckanext/event_audit/migration/event_audit/script.py.mako b/ckanext/event_audit/migration/event_audit/script.py.mako new file mode 100644 index 0000000..2c01563 --- /dev/null +++ b/ckanext/event_audit/migration/event_audit/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/ckanext/event_audit/migration/event_audit/versions/9256fa265b84_create_event_table.py b/ckanext/event_audit/migration/event_audit/versions/9256fa265b84_create_event_table.py new file mode 100644 index 0000000..e32f5cd --- /dev/null +++ b/ckanext/event_audit/migration/event_audit/versions/9256fa265b84_create_event_table.py @@ -0,0 +1,43 @@ +"""Create Event table + +Revision ID: 9256fa265b84 +Revises: +Create Date: 2024-10-23 12:03:33.876737 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "9256fa265b84" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "event_audit_event", + sa.Column("id", sa.String(), primary_key=True), + sa.Column("category", sa.String(), nullable=False), + sa.Column("action", sa.String(), nullable=False), + sa.Column("actor", sa.String()), + sa.Column("action_object", sa.String()), + sa.Column("action_object_id", sa.String()), + sa.Column("target_type", sa.String()), + sa.Column("target_id", sa.String()), + sa.Column("timestamp", sa.TIMESTAMP(timezone=True), nullable=False), + sa.Column("result", sa.JSON(), default={}), + sa.Column("payload", sa.JSON(), default={}), + ) + + op.create_index("ix_event_category", "event_audit_event", ["category"]) + op.create_index("ix_event_action", "event_audit_event", ["action"]) + op.create_index("ix_event_actor", "event_audit_event", ["actor"]) + op.create_index("ix_event_action_object", "event_audit_event", ["action_object"]) + op.create_index("ix_event_timestamp", "event_audit_event", ["timestamp"]) + op.create_index("ix_event_actor_action", "event_audit_event", ["actor", "action"]) + + +def downgrade(): + op.drop_table("event_audit_event") diff --git a/ckanext/event_audit/model.py b/ckanext/event_audit/model.py new file mode 100644 index 0000000..38120e0 --- /dev/null +++ b/ckanext/event_audit/model.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from sqlalchemy import TIMESTAMP, Column, Index, String +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.ext.mutable import MutableDict + +import ckan.plugins.toolkit as tk +from ckan import model + + +class EventModel(tk.BaseModel): + __tablename__ = "event_audit_event" + + id = Column(String, primary_key=True) + category = Column(String, nullable=False, index=True) + action = Column(String, nullable=False, index=True) + actor = Column(String, index=True) + action_object = Column(String, index=True) + action_object_id = Column(String, index=True) + target_type = Column(String, index=True) + target_id = Column(String, index=True) + timestamp = Column(TIMESTAMP(timezone=True), nullable=False, index=True) + result = Column(MutableDict.as_mutable(JSONB), default="{}") + payload = Column(MutableDict.as_mutable(JSONB), default="{}") + + __table_args__ = (Index("ix_event_actor_action", "actor", "action"),) + + def save(self) -> None: + model.Session.add(self) + model.Session.commit() diff --git a/ckanext/event_audit/plugin.py b/ckanext/event_audit/plugin.py index a837fbc..13efde5 100644 --- a/ckanext/event_audit/plugin.py +++ b/ckanext/event_audit/plugin.py @@ -1,18 +1,19 @@ from __future__ import annotations import os + import yaml -import ckan.plugins as plugins import ckan.plugins.toolkit as tk +from ckan import plugins as p from ckan.config.declaration import Declaration, Key from ckan.logic import clear_validators_cache @tk.blanket.config_declarations @tk.blanket.validators -class EventAuditPlugin(plugins.SingletonPlugin): - plugins.implements(plugins.IConfigDeclaration) +class EventAuditPlugin(p.SingletonPlugin): + p.implements(p.IConfigDeclaration) # IConfigDeclaration diff --git a/ckanext/event_audit/repositories/__init__.py b/ckanext/event_audit/repositories/__init__.py index 56908bb..f34c8cc 100644 --- a/ckanext/event_audit/repositories/__init__.py +++ b/ckanext/event_audit/repositories/__init__.py @@ -1,4 +1,5 @@ -from .redis import RedisRepository from .base import AbstractRepository +from .postgres import PostgresRepository +from .redis import RedisRepository -__all__ = ["RedisRepository", "AbstractRepository"] +__all__ = ["RedisRepository", "AbstractRepository", "PostgresRepository"] diff --git a/ckanext/event_audit/repositories/base.py b/ckanext/event_audit/repositories/base.py index 0b3b3f3..80c0b98 100644 --- a/ckanext/event_audit/repositories/base.py +++ b/ckanext/event_audit/repositories/base.py @@ -11,20 +11,23 @@ class AbstractRepository(ABC): @classmethod @abstractmethod def get_name(cls) -> str: - """Return the name of the repository""" + """Return the name of the repository.""" @abstractmethod def write_event(self, event: types.Event) -> types.WriteStatus: - """Write an event to the repository. This method accepts an Event object - and writes it to the repository. The Event object validates the input.""" + """Write an event to the repository. + + This method accepts an Event object and writes it to the repository. + The Event object validates the input. + """ def build_event(self, event_data: types.EventData) -> types.Event: return types.Event(**event_data) @abstractmethod def get_event(self, event_id: str) -> types.Event | None: - """Get a single event by its ID""" + """Get a single event by its ID.""" @abstractmethod def filter_events(self, filters: types.Filters) -> list[types.Event]: - """Filter events based on the provided kwargs""" + """Filter events based on the provided kwargs.""" diff --git a/ckanext/event_audit/repositories/postgres.py b/ckanext/event_audit/repositories/postgres.py new file mode 100644 index 0000000..a9a7b2d --- /dev/null +++ b/ckanext/event_audit/repositories/postgres.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from typing import List + +from sqlalchemy import select + +from ckan.model import Session + +from ckanext.event_audit import model, types + + +class PostgresRepository: + def __init__(self): + self.session = Session + + @classmethod + def get_name(cls) -> str: + return "postgres" + + def write_event(self, event: types.Event) -> types.WriteStatus: + db_event = model.EventModel(**event.model_dump()) + db_event.save() + + return types.WriteStatus(status=True) + + def get_event(self, event_id: str) -> types.Event | None: + result = self.session.execute( + select(model.EventModel).where(model.EventModel.id == event_id) + ).scalar_one_or_none() + + if result: + return types.Event.model_validate(result) + + return None + + def filter_events(self, filters: types.Filters) -> List[types.Event]: + """Filters events based on provided filter criteria.""" + if not isinstance(filters, types.Filters): + raise TypeError( + f"Expected 'filters' to be an instance of Filters, got {type(filters)}" + ) + + query = select(model.EventModel) + + filterable_fields = [ + "category", + "action", + "actor", + "action_object", + "action_object_id", + "target_type", + "target_id", + ] + + for field in filterable_fields: + value = getattr(filters, field, None) + if value: + query = query.where(getattr(model.EventModel, field) == value) + + if filters.time_from: + query = query.where(model.EventModel.timestamp >= filters.time_from) + if filters.time_to: + query = query.where(model.EventModel.timestamp <= filters.time_to) + + result = self.session.execute(query).scalars().all() + return [types.Event.model_validate(event) for event in result] diff --git a/ckanext/event_audit/repositories/redis.py b/ckanext/event_audit/repositories/redis.py index 60ddf90..556b219 100644 --- a/ckanext/event_audit/repositories/redis.py +++ b/ckanext/event_audit/repositories/redis.py @@ -1,6 +1,7 @@ from __future__ import annotations -from datetime import datetime as dt, timezone as tz +from datetime import datetime as dt +from datetime import timezone as tz from ckan.lib.redis import connect_to_redis @@ -28,9 +29,10 @@ def write_event(self, event: types.Event) -> types.WriteStatus: return types.WriteStatus(status=True) def _build_event_key(self, event: types.Event) -> str: - """Builds the key for the event in Redis, storing all the info inside - the key to be able to search for it fast.""" + """Builds the key for the event in Redis. + We store all the info inside the key to be able to search for it fast. + """ return ( f"id:{event.id}|" f"category:{event.category}|" @@ -46,7 +48,7 @@ def _build_event_key(self, event: types.Event) -> str: def get_event(self, event_id: float) -> types.Event | None: _, result = self.conn.hscan(REDIS_SET_KEY, match=f"id:{event_id}|*") # type: ignore - for _, event_data in result.items(): + for event_data in result.values(): return types.Event.model_validate_json(event_data) def filter_events(self, filters: types.Filters) -> list[types.Event]: @@ -68,8 +70,12 @@ def filter_events(self, filters: types.Filters) -> list[types.Event]: REDIS_SET_KEY, cursor=cursor, match=pattern # type: ignore ) - for _, event_data in result.items(): - matching_events.append(types.Event.model_validate_json(event_data)) + matching_events.extend( + [ + types.Event.model_validate_json(event_data) + for event_data in result.values() + ] + ) if cursor == 0: break @@ -99,20 +105,14 @@ def _filter_by_time( if not time_from and not time_to: return events - def is_within_time_range(event_time: dt) -> bool: - if time_from and time_to: - return time_from <= event_time <= time_to - if time_from: - return time_from <= event_time <= dt.now(tz.utc) - if time_to: - return event_time <= time_to - return True + self.time_from = time_from + self.time_to = time_to if events: return [ event for event in events - if is_within_time_range(dt.fromisoformat(event.timestamp)) + if self.is_within_time_range(dt.fromisoformat(event.timestamp)) ] filtered_events: list[types.Event] = [] @@ -122,14 +122,23 @@ def is_within_time_range(event_time: dt) -> bool: while True: cursor, result = self.conn.hscan(REDIS_SET_KEY, cursor=cursor) # type: ignore - for _, event_data in result.items(): + for event_data in result.values(): event = types.Event.model_validate_json(event_data) event_time = dt.fromisoformat(event.timestamp) - if is_within_time_range(event_time): + if self.is_within_time_range(event_time): filtered_events.append(event) if cursor == 0: break return filtered_events + + def is_within_time_range(self, event_time: dt) -> bool: + if self.time_from and self.time_to: + return self.time_from <= event_time <= self.time_to + if self.time_from: + return self.time_from <= event_time <= dt.now(tz.utc) + if self.time_to: + return event_time <= self.time_to + return True diff --git a/ckanext/event_audit/tests/conftest.py b/ckanext/event_audit/tests/conftest.py index 5f2ec55..e7b9c33 100644 --- a/ckanext/event_audit/tests/conftest.py +++ b/ckanext/event_audit/tests/conftest.py @@ -5,7 +5,14 @@ from ckanext.event_audit import types -@pytest.fixture +@pytest.fixture() +def clean_db(reset_db, migrate_db_for): + reset_db() + + migrate_db_for("event_audit") + + +@pytest.fixture() def event(): return types.ModelEvent( action="created", @@ -14,7 +21,7 @@ def event(): ) -@pytest.fixture +@pytest.fixture() def event_factory(): def factory(**kwargs): kwargs.setdefault("action", "created") diff --git a/ckanext/event_audit/tests/repositories/test_postgres.py b/ckanext/event_audit/tests/repositories/test_postgres.py new file mode 100644 index 0000000..af11200 --- /dev/null +++ b/ckanext/event_audit/tests/repositories/test_postgres.py @@ -0,0 +1,119 @@ +from datetime import datetime as dt +from datetime import timedelta as td +from datetime import timezone as tz +from typing import Callable + +import pytest + +from ckanext.event_audit import types +from ckanext.event_audit.repositories import PostgresRepository + + +@pytest.mark.usefixtures("with_plugins", "clean_db") +class TestPostgresRepo: + def test_write_event(self, event: types.Event): + postgres_repo = PostgresRepository() + + status = postgres_repo.write_event(event) + assert status.status + + def test_get_event(self, event: types.Event): + postgres_repo = PostgresRepository() + + postgres_repo.write_event(event) + loaded_event = postgres_repo.get_event(event.id) + + assert isinstance(loaded_event, types.Event) + assert event.model_dump() == loaded_event.model_dump() + + def test_get_event_not_found(self): + postgres_repo = PostgresRepository() + assert postgres_repo.get_event("xxx") is None + + def test_filter_by_category(self, event: types.Event): + postgres_repo = PostgresRepository() + + postgres_repo.write_event(event) + events = postgres_repo.filter_events(types.Filters(category="model")) + + assert len(events) == 1 + assert events[0].model_dump() == event.model_dump() + + def test_filter_by_action(self, event: types.Event): + postgres_repo = PostgresRepository() + + postgres_repo.write_event(event) + events = postgres_repo.filter_events(types.Filters(action="created")) + + assert len(events) == 1 + assert events[0].model_dump() == event.model_dump() + + def test_filter_by_action_and_action_object(self, event: types.Event): + postgres_repo = PostgresRepository() + + postgres_repo.write_event(event) + events = postgres_repo.filter_events( + types.Filters(category="model", action_object="package") + ) + + assert len(events) == 1 + assert events[0].model_dump() == event.model_dump() + + def test_filter_by_time_from(self, event_factory: Callable[..., types.Event]): + postgres_repo = PostgresRepository() + + event = event_factory(timestamp=(dt.now(tz.utc) - td(days=365)).isoformat()) + postgres_repo.write_event(event) + + events = postgres_repo.filter_events(types.Filters(time_from=dt.now(tz.utc))) + assert len(events) == 0 + + events = postgres_repo.filter_events( + types.Filters(time_from=dt.now(tz.utc) - td(days=366)) + ) + assert len(events) == 1 + assert events[0].model_dump() == event.model_dump() + + def test_filter_by_time_to(self, event: types.Event): + postgres_repo = PostgresRepository() + + postgres_repo.write_event(event) + + events = postgres_repo.filter_events( + types.Filters(time_to=dt.now(tz.utc) - td(days=1)) + ) + assert len(events) == 0 + + events = postgres_repo.filter_events(types.Filters(time_to=dt.now(tz.utc))) + assert len(events) == 1 + assert events[0].model_dump() == event.model_dump() + + def test_filter_by_time_between(self, event_factory: Callable[..., types.Event]): + postgres_repo = PostgresRepository() + + event = event_factory(timestamp=(dt.now(tz.utc) - td(days=365)).isoformat()) + postgres_repo.write_event(event) + + events = postgres_repo.filter_events( + types.Filters( + time_from=dt.now(tz.utc) - td(days=366), + time_to=dt.now(tz.utc), + ) + ) + assert len(events) == 1 + assert events[0].model_dump() == event.model_dump() + + def test_filter_by_multiple(self, event_factory: Callable[..., types.Event]): + postgres_repo = PostgresRepository() + + for _ in range(5): + postgres_repo.write_event(event_factory()) + + events = postgres_repo.filter_events( + types.Filters( + category="model", + action="created", + ) + ) + + assert len(events) == 5 diff --git a/ckanext/event_audit/tests/test_repositories.py b/ckanext/event_audit/tests/repositories/test_redis.py similarity index 79% rename from ckanext/event_audit/tests/test_repositories.py rename to ckanext/event_audit/tests/repositories/test_redis.py index 1ff461a..8909647 100644 --- a/ckanext/event_audit/tests/test_repositories.py +++ b/ckanext/event_audit/tests/repositories/test_redis.py @@ -1,10 +1,12 @@ -import pytest - +from datetime import datetime as dt +from datetime import timedelta as td +from datetime import timezone as tz from typing import Callable -from datetime import datetime as dt, timedelta as td, timezone as tz -from ckanext.event_audit.repositories import RedisRepository +import pytest + from ckanext.event_audit import types +from ckanext.event_audit.repositories import RedisRepository @pytest.mark.usefixtures("clean_redis") @@ -62,9 +64,7 @@ def test_filter_by_time_from(self, event_factory: Callable[..., types.Event]): result = redis_repo.write_event(event) assert result.status is True - events = redis_repo.filter_events( - types.Filters(time_from=dt.now(tz.utc)) - ) + events = redis_repo.filter_events(types.Filters(time_from=dt.now(tz.utc))) assert len(events) == 0 events = redis_repo.filter_events( @@ -102,27 +102,3 @@ def test_filter_by_time_between(self, event_factory: Callable[..., types.Event]) ) assert len(events) == 1 assert events[0].model_dump() == event.model_dump() - - -class TestEvent: - def test_event(self): - event = types.Event(category="model", action="created") - - assert event.category == "model" - assert event.action == "created" - - def test_empty_category(self): - with pytest.raises(ValueError): - types.Event(category="", action="created") - - def test_empty_action(self): - with pytest.raises(ValueError): - types.Event(category="model", action="") - - def test_category_not_string(self): - with pytest.raises(ValueError): - types.Event(category=1, action="created") - - def test_action_not_string(self): - with pytest.raises(ValueError): - types.Event(category="model", action=1) diff --git a/ckanext/event_audit/tests/test_config.py b/ckanext/event_audit/tests/test_config.py index 4c5f15c..d8dcdae 100644 --- a/ckanext/event_audit/tests/test_config.py +++ b/ckanext/event_audit/tests/test_config.py @@ -6,6 +6,6 @@ @pytest.mark.usefixtures("with_plugins") -class TestEventAuditConfig(object): +class TestEventAuditConfig: def test_get_active_repo_default(self): assert config.get_active_repo() == "redis" diff --git a/ckanext/event_audit/tests/test_interface.py b/ckanext/event_audit/tests/test_interface.py index e835242..01dac3b 100644 --- a/ckanext/event_audit/tests/test_interface.py +++ b/ckanext/event_audit/tests/test_interface.py @@ -4,9 +4,9 @@ import ckan.plugins as p +from ckanext.event_audit import types, utils from ckanext.event_audit.interfaces import IEventAudit from ckanext.event_audit.repositories import AbstractRepository -from ckanext.event_audit import types, utils class MyRepository(AbstractRepository): @@ -35,7 +35,7 @@ def register_repository(self) -> dict[str, type[AbstractRepository]]: @pytest.mark.ckan_config("ckan.plugins", "event_audit test_event_audit") @pytest.mark.usefixtures("non_clean_db", "with_plugins") -class TestOTLInterace(object): +class TestOTLInterace: def test_get_available_repos(self, app, user, sysadmin): repos = utils.get_available_repos() assert MyRepository.get_name() in repos diff --git a/ckanext/event_audit/tests/test_types.py b/ckanext/event_audit/tests/test_types.py new file mode 100644 index 0000000..6496f0e --- /dev/null +++ b/ckanext/event_audit/tests/test_types.py @@ -0,0 +1,176 @@ +from datetime import datetime, timedelta, timezone + +import pytest +from pydantic_core import ValidationError + +from ckanext.event_audit import types + + +class TestEvent: + def test_valid_event(self): + """Test creation of a valid event with required fields.""" + event = types.Event(category="model", action="created") + + assert event.category == "model" + assert event.action == "created" + + assert isinstance(event.id, str) + assert isinstance(event.timestamp, str) + + def test_event_with_optional_fields(self, user): + """Test creating an event with all fields filled.""" + timestamp = datetime.now(timezone.utc).isoformat() + event = types.Event( + category="model", + action="created", + actor=user["id"], + action_object="package", + action_object_id="123", + target_type="dataset", + target_id="456", + timestamp=timestamp, + result={"status": "success"}, + payload={"key": "value"}, + ) + + assert event.actor == user["id"] + assert event.action_object == "package" + assert event.target_type == "dataset" + assert event.timestamp == timestamp + assert event.result["status"] == "success" + assert event.payload["key"] == "value" + + def test_empty_category(self): + """Test that an empty category raises a ValidationError.""" + with pytest.raises( + ValidationError, match="The `category` field must be a non-empty string." + ): + types.Event(category="", action="created") + + def test_empty_action(self): + """Test that an empty action raises a ValidationError.""" + with pytest.raises( + ValidationError, match="The `action` field must be a non-empty string." + ): + types.Event(category="model", action="") + + def test_category_not_string(self): + """Test that non-string category raises a ValidationError.""" + with pytest.raises(ValidationError, match="Input should be a valid string."): + types.Event(category=1, action="created") + + def test_action_not_string(self): + """Test that non-string action raises a ValidationError.""" + with pytest.raises(ValidationError, match="Input should be a valid string"): + types.Event(category="model", action=1) + + def test_actor_not_string(self): + """Test that a non-string actor raises a ValidationError.""" + with pytest.raises(ValidationError): + types.Event(category="model", action="created", actor=123) + + def test_invalid_timestamp_format(self): + """Test that an invalid timestamp format raises a ValidationError.""" + with pytest.raises(ValidationError, match="Date format incorrect"): + types.Event(category="model", action="created", timestamp="invalid-date") + + def test_future_timestamp(self): + """Test handling of future timestamps.""" + future_timestamp = (datetime.now(timezone.utc) + timedelta(days=1)).isoformat() + event = types.Event( + category="model", action="created", timestamp=future_timestamp + ) + assert event.timestamp == future_timestamp + + def test_default_timestamp(self): + """Test that the default timestamp is set to the current time.""" + event = types.Event(category="model", action="created") + timestamp = datetime.now(timezone.utc).isoformat() + + # Allowing a small difference in time to handle execution delays + assert event.timestamp[:19] == timestamp[:19] + + def test_empty_result_and_payload(self): + """Test that result and payload default to empty dictionaries.""" + event = types.Event(category="model", action="created") + assert event.result == {} + assert event.payload == {} + + def test_user_doesnt_exist(self): + """Test that invalid actor reference raises a ValidationError.""" + with pytest.raises(ValidationError, match="Not found: User"): + types.Event(category="model", action="created", actor="non-existent-user") + + def test_custom_id_generation(self): + """Test that a custom id can be provided.""" + custom_id = "12345" + event = types.Event(category="model", action="created", id=custom_id) + assert event.id == custom_id + + def test_invalid_field_assignment(self): + """Test that assigning invalid data types to fields raises an error.""" + with pytest.raises(ValidationError): + types.Event(category="model", action="created", result="not-a-dict") + + with pytest.raises(ValidationError): + types.Event(category="model", action="created", payload="not-a-dict") + + +class TestFilters: + def test_empty_filters(self): + """Test creating a Filters object with no fields.""" + assert types.Filters() + + def test_valid_filters(self, user): + """Test creating a valid Filters object.""" + filters = types.Filters( + category="api", + action="created", + actor=user["id"], + action_object="package", + action_object_id="123", + target_type="organization", + target_id="456", + time_from=datetime.now() - timedelta(days=1), + time_to=datetime.now(), + ) + assert filters.category == "api" + assert filters.action == "created" + assert filters.actor == user["id"] + + def test_empty_optional_strings(self): + """Test that optional strings can be None or empty.""" + filters = types.Filters(category=None, action="") + assert filters.category is None + assert filters.action == "" + + def test_whitespace_trimming(self): + """Test that leading and trailing spaces are removed from string fields.""" + filters = types.Filters(category=" api ", action=" created ") + assert filters.category == "api" + assert filters.action == "created" + + def test_time_range_validation(self): + """Test that `time_from` must be earlier than `time_to`.""" + with pytest.raises( + ValueError, match="`time_from` must be earlier than `time_to`." + ): + types.Filters( + time_from=datetime.now(), + time_to=datetime.now() - timedelta(days=1), + ) + + def test_invalid_time_from_type(self): + """Test that invalid datetime fields raise a validation error.""" + with pytest.raises(ValueError, match="Input should be a valid datetime"): + types.Filters(time_from="xxx") + + def test_invalid_actor_type(self): + """Test that passing incorrect field types raises an error.""" + with pytest.raises(ValueError, match="Input should be a valid string"): + types.Filters(actor=123) # Actor must be a string + + def test_actor_doesnt_exist(self): + """Test that an invalid actor reference raises a ValidationError.""" + with pytest.raises(ValidationError, match="Not found: User"): + types.Filters(actor="non-existent-user") diff --git a/ckanext/event_audit/tests/test_utils.py b/ckanext/event_audit/tests/test_utils.py index 76fa025..2fc5608 100644 --- a/ckanext/event_audit/tests/test_utils.py +++ b/ckanext/event_audit/tests/test_utils.py @@ -1,8 +1,6 @@ from __future__ import annotations -import pytest - -from ckanext.event_audit import utils, repositories, config +from ckanext.event_audit import repositories, utils class TestEventAuditUtils: diff --git a/ckanext/event_audit/types.py b/ckanext/event_audit/types.py index de26e2a..7ec657f 100644 --- a/ckanext/event_audit/types.py +++ b/ckanext/event_audit/types.py @@ -1,14 +1,14 @@ from __future__ import annotations import uuid -from typing import Optional, TypedDict, Any, Dict, Literal -from datetime import datetime, timezone from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any, Dict, Literal, Optional, TypedDict, Union -from pydantic import BaseModel, Field, validator +from pydantic import BaseModel, ConfigDict, Field, validator -import ckan.model as model import ckan.plugins.toolkit as tk +from ckan import model @dataclass @@ -26,13 +26,18 @@ class EventData(TypedDict): action_object_id: str target_type: str target_id: str - timestamp: str + timestamp: Union[str, datetime] result: Dict[Any, Any] payload: Dict[Any, Any] class Event(BaseModel): - """TODO: test this""" + """Event model. + + This model represents an event that occurred in the system. + """ + + model_config = ConfigDict(from_attributes=True) id: Any = Field(default_factory=lambda: str(uuid.uuid4())) category: str @@ -42,48 +47,52 @@ class Event(BaseModel): action_object_id: str = "" target_type: str = "" target_id: str = "" - timestamp: str = Field( + timestamp: Union[str, datetime] = Field( default_factory=lambda: datetime.now(timezone.utc).isoformat(), ) result: Dict[Any, Any] = Field(default_factory=dict) payload: Dict[Any, Any] = Field(default_factory=dict) @validator("category") - def validate_category(cls, v): - if not v or not isinstance(v, str): + @classmethod + def validate_category(cls, v: str) -> str: + if not v: raise ValueError("The `category` field must be a non-empty string.") return v @validator("action") - def validate_action(cls, v): - if not v or not isinstance(v, str): + @classmethod + def validate_action(cls, v: str) -> str: + if not v: raise ValueError("The `action` field must be a non-empty string.") return v @validator("actor") - def validate_actor(cls, v: str): + @classmethod + def validate_actor(cls, v: str) -> str: if not v: return v - if not isinstance(v, str): - raise ValueError("The `actor` field must be a string.") - if not model.Session.query(model.User).get(v): - raise ValueError("%s: %s" % (tk._("Not found"), tk._("User"))) + raise ValueError("{}: {}".format(tk._("Not found"), tk._("User"))) return v @validator("timestamp") - def validate_timestamp(cls, v): + @classmethod + def validate_timestamp(cls, v: Union[str, datetime]) -> str: + if v and isinstance(v, datetime): + return v.isoformat() + if not v or not isinstance(v, str): raise ValueError("The `timestamp` field must be a non-empty string.") try: tk.h.date_str_to_datetime(v) - except (TypeError, ValueError): - raise ValueError(tk._("Date format incorrect")) + except (TypeError, ValueError) as e: + raise ValueError(tk._("Date format incorrect")) from e return v @@ -101,9 +110,9 @@ class ApiEvent(Event): class Filters(BaseModel): - """TODO: test this - Filters for querying events. This model is used to filter events based on - different criteria. + """Filters for querying events. + + This model is used to filter events based on different criteria. """ category: Optional[str] = Field( @@ -134,3 +143,34 @@ class Filters(BaseModel): time_to: Optional[datetime] = Field( default=None, description="End time for filtering (defaults to now)" ) + + @validator("actor") + @classmethod + def validate_actor(cls, v: str) -> str: + if not v: + return v + + if not model.Session.query(model.User).get(v): + raise ValueError("{}: {}".format(tk._("Not found"), tk._("User"))) + + return v + + @validator("time_to") + @classmethod + def validate_time_range(cls, time_to: datetime, values: dict[str, Any]): + """Ensure `time_from` is before `time_to`.""" + time_from = values.get("time_from") + + if time_from and time_to and time_from > time_to: + raise ValueError("`time_from` must be earlier than `time_to`.") + + return time_to + + @validator("*", pre=True) + @classmethod + def strip_strings(cls, v: Any) -> Any: + """Strip leading and trailing spaces from all string fields.""" + if isinstance(v, str): + return v.strip() + + return v diff --git a/ckanext/event_audit/utils.py b/ckanext/event_audit/utils.py index 97fa23e..c680fd8 100644 --- a/ckanext/event_audit/utils.py +++ b/ckanext/event_audit/utils.py @@ -1,15 +1,14 @@ from __future__ import annotations - import ckan.plugins as p -import ckanext.event_audit.repositories as repos import ckanext.event_audit.config as audit_config +import ckanext.event_audit.repositories as repos from ckanext.event_audit.interfaces import IEventAudit def get_available_repos() -> dict[str, type[repos.AbstractRepository]]: - """Get the available repositories + """Get the available repositories. Returns: dict[str, type[repos.AbstractRepository]]: The available repositories diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..64ddeb4 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,169 @@ +[tool.ruff] +target-version = "py38" +exclude = [ + "**/migration", +] + +[tool.ruff.lint] +select = [ + "ANN0", # type annotations for function arguments + "B", # likely bugs and design problems + "BLE", # do not catch blind exception + "C4", # better list/set/dict comprehensions + "C90", # check McCabe complexity + # "DTZ", # enforce timezone in date objects + "E", # pycodestyle error + "W", # pycodestyle warning + "F", # pyflakes + "FA", # verify annotations from future + "G", # format strings for logging statements + "N", # naming conventions + "I", # isort + "ICN", # import conventions + # "D1", # require doc + "D2", # doc formatting + "D4", # doc convention + "PL", # pylint + "PERF", # performance anti-patterns + "PT", # pytest style + # "PTH", # replace os.path with pathlib + "PIE", # misc lints + "RET", # improvements for return statements + "RSE", # improvements for rise statements + "S", # security testing + "SIM", # simplify code + "T10", # debugging statements + "T20", # print statements + "TID", # tidier imports + "TRY", # better exceptions + "UP", # upgrade syntax for newer versions of the language +] + +ignore = [ + "E712", # comparison to bool: violated by SQLAlchemy filters + "PLC1901", # simplify comparison to empty string: violated by SQLAlchemy filters + "PT004", # fixture does not return anything, add leading underscore: violated by clean_db + "RET503", # don't enforce return-None, + "TRY003", # allow specifying the error message, + "UP006", + "UP007" +] + +[tool.ruff.lint.per-file-ignores] +"ckanext/event_audit/tests*" = ["S", "PL", "ANN"] + +[tool.ruff.lint.flake8-import-conventions.aliases] +"ckan.plugins" = "p" +"ckan.plugins.toolkit" = "tk" +sqlalchemy = "sa" + +[tool.ruff.lint.pydocstyle] +convention = "google" + +[tool.ruff.lint.isort] +section-order = [ + "future", + "standard-library", + "first-party", + "third-party", + "ckan", + "ckanext", + "self", + "local-folder", +] + +[tool.ruff.lint.isort.sections] +# Group all Django imports into a separate section. +ckan = ["ckan"] +ckanext = ["ckanext"] +self = ["ckanext.event_audit"] + +[tool.pytest.ini_options] +addopts = "--ckan-ini test.ini -m 'not benchmark'" +filterwarnings = [ + "ignore::sqlalchemy.exc.SADeprecationWarning", + "ignore::sqlalchemy.exc.SAWarning", + "ignore::DeprecationWarning", +] + +[tool.git-changelog] +output = "CHANGELOG.md" +convention = "conventional" +parse-trailers = true + +[tool.pyright] +pythonVersion = "3.8" +include = ["ckanext"] +exclude = [ + "**/test*", + "**/migration", +] +strict = [] + +strictParameterNoneValue = true + +# Check the meaning of rules here +# https://github.com/microsoft/pyright/blob/main/docs/configuration.md +reportFunctionMemberAccess = true # non-standard member accesses for functions +reportMissingImports = true +reportMissingModuleSource = true +reportMissingTypeStubs = false +reportImportCycles = true +reportUnusedImport = true +reportUnusedClass = true +reportUnusedFunction = true +reportUnusedVariable = true +reportDuplicateImport = true +reportOptionalSubscript = true +reportOptionalMemberAccess = true +reportOptionalCall = true +reportOptionalIterable = true +reportOptionalContextManager = true +reportOptionalOperand = true +reportTypedDictNotRequiredAccess = false # Context won't work with this rule +reportConstantRedefinition = true +reportIncompatibleMethodOverride = true +reportIncompatibleVariableOverride = true +reportOverlappingOverload = true +reportUntypedFunctionDecorator = false +reportUnknownParameterType = true +reportUnknownArgumentType = false +reportUnknownLambdaType = false +reportUnknownMemberType = false +reportMissingTypeArgument = true +reportInvalidTypeVarUse = true +reportCallInDefaultInitializer = true +reportUnknownVariableType = true +reportUntypedBaseClass = true +reportUnnecessaryIsInstance = true +reportUnnecessaryCast = true +reportUnnecessaryComparison = true +reportAssertAlwaysTrue = true +reportSelfClsParameterName = true +reportUnusedCallResult = false # allow function calls for side-effect only +useLibraryCodeForTypes = true +reportGeneralTypeIssues = true +reportPropertyTypeMismatch = true +reportWildcardImportFromLibrary = true +reportUntypedClassDecorator = false +reportUntypedNamedTuple = true +reportPrivateUsage = true +reportPrivateImportUsage = true +reportInconsistentConstructor = true +reportMissingSuperCall = false +reportUninitializedInstanceVariable = true +reportInvalidStringEscapeSequence = true +reportMissingParameterType = true +reportImplicitStringConcatenation = false +reportUndefinedVariable = true +reportUnboundVariable = true +reportInvalidStubStatement = true +reportIncompleteStub = true +reportUnsupportedDunderAll = true +reportUnusedCoroutine = true +reportUnnecessaryTypeIgnoreComment = true +reportMatchNotExhaustive = true + +[tool.coverage.run] +branch = true +omit = ["ckanext/event_audit/tests/*"] diff --git a/setup.py b/setup.py index 4d26df7..288c82d 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from setuptools import setup setup( @@ -7,10 +6,10 @@ # message extraction at # http://babel.pocoo.org/docs/messages/#extraction-method-mapping-and-configuration message_extractors={ - 'ckanext': [ - ('**.py', 'python', None), - ('**.js', 'javascript', None), - ('**/templates/**.html', 'ckan', None), + "ckanext": [ + ("**.py", "python", None), + ("**.js", "javascript", None), + ("**/templates/**.html", "ckan", None), ], } )