From 64865226b9ad6242d4c8b2f017488f331570d17c Mon Sep 17 00:00:00 2001 From: Stas Davydov Date: Thu, 11 Jul 2024 19:20:57 +0800 Subject: [PATCH] Updated decorator implementation with parameter for default ID generation function. --- README.md | 45 ++++++++++++------------ benchmark.py | 15 ++++---- pys/__init__.py | 91 +++++++++++++++++++++++++++++-------------------- pys/pydantic.py | 19 ++--------- setup.py | 2 +- 5 files changed, 89 insertions(+), 83 deletions(-) diff --git a/README.md b/README.md index ddb793b..b31f58d 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,10 @@ Simple fast JSON file storage for Python dataclasses and Pydantic models, thread and multiprocess safe. ---- +It's standard to use SQL or NoSQL database servers as data backend, but sometimes it's more +convenient to have data persisted as file(s) locally on backend application side. If you still +need to use SQL for data retrieval the best option is SQLite, but for simple REST APIs it +could be better to work with objects as is. So here we go. ## Installation ```shell @@ -10,7 +14,7 @@ pip install pysdato ``` ## Usage -The library is intended to store Python dataclasses or Pydantic models as JSON-files referenced by ID +The library is intended to store Python `dataclasses`, `msqspec.Struct` or Pydantic models as JSON-files referenced by ID and supports object hierarchy. Let's say we have `Author` model. Object's ID is key point for persistence -- it will be used as name of @@ -23,7 +27,7 @@ from dataclasses import dataclass import pys # Initialize storage with path where files will be saved -storage = pys.storage('.storage') +storage = pys.storage('storage.db') @pys.saveable @dataclass @@ -59,7 +63,7 @@ class Author(BaseModel): class Book(BaseModel): title: str -storage = pys.storage('.storage') +storage = pys.storage('storage.db') # A few books of Leo Tolstoy leo = Author(name='Leo Tolstoy') @@ -98,7 +102,7 @@ class Author(BaseModel): class Book(BaseModel): title: str -storage = pys.storage('.storage') +storage = pys.storage('storage.db') # A few books of Leo Tolstoy leo = Author(name='Leo Tolstoy') @@ -115,7 +119,14 @@ assert war_and_peace in leo_books assert for_kids in leo_books ``` -## Reference +## Storages +Library supports two storages implementation: +- `sqlite_storage()` - SQLite based -- really fast, uses one file for all objects. +- `file_storage()` - JSON file per object storage, it is slower, but saves each object in a separate JSON file. + +The default storage is SQLite based. + +## Library Reference ```python import pys @@ -144,20 +155,12 @@ storage.destroy() ## Release Notes -### 0.0.5 -Performance is dramatically improved with SQLite storage implementation. +- **0.0.6** `saveable` decorator reworked, added `default_id` parameter that can be used for +changing ID generation behaviour. By default, we use `str(id(self))` as ID (and `str(uuid.uuid4())` +for `pys.pydantic.ModelWithID`), but it can be changed now. +- **0.0.5** Performance is dramatically improved with SQLite storage implementation. Default storage is SQLite storage now. - -### 0.0.4 -SQLite storage is added. -Support of `msqspec` JSON and structures is added. - -### 0.0.3 -Benchmark is added, performance is improved. -Fixed dependency set up. - -### 0.0.2 -Added support for Python 3.x < 3.10 - -### 0.0.1 -Initial public release +- **0.0.4** SQLite storage is added. Support of `msqspec` JSON and structures is added. +- **0.0.3** Benchmark is added, performance is improved. Fixed dependency set up. +- **0.0.2** Added support for Python 3.x < 3.10 +- **0.0.1** Initial public release diff --git a/benchmark.py b/benchmark.py index 738fcca..a0bea2c 100644 --- a/benchmark.py +++ b/benchmark.py @@ -83,13 +83,12 @@ class Book(msgspec.Struct): # T4: 1.25 sec # SQLite storage is added -# Storage: file.Storage(base_path=benchmark.storage) -# T1: 651.90 ms -# T2: 266.05 ms -# T3: 1428.29 ms -# T4: 1234.20 ms +# T1: 656.80 ms +# T2: 264.35 ms +# T3: 1368.96 ms +# T4: 1182.48 ms # Storage: sqlite.Storage(base_path=benchmark.db) -# T1: 19.98 ms -# T2: 0.00 ms -# T3: 7.00 ms +# T1: 20.00 ms +# T2: 1.00 ms +# T3: 6.51 ms # T4: 1.00 ms diff --git a/pys/__init__.py b/pys/__init__.py index ef0e3a7..7d63ee2 100644 --- a/pys/__init__.py +++ b/pys/__init__.py @@ -1,7 +1,7 @@ -import uuid +import functools from dataclasses import is_dataclass, asdict from pathlib import Path -from typing import Union +from typing import Union, Callable, Any import msgspec @@ -9,42 +9,59 @@ from . import file, sqlite -def saveable(cls=None, /, field_as_id: str = 'id'): - def wrapper(cls): - def my_id_or_default(self): - if hasattr(self, field_as_id): - if getattr(self, field_as_id, None) is None: - setattr(self, field_as_id, str(uuid.uuid4())) - return getattr(self, field_as_id) - else: - return str(id(self)) - - setattr(cls, '__my_id__', my_id_or_default) - if not hasattr(cls, '__json__'): - if issubclass(cls, msgspec.Struct): - setattr(cls, '__json__', - lambda self: - msgspec.json.encode(self).decode(encoding='UTF-8') - ) - elif is_dataclass(cls): - setattr(cls, '__json__', - lambda self: - msgspec.json.encode( - asdict(self), - ).decode(encoding='UTF-8') - ) - elif is_pydantic(cls): - setattr(cls, '__json__', lambda self: self.model_dump_json()) - else: - raise NotImplementedError( - f'The class {cls} is not @dataclass nor Pydantic Model and does not have __json__() method.' - f'Please implement __json__() method by yourself.') - return cls - - if cls is None: - return wrapper +def saveable(cls=None, *, + field_as_id: str = 'id', + default_id: Callable[[Any], str] = lambda self: str(id(self))): + """ + Decorate the given `cls` with `__my_id__()` and `__json__()` methods + required for persistence. + :param cls: Class to decorate. + :param field_as_id: existing class field to be used as object ID. + :param default_id: Default ID value function (id(self) by default). + :return: Decorated class + """ + if cls: + @functools.wraps(cls, updated=()) + class _Persisted(cls): + def __my_id__(self): + """ + Get object ID to be used for persisting. If `field_as_id` is specified + then use this field as ID, otherwise use `id()` + :return: Object's ID + """ + if field_as_id and hasattr(self, field_as_id): + if not getattr(self, field_as_id, None): + setattr(self, field_as_id, default_id(self)) + _id = getattr(self, field_as_id) + else: + _id = default_id(self) + if not _id: + raise ValueError('ID shall not be empty') + return _id + + def __json__(self): + """ + Get JSON representation of the object + :return: JSON representation + """ + if issubclass(cls, msgspec.Struct): + return msgspec.json.encode(self).decode(encoding='UTF-8') + elif is_dataclass(cls): + return msgspec.json.encode(asdict(self)).decode(encoding='UTF-8') + elif is_pydantic(cls): + return self.model_dump_json() + elif hasattr(cls, '__json__'): + return cls.__json__(self) + else: + raise NotImplementedError( + f'The class {cls} is not msgspec.Struct, @dataclass nor Pydantic Model ' + f'and does not have __json__() method. Please implement __json__() method by yourself.') + + return _Persisted else: - return wrapper(cls) + def wrapper(decor_cls): + return saveable(decor_cls, field_as_id=field_as_id) + return wrapper def file_storage(base_path: Union[str, Path]): diff --git a/pys/pydantic.py b/pys/pydantic.py index 771304e..708e81b 100644 --- a/pys/pydantic.py +++ b/pys/pydantic.py @@ -1,26 +1,13 @@ import uuid -from typing import Any from pydantic import BaseModel +from . import saveable + +@saveable(field_as_id='id', default_id=lambda _: str(uuid.uuid4())) class ModelWithID(BaseModel): """ Base class for models with `id` field prefilled with random UUID if not initialized. """ id: str = None - - def __init__(self, **data: Any) -> None: - """ - Initialize a model - :param data: model fields - """ - super().__init__(**data) - if self.id is None: - self.id = str(uuid.uuid4()) - - def __my_id__(self): - return self.id - - def __json__(self): - return self.model_dump_json() diff --git a/setup.py b/setup.py index b923507..e421695 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ README = (HERE / "README.md").read_text() setup(name='pysdato', - version='0.0.5', + version='0.0.6', description='Simple JSON file storage for Python dataclasses and pydantic models, thread and multiprocess safe', long_description=README, long_description_content_type="text/markdown",