From 1930d8e4f886772f97328a1095e965ab0aef7705 Mon Sep 17 00:00:00 2001 From: Alex Hadley Date: Fri, 5 Apr 2024 18:00:14 -0700 Subject: [PATCH 1/4] #170 Update last update timestamp functionality and tests --- paramdb/__init__.py | 16 +--- paramdb/_database.py | 28 +++--- paramdb/_keys.py | 20 ---- paramdb/_param_data/_collections.py | 46 ++++----- paramdb/_param_data/_dataclasses.py | 128 ++++++++++--------------- paramdb/_param_data/_param_data.py | 130 +++++++++++++++----------- paramdb/_param_data/_type_mixins.py | 2 - tests/_param_data/test_collections.py | 93 +++++++++++------- tests/_param_data/test_dataclasses.py | 121 +++++++++++++++--------- tests/_param_data/test_param_data.py | 65 +++++-------- tests/conftest.py | 125 +++++++++++++++++++------ tests/helpers.py | 26 ++++-- tests/test_database.py | 87 +++++++++-------- 13 files changed, 489 insertions(+), 398 deletions(-) delete mode 100644 paramdb/_keys.py diff --git a/paramdb/__init__.py b/paramdb/__init__.py index 6247871..7232745 100644 --- a/paramdb/__init__.py +++ b/paramdb/__init__.py @@ -1,25 +1,19 @@ -""" -Database for storing and retrieving QPU parameters during quantum control experiments. -""" +"""Python package for storing and retrieving experiment parameters.""" -from paramdb._keys import CLASS_NAME_KEY, PARAMLIST_ITEMS_KEY, LAST_UPDATED_KEY from paramdb._param_data._param_data import ParamData -from paramdb._param_data._dataclasses import Param, Struct +from paramdb._param_data._dataclasses import ParamDataclass from paramdb._param_data._collections import ParamList, ParamDict from paramdb._param_data._type_mixins import ParentType, RootType -from paramdb._database import ParamDB, CommitEntry, CommitEntryWithData +from paramdb._database import CLASS_NAME_KEY, ParamDB, CommitEntry, CommitEntryWithData __all__ = [ - "CLASS_NAME_KEY", - "PARAMLIST_ITEMS_KEY", - "LAST_UPDATED_KEY", "ParamData", - "Param", - "Struct", + "ParamDataclass", "ParamList", "ParamDict", "ParentType", "RootType", + "CLASS_NAME_KEY", "ParamDB", "CommitEntry", "CommitEntryWithData", diff --git a/paramdb/_database.py b/paramdb/_database.py index 4a3062b..a404e7e 100644 --- a/paramdb/_database.py +++ b/paramdb/_database.py @@ -14,7 +14,6 @@ Mapped, mapped_column, ) -from paramdb._keys import CLASS_NAME_KEY from paramdb._param_data._param_data import ParamData, get_param_class try: @@ -27,6 +26,12 @@ T = TypeVar("T") SelectT = TypeVar("SelectT", bound=Select[Any]) +CLASS_NAME_KEY = "__type" +""" +Dictionary key corresponding to an object's class name in the JSON representation of a +ParamDB commit. +""" + def _compress(text: str) -> bytes: """Compress the given text using Zstandard.""" @@ -48,10 +53,10 @@ def _full_class_name(cls: type) -> str: def _to_dict(obj: Any) -> Any: """ - Convert the given object into a dictionary to be passed to `json.dumps`. + Convert the given object into a dictionary to be passed to ``json.dumps()``. Note that objects within the dictionary do not need to be JSON serializable, - since they will be recursively processed by `json.dumps`. + since they will be recursively processed by ``json.dumps()``. """ class_full_name = _full_class_name(type(obj)) class_full_name_dict = {CLASS_NAME_KEY: class_full_name} @@ -69,7 +74,7 @@ def _to_dict(obj: Any) -> Any: def _from_dict(json_dict: dict[str, Any]) -> Any: """ - If the given dictionary created by ``json.loads`` has the key ``CLASS_NAME_KEY``, + If the given dictionary created by ``json.loads()`` has the key ``CLASS_NAME_KEY``, attempt to construct an object of the named type from it. Otherwise, return the dictionary unchanged. @@ -122,12 +127,10 @@ class _Snapshot(_Base): """Compressed data.""" id: Mapped[int] = mapped_column(init=False, primary_key=True) """Commit ID.""" - # datetime.utcnow() is wrapped in a lambda function to allow it to be mocked in - # tests where we want to control the time. timestamp: Mapped[datetime] = mapped_column( - default_factory=lambda: datetime.utcnow() # pylint: disable=unnecessary-lambda + default_factory=lambda: datetime.now(timezone.utc) ) - """Naive datetime in UTC time (since this is how SQLite stores datetimes).""" + """Datetime in UTC time (since this is how SQLite stores datetimes).""" @dataclass(frozen=True) @@ -165,10 +168,10 @@ class ParamDB(Generic[T]): not exist. To work with type checking, this class can be parameterized with a root data type ``T``. For example:: - from paramdb import Struct, ParamDB + from paramdb import ParamDataclass, ParamDB - class Root(Struct): - pass + class Root(ParamDataclass): + ... param_db = ParamDB[Root]("path/to/param.db") """ @@ -276,8 +279,7 @@ def load(self, commit_id: int | None = None, *, load_classes: bool = True) -> An reconstructed. The relevant parameter data classes must be defined in the current program. However, if ``load_classes`` is False, classes are loaded directly from the database as dictionaries with the class name in the key - :py:const:`~paramdb._keys.CLASS_NAME_KEY` and, if they are parameters, the last - updated time in the key :py:const:`~paramdb._keys.LAST_UPDATED_KEY`. + :py:const:`CLASS_NAME_KEY`. """ select_stmt = self._select_commit(select(_Snapshot.data), commit_id) with self._Session() as session: diff --git a/paramdb/_keys.py b/paramdb/_keys.py deleted file mode 100644 index 1641f21..0000000 --- a/paramdb/_keys.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Keys for special properties in dictionary representations of parameter data.""" - -CLASS_NAME_KEY = "__type" -""" -Dictionary key corresponding to the parameter class name when :py:meth:`.ParamDB.load` -or :py:meth:`.ParamDB.commit_history_with_data` is called with ``load_classes`` as -False. -""" - -PARAMLIST_ITEMS_KEY = "__items" -""" -Dictionary key corresponding to the list of items in the dictionary representation of a -:py:class:`.ParamList` object. -""" - -LAST_UPDATED_KEY = "__last_updated" -""" -Dictionary key corresponding to the last updated time in the dictionary representation -of a :py:class:`.Param` object. -""" diff --git a/paramdb/_param_data/_collections.py b/paramdb/_param_data/_collections.py index dfaeccc..c7439ba 100644 --- a/paramdb/_param_data/_collections.py +++ b/paramdb/_param_data/_collections.py @@ -13,19 +13,19 @@ ValuesView, ItemsView, ) -from datetime import datetime +from abc import abstractmethod from typing_extensions import Self -from paramdb._keys import PARAMLIST_ITEMS_KEY from paramdb._param_data._param_data import ParamData T = TypeVar("T") +CollectionT = TypeVar("CollectionT", bound=Collection[Any]) # pylint: disable-next=abstract-method -class _ParamCollection(ParamData): +class _ParamCollection(ParamData, Generic[CollectionT]): """Base class for parameter collections.""" - _contents: Collection[Any] + _contents: CollectionT def __len__(self) -> int: return len(self._contents) @@ -41,12 +41,15 @@ def __eq__(self, other: Any) -> bool: def __repr__(self) -> str: return f"{type(self).__name__}({self._contents})" - @property - def last_updated(self) -> datetime | None: - return self._get_last_updated(self._contents) + def _to_json(self) -> CollectionT: + return self._contents + + @classmethod + @abstractmethod + def _from_json(cls, json_data: CollectionT) -> Self: ... -class ParamList(_ParamCollection, MutableSequence[T], Generic[T]): +class ParamList(_ParamCollection[list[T]], MutableSequence[T], Generic[T]): """ Subclass of :py:class:`ParamData` and ``MutableSequence``. @@ -54,9 +57,8 @@ class ParamList(_ParamCollection, MutableSequence[T], Generic[T]): iterable (like builtin ``list``). """ - _contents: list[T] - def __init__(self, iterable: Iterable[T] | None = None) -> None: + super().__init__() self._contents = [] if iterable is None else list(iterable) if iterable is not None: for item in self._contents: @@ -80,6 +82,7 @@ def __setitem__(self, index: slice, value: Iterable[T]) -> None: ... def __setitem__(self, index: SupportsIndex | slice, value: Any) -> None: old_value: Any = self._contents[index] self._contents[index] = value + self._update_last_updated() if isinstance(index, slice): for item in old_value: self._remove_child(item) @@ -92,21 +95,20 @@ def __setitem__(self, index: SupportsIndex | slice, value: Any) -> None: def __delitem__(self, index: SupportsIndex | slice) -> None: old_value = self._contents[index] del self._contents[index] + self._update_last_updated() self._remove_child(old_value) def insert(self, index: SupportsIndex, value: T) -> None: self._contents.insert(index, value) + self._update_last_updated() self._add_child(value) - def to_dict(self) -> dict[str, list[T]]: - return {PARAMLIST_ITEMS_KEY: self._contents} - @classmethod - def from_dict(cls, json_dict: dict[str, list[T]]) -> Self: - return cls(json_dict[PARAMLIST_ITEMS_KEY]) + def _from_json(cls, json_data: list[T]) -> Self: + return cls(json_data) -class ParamDict(_ParamCollection, MutableMapping[str, T], Generic[T]): +class ParamDict(_ParamCollection[dict[str, T]], MutableMapping[str, T], Generic[T]): """ Subclass of :py:class:`ParamData` and ``MutableMapping``. @@ -117,9 +119,8 @@ class ParamDict(_ParamCollection, MutableMapping[str, T], Generic[T]): and items are returned as dict_keys, dict_values, and dict_items objects. """ - _contents: dict[str, T] - def __init__(self, mapping: Mapping[str, T] | None = None, /, **kwargs: T): + super().__init__() self._contents = ({} if mapping is None else dict(mapping)) | kwargs for item in self._contents.values(): self._add_child(item) @@ -138,12 +139,14 @@ def __getitem__(self, key: str) -> T: def __setitem__(self, key: str, value: T) -> None: old_value = self._contents[key] if key in self._contents else None self._contents[key] = value + self._update_last_updated() self._remove_child(old_value) self._add_child(value) def __delitem__(self, key: str) -> None: old_value = self._contents[key] if key in self._contents else None del self._contents[key] + self._update_last_updated() self._remove_child(old_value) def __iter__(self) -> Iterator[str]: @@ -194,9 +197,6 @@ def items(self) -> ItemsView[str, T]: # Use dict_items so items print nicely return self._contents.items() - def to_dict(self) -> dict[str, T]: - return self._contents - @classmethod - def from_dict(cls, json_dict: dict[str, T]) -> Self: - return cls(json_dict) + def _from_json(cls, json_data: dict[str, T]) -> Self: + return cls(json_data) diff --git a/paramdb/_param_data/_dataclasses.py b/paramdb/_param_data/_dataclasses.py index b39cccd..408510d 100644 --- a/paramdb/_param_data/_dataclasses.py +++ b/paramdb/_param_data/_dataclasses.py @@ -2,111 +2,77 @@ from __future__ import annotations from typing import Any -from abc import abstractmethod -from datetime import datetime, timezone from dataclasses import dataclass, is_dataclass, fields from typing_extensions import Self, dataclass_transform -from paramdb._keys import LAST_UPDATED_KEY from paramdb._param_data._param_data import ParamData @dataclass_transform() -class _ParamDataclass(ParamData): +class ParamDataclass(ParamData): """ - Base class for parameter dataclasses. + Subclass of :py:class:`ParamData`. + + Base class for parameter Data Classes. Subclasses are automatically converted to + Data Classes. For example:: + + class CustomParam(ParamDataclass): + value1: float + value2: int - Any keyword arguments given when creating a subclass are passed to the dataclass - constructor. + + Any keyword arguments given when creating a subclass are passed internally to the + standard `@dataclass()` decorator. """ + __field_names: set[str] + + # pylint: disable-next=unused-argument + def __new__(cls, *args: Any, **kwargs: Any) -> Self: + # Prevent instantiating ParamDataclass and call the superclass __init__() here + # since __init__() will be overwritten by dataclass() + if cls is ParamDataclass: + raise TypeError("only subclasses of ParamDataclass can be instantiated") + self = super(ParamDataclass, cls).__new__(cls) + super(ParamDataclass, self).__init__() # pylint: disable=super-with-arguments + return self + def __init_subclass__(cls, /, **kwargs: Any) -> None: # Convert subclasses into dataclasses super().__init_subclass__() # kwargs are passed to dataclass constructor dataclass(**kwargs)(cls) + cls.__field_names = ( + {f.name for f in fields(cls)} if is_dataclass(cls) else set() + ) + + def __post_init__(self) -> None: + # Called by the self.__init__() generated by dataclass() + for field_name in self.__field_names: + self._add_child(getattr(self, field_name)) def __getitem__(self, name: str) -> Any: - # Enable getting attributes via indexing + # Enable getting attributes via square brackets return getattr(self, name) def __setitem__(self, name: str, value: Any) -> None: - # Enable setting attributes via indexing + # Enable setting attributes via square brackets setattr(self, name, value) - @property - @abstractmethod - def last_updated(self) -> datetime | None: ... + def __setattr__(self, name: str, value: Any) -> None: + # If this attribute is a Data Class field, update last updated and children + if name in self.__field_names: + old_value = getattr(self, name) if hasattr(self, name) else None + super().__setattr__(name, value) + self._update_last_updated() + self._remove_child(old_value) + self._add_child(value) + return + super().__setattr__(name, value) - def to_dict(self) -> dict[str, Any]: + def _to_json(self) -> dict[str, Any]: if is_dataclass(self): return {f.name: getattr(self, f.name) for f in fields(self) if f.init} return {} @classmethod - def from_dict(cls, json_dict: dict[str, Any]) -> Self: - return cls(**json_dict) - - -class Param(_ParamDataclass): - """ - Subclass of :py:class:`ParamData`. - - Base class for parameters. Subclasses are automatically converted to dataclasses. - For example:: - - class CustomParam(Param): - value: float - """ - - def __post_init__(self) -> None: - self.__last_updated = datetime.now(timezone.utc).astimezone() - - def __setattr__(self, name: str, value: Any) -> None: - # Set the given attribute and update the last updated time if the object is - # initialized and the variable name does not have a double underscore in it (to - # exclude private variables, like __initialized, and dunder variables). - super().__setattr__(name, value) - if "__" not in name: - self.__last_updated = datetime.now(timezone.utc).astimezone() - - @property - def last_updated(self) -> datetime: - """When this parameter was last updated.""" - return self.__last_updated - - def to_dict(self) -> dict[str, Any]: - return {LAST_UPDATED_KEY: self.__last_updated} | super().to_dict() - - @classmethod - def from_dict(cls, json_dict: dict[str, Any]) -> Self: - last_updated = json_dict.pop(LAST_UPDATED_KEY) - obj = cls(**json_dict) - obj.__last_updated = last_updated # pylint: disable=unused-private-member - return obj - - -class Struct(_ParamDataclass): - """ - Subclass of :py:class:`ParamData`. - - Base class for parameter structures. Subclasses are automatically converted to - dataclasses. For example:: - - class CustomStruct(Struct): - value: float - custom_param: CustomParam - """ - - def __post_init__(self) -> None: - # Add fields as children. - for f in fields(self): - self._add_child(getattr(self, f.name)) - - def __setattr__(self, name: str, value: Any) -> None: - old_value = getattr(self, name) if hasattr(self, name) else None - super().__setattr__(name, value) - self._remove_child(old_value) - self._add_child(value) - - @property - def last_updated(self) -> datetime | None: - return self._get_last_updated(getattr(self, f.name) for f in fields(self)) + def _from_json(cls, json_data: dict[str, Any]) -> Self: + return cls(**json_data) diff --git a/paramdb/_param_data/_param_data.py b/paramdb/_param_data/_param_data.py index d7a80ff..a195811 100644 --- a/paramdb/_param_data/_param_data.py +++ b/paramdb/_param_data/_param_data.py @@ -2,14 +2,18 @@ from __future__ import annotations from typing import Any -from collections.abc import Iterable, Mapping from abc import ABC, abstractmethod from weakref import WeakValueDictionary -from datetime import datetime +from datetime import datetime, timezone from typing_extensions import Self -# Stores weak references to existing parameter classes +_LAST_UPDATED_KEY = "last_updated" +"""Dictionary key corresponding to a ``ParamData`` object's last updated time.""" +_DATA_KEY = "data" +"""Dictionary key corresponding to a ``ParamData`` object's data.""" + _param_classes: WeakValueDictionary[str, type[ParamData]] = WeakValueDictionary() +"""Dictionary of weak references to existing ``ParamData`` classes.""" def get_param_class(class_name: str) -> type[ParamData] | None: @@ -20,53 +24,91 @@ def get_param_class(class_name: str) -> type[ParamData] | None: class ParamData(ABC): """Abstract base class for all parameter data.""" - # Most recently initialized structure that contains this parameter data - _parent: ParamData | None = None + __parent: ParamData | None = None + __last_updated: datetime def __init_subclass__(cls, /, **kwargs: Any) -> None: - # Add subclass to dictionary of parameter data classes super().__init_subclass__(**kwargs) + + # Add subclass to dictionary of ParamData classes _param_classes[cls.__name__] = cls + def __init__(self) -> None: + self.__last_updated = datetime.now(timezone.utc).astimezone() + def _add_child(self, child: Any) -> None: """Add the given object as a child, if it is ``ParamData``.""" - if isinstance(child, ParamData): - # Use ParamData __setattr__ to avoid updating _last_updated - ParamData.__setattr__(child, "_parent", self) + # pylint: disable-next=protected-access,unused-private-member + child.__parent = self def _remove_child(self, child: Any) -> None: """Remove the given object as a child, if it is ``ParamData``.""" - if isinstance(child, ParamData): - # Use ParamData __setattr__ to avoid updating _last_updated - ParamData.__setattr__(child, "_parent", None) + # pylint: disable-next=protected-access,unused-private-member + child.__parent = None + + def _update_last_updated(self) -> None: + """Update last updated for this object and its chain of parents.""" + # pylint: disable=protected-access,unused-private-member + new_last_updated = datetime.now(timezone.utc).astimezone() + current: ParamData | None = self + + # Continue up the chain of parents, stopping if we reach a last updated + # timestamp that is more recent than the new one + while current and not ( + current.__last_updated and current.__last_updated >= new_last_updated + ): + current.__last_updated = new_last_updated + current = current.__parent - def _get_last_updated(self, obj: Any) -> datetime | None: + @abstractmethod + def _to_json(self) -> Any: """ - Get the last updated time from a ``ParamData`` object, or recursively search - through any iterable type to find the latest last updated time. + Convert the data stored in this object into a JSON serializable format, which + will later be passed to ``self._data_from_json()`` to reconstruct this object. + + The last updated timestamp is handled separately and does not need to be saved + here. + + Note that objects within the dictionary do not need to be JSON serializable, + since they will be recursively processed by ``json.dumps()``. """ - if isinstance(obj, ParamData): - return obj.last_updated - if isinstance(obj, Iterable) and not isinstance(obj, str): - # Strings are excluded because they will never contain ParamData and contain - # strings, leading to infinite recursion. - values = obj.values() if isinstance(obj, Mapping) else obj - return max( - filter(None, (self._get_last_updated(v) for v in values)), - default=None, - ) - return None - @property + @classmethod @abstractmethod - def last_updated(self) -> datetime | None: + def _from_json(cls, json_data: Any) -> Self: """ - When any parameter within this parameter data were last updated, or ``None`` if - this object contains no parameters. + Construct a parameter data object from the given JSON data, usually created by + ``json.loads()`` and originally constructed by ``self._data_to_json()``. + + The last updated timestamp is handled separately and does not need to be set + here. """ + def to_dict(self) -> dict[str, Any]: + """ + Return a dictionary representation of this parameter data object, which can be + used to reconstruct the object by passing it to :py:meth:`from_dict`. + """ + return {_LAST_UPDATED_KEY: self.last_updated, _DATA_KEY: self._to_json()} + + @classmethod + def from_dict(cls, data_dict: dict[str, Any]) -> Self: + """ + Construct a parameter data object from the given dictionary, usually created by + ``json.loads()`` and originally constructed by :py:meth:`from_dict`. + """ + param_data = cls._from_json(data_dict[_DATA_KEY]) + # pylint: disable-next=protected-access,unused-private-member + param_data.__last_updated = data_dict[_LAST_UPDATED_KEY] + return param_data + + @property + def last_updated(self) -> datetime: + """When any parameter within this parameter data was last updated.""" + return self.__last_updated + @property def parent(self) -> ParamData: """ @@ -77,12 +119,12 @@ def parent(self) -> ParamData: Raises a ``ValueError`` if there is currently no parent, which can occur if the parent is still being initialized. """ - if self._parent is None: + if self.__parent is None: raise ValueError( f"'{type(self).__name__}' object has no parent, or its parent has not" " been initialized yet" ) - return self._parent + return self.__parent @property def root(self) -> ParamData: @@ -90,26 +132,8 @@ def root(self) -> ParamData: Root of this parameter data. The root is defined to be the first object with no parent when going up the chain of parents. """ + # pylint: disable=protected-access root = self - while root._parent is not None: # pylint: disable=protected-access - root = root._parent # pylint: disable=protected-access + while root.__parent is not None: + root = root.__parent return root - - @abstractmethod - def to_dict(self) -> dict[str, Any]: - """ - Convert this parameter data object into a dictionary to be passed to - ``json.dumps``. This dictionary will later be passed to :py:meth:`from_dict` - to reconstruct the object. - - Note that objects within the dictionary do not need to be JSON serializable, - since they will be recursively processed by ``json.dumps``. - """ - - @classmethod - @abstractmethod - def from_dict(cls, json_dict: dict[str, Any]) -> Self: - """ - Construct a parameter data object from the given dictionary, usually created by - `json.loads` and originally constructed by :py:meth:`from_dict`. - """ diff --git a/paramdb/_param_data/_type_mixins.py b/paramdb/_param_data/_type_mixins.py index 52c59a2..8457d53 100644 --- a/paramdb/_param_data/_type_mixins.py +++ b/paramdb/_param_data/_type_mixins.py @@ -7,7 +7,6 @@ PT = TypeVar("PT", bound=ParamData) -# pylint: disable-next=abstract-method class ParentType(ParamData, Generic[PT]): """ Mixin for :py:class:`ParamData` that sets the type hint for @@ -25,7 +24,6 @@ def parent(self) -> PT: return cast(PT, super().parent) -# pylint: disable-next=abstract-method class RootType(ParamData, Generic[PT]): """ Mixin for :py:class:`ParamData` that sets the type hint for diff --git a/tests/_param_data/test_collections.py b/tests/_param_data/test_collections.py index ce1878e..feb3ea7 100644 --- a/tests/_param_data/test_collections.py +++ b/tests/_param_data/test_collections.py @@ -3,7 +3,11 @@ from typing import Union, Any from copy import deepcopy import pytest -from tests.helpers import CustomParamList, CustomParamDict, Times +from tests.helpers import ( + CustomParamList, + CustomParamDict, + capture_start_end_times, +) from paramdb import ParamData, ParamList, ParamDict ParamCollection = Union[ParamList[Any], ParamDict[Any]] @@ -199,31 +203,6 @@ def test_param_collection_repr_subclass( ) -def test_param_collection_no_last_updated( - param_collection_type: type[ParamCollection], -) -> None: - """Empty parameter collection has no last updated time.""" - assert param_collection_type().last_updated is None - - -def test_param_list_last_updated( - param_list: ParamList[Any], updated_param_data: ParamData, updated_times: Times -) -> None: - """Parameter list can correctly get the last updated time from its contents.""" - param_list.append(updated_param_data) - assert param_list.last_updated is not None - assert updated_times.start < param_list.last_updated.timestamp() < updated_times.end - - -def test_param_dict_last_updated( - param_dict: ParamDict[Any], updated_param_data: ParamData, updated_times: Times -) -> None: - """Parameter list can correctly get the last updated time from its contents.""" - param_dict["param_data"] = updated_param_data - assert param_dict.last_updated is not None - assert updated_times.start < param_dict.last_updated.timestamp() < updated_times.end - - def test_param_list_get_index( param_list: ParamList[Any], param_list_contents: list[Any] ) -> None: @@ -259,10 +238,21 @@ def test_param_list_set_index(param_list: ParamList[Any]) -> None: assert param_list[0] == new_number +def test_param_list_set_index_last_updated(param_list: ParamList[Any]) -> None: + """ + Parameter list updates its last updated timestamp when an item is set by index. + """ + with capture_start_end_times() as times: + param_list[0] = 4.56 + assert times.start < param_list.last_updated.timestamp() < times.end + + def test_param_list_set_index_parent( param_list: ParamList[Any], param_data: ParamData ) -> None: - """Parameter data added to a parameter list via indexing has the correct parent.""" + """ + A parameter data added to a parameter list via indexing has the correct parent. + """ with pytest.raises(ValueError): _ = param_data.parent for _ in range(2): # Run twice to check reassigning the same parameter data @@ -281,10 +271,19 @@ def test_param_list_set_slice(param_list: ParamList[Any]) -> None: assert param_list[0:2] == new_numbers +def test_param_list_set_slice_last_updated(param_list: ParamList[Any]) -> None: + """ + A parameter list updates its last updated timestamp when an item is set by slice. + """ + with capture_start_end_times() as times: + param_list[0:2] = [4.56, 7.89] + assert times.start < param_list.last_updated.timestamp() < times.end + + def test_param_list_set_slice_parent( param_list: ParamList[Any], param_data: ParamData ) -> None: - """Parameter data added to a parameter list via slicing has the correct parent.""" + """A parameter data added to a parameter list via slicing has the correct parent.""" for _ in range(2): # Run twice to check reassigning the same parameter data param_list[0:2] = [None, param_data] assert param_data.parent is param_list @@ -300,6 +299,13 @@ def test_param_list_insert(param_list: ParamList[Any]) -> None: assert param_list[1] == new_number +def test_param_list_insert_last_updated(param_list: ParamList[Any]) -> None: + """A parameter list updates its last updated timestamp when an item is inserted.""" + with capture_start_end_times() as times: + param_list.insert(1, 4.56) + assert times.start < param_list.last_updated.timestamp() < times.end + + def test_param_list_insert_parent( param_list: ParamList[Any], param_data: ParamData ) -> None: @@ -317,6 +323,13 @@ def test_param_list_del( assert list(param_list) == param_list_contents[1:] +def test_param_list_del_last_updated(param_list: ParamList[Any]) -> None: + """A parameter list updates its last updated timestamp when an item is deleted.""" + with capture_start_end_times() as times: + del param_list[0] + assert times.start < param_list.last_updated.timestamp() < times.end + + def test_param_list_del_parent( param_list: ParamList[Any], param_data: ParamData ) -> None: @@ -373,12 +386,12 @@ def test_param_dict_get( def test_param_dict_get_parent(param_dict: ParamDict[Any]) -> None: """An item gotten from a parameter dictionary has the correct parent.""" - assert param_dict["param"].parent is param_dict - assert param_dict.param.parent is param_dict + assert param_dict["simple_param"].parent is param_dict + assert param_dict.simple_param.parent is param_dict def test_param_dict_set(param_dict: ParamDict[Any]) -> None: - """Can set an item in a parameter list.""" + """Can set an item in a parameter dictionary using index bracket or dot notation.""" new_number_1 = 4.56 new_number_2 = 7.89 assert param_dict["number"] != new_number_1 @@ -388,6 +401,13 @@ def test_param_dict_set(param_dict: ParamDict[Any]) -> None: assert param_dict["number"] == new_number_2 +def test_param_dict_set_last_updated(param_dict: ParamDict[Any]) -> None: + """A parameter dictionary updates its last updated timestamp when an item is set.""" + with capture_start_end_times() as times: + param_dict["number"] = 4.56 + assert times.start < param_dict.last_updated.timestamp() < times.end + + def test_param_dict_set_parent( param_dict: ParamDict[Any], param_data: ParamData ) -> None: @@ -405,7 +425,9 @@ def test_param_dict_set_parent( def test_param_dict_del( param_dict: ParamDict[Any], param_dict_contents: dict[str, Any] ) -> None: - """Can delete items from a parameter dictionary.""" + """ + Can delete items from a parameter dictionary using index bracket or dot notation. + """ assert dict(param_dict) == param_dict_contents del param_dict["number"] del param_dict.string @@ -414,6 +436,13 @@ def test_param_dict_del( assert dict(param_dict) == param_dict_contents +def test_param_dict_del_last_updated(param_dict: ParamDict[Any]) -> None: + """Parameter dictionary updates last updated timestamp when an item is deleted.""" + with capture_start_end_times() as times: + del param_dict["number"] + assert times.start < param_dict.last_updated.timestamp() < times.end + + def test_param_dict_del_parent( param_dict: ParamDict[Any], param_data: ParamData ) -> None: diff --git a/tests/_param_data/test_dataclasses.py b/tests/_param_data/test_dataclasses.py index f4aa129..09b8b56 100644 --- a/tests/_param_data/test_dataclasses.py +++ b/tests/_param_data/test_dataclasses.py @@ -3,79 +3,108 @@ from typing import Union from copy import deepcopy import pytest -from tests.helpers import CustomParam, CustomStruct, Times, capture_start_end_times -from paramdb import ParamData +from tests.helpers import ( + SimpleParam, + SubclassParam, + ComplexParam, + capture_start_end_times, +) +from paramdb import ParamDataclass, ParamData -ParamDataclass = Union[CustomParam, CustomStruct] +ParamDataclassObject = Union[SimpleParam, SubclassParam, ComplexParam] @pytest.fixture( - name="param_dataclass", - params=["param", "struct"], + name="param_dataclass_object", + params=["simple_param", "subclass_param", "complex_param"], ) -def fixture_param_dataclass(request: pytest.FixtureRequest) -> ParamDataclass: - """Parameter dataclass.""" - param_dataclass: ParamDataclass = deepcopy(request.getfixturevalue(request.param)) - return param_dataclass +def fixture_param_dataclass_object( + request: pytest.FixtureRequest, +) -> ParamDataclassObject: + """Parameter dataclass object.""" + param_dataclass_object: ParamDataclassObject = deepcopy( + request.getfixturevalue(request.param) + ) + return param_dataclass_object + + +def test_param_data_direct_instantiation_fails() -> None: + """Fails to instantiate an object of type ``ParamDataclass``.""" + with pytest.raises(TypeError) as exc_info: + ParamDataclass() + assert ( + str(exc_info.value) == "only subclasses of ParamDataclass can be instantiated" + ) def test_param_dataclass_get( - param_dataclass: ParamDataclass, number: float, string: str + param_dataclass_object: ParamDataclassObject, number: float, string: str ) -> None: """ Parameter dataclass properties can be accessed via dot notation and index brackets. """ - assert param_dataclass.number == number - assert param_dataclass.string == string - assert param_dataclass["number"] == number - assert param_dataclass["string"] == string + assert param_dataclass_object.number == number + assert param_dataclass_object.string == string + assert param_dataclass_object["number"] == number + assert param_dataclass_object["string"] == string -def test_param_dataclass_set(param_dataclass: ParamDataclass, number: float) -> None: - """Parameter data properties can be updated via dot notation and index brackets.""" - param_dataclass.number += 1 - assert param_dataclass.number == number + 1 - param_dataclass["number"] -= 1 - assert param_dataclass.number == number +def test_param_dataclass_set( + param_dataclass_object: ParamDataclassObject, number: float +) -> None: + """ + Parameter dataclass properties can be updated via dot notation and index brackets. + """ + param_dataclass_object.number += 1 + assert param_dataclass_object.number == number + 1 + param_dataclass_object["number"] -= 1 + assert param_dataclass_object.number == number -def test_param_default_last_updated() -> None: - """Parameter object initializes the last updated time to the current time.""" +def test_param_dataclass_set_last_updated( + param_dataclass_object: ParamDataclassObject, +) -> None: + """ + A parameter dataclass object updates last updated timestamp when a field is set. + """ with capture_start_end_times() as times: - param = CustomParam() - assert times.start < param.last_updated.timestamp() < times.end + param_dataclass_object.number = 4.56 + assert times.start < param_dataclass_object.last_updated.timestamp() < times.end -def test_struct_no_last_updated() -> None: - """Structure object that contains no parameters has no last updated time.""" - struct = CustomStruct() - assert struct.last_updated is None - - -def test_struct_last_updated( - struct: CustomStruct, updated_param_data: ParamData, updated_times: Times +def test_param_dataclass_set_last_updated_non_field( + param_dataclass_object: ParamDataclassObject, ) -> None: - """Structure can correctly get the last updated time from its contents.""" - struct.param_data = updated_param_data - assert struct.last_updated is not None - assert updated_times.start < struct.last_updated.timestamp() < updated_times.end + """ + A parameter dataclass object does not update last updated timestamp when a non-field + parameter is set. + """ + with capture_start_end_times() as times: + param_dataclass_object.non_field = 1.23 + assert param_dataclass_object.last_updated.timestamp() < times.start -def test_struct_init_parent(struct: CustomStruct) -> None: - """Structure children correctly identify it as a parent after initialization.""" - assert struct.param is not None - assert struct.struct is not None - assert struct.param.parent is struct - assert struct.struct.parent is struct +def test_param_dataclass_init_parent(complex_param: ComplexParam) -> None: + """ + Parameter dataclass children correctly identify their parent after initialization. + """ + assert complex_param.simple_param is not None + assert complex_param.param_list is not None + assert complex_param.param_dict is not None + assert complex_param.simple_param.parent is complex_param + assert complex_param.param_list.parent is complex_param + assert complex_param.param_dict.parent is complex_param -def test_struct_set_parent(struct: CustomStruct, param_data: ParamData) -> None: +def test_param_dataclass_set_parent( + complex_param: ComplexParam, param_data: ParamData +) -> None: """Parameter data added to a structure has the correct parent.""" with pytest.raises(ValueError): _ = param_data.parent for _ in range(2): # Run twice to check reassigning the same parameter data - struct.param_data = param_data - assert param_data.parent is struct - struct.param_data = None + complex_param.param_data = param_data + assert param_data.parent is complex_param + complex_param.param_data = None with pytest.raises(ValueError): _ = param_data.parent diff --git a/tests/_param_data/test_param_data.py b/tests/_param_data/test_param_data.py index 623a3d4..abe63b8 100644 --- a/tests/_param_data/test_param_data.py +++ b/tests/_param_data/test_param_data.py @@ -3,21 +3,26 @@ from dataclasses import is_dataclass from copy import deepcopy import pytest -from tests.helpers import CustomStruct, Times, capture_start_end_times +from tests.helpers import ComplexParam, Times, capture_start_end_times from paramdb import ParamData from paramdb._param_data._param_data import get_param_class -def test_custom_subclass_extra_kwarg(param_data: ParamData) -> None: +@pytest.fixture(name="param_data_type") +def fixture_param_data_type(param_data: ParamData) -> type[ParamData]: + """Parameter data type.""" + return type(param_data) + + +def test_custom_subclass_extra_kwarg_fails(param_data_type: type[ParamData]) -> None: """Extra keyword arugments in a custom parameter data subclass raise a TypeError.""" - cls = type(param_data) with pytest.raises(TypeError) as exc_info: # pylint: disable-next=unused-variable - class CustomParamData(cls, extra_kwarg="test"): # type: ignore + class CustomParamData(param_data_type, extra_kwarg="test"): # type: ignore """Custom parameter data class with an extra keyword arugment.""" error_message = str(exc_info.value) - if is_dataclass(cls): + if is_dataclass(param_data_type): assert ( error_message == "dataclass() got an unexpected keyword argument 'extra_kwarg'" @@ -37,44 +42,22 @@ def test_get_param_class(param_data: ParamData) -> None: assert get_param_class(param_class.__name__) is param_class -def test_param_data_last_updated( - updated_param_data: ParamData, updated_times: Times -) -> None: - """Updating simple parameter data updates the last updated time.""" - assert updated_param_data.last_updated is not None - assert ( - updated_times.start - < updated_param_data.last_updated.timestamp() - < updated_times.end - ) +def test_param_data_initial_last_updated(param_data_type: type[ParamData]) -> None: + """New parameter data objects are initialized with a last updated timestamp.""" + with capture_start_end_times() as times: + new_param_data = param_data_type() + assert new_param_data.last_updated is not None + assert times.start < new_param_data.last_updated.timestamp() < times.end -def test_list_or_dict_last_updated( +def test_param_data_updates_last_updated( updated_param_data: ParamData, updated_times: Times ) -> None: - """Can get last updated from a Python list or dictionary.""" - # Can get last updated time from within a list - struct_with_list = CustomStruct( - list=[CustomStruct(), [updated_param_data, CustomStruct()]] - ) - assert struct_with_list.last_updated is not None - assert ( - updated_times.start - < struct_with_list.last_updated.timestamp() - < updated_times.end - ) - - # Can get last updated time from within a dictionary - struct_with_dict = CustomStruct( - dict={ - "p1": CustomStruct(), - "p2": {"p1": updated_param_data, "p2": CustomStruct()}, - } - ) - assert struct_with_dict.last_updated is not None + """Updating parameter data updates the last updated time.""" + assert updated_param_data.last_updated is not None assert ( updated_times.start - < struct_with_list.last_updated.timestamp() + < updated_param_data.last_updated.timestamp() < updated_times.end ) @@ -83,11 +66,11 @@ def test_child_does_not_change(param_data: ParamData) -> None: """ Including a parameter data object as a child within a parent structure does not change the parameter in terms of equality comparison (i.e. public properties, - importantly last_updated, have not changed). + importantly ``last_updated``, have not changed). """ param_data_original = deepcopy(param_data) with capture_start_end_times(): - _ = CustomStruct(param_data=param_data) + _ = ComplexParam(param_data=param_data) assert param_data == param_data_original @@ -122,7 +105,7 @@ def test_parent_is_root(param_data: ParamData) -> None: Parameter data object with a parent that has no parent returns the parent as the root. """ - parent = CustomStruct(param_data=param_data) + parent = ComplexParam(param_data=param_data) assert param_data.root is parent @@ -131,5 +114,5 @@ def test_parent_of_parent_is_root(param_data: ParamData) -> None: Parameter data object with a parent that has a parent returns the highest level parent as the root. """ - root = CustomStruct(struct=CustomStruct(param_data=param_data)) + root = ComplexParam(complex_param=ComplexParam(param_data=param_data)) assert param_data.root is root diff --git a/tests/conftest.py b/tests/conftest.py index 0df6526..3fd6d0c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,8 +7,10 @@ from tests.helpers import ( DEFAULT_NUMBER, DEFAULT_STRING, - CustomStruct, - CustomParam, + EmptyParam, + SimpleParam, + SubclassParam, + ComplexParam, Times, capture_start_end_times, ) @@ -26,20 +28,34 @@ def fixture_string() -> str: return DEFAULT_STRING -@pytest.fixture(name="param") -def fixture_param(number: float, string: str) -> CustomParam: - """Parameter.""" - return CustomParam(number=number, string=string) +@pytest.fixture(name="empty_param") +def fixture_empty_param() -> EmptyParam: + """Empty parameter dataclass object.""" + return EmptyParam() -@pytest.fixture(name="struct") -def fixture_struct(number: float, string: str) -> CustomStruct: - """Structure.""" - return CustomStruct( +@pytest.fixture(name="simple_param") +def fixture_simple_param(number: float, string: str) -> SimpleParam: + """Simple parameter dataclass object.""" + return SimpleParam(number=number, string=string) + + +@pytest.fixture(name="subclass_param") +def fixture_subclass_param(number: float, string: str) -> SubclassParam: + """Parameter dataclass object that is a subclass of another parameter dataclass.""" + return SubclassParam(number=number, string=string, second_number=number) + + +@pytest.fixture(name="complex_param") +def fixture_complex_param(number: float, string: str) -> ComplexParam: + """Complex parameter dataclass object.""" + return ComplexParam( number=number, string=string, - param=CustomParam(), - struct=CustomStruct(), + empty_param=EmptyParam(), + simple_param=SimpleParam(), + subclass_param=SubclassParam(), + complex_param=ComplexParam(), param_list=ParamList(), param_dict=ParamDict(), ) @@ -48,30 +64,59 @@ def fixture_struct(number: float, string: str) -> CustomStruct: @pytest.fixture(name="param_list_contents") def fixture_param_list_contents(number: float, string: str) -> list[Any]: """Contents to initialize a parameter list.""" - return [number, string, CustomParam(), CustomStruct(), ParamList(), ParamDict()] + return [ + number, + string, + EmptyParam(), + SimpleParam(), + SubclassParam(), + ComplexParam(), + ParamList(), + ParamDict(), + ] @pytest.fixture(name="param_dict_contents") +# pylint: disable-next=too-many-arguments def fixture_param_dict_contents( - number: float, string: str, param: CustomParam, struct: CustomStruct + number: float, + string: str, + empty_param: EmptyParam, + simple_param: SimpleParam, + subclass_param: SubclassParam, + complex_param: ComplexParam, ) -> dict[str, Any]: """Contents to initialize a parameter dictionary.""" return { "number": number, "string": string, - "param": deepcopy(param), - "struct": deepcopy(struct), + "empty_param": deepcopy(empty_param), + "simple_param": deepcopy(simple_param), + "subclass_param": deepcopy(subclass_param), + "complex_param": deepcopy(complex_param), "param_list": ParamList(), "param_dict": ParamDict(), } +@pytest.fixture(name="empty_param_list") +def fixture_empty_param_list() -> ParamList[Any]: + """Empty parameter list.""" + return ParamList() + + @pytest.fixture(name="param_list") def fixture_param_list(param_list_contents: list[Any]) -> ParamList[Any]: """Parameter list.""" return ParamList(deepcopy(param_list_contents)) +@pytest.fixture(name="empty_param_dict") +def fixture_empty_param_dict() -> ParamDict[Any]: + """Empty parameter dictionary.""" + return ParamDict() + + @pytest.fixture(name="param_dict") def fixture_param_dict(param_dict_contents: dict[str, Any]) -> ParamDict[Any]: """Parameter dictionary.""" @@ -80,11 +125,18 @@ def fixture_param_dict(param_dict_contents: dict[str, Any]) -> ParamDict[Any]: @pytest.fixture( name="param_data", - params=["param", "struct", "param_list", "param_dict"], + params=[ + "empty_param", + "simple_param", + "subclass_param", + "complex_param", + "empty_param_list", + "param_list", + "empty_param_dict", + "param_dict", + ], ) -def fixture_param_data( - request: pytest.FixtureRequest, -) -> ParamData: +def fixture_param_data(request: pytest.FixtureRequest) -> ParamData: """Parameter data.""" param_data: ParamData = deepcopy(request.getfixturevalue(request.param)) return param_data @@ -92,7 +144,7 @@ def fixture_param_data( @pytest.fixture(name="updated_param_data_and_times") def fixture_updated_param_data_and_times( - param_data: ParamData, + param_data: ParamData, number: float ) -> tuple[ParamData, Times]: """ Parameter data that has been updated between the returned Times. Broken down into @@ -100,15 +152,26 @@ def fixture_updated_param_data_and_times( """ updated_param_data = deepcopy(param_data) with capture_start_end_times() as times: - if isinstance(updated_param_data, CustomParam): + if isinstance(updated_param_data, EmptyParam): + # pylint: disable-next=protected-access + updated_param_data._update_last_updated() + elif isinstance(updated_param_data, SimpleParam): updated_param_data.number += 1 - if isinstance(updated_param_data, CustomStruct): - assert updated_param_data.param is not None - updated_param_data.param.number += 1 - if isinstance(updated_param_data, ParamList): - updated_param_data[2].number += 1 - if isinstance(updated_param_data, ParamDict): - updated_param_data.param.number += 1 + elif isinstance(updated_param_data, SubclassParam): + updated_param_data.second_number += 1 + elif isinstance(updated_param_data, ComplexParam): + assert updated_param_data.simple_param is not None + updated_param_data.simple_param.number += 1 + elif isinstance(updated_param_data, ParamList): + if len(updated_param_data) == 0: + updated_param_data.append(number) + else: + updated_param_data[3].number += 1 + elif isinstance(updated_param_data, ParamDict): + if len(updated_param_data) == 0: + updated_param_data["number"] = number + else: + updated_param_data.simple_param.number += 1 return updated_param_data, times @@ -121,6 +184,8 @@ def fixture_updated_param_data( @pytest.fixture(name="updated_times") -def fixture_start(updated_param_data_and_times: tuple[ParamData, Times]) -> Times: +def fixture_updated_times( + updated_param_data_and_times: tuple[ParamData, Times] +) -> Times: """Times before and after param_data fixture was updated.""" return updated_param_data_and_times[1] diff --git a/tests/helpers.py b/tests/helpers.py index 1f1f447..d747ada 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -11,14 +11,18 @@ from contextlib import contextmanager import time from astropy.units import Quantity # type: ignore # pylint: disable=import-error -from paramdb import ParamData, Param, Struct, ParamList, ParamDict +from paramdb import ParamData, ParamDataclass, ParamList, ParamDict DEFAULT_NUMBER = 1.23 DEFAULT_STRING = "test" -class CustomParam(Param): - """Custom parameter.""" +class EmptyParam(ParamDataclass): + """Empty parameter dataclass""" + + +class SimpleParam(ParamDataclass): + """Simple parameter dataclass.""" number: float = DEFAULT_NUMBER number_init_false: float = field(init=False, default=DEFAULT_NUMBER) @@ -26,16 +30,24 @@ class CustomParam(Param): string: str = DEFAULT_STRING -class CustomStruct(Struct): - """Custom parameter structure.""" +class SubclassParam(SimpleParam): + """Parameter dataclass that is a subclass of another parameter dataclass.""" + + second_number: float = DEFAULT_NUMBER + + +class ComplexParam(ParamDataclass): + """Complex parameter dataclass.""" number: float = DEFAULT_NUMBER number_init_false: float = field(init=False, default=DEFAULT_NUMBER) string: str = DEFAULT_STRING list: list[Any] = field(default_factory=list) dict: dict[str, Any] = field(default_factory=dict) - param: CustomParam | None = None - struct: CustomStruct | None = None + empty_param: EmptyParam | None = None + simple_param: SimpleParam | None = None + subclass_param: SubclassParam | None = None + complex_param: ComplexParam | None = None param_list: ParamList[Any] = field(default_factory=ParamList) param_dict: ParamDict[Any] = field(default_factory=ParamDict) param_data: ParamData | None = None diff --git a/tests/test_database.py b/tests/test_database.py index 68931fa..0c5fa40 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -11,8 +11,10 @@ from datetime import datetime, timezone import pytest from tests.helpers import ( - CustomStruct, - CustomParam, + EmptyParam, + SimpleParam, + SubclassParam, + ComplexParam, CustomParamList, CustomParamDict, Times, @@ -20,7 +22,7 @@ ) from paramdb import ( ParamData, - Struct, + ParamDataclass, ParamList, ParamDict, ParamDB, @@ -31,7 +33,7 @@ from paramdb._param_data._param_data import _param_classes -class Unknown(Struct): +class Unknown(ParamDataclass): """ Class that is unknown to ParamDB. By default, it will get added to the private param class dictionary when created, but on the next line we manually delete it. @@ -162,15 +164,16 @@ def test_commit_and_load(db_path: str, param_data: ParamData) -> None: assert commit_entry_from_history == commit_entry -def test_commit_and_load_timestamp(db_path: str, param: CustomParam) -> None: +def test_commit_and_load_timestamp(db_path: str, simple_param: SimpleParam) -> None: """Can make a commit using a specific timestamp and load it back.""" param_db = ParamDB[ParamData](db_path) - utc_timestamp = datetime.utcnow() - aware_timestamp = utc_timestamp.replace(tzinfo=timezone.utc).astimezone() + utc_timestamp = datetime.now(timezone.utc) + naive_timestamp = utc_timestamp.replace(tzinfo=None) + aware_timestamp = utc_timestamp.astimezone() - for i, timestamp in enumerate((utc_timestamp, aware_timestamp)): + for i, timestamp in enumerate((utc_timestamp, naive_timestamp, aware_timestamp)): with capture_start_end_times() as times: - commit_entry = param_db.commit(f"Commit {i}", param, timestamp) + commit_entry = param_db.commit(f"Commit {i}", simple_param, timestamp) # Given timestamp was used, not the current time assert commit_entry.timestamp.timestamp() < times.start @@ -238,29 +241,33 @@ def test_load_classes_false_unknown_class(db_path: str) -> None: assert data_from_history.pop(CLASS_NAME_KEY) == Unknown.__name__ -# pylint: disable-next=too-many-arguments +# pylint: disable-next=too-many-arguments,too-many-locals def test_commit_and_load_complex( db_path: str, number: float, string: str, param_list_contents: list[Any], param_dict_contents: dict[str, Any], - param: CustomParam, - struct: CustomStruct, + empty_param: EmptyParam, + simple_param: SimpleParam, + subclass_param: SubclassParam, + complex_param: ComplexParam, param_list: ParamList[Any], param_dict: ParamDict[Any], ) -> None: """Can commit and load a complex parameter structure.""" - class Root(Struct): + class Root(ParamDataclass): """Complex root structure to test the database.""" number: float string: str list: list[Any] dict: dict[str, Any] - param: CustomParam - struct: CustomStruct + empty_param: EmptyParam + simple_param: SimpleParam + subclass_param: SubclassParam + complex_param: ComplexParam param_list: ParamList[Any] param_dict: ParamDict[Any] custom_param_list: CustomParamList @@ -271,8 +278,10 @@ class Root(Struct): string=string, list=param_list_contents, dict=param_dict_contents, - param=param, - struct=struct, + empty_param=empty_param, + simple_param=simple_param, + subclass_param=subclass_param, + complex_param=complex_param, param_list=param_list, param_dict=param_dict, custom_param_list=CustomParamList(deepcopy(param_list_contents)), @@ -290,11 +299,11 @@ class Root(Struct): def test_commit_load_latest(db_path: str) -> None: """The database can load the latest data and commit entry after each commit.""" - param_db = ParamDB[CustomParam](db_path) + param_db = ParamDB[SimpleParam](db_path) for i in range(10): # Make the commit message = f"Commit {i}" - param = CustomParam(number=i) + param = SimpleParam(number=i) with capture_start_end_times(): commit_entry = param_db.commit(message, param) @@ -305,11 +314,11 @@ def test_commit_load_latest(db_path: str) -> None: def test_commit_load_multiple(db_path: str) -> None: """Can commit multiple times and load previous commits.""" - param_db = ParamDB[CustomParam](db_path) + param_db = ParamDB[SimpleParam](db_path) commit_entries: list[CommitEntry] = [] # Make 10 commits - params = [CustomParam(number=i + 1) for i in range(10)] + params = [SimpleParam(number=i + 1) for i in range(10)] for i, param in enumerate(params): with capture_start_end_times(): commit_entry = param_db.commit(f"Commit {i + 1}", param) @@ -338,47 +347,47 @@ def test_commit_load_multiple(db_path: str) -> None: assert param_from_history.last_updated == param.last_updated -def test_separate_connections(db_path: str, param: CustomParam) -> None: +def test_separate_connections(db_path: str, simple_param: SimpleParam) -> None: """ Can commit and load using separate connections. This simulates committing to the database in one program and loading in another program at a later time. """ # Commit using one connection - param_db1 = ParamDB[CustomParam](db_path) - param_db1.commit("Initial commit", param) + param_db1 = ParamDB[SimpleParam](db_path) + param_db1.commit("Initial commit", simple_param) del param_db1 # Load back using another connection - param_db2 = ParamDB[CustomParam](db_path) + param_db2 = ParamDB[SimpleParam](db_path) param_loaded = param_db2.load() - assert param == param_loaded - assert param.last_updated == param_loaded.last_updated + assert simple_param == param_loaded + assert simple_param.last_updated == param_loaded.last_updated def test_empty_num_commits(db_path: str) -> None: """An empty database has no commits according to num_commits.""" - param_db = ParamDB[CustomStruct](db_path) + param_db = ParamDB[SimpleParam](db_path) assert param_db.num_commits == 0 -def test_num_commits(db_path: str, param: CustomParam) -> None: +def test_num_commits(db_path: str, simple_param: SimpleParam) -> None: """A database with multiple commits has the correct value for num_commits.""" - param_db = ParamDB[CustomParam](db_path) + param_db = ParamDB[SimpleParam](db_path) for i in range(10): - param_db.commit(f"Commit {i}", param) + param_db.commit(f"Commit {i}", simple_param) assert param_db.num_commits == 10 def test_empty_commit_history(db_path: str) -> None: """Loads an empty commit history from an empty database.""" - param_db = ParamDB[CustomStruct](db_path) + param_db = ParamDB[SimpleParam](db_path) for history_func in param_db.commit_history, param_db.commit_history_with_data: assert history_func() == [] # type: ignore def test_empty_commit_history_slice(db_path: str) -> None: """Correctly slices an empty commit history.""" - param_db = ParamDB[CustomStruct](db_path) + param_db = ParamDB[SimpleParam](db_path) for history_func in param_db.commit_history, param_db.commit_history_with_data: assert history_func(0) == [] # type: ignore assert history_func(0, 10) == [] # type: ignore @@ -386,19 +395,19 @@ def test_empty_commit_history_slice(db_path: str) -> None: assert history_func(-10, -5) == [] # type: ignore -def test_commit_history(db_path: str, param: CustomParam) -> None: +def test_commit_history(db_path: str, simple_param: SimpleParam) -> None: """ Loads the commit history with the correct messages and timestamps for a series of commits. """ - param_db = ParamDB[CustomParam](db_path) + param_db = ParamDB[SimpleParam](db_path) commit_times: list[Times] = [] # Make 10 commits for i in range(10): with capture_start_end_times() as times: commit_times.append(times) - param_db.commit(f"Commit {i}", param) + param_db.commit(f"Commit {i}", simple_param) # Load commit history commit_history = param_db.commit_history() @@ -414,13 +423,13 @@ def test_commit_history(db_path: str, param: CustomParam) -> None: assert times.start < commit_entry_with_data.timestamp.timestamp() < times.end -def test_commit_history_slice(db_path: str, param: CustomParam) -> None: +def test_commit_history_slice(db_path: str, simple_param: SimpleParam) -> None: """Can retrieve a slice of a commit history, using Python slicing rules.""" - param_db = ParamDB[CustomParam](db_path) + param_db = ParamDB[SimpleParam](db_path) # Make 10 commits for i in range(10): - param_db.commit(f"Commit {i}", param) + param_db.commit(f"Commit {i}", simple_param) # Load slices of commit history commit_history = param_db.commit_history() From b62897fb6827f9b8cac189d3147ea8d8ee7ca6f0 Mon Sep 17 00:00:00 2001 From: Alex Hadley Date: Sun, 21 Apr 2024 15:48:06 -0700 Subject: [PATCH 2/4] Update docs with ParamDataclass changes --- docs/api-reference.md | 17 +-- docs/database.md | 6 +- docs/parameter-data.md | 150 ++++++++++++++------------ paramdb/_database.py | 2 +- paramdb/_param_data/_dataclasses.py | 6 +- paramdb/_param_data/_type_mixins.py | 4 +- tests/_param_data/test_dataclasses.py | 2 +- 7 files changed, 96 insertions(+), 91 deletions(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index 5062c73..93b668b 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -10,8 +10,7 @@ All of the following can be imported from `paramdb`. ```{eval-rst} .. autoclass:: ParamData -.. autoclass:: Param -.. autoclass:: Struct +.. autoclass:: ParamDataclass .. autoclass:: ParamList .. autoclass:: ParamDict .. autoclass:: ParentType @@ -26,19 +25,13 @@ All of the following can be imported from `paramdb`. .. autoclass:: CommitEntryWithData ``` -## Keys - -```{py:currentmodule} paramdb._keys - -``` - ```{eval-rst} +.. py:currentmodule:: paramdb._database .. autodata:: CLASS_NAME_KEY -.. autodata:: PARAMLIST_ITEMS_KEY -.. autodata:: LAST_UPDATED_KEY +.. py:currentmodule:: paramdb ``` diff --git a/docs/database.md b/docs/database.md index 5b9cdc9..0913eb4 100644 --- a/docs/database.md +++ b/docs/database.md @@ -23,12 +23,12 @@ the root data type in order for its methods (e.g. {py:meth}`ParamDB.commit`) wor with type checking. For example: ```{jupyter-execute} -from paramdb import Struct, Param, ParamDB +from paramdb import ParamDataclass, ParamDB -class Root(Struct): +class Root(ParamDataclass): param: CustomParam -class CustomParam(Param): +class CustomParam(ParamDataclass): value: float param_db = ParamDB[Root]("path/to/param.db") diff --git a/docs/parameter-data.md b/docs/parameter-data.md index 9111975..9f5fd53 100644 --- a/docs/parameter-data.md +++ b/docs/parameter-data.md @@ -16,8 +16,8 @@ A ParamDB database stores parameter data. The abstract base class {py:class}`Par defines some core functionality for this data, including the {py:class}`~ParamData.last_updated`, {py:class}`~ParamData.parent`, and {py:class}`~ParamData.root` properties. Internally, any subclasses of -{py:class}`ParamData` automatically registered with ParamDB so that they can be loaded -to and from JSON, which is how they are stored in the database. +{py:class}`ParamData` are automatically registered with ParamDB so that they can be +loaded to and from JSON, which is how they are stored in the database. All of the classes described on this page are subclasses of {py:class}`ParamData`. @@ -28,33 +28,34 @@ type (`str`, `int`, `float`, `bool`, `None`, `dict`, or `list`), a [`datetime`], a `TypeError` will be raised when they are committed to the database. ``` -## Parameters +## Data Classes -A parameter is defined from the base class {py:class}`Param`. This custom class is -automatically converted into a [`dataclass`], meaning that class variables with type -annotations become object properties and the corresponding [`__init__`] function is -generated. An example of a defining a custom parameter is shown below. +A parameter data class is defined from the base class {py:class}`ParamDataclass`. This +custom class is automatically converted into a [data class], meaning that class variables +with type annotations become object properties and the corresponding [`__init__`] +function is generated. An example of a defining a custom parameter Data Class is shown +below. ```{jupyter-execute} -from paramdb import Param +from paramdb import ParamDataclass -class CustomParam(Param): +class CustomParam(ParamDataclass): value: float -param = CustomParam(value=1.23) +custom_param = CustomParam(value=1.23) ``` These properties can then be accessed and updated. ```{jupyter-execute} -param.value += 0.004 -param.value +custom_param.value += 0.004 +custom_param.value ``` -The dataclass aspects of the subclass can be customized by passing keyword arguments when +The data class aspects of the subclass can be customized by passing keyword arguments when defining the custom class (the same arguments that would be passed to the [`@dataclass`] decorator), and by using the dataclass [`field`] function. The class arguments have the -same default values as in [`@dataclass`]. An example of dataclass customization is shown +same default values as in [`@dataclass`]. An example of data class customization is shown below. ```{note} @@ -66,24 +67,24 @@ when building up dataclasses through inheritance. ```{jupyter-execute} from dataclasses import field -class CustomizedDataclassParam(Param, kw_only=True): +class KeywordOnlyParam(ParamDataclass, kw_only=True): values: list[int] = field(default_factory=list) count: int -customized_dataclass_param = CustomizedDataclassParam(count=123) -customized_dataclass_param +keyword_only_param = KeywordOnlyParam(count=123) +keyword_only_param ``` ```{warning} -For mutable default values, `default_factory` should generally be used. See Python -dataclass documentation for [mutable default values] for more information. +For mutable default values, `default_factory` should generally be used. See the Python +data class documentation on [mutable default values] for more information. ``` -Methods can also be added, including dynamic read-only properties using the -[`@property`] decorator. For example: +Custom methods can also be added, including dynamic properties using the [`@property`] +decorator. For example: ```{jupyter-execute} -class ParamWithProperty(Param): +class ParamWithProperty(ParamDataclass): value: int @property @@ -95,13 +96,13 @@ param_with_property.value_cubed ``` ````{important} -Since [`__init__`] is generated for dataclasses, other initialization must be done using +Since [`__init__`] is generated for data classes, other initialization must be done using the [`__post_init__`] function. Furthermore, since [`__post_init__`] is used internally by -{py:class}`ParamData`, {py:class}`Param`, and {py:class}`Struct` to perform -initialization, always call the superclass's [`__post_init__`] at the end. For example: +{py:class}`ParamDataclass` to perform initialization, always call the superclass's +[`__post_init__`]. For example: ```{jupyter-execute} -class ParamCustomInit(Param): +class ParamCustomInit(ParamDataclass): def __post_init__(self) -> None: print("Initializing...") # Replace with custom initialization code super().__post_init__() @@ -110,63 +111,52 @@ param_custom_init = ParamCustomInit() ``` ```` -```{tip} -Since the base class of all parameter classes, {py:class}`ParamData`, is an abstract class -that inherits from [`abc.ABC`], you can use abstract decorators in parameter and structure -classes without inheriting from [`abc.ABC`] again. -``` - -Parameters track when any of their properties was last updated in the read-only -{py:attr}`~Param.last_updated` property. For example: +Parameter data track when any of their properties were last updated, and this value can be +accessed by the read-only {py:attr}`~ParamData.last_updated` property. For example: ```{jupyter-execute} -param.last_updated +custom_param.last_updated ``` ```{jupyter-execute} import time time.sleep(1) -param.value += 1 -param.last_updated +custom_param.value += 1 +custom_param.last_updated ``` -## Structures - -A structure is defined from the base class {py:class}`Struct` and is intended -to be defined as a dataclass. The key difference from {py:class}`Param` is that -structures do not store their own last updated time; instead, the -{py:attr}`ParamData.last_updated` property returns the most recent last updated time -of any {py:class}`ParamData` they contain. For example: +Parameter dataclasses can also be nested, in which case the +{py:attr}`ParamData.last_updated` property returns the most recent last updated time stamp +among its own last updated time and the last updated times of any {py:class}`ParamData` +it contains. For example: ```{jupyter-execute} -from paramdb import Struct, ParamDict - -class CustomStruct(Struct): +class NestedParam(ParamDataclass): value: float - param: CustomParam + child_param: CustomParam -struct = CustomStruct(value=1.23, param=CustomParam(value=4.56)) -struct.last_updated +nested_param = NestedParam(value=1.23, child_param=CustomParam(value=4.56)) +nested_param.last_updated ``` ```{jupyter-execute} time.sleep(1) -struct.param.value += 1 -struct.last_updated +nested_param.child_param.value += 1 +nested_param.last_updated ``` You can access the parent of any parameter data using the {py:attr}`ParamData.parent` property. For example: ```{jupyter-execute} -struct.param.parent == struct +nested_param.child_param.parent is nested_param ``` Similarly, the root can be accessed via {py:attr}`ParamData.root`: ```{jupyter-execute} -struct.param.root == struct +nested_param.child_param.root is nested_param ``` See [Type Mixins](#type-mixins) for information on how to get the parent and root @@ -175,28 +165,40 @@ properties to work better with static type checkers. ## Collections Ordinary lists and dictionaries can be used within parameter data; however, any -parameter data objects they contain will not have a parent object. This is because -internally, the parent is set by the {py:class}`ParamData` object that most recently -added the given parameter data as a child. Therefore, it is not recommended to use -ordinary lists and dictionaries to store parameter data. Instead, {py:class}`ParamList` -and {py:class}`ParamDict` can be used. +parameter data objects they contain will not have a last updated time or a parent object. +Therefore, it is not recommended to use ordinary lists and dictionaries to store parameter +data. Instead, {py:class}`ParamList` and {py:class}`ParamDict` can be used. + +### Parameter Lists {py:class}`ParamList` implements the abstract base class `MutableSequence` from [`collections.abc`], so it behaves similarly to a list. It is also a subclass of -{py:class}`ParamData`, so the parent and root properties will work properly. For -example, +{py:class}`ParamData`, so the last updated, parent, and root properties will work +properly. For example: ```{jupyter-execute} from paramdb import ParamList param_list = ParamList([CustomParam(value=1), CustomParam(value=2), CustomParam(value=3)]) -param_list[1].parent == param_list +param_list[1].parent is param_list +``` + +```{jupyter-execute} +param_list.last_updated +``` + +```{jupyter-execute} +time.sleep(1) +param_list[1].value += 1 +param_list.last_updated ``` +### Parameter Dictionaries + Similarly, {py:class}`ParamDict` implements `MutableMapping` from [`collections.abc`], so it behaves similarly to a dictionary. Additionally, its items can be accessed via dot notation in addition to index brackets (unless they begin with an underscore). For -example, +example: ```{jupyter-execute} from paramdb import ParamDict @@ -209,6 +211,16 @@ param_dict = ParamDict( param_dict.p2.root == param_dict ``` +```{jupyter-execute} +param_list.last_updated +``` + +```{jupyter-execute} +time.sleep(1) +param_list[1].value += 1 +param_list.last_updated +``` + Parameter collections can also be subclassed to provide custom functionality. For example: ```{jupyter-execute} @@ -232,21 +244,21 @@ example: ```{jupyter-execute} from paramdb import ParentType -class ParentStruct(Struct): - param: Child +class ParentParam(ParamDataclass): + child_param: ChildParam -class ChildParam(Param, ParentType[ParentStruct]): +class ChildParam(ParamDataclass, ParentType[ParentParam]): value: float -struct = ParentStruct(param=ChildParam(value=1.23)) +parent_param = ParentParam(child_param=ChildParam(value=1.23)) ``` This does nothing to the functionality, but static type checkers will now know that -`struct.param.parent` in the example above is a `ParentStruct` object. +`parent_param.child_param.parent` in the example above is a `ParentParam` object. [`datetime`]: https://docs.python.org/3/library/datetime.html#datetime-objects [`astropy.units.quantity`]: https://docs.astropy.org/en/stable/api/astropy.units.Quantity.html#astropy.units.Quantity -[`dataclass`]: https://docs.python.org/3/library/dataclasses.html +[data class]: https://docs.python.org/3/library/dataclasses.html [`@dataclass`]: https://docs.python.org/3/library/dataclasses.html#dataclasses.dataclass [`field`]: https://docs.python.org/3/library/dataclasses.html#dataclasses.field [`__init__`]: https://docs.python.org/3/reference/datamodel.html#object.__init__ diff --git a/paramdb/_database.py b/paramdb/_database.py index a404e7e..0023c65 100644 --- a/paramdb/_database.py +++ b/paramdb/_database.py @@ -279,7 +279,7 @@ def load(self, commit_id: int | None = None, *, load_classes: bool = True) -> An reconstructed. The relevant parameter data classes must be defined in the current program. However, if ``load_classes`` is False, classes are loaded directly from the database as dictionaries with the class name in the key - :py:const:`CLASS_NAME_KEY`. + :py:const:`~paramdb._database.CLASS_NAME_KEY`. """ select_stmt = self._select_commit(select(_Snapshot.data), commit_id) with self._Session() as session: diff --git a/paramdb/_param_data/_dataclasses.py b/paramdb/_param_data/_dataclasses.py index 408510d..30dd256 100644 --- a/paramdb/_param_data/_dataclasses.py +++ b/paramdb/_param_data/_dataclasses.py @@ -12,8 +12,8 @@ class ParamDataclass(ParamData): """ Subclass of :py:class:`ParamData`. - Base class for parameter Data Classes. Subclasses are automatically converted to - Data Classes. For example:: + Base class for parameter data classes. Subclasses are automatically converted to + data classes. For example:: class CustomParam(ParamDataclass): value1: float @@ -21,7 +21,7 @@ class CustomParam(ParamDataclass): Any keyword arguments given when creating a subclass are passed internally to the - standard `@dataclass()` decorator. + standard ``@dataclass()`` decorator. """ __field_names: set[str] diff --git a/paramdb/_param_data/_type_mixins.py b/paramdb/_param_data/_type_mixins.py index 8457d53..d51ae5f 100644 --- a/paramdb/_param_data/_type_mixins.py +++ b/paramdb/_param_data/_type_mixins.py @@ -12,7 +12,7 @@ class ParentType(ParamData, Generic[PT]): Mixin for :py:class:`ParamData` that sets the type hint for :py:attr:`ParamData.parent` to type parameter ``PT``. For example:: - class CustomParam(ParentType[ParentStruct], Param): + class CustomParam(ParentType[ParentParam], Param): ... Note that if the parent actually has a different type, the type hint will be @@ -29,7 +29,7 @@ class RootType(ParamData, Generic[PT]): Mixin for :py:class:`ParamData` that sets the type hint for :py:attr:`ParamData.root` to type parameter ``PT``. For example:: - class CustomParam(RootType[RootStruct], Param): + class CustomParam(RootType[RootParam], Param): ... Note that if the root actually has a different type, the type hint will be diff --git a/tests/_param_data/test_dataclasses.py b/tests/_param_data/test_dataclasses.py index 09b8b56..07d913d 100644 --- a/tests/_param_data/test_dataclasses.py +++ b/tests/_param_data/test_dataclasses.py @@ -28,7 +28,7 @@ def fixture_param_dataclass_object( return param_dataclass_object -def test_param_data_direct_instantiation_fails() -> None: +def test_param_dataclass_direct_instantiation_fails() -> None: """Fails to instantiate an object of type ``ParamDataclass``.""" with pytest.raises(TypeError) as exc_info: ParamDataclass() From a1862db2a9a05c9cbf1c6c269d34344ae2b747fd Mon Sep 17 00:00:00 2001 From: Alex Hadley Date: Sun, 21 Apr 2024 15:50:08 -0700 Subject: [PATCH 3/4] Add badges to documentation website --- README.md | 4 ++++ docs/index.md | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/README.md b/README.md index 1ffdd25..c686050 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # ParamDB + + [![PyPI Latest Release](https://img.shields.io/pypi/v/paramdb)](https://pypi.org/project/paramdb/) [![PyPI Python Versions](https://img.shields.io/pypi/pyversions/paramdb)](https://pypi.org/project/paramdb/) [![License](https://img.shields.io/pypi/l/paramdb)](https://github.com/PainterQubits/paramdb/blob/main/LICENSE) @@ -7,6 +9,8 @@ [![Codecov](https://codecov.io/github/PainterQubits/paramdb/branch/main/graph/badge.svg?token=PQEJWLBTBK)](https://codecov.io/github/PainterQubits/paramdb) [![Documentation Status](https://readthedocs.org/projects/paramdb/badge/?version=stable)](https://paramdb.readthedocs.io/en/stable/?badge=stable) + + Python package for storing and retrieving experiment parameters. diff --git a/docs/index.md b/docs/index.md index 603de73..d4a133f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,5 +1,10 @@ # ParamDB +```{include} ../README.md +:start-after: +:end-before: +``` + ```{include} ../README.md :start-after: :end-before: From 372b65fb996909f6e1209e24d5c82c05680746f8 Mon Sep 17 00:00:00 2001 From: Alex Hadley Date: Sun, 21 Apr 2024 16:19:18 -0700 Subject: [PATCH 4/4] Update CHANGELOG and readthedocs config --- .readthedocs.yaml | 6 +++--- CHANGELOG.md | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 76d93ad..b528631 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -7,10 +7,10 @@ build: jobs: post_create_environment: - pip install poetry==1.8.2 - - poetry config virtualenvs.create false post_install: - - poetry install --without dev - - poetry run python -m ipykernel install --user + # See https://docs.readthedocs.io/en/stable/build-customization.html#install-dependencies-with-poetry + - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH poetry install --without dev + - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH poetry run python -m ipykernel install --user sphinx: configuration: docs/conf.py diff --git a/CHANGELOG.md b/CHANGELOG.md index dd7e111..07afd4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Changed + +- All `ParamData` objects now internally track the latest time that they or any of their + children were last updated, which is returned by `ParamData.last_updated`. +- `Param` and `Struct` are combined into a single class `ParamDataclass`. + ## [0.11.0] (Jan 31 2024) ### Added