Merge pull request #197 from PainterQubits/#196-add-raw_json-option

#196 Add raw_json option

.github/dependabot.yml

@@ -2,7 +2,7 @@ version: 2
 updates:
   - package-ecosystem: "github-actions"
     directory: "/"
-    target-branch: "develop"
+    target-branch: "main"
     schedule:
       interval: "monthly"
     groups:
@@ -11,7 +11,7 @@ updates:
 
   - package-ecosystem: "pip"
     directory: "/"
-    target-branch: "develop"
+    target-branch: "main"
     schedule:
       interval: "monthly"
     groups:

.github/workflows/ci.yml

@@ -4,11 +4,9 @@ on:
   push:
     branches:
       - main
-      - develop
   pull_request:
     branches:
       - main
-      - develop
 
 env:
   POETRY_VERSION: "1.8.3"

.github/workflows/publish.yml

@@ -6,6 +6,7 @@ on:
 
 env:
   POETRY_VERSION: "1.8.3"
+  PACKAGE_NAME: "paramdb"
 
 jobs:
   build:
@@ -23,21 +24,21 @@ jobs:
       - name: Upload package artifact
         uses: actions/upload-artifact@v4
         with:
-          name: paramdb-release
+          name: ${{ env.PACKAGE_NAME }}-release
           path: dist
 
   publish:
     needs: build
     runs-on: ubuntu-latest
     environment:
       name: pypi
-      url: https://pypi.org/project/paramdb
+      url: https://pypi.org/project/${{ env.PACKAGE_NAME }}
     permissions:
       id-token: write
     steps:
       - uses: actions/download-artifact@v4
         with:
-          name: paramdb-release
+          name: ${{ env.PACKAGE_NAME }}-release
           path: dist
 
       - name: Publish to PyPI

CHANGELOG.md

@@ -7,6 +7,20 @@ project adheres to clauses 1–8 of [Semantic Versioning](https://semver.org/spe
 
 ## [Unreleased]
 
+## [0.14.0] (Jun 18 2024)
+
+### Changed
+
+- The option `decode_json` in `ParamDB.load()` was replaced with `raw_json`, which
+  allows loading the raw JSON string from the database.
+- The order of data for `ParamData` objects in the underlying JSON representation was
+  changed; see `ParamDB.load()` for the new order.
+
+### Removed
+
+- `ParamDBKey.WRAPPER` was removed in favor of encoding these values using
+  `ParamDBKey.PARAM` with a class name of `None`.
+
 ## [0.13.0] (Jun 14 2024)
 
 ### Added
@@ -191,7 +205,8 @@ project adheres to clauses 1–8 of [Semantic Versioning](https://semver.org/spe
 - Database class `ParamDB` to store parameters in a SQLite file
 - Ability to retrieve the commit history as `CommitEntry` objects
 
-[unreleased]: https://github.com/PainterQubits/paramdb/compare/v0.13.0...develop
+[unreleased]: https://github.com/PainterQubits/paramdb/compare/v0.14.0...main
+[0.14.0]: https://github.com/PainterQubits/paramdb/releases/tag/v0.14.0
 [0.13.0]: https://github.com/PainterQubits/paramdb/releases/tag/v0.13.0
 [0.12.0]: https://github.com/PainterQubits/paramdb/releases/tag/v0.12.0
 [0.11.0]: https://github.com/PainterQubits/paramdb/releases/tag/v0.11.0

CITATION.cff

@@ -4,6 +4,6 @@ authors:
   - family-names: "Hadley"
     given-names: "Alex"
 title: "ParamDB"
-version: 0.13.0
-date-released: 2024-06-14
+version: 0.14.0
+date-released: 2024-06-18
 url: "https://github.com/PainterQubits/paramdb"

docs/conf.py

@@ -4,7 +4,7 @@
 project = "ParamDB"
 copyright = "2023–2024, California Institute of Technology"
 author = "Alex Hadley"
-release = "0.13.0"
+release = "0.14.0"
 
 # General configuration
 extensions = [

paramdb/_database.py

@@ -41,13 +41,13 @@ class ParamDBKey:
     """Key for ordinary lists."""
     DICT = "d"
     """Key for ordinary dictionaries."""
-    WRAPPER = "w"
+    PARAM = "p"
     """
-    Key for non-:py:class:`ParamData` children of :py:class:`ParamData` objects, since
-    they are wrapped with additional metadata, such as a last updated time.
+    Key for :py:class:`ParamData` objects.
+
+    The JSON object should either include a parameter class name, or be None if wrapping
+    a non-:py:class:`ParamData` with parameter metadata (e.g. a last updated time).
     """
-    PARAM = "p"
-    """Key for :py:class:`ParamData` objects."""
 
 
 def _compress(text: str) -> bytes:
@@ -80,13 +80,12 @@ def _encode_json(obj: Any) -> Any:
             {key: _encode_json(value) for key, value in obj.items()},
         ]
     if isinstance(obj, ParamData):
-        timestamp_and_json = [
-            obj.last_updated.timestamp(),
+        return [
+            ParamDBKey.PARAM,
             _encode_json(obj.to_json()),
+            None if isinstance(obj, _ParamWrapper) else type(obj).__name__,
+            obj.last_updated.timestamp(),
         ]
-        if isinstance(obj, _ParamWrapper):
-            return [ParamDBKey.WRAPPER, *timestamp_and_json]
-        return [ParamDBKey.PARAM, type(obj).__name__, *timestamp_and_json]
     raise TypeError(
         f"'{type(obj).__name__}' object {repr(obj)} is not JSON serializable, so the"
         " commit failed"
@@ -105,13 +104,13 @@ def _decode_json(json_data: Any) -> Any:
             return [_decode_json(item) for item in data[0]]
         if key == ParamDBKey.DICT:
             return {key: _decode_json(value) for key, value in data[0].items()}
-        if key == ParamDBKey.WRAPPER:
-            return _ParamWrapper.from_json(data[0], _decode_json(data[1]))
         if key == ParamDBKey.PARAM:
-            class_name = data[0]
-            param_class = get_param_class(class_name)
+            json_data, class_name, timestamp = data
+            param_class = (
+                _ParamWrapper if class_name is None else get_param_class(class_name)
+            )
             if param_class is not None:
-                return param_class.from_json(data[1], _decode_json(data[2]))
+                return param_class.from_json(_decode_json(json_data), timestamp)
             raise ValueError(
                 f"ParamData class '{class_name}' is not known to ParamDB, so the load"
                 " failed"
@@ -122,16 +121,19 @@ def _decode_json(json_data: Any) -> Any:
 def _encode(obj: Any) -> bytes:
     """Encode the given object into bytes that will be stored in the database."""
     # pylint: disable=no-member
-    return _compress(json.dumps(_encode_json(obj)))
+    return _compress(json.dumps(_encode_json(obj), separators=(",", ":")))
 
 
-def _decode(data: bytes, decode_json: bool) -> Any:
+def _decode(data: bytes, raw_json: bool) -> Any:
     """
-    Decode an object from the given data from the database. Classes will be loaded in
-    if ``load_classes`` is True; otherwise, classes will be loaded as dictionaries.
+    Decode an object from the given data from the database.
+
+    If ``raw_json`` is True, the raw JSON string will from the database will be
+    returned; otherwise, the JSON data will be parsed and decoded into the corresponding
+    classes.
     """
-    json_data = json.loads(_decompress(data))
-    return _decode_json(json_data) if decode_json else json_data
+    json_str = _decompress(data)
+    return json_str if raw_json else _decode_json(json.loads(json_str))
 
 
 class _Base(MappedAsDataclass, DeclarativeBase):
@@ -283,24 +285,23 @@ def num_commits(self) -> int:
 
     @overload
     def load(
-        self, commit_id: int | None = None, *, decode_json: Literal[True] = True
+        self, commit_id: int | None = None, *, raw_json: Literal[False] = False
     ) -> DataT: ...
 
     @overload
-    def load(
-        self, commit_id: int | None = None, *, decode_json: Literal[False]
-    ) -> Any: ...
+    def load(self, commit_id: int | None = None, *, raw_json: Literal[True]) -> str: ...
 
-    def load(self, commit_id: int | None = None, *, decode_json: bool = True) -> Any:
+    def load(self, commit_id: int | None = None, *, raw_json: bool = False) -> Any:
         """
         Load and return data from the database. If a commit ID is given, load from that
         commit; otherwise, load from the most recent commit. Raise an ``IndexError`` if
         the specified commit does not exist. Note that commit IDs begin at 1.
 
         By default, objects are reconstructed, which requires the relevant parameter
-        data classes to be defined in the current program. However, if ``decode_json``
-        is False, the encoded JSON data is loaded directly from the database. The format
-        of the encoded data is as follows (see :py:class:`ParamDBKey` for key codes)::
+        data classes to be defined in the current program. However, if ``raw_json``
+        is True, the JSON data is returned directly from the database as a string.
+        The format of the JSON data is as follows (see :py:class:`ParamDBKey` for key
+        codes)::
 
             json_data:
                 | int
@@ -312,15 +313,14 @@ def load(self, commit_id: int | None = None, *, decode_json: bool = True) -> Any
                 | [ParamDBKey.QUANTITY, float<value>, str<unit>]
                 | [ParamDBKey.LIST, [json_data<item>, ...]]
                 | [ParamDBKey.DICT, {str<key>: json_data<value>, ...}]
-                | [ParamDBKey.WRAPPED, float<timestamp>, json_data<data>]
-                | [ParamDBKey.PARAM, str<class_name>, float<timestamp>, json_data<data>]
-        """
+                | [ParamDBKey.PARAM, json_data<data>, str<class_name> | None, float<timestamp>]
+        """  # noqa: E501
         select_stmt = self._select_commit(select(_Snapshot.data), commit_id)
         with self._Session() as session:
             data = session.scalar(select_stmt)
         if data is None:
             raise self._index_error(commit_id)
-        return _decode(data, decode_json)
+        return _decode(data, raw_json)
 
     def load_commit_entry(self, commit_id: int | None = None) -> CommitEntry:
         """
@@ -358,7 +358,7 @@ def commit_history_with_data(
         start: int | None = None,
         end: int | None = None,
         *,
-        decode_json: Literal[True] = True,
+        raw_json: Literal[False] = False,
     ) -> list[CommitEntryWithData[DataT]]: ...
 
     @overload
@@ -367,15 +367,15 @@ def commit_history_with_data(
         start: int | None = None,
         end: int | None = None,
         *,
-        decode_json: Literal[False],
-    ) -> list[CommitEntryWithData[Any]]: ...
+        raw_json: Literal[True],
+    ) -> list[CommitEntryWithData[str]]: ...
 
     def commit_history_with_data(
         self,
         start: int | None = None,
         end: int | None = None,
         *,
-        decode_json: bool = True,
+        raw_json: bool = False,
     ) -> list[CommitEntryWithData[Any]]:
         """
         Retrieve the commit history with data as a list of
@@ -392,7 +392,7 @@ def commit_history_with_data(
                     snapshot.id,
                     snapshot.message,
                     snapshot.timestamp,
-                    _decode(snapshot.data, decode_json),
+                    _decode(snapshot.data, raw_json),
                 )
                 for snapshot in snapshots
             ]

paramdb/_param_data/_param_data.py

@@ -177,7 +177,7 @@ def _init_from_json(self, json_data: Any) -> None:
         self.__init__(json_data)  # type: ignore[misc]
 
     @classmethod
-    def from_json(cls, last_updated_timestamp: float, json_data: list[Any]) -> Self:
+    def from_json(cls, json_data: list[Any], last_updated_timestamp: float) -> Self:
         """
         Construct a parameter data object from the given last updated timestamp and JSON
         data originally constructed by :py:meth:`to_json`.