Enable metadata access for remote datasets. (#1163)

* Enable metadata access for remote datasets.

* Update model for remote folder.

* Change attr decorators.

* Update changelog.

* Implement requested feedback.

* Add warning when there are duplicates within the keys.

* Add entry for RemoteFolder in wk-libs docs.

* Implement metadata objects that fetch their data on access.

* Implement Metadata class as MutableMapping.

* Fix typing.

* remove unused imports

---------

Co-authored-by: Florian M <[email protected]>
Co-authored-by: Florian M <[email protected]>

docs/mkdocs.yml

@@ -125,6 +125,7 @@ nav:
               - Layer: api/webknossos/dataset/layer.md
               - MagView: api/webknossos/dataset/mag_view.md
               - View: api/webknossos/dataset/view.md
+              - RemoteFolder: api/webknossos/dataset/remote_folder.md
           - Annotation: api/webknossos/annotation/annotation.md
           - Skeleton:
               - Skeleton: api/webknossos/skeleton/skeleton.md

webknossos/Changelog.md

@@ -15,6 +15,7 @@ For upgrade instructions, please check the respective _Breaking Changes_ section
 ### Breaking Changes
 
 ### Added
+- Enable metadata access for remote datasets. [#1163](https://github.com/scalableminds/webknossos-libs/pull/1163)
 
 ### Changed
 

webknossos/examples/accessing_metadata.py

@@ -0,0 +1,21 @@
+import webknossos as wk
+
+
+def main() -> None:
+    with wk.webknossos_context(url="https://webknossos.org/"):
+        l4_sample_dataset = wk.Dataset.open_remote("l4_sample")
+        # Access the metadata of the dataset
+        print(l4_sample_dataset.metadata)
+
+        # Edit the metadata of the dataset
+        l4_sample_dataset.metadata["new_key"] = "new_value"
+
+        # Access metadata of a folder
+        print(l4_sample_dataset.folder.metadata)
+
+        # Edit the metadata of the folder
+        l4_sample_dataset.folder.metadata["new_folder_key"] = "new_folder_value"
+
+
+if __name__ == "__main__":
+    main()

webknossos/webknossos/client/api_client/_abstract_api_client.py

@@ -56,6 +56,10 @@ def _get_json_paginated(
             response, response_type
         ), self._extract_total_count_header(response)
 
+    def _put_json(self, route: str, body_structured: Any) -> None:
+        body_json = self._prepare_for_json(body_structured)
+        self._put(route, body_json)
+
     def _patch_json(self, route: str, body_structured: Any) -> None:
         body_json = self._prepare_for_json(body_structured)
         self._patch(route, body_json)
@@ -125,6 +129,27 @@ def _patch(
             timeout_seconds=timeout_seconds,
         )
 
+    def _put(
+        self,
+        route: str,
+        body_json: Optional[Any] = None,
+        query: Optional[Query] = None,
+        multipart_data: Optional[httpx._types.RequestData] = None,
+        files: Optional[httpx._types.RequestFiles] = None,
+        retry_count: int = 0,
+        timeout_seconds: Optional[float] = None,
+    ) -> httpx.Response:
+        return self._request(
+            "PUT",
+            route,
+            body_json=body_json,
+            multipart_data=multipart_data,
+            files=files,
+            query=query,
+            retry_count=retry_count,
+            timeout_seconds=timeout_seconds,
+        )
+
     def _post(
         self,
         route: str,

webknossos/webknossos/client/api_client/models.py

@@ -49,13 +49,21 @@ class ApiBoundingBox:
     depth: int
 
 
+@attr.s(auto_attribs=True)
+class ApiAdditionalAxis:
+    name: str
+    bounds: Tuple[int, int]
+    index: int
+
+
 @attr.s(auto_attribs=True)
 class ApiDataLayer:
     name: str
     category: str
     element_class: str
     bounding_box: ApiBoundingBox
     resolutions: List[Tuple[int, int, int]]
+    additional_axes: Optional[List[ApiAdditionalAxis]] = None
     largest_segment_id: Optional[int] = None
     default_view_configuration: Optional[Dict[str, Any]] = None
 
@@ -73,6 +81,13 @@ class ApiDataSource:
     scale: Optional[ApiVoxelSize] = None
 
 
+@attr.s(auto_attribs=True)
+class ApiMetadata:
+    key: str
+    type: str
+    value: Any
+
+
 @attr.s(auto_attribs=True)
 class ApiDataset:
     name: str
@@ -82,6 +97,7 @@ class ApiDataset:
     tags: List[str]
     data_store: ApiDataStore
     data_source: ApiDataSource
+    metadata: Optional[List[ApiMetadata]] = None
     display_name: Optional[str] = None
     description: Optional[str] = None
 
@@ -291,3 +307,13 @@ class ApiFolderWithParent:
     id: str
     name: str
     parent: Optional[str] = None
+
+
+@attr.s(auto_attribs=True)
+class ApiFolder:
+    id: str
+    name: str
+    allowed_teams: List[ApiTeam]
+    allowed_teams_cumulative: List[ApiTeam]
+    is_editable: bool
+    metadata: Optional[List[ApiMetadata]] = None

webknossos/webknossos/dataset/_metadata.py

@@ -0,0 +1,119 @@
+from contextlib import contextmanager
+from typing import (
+    Any,
+    Dict,
+    Generator,
+    Iterator,
+    List,
+    MutableMapping,
+    Sequence,
+    TypeVar,
+    Union,
+)
+
+from webknossos.client.api_client.models import ApiDataset, ApiFolder, ApiMetadata
+from webknossos.utils import infer_metadata_type, parse_metadata_value
+
+_T = TypeVar("_T", bound="Metadata")
+
+
+class Metadata(MutableMapping):
+    __slots__ = ()
+    _api_path: str
+    _api_type: Any
+
+    def __init__(self, _id: str, *args: Any, **kwargs: Dict[str, Any]) -> None:
+        if not self._api_path or not self._api_type:
+            raise NotImplementedError(
+                "This class is not meant to be used directly. Please use FolderMetadata or DatasetMetadata."
+            )
+        super().__init__(*args, **kwargs)
+        self._id: str = _id
+        self._has_changed: bool = False
+        self._mapping: Dict = {}
+
+    @contextmanager
+    def _recent_metadata(self: _T) -> Generator[_T, None, None]:
+        from ..client.context import _get_api_client
+
+        try:
+            client = _get_api_client()
+            full_object = client._get_json(
+                f"{self._api_path}{self._id}",
+                self._api_type,  # type: ignore
+            )
+            metadata: List[ApiMetadata] = full_object.metadata
+            if metadata is not None:
+                self._mapping = {
+                    element.key: parse_metadata_value(element.value, element.type)
+                    for element in metadata
+                }
+            else:
+                self._mapping = {}
+            yield self
+        finally:
+            if self._has_changed:
+                api_metadata = [
+                    ApiMetadata(key=k, type=infer_metadata_type(v), value=v)
+                    for k, v in self._mapping.items()
+                ]
+
+                full_object.metadata = api_metadata
+                if self._api_type == ApiDataset:
+                    client._patch_json(f"{self._api_path}{self._id}", full_object)
+                else:
+                    client._put_json(f"{self._api_path}{self._id}", full_object)
+                self._has_changed = False
+
+    def __setitem__(
+        self, key: str, value: Union[str, int, float, Sequence[str]]
+    ) -> None:
+        with self._recent_metadata() as metadata:
+            metadata._has_changed = True
+            metadata._mapping[key] = value
+
+    def __getitem__(self, key: str) -> Union[str, int, float, Sequence[str]]:
+        with self._recent_metadata() as metadata:
+            return metadata._mapping[key]
+
+    def __delitem__(self, key: str) -> None:
+        with self._recent_metadata() as metadata:
+            metadata._has_changed = True
+            del metadata._mapping[key]
+
+    def __contains__(self, key: object) -> bool:
+        with self._recent_metadata() as metadata:
+            return key in metadata._mapping
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, Metadata):
+            raise NotImplementedError(
+                f"Cannot compare {self.__class__.__name__} with {other.__class__.__name__}"
+            )
+        with self._recent_metadata() as metadata:
+            return metadata._mapping == other._mapping
+
+    def __ne__(self, other: object) -> bool:
+        return not self == other
+
+    def __iter__(self) -> Iterator[Any]:
+        with self._recent_metadata() as metadata:
+            return iter(metadata._mapping)
+
+    def __len__(self) -> int:
+        with self._recent_metadata() as metadata:
+            return len(metadata._mapping)
+
+    def __repr__(self) -> str:
+        with self._recent_metadata() as metadata:
+            return f"{self.__class__.__name__}({repr(metadata._mapping)})"
+
+
+class FolderMetadata(Metadata):
+    _api_path = "/folders/"
+    _api_type = ApiFolder
+
+
+class DatasetMetadata(Metadata):
+    _api_path = "/datasets/"
+    _api_type = ApiDataset

webknossos/webknossos/dataset/dataset.py

@@ -35,9 +35,10 @@
 from numpy.typing import DTypeLike
 from upath import UPath
 
+from webknossos.dataset._metadata import DatasetMetadata
 from webknossos.geometry.vec_int import VecIntLike
 
-from ..client.api_client.models import ApiDataset
+from ..client.api_client.models import ApiDataset, ApiMetadata
 from ..geometry.vec3_int import Vec3Int, Vec3IntLike
 from ._array import ArrayException, ArrayInfo, BaseArray
 from ._utils import pims_images
@@ -69,6 +70,7 @@
     copytree,
     count_defined_values,
     get_executor_for_args,
+    infer_metadata_type,
     is_fs_path,
     named_partial,
     rmtree,
@@ -2083,6 +2085,7 @@ def _update_dataset_info(
         is_public: bool = _UNSET,
         folder_id: str = _UNSET,
         tags: List[str] = _UNSET,
+        metadata: Optional[List[ApiMetadata]] = _UNSET,
     ) -> None:
         from ..client.context import _get_api_client
 
@@ -2100,14 +2103,32 @@ def _update_dataset_info(
             info.is_public = is_public
         if folder_id is not _UNSET:
             info.folder_id = folder_id
-        if display_name is not _UNSET:
-            info.display_name = display_name
+        if metadata is not _UNSET:
+            info.metadata = metadata
 
         with self._context:
             _get_api_client().dataset_update(
                 self._organization_id, self._dataset_name, info
             )
 
+    @property
+    def metadata(self) -> DatasetMetadata:
+        return DatasetMetadata(f"{self._organization_id}/{self._dataset_name}")
+
+    @metadata.setter
+    def metadata(
+        self,
+        metadata: Optional[
+            Union[Dict[str, Union[str, int, float, Sequence[str]]], DatasetMetadata]
+        ],
+    ) -> None:
+        if metadata is not None:
+            api_metadata = [
+                ApiMetadata(key=k, type=infer_metadata_type(v), value=v)
+                for k, v in metadata.items()
+            ]
+        self._update_dataset_info(metadata=api_metadata)
+
     @property
     def display_name(self) -> Optional[str]:
         return self._get_dataset_info().display_name

webknossos/webknossos/dataset/remote_folder.py

@@ -1,8 +1,11 @@
-from typing import Iterable, List
+from typing import Dict, Iterable, List, Optional, Sequence, Union
 
 import attr
 
-from ..client.api_client.models import ApiFolderWithParent
+from webknossos.dataset._metadata import FolderMetadata
+from webknossos.utils import infer_metadata_type
+
+from ..client.api_client.models import ApiFolder, ApiFolderWithParent, ApiMetadata
 
 
 def _get_folder_path(
@@ -15,8 +18,10 @@ def _get_folder_path(
         return f"{_get_folder_path(next(f for f in all_folders if f.id == folder.parent), all_folders)}/{folder.name}"
 
 
-@attr.frozen
+@attr.define
 class RemoteFolder:
+    """This class is used to access and edit metadata of a folder on the webknossos server."""
+
     id: str
     name: str
 
@@ -47,3 +52,23 @@ def get_by_path(cls, path: str) -> "RemoteFolder":
                 return cls(name=folder_info.name, id=folder_info.id)
 
         raise KeyError(f"Could not find folder {path}.")
+
+    @property
+    def metadata(self) -> FolderMetadata:
+        return FolderMetadata(self.id)
+
+    @metadata.setter
+    def metadata(
+        self, metadata: Optional[Dict[str, Union[str, int, float, Sequence[str]]]]
+    ) -> None:
+        from ..client.context import _get_api_client
+
+        client = _get_api_client(enforce_auth=True)
+        folder = client._get_json(f"/folders/{self.id}", ApiFolder)
+        if metadata is not None:
+            api_metadata = [
+                ApiMetadata(key=k, type=infer_metadata_type(v), value=v)
+                for k, v in metadata.items()
+            ]
+        folder.metadata = api_metadata
+        client._put_json(f"/folders/{self.id}", folder)

webknossos/webknossos/skeleton/skeleton.py

@@ -12,7 +12,7 @@
 Vector3 = Tuple[float, float, float]
 
 
-@attr.define()
+@attr.define
 class Skeleton(Group):
     """
     Representation of the [skeleton](/webknossos/skeleton_annotation.html) of an `Annotation`.