From e2375153c255808b430cea5cb8e7a450a4e25bec Mon Sep 17 00:00:00 2001 From: Mark Bader Date: Mon, 23 Sep 2024 12:22:04 +0200 Subject: [PATCH] Enable metadata access for remote datasets. (#1163) * Enable metadata access for remote datasets. * Update model for remote folder. * Change attr decorators. * Update changelog. * Implement requested feedback. * Add warning when there are duplicates within the keys. * Add entry for RemoteFolder in wk-libs docs. * Implement metadata objects that fetch their data on access. * Implement Metadata class as MutableMapping. * Fix typing. * remove unused imports --------- Co-authored-by: Florian M Co-authored-by: Florian M --- docs/mkdocs.yml | 1 + webknossos/Changelog.md | 1 + webknossos/examples/accessing_metadata.py | 21 ++++ .../client/api_client/_abstract_api_client.py | 25 ++++ .../webknossos/client/api_client/models.py | 26 ++++ webknossos/webknossos/dataset/_metadata.py | 119 ++++++++++++++++++ webknossos/webknossos/dataset/dataset.py | 27 +++- .../webknossos/dataset/remote_folder.py | 31 ++++- webknossos/webknossos/skeleton/skeleton.py | 2 +- webknossos/webknossos/utils.py | 37 ++++++ 10 files changed, 283 insertions(+), 7 deletions(-) create mode 100644 webknossos/examples/accessing_metadata.py create mode 100644 webknossos/webknossos/dataset/_metadata.py diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 98bfd609e..606cd1040 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -125,6 +125,7 @@ nav: - Layer: api/webknossos/dataset/layer.md - MagView: api/webknossos/dataset/mag_view.md - View: api/webknossos/dataset/view.md + - RemoteFolder: api/webknossos/dataset/remote_folder.md - Annotation: api/webknossos/annotation/annotation.md - Skeleton: - Skeleton: api/webknossos/skeleton/skeleton.md diff --git a/webknossos/Changelog.md b/webknossos/Changelog.md index b8282827f..f426f5f2f 100644 --- a/webknossos/Changelog.md +++ b/webknossos/Changelog.md @@ -15,6 +15,7 @@ For upgrade instructions, please check the respective _Breaking Changes_ section ### Breaking Changes ### Added +- Enable metadata access for remote datasets. [#1163](https://github.com/scalableminds/webknossos-libs/pull/1163) ### Changed diff --git a/webknossos/examples/accessing_metadata.py b/webknossos/examples/accessing_metadata.py new file mode 100644 index 000000000..c8a4f8fb1 --- /dev/null +++ b/webknossos/examples/accessing_metadata.py @@ -0,0 +1,21 @@ +import webknossos as wk + + +def main() -> None: + with wk.webknossos_context(url="https://webknossos.org/"): + l4_sample_dataset = wk.Dataset.open_remote("l4_sample") + # Access the metadata of the dataset + print(l4_sample_dataset.metadata) + + # Edit the metadata of the dataset + l4_sample_dataset.metadata["new_key"] = "new_value" + + # Access metadata of a folder + print(l4_sample_dataset.folder.metadata) + + # Edit the metadata of the folder + l4_sample_dataset.folder.metadata["new_folder_key"] = "new_folder_value" + + +if __name__ == "__main__": + main() diff --git a/webknossos/webknossos/client/api_client/_abstract_api_client.py b/webknossos/webknossos/client/api_client/_abstract_api_client.py index 21c59904e..26e10bd8e 100644 --- a/webknossos/webknossos/client/api_client/_abstract_api_client.py +++ b/webknossos/webknossos/client/api_client/_abstract_api_client.py @@ -56,6 +56,10 @@ def _get_json_paginated( response, response_type ), self._extract_total_count_header(response) + def _put_json(self, route: str, body_structured: Any) -> None: + body_json = self._prepare_for_json(body_structured) + self._put(route, body_json) + def _patch_json(self, route: str, body_structured: Any) -> None: body_json = self._prepare_for_json(body_structured) self._patch(route, body_json) @@ -125,6 +129,27 @@ def _patch( timeout_seconds=timeout_seconds, ) + def _put( + self, + route: str, + body_json: Optional[Any] = None, + query: Optional[Query] = None, + multipart_data: Optional[httpx._types.RequestData] = None, + files: Optional[httpx._types.RequestFiles] = None, + retry_count: int = 0, + timeout_seconds: Optional[float] = None, + ) -> httpx.Response: + return self._request( + "PUT", + route, + body_json=body_json, + multipart_data=multipart_data, + files=files, + query=query, + retry_count=retry_count, + timeout_seconds=timeout_seconds, + ) + def _post( self, route: str, diff --git a/webknossos/webknossos/client/api_client/models.py b/webknossos/webknossos/client/api_client/models.py index 1653d5b83..97e571d9e 100644 --- a/webknossos/webknossos/client/api_client/models.py +++ b/webknossos/webknossos/client/api_client/models.py @@ -49,6 +49,13 @@ class ApiBoundingBox: depth: int +@attr.s(auto_attribs=True) +class ApiAdditionalAxis: + name: str + bounds: Tuple[int, int] + index: int + + @attr.s(auto_attribs=True) class ApiDataLayer: name: str @@ -56,6 +63,7 @@ class ApiDataLayer: element_class: str bounding_box: ApiBoundingBox resolutions: List[Tuple[int, int, int]] + additional_axes: Optional[List[ApiAdditionalAxis]] = None largest_segment_id: Optional[int] = None default_view_configuration: Optional[Dict[str, Any]] = None @@ -73,6 +81,13 @@ class ApiDataSource: scale: Optional[ApiVoxelSize] = None +@attr.s(auto_attribs=True) +class ApiMetadata: + key: str + type: str + value: Any + + @attr.s(auto_attribs=True) class ApiDataset: name: str @@ -82,6 +97,7 @@ class ApiDataset: tags: List[str] data_store: ApiDataStore data_source: ApiDataSource + metadata: Optional[List[ApiMetadata]] = None display_name: Optional[str] = None description: Optional[str] = None @@ -291,3 +307,13 @@ class ApiFolderWithParent: id: str name: str parent: Optional[str] = None + + +@attr.s(auto_attribs=True) +class ApiFolder: + id: str + name: str + allowed_teams: List[ApiTeam] + allowed_teams_cumulative: List[ApiTeam] + is_editable: bool + metadata: Optional[List[ApiMetadata]] = None diff --git a/webknossos/webknossos/dataset/_metadata.py b/webknossos/webknossos/dataset/_metadata.py new file mode 100644 index 000000000..a53f53e39 --- /dev/null +++ b/webknossos/webknossos/dataset/_metadata.py @@ -0,0 +1,119 @@ +from contextlib import contextmanager +from typing import ( + Any, + Dict, + Generator, + Iterator, + List, + MutableMapping, + Sequence, + TypeVar, + Union, +) + +from webknossos.client.api_client.models import ApiDataset, ApiFolder, ApiMetadata +from webknossos.utils import infer_metadata_type, parse_metadata_value + +_T = TypeVar("_T", bound="Metadata") + + +class Metadata(MutableMapping): + __slots__ = () + _api_path: str + _api_type: Any + + def __init__(self, _id: str, *args: Any, **kwargs: Dict[str, Any]) -> None: + if not self._api_path or not self._api_type: + raise NotImplementedError( + "This class is not meant to be used directly. Please use FolderMetadata or DatasetMetadata." + ) + super().__init__(*args, **kwargs) + self._id: str = _id + self._has_changed: bool = False + self._mapping: Dict = {} + + @contextmanager + def _recent_metadata(self: _T) -> Generator[_T, None, None]: + from ..client.context import _get_api_client + + try: + client = _get_api_client() + full_object = client._get_json( + f"{self._api_path}{self._id}", + self._api_type, # type: ignore + ) + metadata: List[ApiMetadata] = full_object.metadata + if metadata is not None: + self._mapping = { + element.key: parse_metadata_value(element.value, element.type) + for element in metadata + } + else: + self._mapping = {} + yield self + finally: + if self._has_changed: + api_metadata = [ + ApiMetadata(key=k, type=infer_metadata_type(v), value=v) + for k, v in self._mapping.items() + ] + + full_object.metadata = api_metadata + if self._api_type == ApiDataset: + client._patch_json(f"{self._api_path}{self._id}", full_object) + else: + client._put_json(f"{self._api_path}{self._id}", full_object) + self._has_changed = False + + def __setitem__( + self, key: str, value: Union[str, int, float, Sequence[str]] + ) -> None: + with self._recent_metadata() as metadata: + metadata._has_changed = True + metadata._mapping[key] = value + + def __getitem__(self, key: str) -> Union[str, int, float, Sequence[str]]: + with self._recent_metadata() as metadata: + return metadata._mapping[key] + + def __delitem__(self, key: str) -> None: + with self._recent_metadata() as metadata: + metadata._has_changed = True + del metadata._mapping[key] + + def __contains__(self, key: object) -> bool: + with self._recent_metadata() as metadata: + return key in metadata._mapping + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Metadata): + raise NotImplementedError( + f"Cannot compare {self.__class__.__name__} with {other.__class__.__name__}" + ) + with self._recent_metadata() as metadata: + return metadata._mapping == other._mapping + + def __ne__(self, other: object) -> bool: + return not self == other + + def __iter__(self) -> Iterator[Any]: + with self._recent_metadata() as metadata: + return iter(metadata._mapping) + + def __len__(self) -> int: + with self._recent_metadata() as metadata: + return len(metadata._mapping) + + def __repr__(self) -> str: + with self._recent_metadata() as metadata: + return f"{self.__class__.__name__}({repr(metadata._mapping)})" + + +class FolderMetadata(Metadata): + _api_path = "/folders/" + _api_type = ApiFolder + + +class DatasetMetadata(Metadata): + _api_path = "/datasets/" + _api_type = ApiDataset diff --git a/webknossos/webknossos/dataset/dataset.py b/webknossos/webknossos/dataset/dataset.py index 0f82d737d..570f301f6 100644 --- a/webknossos/webknossos/dataset/dataset.py +++ b/webknossos/webknossos/dataset/dataset.py @@ -35,9 +35,10 @@ from numpy.typing import DTypeLike from upath import UPath +from webknossos.dataset._metadata import DatasetMetadata from webknossos.geometry.vec_int import VecIntLike -from ..client.api_client.models import ApiDataset +from ..client.api_client.models import ApiDataset, ApiMetadata from ..geometry.vec3_int import Vec3Int, Vec3IntLike from ._array import ArrayException, ArrayInfo, BaseArray from ._utils import pims_images @@ -69,6 +70,7 @@ copytree, count_defined_values, get_executor_for_args, + infer_metadata_type, is_fs_path, named_partial, rmtree, @@ -2083,6 +2085,7 @@ def _update_dataset_info( is_public: bool = _UNSET, folder_id: str = _UNSET, tags: List[str] = _UNSET, + metadata: Optional[List[ApiMetadata]] = _UNSET, ) -> None: from ..client.context import _get_api_client @@ -2100,14 +2103,32 @@ def _update_dataset_info( info.is_public = is_public if folder_id is not _UNSET: info.folder_id = folder_id - if display_name is not _UNSET: - info.display_name = display_name + if metadata is not _UNSET: + info.metadata = metadata with self._context: _get_api_client().dataset_update( self._organization_id, self._dataset_name, info ) + @property + def metadata(self) -> DatasetMetadata: + return DatasetMetadata(f"{self._organization_id}/{self._dataset_name}") + + @metadata.setter + def metadata( + self, + metadata: Optional[ + Union[Dict[str, Union[str, int, float, Sequence[str]]], DatasetMetadata] + ], + ) -> None: + if metadata is not None: + api_metadata = [ + ApiMetadata(key=k, type=infer_metadata_type(v), value=v) + for k, v in metadata.items() + ] + self._update_dataset_info(metadata=api_metadata) + @property def display_name(self) -> Optional[str]: return self._get_dataset_info().display_name diff --git a/webknossos/webknossos/dataset/remote_folder.py b/webknossos/webknossos/dataset/remote_folder.py index 49545c819..36bf389f7 100644 --- a/webknossos/webknossos/dataset/remote_folder.py +++ b/webknossos/webknossos/dataset/remote_folder.py @@ -1,8 +1,11 @@ -from typing import Iterable, List +from typing import Dict, Iterable, List, Optional, Sequence, Union import attr -from ..client.api_client.models import ApiFolderWithParent +from webknossos.dataset._metadata import FolderMetadata +from webknossos.utils import infer_metadata_type + +from ..client.api_client.models import ApiFolder, ApiFolderWithParent, ApiMetadata def _get_folder_path( @@ -15,8 +18,10 @@ def _get_folder_path( return f"{_get_folder_path(next(f for f in all_folders if f.id == folder.parent), all_folders)}/{folder.name}" -@attr.frozen +@attr.define class RemoteFolder: + """This class is used to access and edit metadata of a folder on the webknossos server.""" + id: str name: str @@ -47,3 +52,23 @@ def get_by_path(cls, path: str) -> "RemoteFolder": return cls(name=folder_info.name, id=folder_info.id) raise KeyError(f"Could not find folder {path}.") + + @property + def metadata(self) -> FolderMetadata: + return FolderMetadata(self.id) + + @metadata.setter + def metadata( + self, metadata: Optional[Dict[str, Union[str, int, float, Sequence[str]]]] + ) -> None: + from ..client.context import _get_api_client + + client = _get_api_client(enforce_auth=True) + folder = client._get_json(f"/folders/{self.id}", ApiFolder) + if metadata is not None: + api_metadata = [ + ApiMetadata(key=k, type=infer_metadata_type(v), value=v) + for k, v in metadata.items() + ] + folder.metadata = api_metadata + client._put_json(f"/folders/{self.id}", folder) diff --git a/webknossos/webknossos/skeleton/skeleton.py b/webknossos/webknossos/skeleton/skeleton.py index c4e1c4336..bcc965db8 100644 --- a/webknossos/webknossos/skeleton/skeleton.py +++ b/webknossos/webknossos/skeleton/skeleton.py @@ -12,7 +12,7 @@ Vector3 = Tuple[float, float, float] -@attr.define() +@attr.define class Skeleton(Group): """ Representation of the [skeleton](/webknossos/skeleton_annotation.html) of an `Annotation`. diff --git a/webknossos/webknossos/utils.py b/webknossos/webknossos/utils.py index e773dd749..256fe051d 100644 --- a/webknossos/webknossos/utils.py +++ b/webknossos/webknossos/utils.py @@ -25,8 +25,10 @@ Mapping, Optional, Protocol, + Sequence, Tuple, TypeVar, + Union, ) import numpy as np @@ -119,6 +121,41 @@ def named_partial(func: F, *args: Any, **kwargs: Any) -> F: return partial_func +def infer_metadata_type(value: Union[str, int, float, Sequence[str]]) -> str: + if isinstance(value, str): + return "string" + if isinstance(value, Sequence): + for i in value: + if not isinstance(i, str): + raise ValueError( + f"In lists only str type is allowed, got: {type(value)}" + ) + if all(isinstance(i, str) for i in value): + return "string[]" + raise ValueError(f"Unsupported metadata type: {type(value)}") + if isinstance(value, (int, float)): + return "number" + raise ValueError(f"Unsupported metadata type: {type(value)}") + + +def parse_metadata_value( + value: str, ts_type: str +) -> Union[str, int, float, Sequence[str]]: + if ts_type == "string[]": + result = json.loads(value, parse_int=str, parse_float=str) + assert isinstance(result, list), f"Expected a list, got {type(result)}" + elif ts_type == "number": + try: + result = int(value) + except ValueError: + result = float(value) + elif ts_type == "string": + result = value + else: + raise ValueError(f"Unknown metadata type {ts_type}") + return result + + def wait_and_ensure_success( futures: List[Future], executor: Executor,