From 1ccd6b174f2c56ce530895f15264a9216ea56deb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20V=2E=20Treider?= Date: Mon, 2 Dec 2024 12:16:44 +0100 Subject: [PATCH] add: isnull filter (#2052) --- CHANGELOG.md | 4 + cognite/client/_version.py | 2 +- cognite/client/data_classes/filters.py | 45 +++++++++- cognite/client/utils/useful_types.py | 8 +- docs/source/filters.rst | 89 ++++++++++--------- pyproject.toml | 2 +- .../test_data_models/test_filters.py | 28 ++++++ 7 files changed, 128 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6e6e6f6be..3570316260 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,10 @@ Changes are grouped as follows - `Fixed` for any bug fixes. - `Security` in case of vulnerabilities. +## [7.69.4] - 2024-12-02 +### Added +- An IsNull filter has been added for use in Data Modeling. + ## [7.69.3] - 2024-12-02 ### Added - API endpoints currently accepting relative time strings like `2d-ago` now support a forward-looking syntax, e.g. `2w-ahead` or `15m-ahead`. diff --git a/cognite/client/_version.py b/cognite/client/_version.py index 5e35c2bdfa..5fb03bbbe8 100644 --- a/cognite/client/_version.py +++ b/cognite/client/_version.py @@ -1,4 +1,4 @@ from __future__ import annotations -__version__ = "7.69.3" +__version__ = "7.69.4" __api_subversion__ = "20230101" diff --git a/cognite/client/data_classes/filters.py b/cognite/client/data_classes/filters.py index d6cfd858a6..ca008c3e49 100644 --- a/cognite/client/data_classes/filters.py +++ b/cognite/client/data_classes/filters.py @@ -9,13 +9,13 @@ from cognite.client.data_classes.labels import Label from cognite.client.utils._identifier import InstanceId from cognite.client.utils._text import convert_all_keys_to_camel_case, to_camel_case -from cognite.client.utils.useful_types import SequenceNotStr +from cognite.client.utils.useful_types import SequenceNotStr, is_sequence_not_str if TYPE_CHECKING: from cognite.client.data_classes.data_modeling.ids import ContainerId, ViewId -PropertyReference: TypeAlias = str | tuple[str, ...] | list[str] | EnumProperty +PropertyReference: TypeAlias = str | SequenceNotStr[str] | EnumProperty RawValue: TypeAlias = str | float | bool | Sequence | Mapping[str, Any] | Label | InstanceId @@ -895,7 +895,7 @@ def __init__(self, space: str | SequenceNotStr[str], instance_type: Literal["nod @classmethod def load(cls, filter_: dict[str, Any]) -> NoReturn: - raise NotImplementedError("Custom filter 'SpaceFilter' can not be loaded") + raise NotImplementedError("Custom filter 'SpaceFilter' cannot be loaded") def _filter_body(self, camel_case_property: bool) -> dict[str, Any]: return { @@ -905,3 +905,42 @@ def _filter_body(self, camel_case_property: bool) -> dict[str, Any]: def _involved_filter_types(self) -> set[type[Filter]]: return self._involved_filter + + +class IsNull(Not): # type: ignore [misc] + """Data modeling filter for instances whose property is null, effectively a negated Exists-filter. + + Args: + property (SequenceNotStr[str]): The property to filter on. + + Example: + Filter than can be used to retrieve instances where the property value is not set: + + - A filter using a tuple to reference the property: + + >>> from cognite.client.data_classes.filters import IsNull + >>> flt = IsNull(("space", "view_xid/version", "some_property")) + + - Composing the property reference using the ``View.as_property_ref`` method: + + >>> flt = IsNull(my_view.as_property_ref("some_property")) + + """ + + _filter_name = "__is_null" + + def __init__(self, property: SequenceNotStr[str]) -> None: + if not is_sequence_not_str(property): + raise TypeError( + "The IsNull filter is a Data Modeling filter and expected a sequence of str to describe the property, " + f"like ['node', 'space'] or ['my-space', 'my-view/version', 'my-property'], got: {property}" + ) + super().__init__(Exists(property)) + self._filter_name = Not._filter_name + + @classmethod + def load(cls, filter_: dict[str, Any]) -> NoReturn: + raise NotImplementedError("Custom filter 'IsNull' cannot be loaded") + + def _involved_filter_types(self) -> set[type[Filter]]: + return {Not, Exists} diff --git a/cognite/client/utils/useful_types.py b/cognite/client/utils/useful_types.py index 6adcf6905d..0c89f69455 100644 --- a/cognite/client/utils/useful_types.py +++ b/cognite/client/utils/useful_types.py @@ -1,14 +1,14 @@ from __future__ import annotations from collections.abc import Iterator, Sequence -from typing import Any, Protocol, SupportsIndex, TypeVar, overload, runtime_checkable +from typing import Any, Protocol, SupportsIndex, TypeGuard, TypeVar, overload, runtime_checkable _T_co = TypeVar("_T_co", covariant=True) # Source from https://github.com/python/typing/issues/256#issuecomment-1442633430 # This works because str.__contains__ does not accept object (either in typeshed or at runtime) -@runtime_checkable +@runtime_checkable # TODO: remove; does not accepts tuple, change usage to 'is_sequence_not_str' below class SequenceNotStr(Protocol[_T_co]): @overload def __getitem__(self, index: SupportsIndex, /) -> _T_co: ... @@ -29,5 +29,9 @@ def count(self, value: Any, /) -> int: ... def __reversed__(self) -> Iterator[_T_co]: ... +def is_sequence_not_str(obj: Any) -> TypeGuard[SequenceNotStr]: + return isinstance(obj, Sequence) and not isinstance(obj, str) + + class SupportsRead(Protocol[_T_co]): def read(self, length: int = ..., /) -> _T_co: ... diff --git a/docs/source/filters.rst b/docs/source/filters.rst index 8c0b27bc18..dd63e6032a 100644 --- a/docs/source/filters.rst +++ b/docs/source/filters.rst @@ -3,25 +3,19 @@ Filters The filter language provides a set of classes that can be used to construct complex queries for filtering data. Each filter class represents a specific filtering criterion, -allowing users to tailor their queries to their specific needs. +allowing users to tailor their queries to their specific needs. The filter classes can be shared both by the classic CDF resources (like Assets, Time Series, Events, Files etc) and Data Modelling (Views and Instances). -When filtering on Data Modelling, the filter can be used on any container property. These can be referenced directly with a tuple notation like: +When filtering on Data Modelling, the filter can be used on any property. These can be referenced directly with a tuple notation like: ``('space', 'view_external_id/view_version', 'property')``, or, which is usually more convenient, one can use the ``as_property_ref`` method on the View or ViewID object like: ``myView.as_property_ref('property')``. -Below is an overview of the -available filters: +All filters inherit from the base class ``Filter`` (``cognite.client.data_classes.filters.Filter``). -Filter ------- -Base class for all filters +Below is an overview of the available filters: -.. autoclass:: cognite.client.data_classes.filters.Filter - :members: - :member-order: bysource Logical Operators ----------------- @@ -135,39 +129,6 @@ The `InAssetSubtree` filter checks if an item belongs to a specified asset subtr :members: :member-order: bysource -Geo Filters ------------ -GeoJSON -^^^^^^^ -The `GeoJSON` filter performs geometric queries using GeoJSON representations. - -.. autoclass:: cognite.client.data_classes.filters.GeoJSON - :members: - :member-order: bysource - -GeoJSONDisjoint -^^^^^^^^^^^^^^^ -The `GeoJSONDisjoint` filter checks if two geometric shapes are disjoint. - -.. autoclass:: cognite.client.data_classes.filters.GeoJSONDisjoint - :members: - :member-order: bysource - -GeoJSONIntersects -^^^^^^^^^^^^^^^^^ -The `GeoJSONIntersects` filter checks if two geometric shapes intersect. - -.. autoclass:: cognite.client.data_classes.filters.GeoJSONIntersects - :members: - :member-order: bysource - -GeoJSONWithin -^^^^^^^^^^^^^ -The `GeoJSONWithin` filter checks if one geometric shape is within another. - -.. autoclass:: cognite.client.data_classes.filters.GeoJSONWithin - :members: - :member-order: bysource Data Modeling-Specific Filters ------------------------------ @@ -179,6 +140,14 @@ The `SpaceFilter` filters instances from one or more specific space(s). :members: :member-order: bysource +IsNull +^^^^^^ +The `IsNull` filter checks if a property is null. + +.. autoclass:: cognite.client.data_classes.filters.IsNull + :members: + :member-order: bysource + HasData ^^^^^^^ The `HasData` filter checks if an instance has data for a given property. @@ -202,3 +171,37 @@ The `Nested` filter applies a filter to the node pointed to by a direct relation .. autoclass:: cognite.client.data_classes.filters.Nested :members: :member-order: bysource + +Geo Filters +----------- +GeoJSON +^^^^^^^ +The `GeoJSON` filter performs geometric queries using GeoJSON representations. + +.. autoclass:: cognite.client.data_classes.filters.GeoJSON + :members: + :member-order: bysource + +GeoJSONDisjoint +^^^^^^^^^^^^^^^ +The `GeoJSONDisjoint` filter checks if two geometric shapes are disjoint. + +.. autoclass:: cognite.client.data_classes.filters.GeoJSONDisjoint + :members: + :member-order: bysource + +GeoJSONIntersects +^^^^^^^^^^^^^^^^^ +The `GeoJSONIntersects` filter checks if two geometric shapes intersect. + +.. autoclass:: cognite.client.data_classes.filters.GeoJSONIntersects + :members: + :member-order: bysource + +GeoJSONWithin +^^^^^^^^^^^^^ +The `GeoJSONWithin` filter checks if one geometric shape is within another. + +.. autoclass:: cognite.client.data_classes.filters.GeoJSONWithin + :members: + :member-order: bysource diff --git a/pyproject.toml b/pyproject.toml index 85caa98820..aaea85ea3a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] name = "cognite-sdk" -version = "7.69.3" +version = "7.69.4" description = "Cognite Python SDK" readme = "README.md" documentation = "https://cognite-sdk-python.readthedocs-hosted.com" diff --git a/tests/tests_unit/test_data_classes/test_data_models/test_filters.py b/tests/tests_unit/test_data_classes/test_data_models/test_filters.py index 7fc9a0973a..62a9490fd9 100644 --- a/tests/tests_unit/test_data_classes/test_data_models/test_filters.py +++ b/tests/tests_unit/test_data_classes/test_data_models/test_filters.py @@ -406,3 +406,31 @@ def test_space_filter_loads_as_unknown(self, body: dict[str, str | list[str]]) - def test_space_filter_passes_verification(self, cognite_client: CogniteClient, space_filter: f.SpaceFilter) -> None: cognite_client.data_modeling.instances._validate_filter(space_filter) assert True + + +class TestIsNullFilter: + @pytest.mark.parametrize("prop", (("prop",), ["prop", "more"], tuple("abcd"))) + def test_filter(self, prop: list[str] | tuple[str]) -> None: + flt = f.IsNull(prop).dump() + not_exists = f.Not(f.Exists(prop)).dump() + assert flt == not_exists + assert flt == {"not": {"exists": {"property": prop}}} + + def test_str_not_allowed(self) -> None: + exp_msg = "^The IsNull filter is a Data Modeling filter and expec.*'my-property'], got: prop$" + with pytest.raises(TypeError, match=exp_msg): + f.IsNull("prop") # type: ignore [arg-type] + + def test_is_null_filter_passes_verification(self, cognite_client: CogniteClient) -> None: + cognite_client.data_modeling.instances._validate_filter(f.IsNull(["node", "space"])) + assert True + + def test_is_null_filter_passes_isinstance_checks(self) -> None: + flt = f.IsNull(["node", "space"]) + assert isinstance(flt, Filter) + + def test_is_null_filter_loads_as_unknown(self) -> None: + # IsNull filter is an SDK concept, so it should load as an UnknownFilter: + dumped = {f.IsNull._filter_name: {"not": {"exists": {"property": ["node", "space"]}}}} + loaded_flt = Filter.load(dumped) + assert isinstance(loaded_flt, UnknownFilter)