diff --git a/ckanext/__init__.py b/ckanext/__init__.py index 35ee891..6c0b86d 100644 --- a/ckanext/__init__.py +++ b/ckanext/__init__.py @@ -1,4 +1,3 @@ -# encoding: utf-8 # this is a namespace package try: diff --git a/ckanext/transmute/interfaces.py b/ckanext/transmute/interfaces.py index 8523959..25b14ac 100644 --- a/ckanext/transmute/interfaces.py +++ b/ckanext/transmute/interfaces.py @@ -1,12 +1,12 @@ from __future__ import annotations from typing import Any + from ckan.plugins.interfaces import Interface class ITransmute(Interface): - """Main extension point of ckanext-transmute. - """ + """Main extension point of ckanext-transmute.""" def get_transmutators(self) -> dict[str, Any]: """Register custom transmutation functions. @@ -53,5 +53,4 @@ def get_transmutation_schemas(self): Returns: Mapping with definitions of named schemas. """ - return {} diff --git a/ckanext/transmute/logic/action.py b/ckanext/transmute/logic/action.py index a32ca16..738d58c 100644 --- a/ckanext/transmute/logic/action.py +++ b/ckanext/transmute/logic/action.py @@ -1,21 +1,18 @@ from __future__ import annotations -import logging import contextvars -from typing import Any, Union - -from ckan import types -import ckan.plugins.toolkit as tk +import logging +from typing import Any import ckan.lib.navl.dictization_functions as df -from ckan.logic import validate, ValidationError +import ckan.plugins.toolkit as tk +from ckan import types +from ckan.logic import ValidationError, validate -from ckanext.transmute.types import Field, MODE_COMBINE -from ckanext.transmute.schema import SchemaParser, SchemaField -from ckanext.transmute.schema import transmute_schema from ckanext.transmute.exception import TransmutatorError -from ckanext.transmute.utils import get_transmutator, SENTINEL, get_schema - +from ckanext.transmute.schema import SchemaField, SchemaParser, transmute_schema +from ckanext.transmute.types import MODE_COMBINE, Field +from ckanext.transmute.utils import SENTINEL, get_schema, get_transmutator log = logging.getLogger(__name__) data_ctx = contextvars.ContextVar("data") @@ -60,14 +57,13 @@ def tsm_transmute(context: types.Context, data_dict: dict[str, Any]) -> dict[str def _transmute_data(data, definition, root): - """Mutates an actual data in `data` dict + """Mutates an actual data in `data` dict. Args: data (dict: [str, Any]): a data to mutate definition (SchemaParser): SchemaParser object root (str): a root schema type """ - schema = definition.types[root] if not schema: @@ -166,12 +162,12 @@ def _process_field( def _default_from(data: dict[str, Any], field: SchemaField): - default_from: Union[list[str], str] = field.get_default_from() + default_from: list[str] | str = field.get_default_from() return _get_external_fields(data, default_from, field) def _replace_from(data: dict[str, Any], field: SchemaField): - replace_from: Union[list[str], str] = field.get_replace_from() + replace_from: list[str] | str = field.get_replace_from() return _get_external_fields(data, replace_from, field) @@ -202,7 +198,7 @@ def _combine_from_fields(data: dict[str, Any], external_fields: list[str]): def _get_first_filled(data: dict[str, Any], external_fields: list[str]): - """Return first not-empty field value""" + """Return first not-empty field value.""" for field_name in external_fields: field_value = data[field_name] @@ -211,7 +207,7 @@ def _get_first_filled(data: dict[str, Any], external_fields: list[str]): def _apply_validators(field: Field, validators: list[str | list[str]]): - """Applies validators sequentially to the field value + """Applies validators sequentially to the field value. Args: field (Field): Field object diff --git a/ckanext/transmute/plugin.py b/ckanext/transmute/plugin.py index a21e57a..24514d8 100644 --- a/ckanext/transmute/plugin.py +++ b/ckanext/transmute/plugin.py @@ -2,13 +2,14 @@ import json from typing import Any + import ckan.plugins as p import ckan.plugins.toolkit as tk +from ckanext.transmute.interfaces import ITransmute from ckanext.transmute.logic.action import get_actions from ckanext.transmute.logic.auth import get_auth_functions from ckanext.transmute.transmutators import get_transmutators -from ckanext.transmute.interfaces import ITransmute from . import utils @@ -27,12 +28,12 @@ def update_config(self, config_): # IActions def get_actions(self): - """Registers a list of extension specific actions""" + """Registers a list of extension specific actions.""" return get_actions() # IAuthFunctions def get_auth_functions(self): - """Registers a list of extension specific auth function""" + """Registers a list of extension specific auth function.""" return get_auth_functions() # ITransmute @@ -41,8 +42,10 @@ def get_transmutators(self): def get_transmutation_schemas(self) -> dict[str, Any]: prefix = "ckanext.transmute.schema." - return { - key[len(prefix) :]: json.load(open(tk.config[key])) - for key in tk.config - if key.startswith(prefix) - } + schemas: dict[str, Any] = {} + for key in tk.config: + if not key.startswith(prefix): + continue + with open(tk.config[key]) as src: + schemas[key[len(prefix) :]] = json.load(src) + return schemas diff --git a/ckanext/transmute/schema.py b/ckanext/transmute/schema.py index bc93e6a..f969b70 100644 --- a/ckanext/transmute/schema.py +++ b/ckanext/transmute/schema.py @@ -1,13 +1,13 @@ from __future__ import annotations -from typing import Any, Optional, Union - import copy import dataclasses +from typing import Any -from ckan.logic.schema import validator_args from ckan import types -from ckanext.transmute.exception import SchemaParsingError, SchemaFieldError +from ckan.logic.schema import validator_args + +from ckanext.transmute.exception import SchemaFieldError, SchemaParsingError from ckanext.transmute.utils import SENTINEL @@ -16,15 +16,15 @@ class SchemaField: name: str type: str definition: dict[str, Any] - map: Optional[str] = None + map: str | None = None validators: list[Any] = dataclasses.field(default_factory=list) multiple: bool = False remove: bool = False default: Any = SENTINEL - default_from: Optional[str] = None + default_from: str | None = None value: Any = SENTINEL - replace_from: Optional[str] = None - inherit_mode: Optional[str] = "combine" + replace_from: str | None = None + inherit_mode: str | None = "combine" update: bool = False validate_missing: bool = False weight: int = 0 @@ -39,7 +39,7 @@ def __repr__(self): def is_multiple(self) -> bool: return bool(self.multiple) - def get_default_from(self) -> Union[list[str], str]: + def get_default_from(self) -> list[str] | str: if not self.default_from: raise SchemaFieldError("Field: `default_from` field name is not defined") @@ -51,7 +51,7 @@ def get_default_from(self) -> Union[list[str], str]: return self._get_sibling_field_name(self.default_from) - def get_replace_from(self) -> Union[list[str], str]: + def get_replace_from(self) -> list[str] | str: if not self.replace_from: raise SchemaFieldError("Field: `replace_from` field name is not defined") @@ -82,7 +82,7 @@ def get_root_type(self): if not root_type: raise SchemaParsingError("Schema: root type is missing") - if not root_type in self.schema.get("types", []): + if root_type not in self.schema.get("types", []): raise SchemaParsingError("Schema: root_type is declared but not defined") return root_type @@ -104,7 +104,7 @@ def _parse_field( self, field_name: str, field_meta: dict[str, Any], _type: str ) -> SchemaField: """Create a SchemaField combining all the - information about field + information about field. Args: field_name (str): current field original name @@ -114,7 +114,6 @@ def _parse_field( Returns: SchemaField: SchemaField object """ - params: dict[str, Any] = dict({"type": _type}, **field_meta) return SchemaField(name=field_name, definition=self.types[_type], **params) diff --git a/ckanext/transmute/tests/conftest.py b/ckanext/transmute/tests/conftest.py index 841a777..bce04e3 100644 --- a/ckanext/transmute/tests/conftest.py +++ b/ckanext/transmute/tests/conftest.py @@ -1,7 +1,7 @@ import pytest -@pytest.fixture +@pytest.fixture() def tsm_schema(): return { "root": "Dataset", diff --git a/ckanext/transmute/tests/logic/test_action.py b/ckanext/transmute/tests/logic/test_action.py index 83e23e3..76d58df 100644 --- a/ckanext/transmute/tests/logic/test_action.py +++ b/ckanext/transmute/tests/logic/test_action.py @@ -1,15 +1,14 @@ from __future__ import annotations from typing import Any -from ckanext.transmute.exception import SchemaParsingError, SchemaFieldError import pytest import ckan.lib.helpers as h -import ckan.tests.factories as factories from ckan.logic import ValidationError from ckan.tests.helpers import call_action +from ckanext.transmute.exception import SchemaParsingError from ckanext.transmute.tests.helpers import build_schema from ckanext.transmute.types import MODE_FIRST_FILLED @@ -17,7 +16,7 @@ @pytest.mark.usefixtures("with_plugins") class TestTransmuteAction: def test_custom_root(self): - """Action allows using a root different from "Dataset" """ + """Action allows using a root different from "Dataset".""" result = call_action( "tsm_transmute", data={}, @@ -31,7 +30,7 @@ def test_custom_root(self): def test_transmute_default(self): """If the origin evaluates to False it must be replaced - with the default value + with the default value. """ data: dict[str, Any] = { "metadata_created": "", @@ -59,7 +58,7 @@ def test_transmute_default(self): ) def test_transmute_default_with_origin_value(self): - """The default value mustn't replace the origin value""" + """The default value mustn't replace the origin value.""" metadata_created: str = "2024-02-03" metadata_created_default: str = "2022-02-03" @@ -87,7 +86,7 @@ def test_transmute_default_with_origin_value(self): def test_transmute_default_from_without_origin_value(self, tsm_schema): """The `default_from` must copy value from target field if the origin - value is empty + value is empty. """ data: dict[str, Any] = { "metadata_created": "", @@ -124,7 +123,7 @@ def test_transmute_default_from_with_origin_value(self, tsm_schema): assert result["metadata_modified"] == h.date_str_to_datetime(metadata_modified) def test_transmute_default_from_with_empty_target(self): - """The target field value could be empty""" + """The target field value could be empty.""" data: dict[str, Any] = { "metadata_created": "", "metadata_modified": "", @@ -150,7 +149,7 @@ def test_transmute_default_from_with_empty_target(self): def test_transmute_replace_from(self): """The `replace_from` must copy value from target field and replace - the origin value whether it is empty or not + the origin value whether it is empty or not. """ metadata_created: str = "2024-02-03" metadata_modified: str = "2022-02-03" @@ -178,8 +177,7 @@ def test_transmute_replace_from(self): assert result["metadata_modified"] == result["metadata_created"] def test_transmute_replace_from_multiple(self): - """Replace from multiple fields must combine values of those fields""" - + """Replace from multiple fields must combine values of those fields.""" data = {"field_1": [1, 2, 3], "field_2": [3, 4, 5], "field_3": ""} tsm_schema = build_schema( @@ -202,8 +200,7 @@ def test_transmute_replace_from_multiple(self): assert result["field_3"] == data["field_1"] + data["field_2"] def test_transmute_replace_from_multiple_different_types(self): - """Replace from multiple fields must combine values of those fields""" - + """Replace from multiple fields must combine values of those fields.""" data = { "field_1": [1, 2, 3], "field_2": 1, @@ -234,8 +231,7 @@ def test_transmute_replace_from_multiple_different_types(self): ] def test_transmute_default_from_multiple(self): - """Default from multiple fields must combine values of those fields""" - + """Default from multiple fields must combine values of those fields.""" data = {"field_1": [1, 2, 3], "field_2": [3, 4, 5], "field_3": ""} tsm_schema = build_schema( @@ -258,8 +254,7 @@ def test_transmute_default_from_multiple(self): assert result["field_3"] == data["field_1"] + data["field_2"] def test_transmute_default_from_multiple_different_types(self): - """Default from multiple fields must combine values of those fields""" - + """Default from multiple fields must combine values of those fields.""" data = { "field_1": [1, 2, 3], "field_2": 1, @@ -324,7 +319,7 @@ def test_transmute_replace_from_nested(self): result["title_ar"] == data["title_translated"][0]["nested_field"]["ar"] def test_transmute_remove_field(self): - """Field with `remove` must be excluded from the result""" + """Field with `remove` must be excluded from the result.""" data: dict[str, Any] = { "metadata_created": "2024-02-03", "metadata_modified": "2022-02-03", @@ -350,7 +345,8 @@ def test_transmute_remove_field(self): def test_transmute_value(self): """The`value` must replace the origin value, whenever - it's empty or not""" + it's empty or not. + """ data: dict[str, Any] = { "field1": "", "field2": "hello-world", @@ -400,7 +396,7 @@ def test_transmute_deep_nested(self, tsm_schema): ], } - result = call_action( + call_action( "tsm_transmute", data=data, schema=tsm_schema, @@ -430,7 +426,7 @@ def test_transmute_deep_nested(self, tsm_schema): } def test_transmute_no_field_schema(self): - """If no fields specified, there is nothing to do""" + """If no fields specified, there is nothing to do.""" result = call_action( "tsm_transmute", data={"title": "test"}, @@ -440,7 +436,7 @@ def test_transmute_no_field_schema(self): assert result == {"title": "test"} def test_transmute_no_data(self): - """Data is required""" + """Data is required.""" with pytest.raises(ValidationError): call_action( "tsm_transmute", @@ -448,12 +444,12 @@ def test_transmute_no_data(self): ) def test_transmute_no_schema(self): - """Schema is required""" + """Schema is required.""" with pytest.raises(ValidationError): call_action("tsm_transmute", data={"title": "test"}) def test_transmute_empty_data(self): - """If there is no data, there is no sense to do anything""" + """If there is no data, there is no sense to do anything.""" result = call_action( "tsm_transmute", data={}, @@ -463,7 +459,7 @@ def test_transmute_empty_data(self): assert len(result) == 0 def test_transmute_empty_schema(self): - """Schema root type is required""" + """Schema root type is required.""" with pytest.raises(SchemaParsingError) as e: call_action("tsm_transmute", data={"title": "test"}, schema={}) @@ -471,7 +467,7 @@ def test_transmute_empty_schema(self): def test_transmute_new_field_inherit(self): """We can define a new field in schema and it will be - added to the result data + added to the result data. """ data: dict[str, Any] = { "metadata_created": "", @@ -497,7 +493,7 @@ def test_transmute_new_field_inherit(self): assert result["metadata_modified"] == result["metadata_created"] def test_transmute_new_field_from_default_and_value(self): - """Default runs after value""" + """Default runs after value.""" data: dict[str, Any] = {} tsm_schema = build_schema({"field1": {"default": 101, "value": 102}}) @@ -514,7 +510,7 @@ def test_transmute_new_field_from_default_and_value(self): def test_transmute_new_field_from_value(self): """We can define a new field in schema and it will be - added to the result data + added to the result data. """ data: dict[str, Any] = {} @@ -535,7 +531,7 @@ def test_transmute_run_multiple_times(self): tsm_schema = build_schema({"field1": {"value": 101}}) - for i in range(10): + for _i in range(10): result = call_action( "tsm_transmute", data=data, @@ -652,8 +648,7 @@ def test_transmute_update_different_types(self): assert e.value.error_dict["extras"] == ["Original value has different type"] def test_transmute_replace_from_inherit_first_filled_first_true(self): - """Replace from multiple fields must combine values of those fields""" - + """Replace from multiple fields must combine values of those fields.""" data = {"field_1": [1, 2, 3], "field_2": [3, 4, 5], "field_3": ""} tsm_schema = build_schema( @@ -677,8 +672,7 @@ def test_transmute_replace_from_inherit_first_filled_first_true(self): assert result["field_3"] == data["field_1"] def test_transmute_replace_from_inherit_first_filled_last_true(self): - """Replace from multiple fields must combine values of those fields""" - + """Replace from multiple fields must combine values of those fields.""" data = {"field_1": "", "field_2": [3, 4, 5], "field_3": ""} tsm_schema = build_schema( @@ -702,8 +696,7 @@ def test_transmute_replace_from_inherit_first_filled_last_true(self): assert result["field_3"] == data["field_2"] def test_transmute_default_from_inherit_first_filled_last_true(self): - """Replace from multiple fields must combine values of those fields""" - + """Replace from multiple fields must combine values of those fields.""" data = {"field_1": "", "field_2": [3, 4, 5], "field_3": ""} tsm_schema = build_schema( diff --git a/ckanext/transmute/tests/test_transmutators.py b/ckanext/transmute/tests/test_transmutators.py index 9f2a69c..c80fc09 100644 --- a/ckanext/transmute/tests/test_transmutators.py +++ b/ckanext/transmute/tests/test_transmutators.py @@ -1,15 +1,14 @@ from __future__ import annotations -from re import T from typing import Any import pytest -from ckan.tests.helpers import call_action from ckan.logic import ValidationError +from ckan.tests.helpers import call_action -from ckanext.transmute.tests.helpers import build_schema from ckanext.transmute.exception import TransmutatorError +from ckanext.transmute.tests.helpers import build_schema @pytest.mark.usefixtures("with_plugins") @@ -35,8 +34,7 @@ def test_transmute_validator_without_args(self): @pytest.mark.parametrize("default", [False, 0, "", [], {}, None]) def test_default_allows_falsy_values(self, default): - """False, 0, "", etc. can be used as a default value""" - + """False, 0, "", etc. can be used as a default value.""" tsm_schema = build_schema( { "field_name": {"default": default}, @@ -100,8 +98,8 @@ def test_trim_string_transmutator_with_two_args(self): {"field_name": {"validators": [["tsm_trim_string", 0, 1]]}} ) - with pytest.raises(TransmutatorError) as e: - result = call_action( + with pytest.raises(TransmutatorError): + call_action( "tsm_transmute", data=data, schema=tsm_schema, @@ -161,7 +159,7 @@ def test_concat_transmutator_with_self(self): assert result["field_name"] == new_field_value def test_concat_transmutator_without_self(self): - """You can skip using $self if you want for some reason""" + """You can skip using $self if you want for some reason.""" data: dict[str, Any] = { "identifier": "right-to-the-night-results", } @@ -189,11 +187,11 @@ def test_concat_transmutator_without_self(self): root="Dataset", ) - new_field_value = f"https://ckan.url/dataset/information" + new_field_value = "https://ckan.url/dataset/information" assert result["field_name"] == new_field_value def test_concat_transmutator_with_not_string_arg(self): - """You can skip using $self if you want for some reason""" + """You can skip using $self if you want for some reason.""" data: dict[str, Any] = { "identifier": "right-to-the-night-results", } @@ -221,12 +219,11 @@ def test_concat_transmutator_with_not_string_arg(self): root="Dataset", ) - new_field_value = f"https://ckan.url/dataset/1" + new_field_value = "https://ckan.url/dataset/1" assert result["field_name"] == new_field_value def test_concat_transmutator_with_field_link(self): - """We are able to use fields from schema as a concat item""" - + """We are able to use fields from schema as a concat item.""" data: dict[str, Any] = { "identifier": "right-to-the-night-results", "url": "https://ckan.url/dataset/", @@ -259,8 +256,8 @@ def test_concat_transmutator_with_field_link(self): def test_concat_transmutator_with_field_link_nested(self): """We are able to use fields from schema as a concat - item from within nested structure""" - + item from within nested structure. + """ data: dict[str, Any] = { "title": "Package title", "resources": [ @@ -317,7 +314,7 @@ def test_concat_transmutator_with_field_link_nested(self): @pytest.mark.usefixtures("with_plugins") class TestUniqueOnlyTransmutator: def test_unique_only(self): - """You can skip using $self if you want for some reason""" + """You can skip using $self if you want for some reason.""" data: dict[str, Any] = {"field_name": [1, 2, 3, 3, 4, 5, 6, 6]} tsm_schema = build_schema( @@ -338,7 +335,7 @@ def test_unique_only(self): assert result["field_name"] == [1, 2, 3, 4, 5, 6] def test_unique_only_for_not_list(self): - """You can skip using $self if you want for some reason""" + """You can skip using $self if you want for some reason.""" data: dict[str, Any] = {"field_name": 1} tsm_schema = build_schema( @@ -348,7 +345,7 @@ def test_unique_only_for_not_list(self): }, } ) - with pytest.raises(ValidationError) as e: + with pytest.raises(ValidationError): call_action( "tsm_transmute", data=data, @@ -357,7 +354,7 @@ def test_unique_only_for_not_list(self): ) def test_unique_only_empty_list(self): - """You can skip using $self if you want for some reason""" + """You can skip using $self if you want for some reason.""" data: dict[str, Any] = {"field_name": []} tsm_schema = build_schema( diff --git a/ckanext/transmute/transmutators.py b/ckanext/transmute/transmutators.py index 21e0523..dac8a05 100644 --- a/ckanext/transmute/transmutators.py +++ b/ckanext/transmute/transmutators.py @@ -1,11 +1,12 @@ from __future__ import annotations -from typing import Callable, Any, Optional + from datetime import datetime +from typing import Any, Callable -from dateutil.parser import parse, ParserError +from dateutil.parser import ParserError, parse -import ckan.plugins.toolkit as tk import ckan.lib.navl.dictization_functions as df +import ckan.plugins.toolkit as tk from ckanext.transmute.types import Field @@ -24,7 +25,7 @@ def transmutator(func): @transmutator def name_validator(field: Field) -> Field: - """Wrapper over CKAN default `name_validator` validator + """Wrapper over CKAN default `name_validator` validator. Args: field (Field): Field object @@ -43,7 +44,7 @@ def name_validator(field: Field) -> Field: @transmutator def to_lowercase(field: Field) -> Field: - """Casts string value to lowercase + """Casts string value to lowercase. Args: field (Field): Field object @@ -57,7 +58,7 @@ def to_lowercase(field: Field) -> Field: @transmutator def to_uppercase(field: Field) -> Field: - """Casts string value to uppercase + """Casts string value to uppercase. Args: field (Field): Field object @@ -71,10 +72,10 @@ def to_uppercase(field: Field) -> Field: @transmutator def string_only(field: Field) -> Field: - """Validates if field.value is string + """Validates if field.value is string. Args: - value (Field): Field object + field (Field): Field object Raises: df.Invalid: raises is the field.value is not string @@ -90,7 +91,7 @@ def string_only(field: Field) -> Field: @transmutator def isodate(field: Field) -> Field: """Validates datetime string - Mutates an iso-like string to datetime object + Mutates an iso-like string to datetime object. Args: field (Field): Field object @@ -101,7 +102,6 @@ def isodate(field: Field) -> Field: Returns: Field: the same Field with casted value """ - if isinstance(field.value, datetime): return field @@ -115,7 +115,7 @@ def isodate(field: Field) -> Field: @transmutator def to_string(field: Field) -> Field: - """Casts field.value to str + """Casts field.value to str. Args: field (Field): Field object @@ -130,7 +130,7 @@ def to_string(field: Field) -> Field: @transmutator def stop_on_empty(field: Field) -> Field: - """Stop transmutation if field is empty + """Stop transmutation if field is empty. Args: field (Field): Field object @@ -145,11 +145,12 @@ def stop_on_empty(field: Field) -> Field: @transmutator -def get_nested(field: Field, *path) -> Field: - """Fetches a nested value from a field +def get_nested(field: Field, *path: str) -> Field: + """Fetches a nested value from a field. Args: field (Field): Field object + path: Iterable with path segments Raises: df.Invalid: raises if path doesn't exist @@ -166,17 +167,16 @@ def get_nested(field: Field, *path) -> Field: @transmutator -def trim_string(field: Field, max_length) -> Field: - """Trim string lenght +def trim_string(field: Field, max_length: int) -> Field: + """Trim string lenght. Args: - value (Field): Field object + field (Field): Field object max_length (int): String max length Returns: Field: the same Field object if it's valid """ - if not isinstance(max_length, int): raise df.Invalid(tk._("max_length must be integer")) @@ -187,7 +187,7 @@ def trim_string(field: Field, max_length) -> Field: @transmutator def concat(field: Field, *strings: Any) -> Field: """Concat strings to build a new one - Use $self to point on field value + Use $self to point on field value. Args: field (Field): Field object @@ -222,7 +222,7 @@ def concat(field: Field, *strings: Any) -> Field: @transmutator def unique_only(field: Field) -> Field: - """Preserve only unique values from list + """Preserve only unique values from list. Args: field (Field): Field object @@ -238,7 +238,7 @@ def unique_only(field: Field) -> Field: @transmutator def mapper( - field: Field, mapping: dict[Any, Any], default: Optional[Any] = None + field: Field, mapping: dict[Any, Any], default: Any | None = None ) -> Field: """Map a value with a new value. The initial value must serve as a key within a mapping dictionary, while the dict value will represent the updated value. @@ -246,7 +246,7 @@ def mapper( Args: field (Field): Field object mapping (dict[Any, Any]): A dictionary representing the mapping of values. - default (Any): The default value to be used when the key is not found in the mapping. + default (Any): The default value to be used when the key is not found. If the default value is not provided, the current value will be used as it. Returns: @@ -263,10 +263,9 @@ def mapper( def list_mapper( field: Field, mapping: dict[Any, Any], - remove: Optional[bool] = False, + remove: bool | None = False, ) -> Field: - """ - Maps values within a list to their corresponding values in a provided mapping dictionary. + """Maps values within a list to corresponding values from the provided dictionary. Args: field (Field): Field object diff --git a/ckanext/transmute/types.py b/ckanext/transmute/types.py index f2c4af5..c794052 100644 --- a/ckanext/transmute/types.py +++ b/ckanext/transmute/types.py @@ -2,6 +2,7 @@ import dataclasses from typing import Any + from typing_extensions import TypedDict diff --git a/ckanext/transmute/utils.py b/ckanext/transmute/utils.py index 809417e..2ae5478 100644 --- a/ckanext/transmute/utils.py +++ b/ckanext/transmute/utils.py @@ -41,7 +41,9 @@ def get_all_transmutators() -> list[str]: for plugin in reversed(list(p.PluginImplementations(ITransmute))): for name, fn in plugin.get_transmutators().items(): log.debug( - f"Transmutator function {name} from plugin {plugin.name} was inserted" + "Transmutator function %s from plugin %s was inserted", + name, + plugin.name, ) _transmutator_cache[name] = fn @@ -50,6 +52,76 @@ def get_all_transmutators() -> list[str]: def get_json_schema() -> dict[str, Any]: transmutators = get_all_transmutators() + fields_definition = { + "type": "object", + "additionalProperties": False, + "properties": { + "validators": { + "type": "array", + "minItems": 1, + "items": { + "oneOf": [ + { + "type": "string", + "enum": transmutators, + }, + { + "type": "array", + "minItems": 2, + "items": [ + { + "type": "string", + "enum": transmutators, + } + ], + "additionalItems": {"$ref": "#/$defs/anytype"}, + }, + ] + }, + }, + "map": {"type": "string"}, + "default": {"$ref": "#/$defs/anytype"}, + "default_from": { + "anyOf": [ + { + "type": "array", + "minItems": 1, + "items": {"type": "string"}, + }, + {"type": "string"}, + ] + }, + "replace_from": { + "anyOf": [ + { + "type": "array", + "minItems": 1, + "items": {"type": "string"}, + }, + {"type": "string"}, + ] + }, + "inherit_mode": { + "type": "string", + "items": { + "oneOf": [ + { + "type": "string", + "enum": [ + MODE_COMBINE, + MODE_FIRST_FILLED, + ], + } + ] + }, + }, + "value": {"$ref": "#/$defs/anytype"}, + "multiple": {"type": "boolean"}, + "remove": {"type": "boolean"}, + "type": {"type": "string"}, + "update": {"type": "boolean"}, + }, + } return { "$schema": "http://json-schema.org/draft-04/schema", "type": "object", @@ -74,78 +146,7 @@ def get_json_schema() -> dict[str, Any]: "type": "object", "minProperties": 1, "propertyNames": {"pattern": "^[A-Za-z_-]*$"}, - "additionalProperties": { - "type": "object", - "additionalProperties": False, - "properties": { - "validators": { - "type": "array", - "minItems": 1, - "items": { - "oneOf": [ - { - "type": "string", - "enum": transmutators, - }, - { - "type": "array", - "minItems": 2, - "items": [ - { - "type": "string", - "enum": transmutators, - } - ], - "additionalItems": { - "$ref": "#/$defs/anytype" - }, - }, - ] - }, - }, - "map": {"type": "string"}, - "default": {"$ref": "#/$defs/anytype"}, - "default_from": { - "anyOf": [ - { - "type": "array", - "minItems": 1, - "items": {"type": "string"}, - }, - {"type": "string"}, - ] - }, - "replace_from": { - "anyOf": [ - { - "type": "array", - "minItems": 1, - "items": {"type": "string"}, - }, - {"type": "string"}, - ] - }, - "inherit_mode": { - "type": "string", - "items": { - "oneOf": [ - { - "type": "string", - "enum": [ - MODE_COMBINE, - MODE_FIRST_FILLED, - ], - } - ] - }, - }, - "value": {"$ref": "#/$defs/anytype"}, - "multiple": {"type": "boolean"}, - "remove": {"type": "boolean"}, - "type": {"type": "string"}, - "update": {"type": "boolean"}, - }, - }, + "additionalProperties": fields_definition, } }, }, diff --git a/docs/api.md b/docs/api.md index bf498fe..15c7804 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1,4 +1,4 @@ # API -::: transmute.logic.action.get.tsm_transmute +::: transmute.logic.action.tsm_transmute diff --git a/pyproject.toml b/pyproject.toml index 088d194..1337e53 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,33 +65,33 @@ select = [ # "ANN0", # type annotations for function arguments # "B", # likely bugs and design problems # "BLE", # do not catch blind exception - # "C4", # better list/set/dict comprehensions + "C4", # better list/set/dict comprehensions # "C90", # check McCabe complexity # "DTZ", # enforce timezone in date objects - # "E", # pycodestyle error + "E", # pycodestyle error # "W", # pycodestyle warning - # "F", # pyflakes - # "FA", # verify annotations from future - # "G", # format strings for logging statements + "F", # pyflakes + "FA", # verify annotations from future + "G", # format strings for logging statements # "N", # naming conventions - # "I", # isort - # "ICN", # import conventions + "I", # isort + "ICN", # import conventions # # "D1", # require doc # "D2", # doc formatting - # "D4", # doc convention + "D4", # doc convention # "PL", # pylint # "PERF", # performance anti-patterns - # "PT", # pytest style - # "PIE", # misc lints + "PT", # pytest style + "PIE", # misc lints # "RET", # improvements for return statements # "RSE", # improvements for rise statements # "S", # security testing - # "SIM", # simplify code - # "T10", # debugging statements - # "T20", # print statements - # "TID", # tidier imports + "SIM", # simplify code + "T10", # debugging statements + "T20", # print statements + "TID", # tidier imports # "TRY", # better exceptions - # "UP", # upgrade syntax for newer versions of the language + "UP", # upgrade syntax for newer versions of the language ] ignore = [