From 53cf27ce601854904352453d57639ceaf06b5b8e Mon Sep 17 00:00:00 2001 From: Ahmad Nofal Date: Thu, 7 Sep 2023 17:13:15 +0400 Subject: [PATCH 01/13] Fix issue for extra fields not appearing in the errors --- ninja/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ninja/schema.py b/ninja/schema.py index 75735204d..0036d93e4 100644 --- a/ninja/schema.py +++ b/ninja/schema.py @@ -198,7 +198,7 @@ class Schema(BaseModel, metaclass=ResolverMetaclass): class Config: from_attributes = True # aka orm_mode - @model_validator(mode="before") + @model_validator(mode="after") def _run_root_validator(cls, values: Any, info: ValidationInfo) -> Any: values = DjangoGetter(values, cls, info.context) return values From cdd524748727e75dcbb646b1cea0ae65e147da54 Mon Sep 17 00:00:00 2001 From: Ahmad Nofal Date: Fri, 8 Sep 2023 13:10:58 +0400 Subject: [PATCH 02/13] fixes and make the tests pass --- ninja/schema.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/ninja/schema.py b/ninja/schema.py index 0036d93e4..fb35297a0 100644 --- a/ninja/schema.py +++ b/ninja/schema.py @@ -20,6 +20,7 @@ def resolve_initials(self, obj): return "".join(n[:1] for n in self.name.split()) """ +import copy import warnings from typing import Any, Callable, Dict, Type, TypeVar, Union, no_type_check @@ -198,14 +199,17 @@ class Schema(BaseModel, metaclass=ResolverMetaclass): class Config: from_attributes = True # aka orm_mode - @model_validator(mode="after") - def _run_root_validator(cls, values: Any, info: ValidationInfo) -> Any: - values = DjangoGetter(values, cls, info.context) - return values + @model_validator(mode="wrap") + def _run_root_validator2(cls, values: Any, handler: Callable, info: ValidationInfo) -> Any: + values_clone = copy.deepcopy(values) + if not (info and info.context and info.context.get("from_orm", False)): + handler(values) + values_final = DjangoGetter(values_clone, cls, info.context) + return handler(values_final) @classmethod def from_orm(cls: Type[S], obj: Any) -> S: - return cls.model_validate(obj) + return cls.model_validate(obj, context={"from_orm":True}) def dict(self, *a: Any, **kw: Any) -> DictStrAny: "Backward compatibility with pydantic 1.x" From 7bdee9524895329bfd45e02646535af04963fe03 Mon Sep 17 00:00:00 2001 From: Ahmad Nofal Date: Fri, 8 Sep 2023 13:11:23 +0400 Subject: [PATCH 03/13] deep copy is actually not needed --- ninja/schema.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ninja/schema.py b/ninja/schema.py index fb35297a0..c795350e8 100644 --- a/ninja/schema.py +++ b/ninja/schema.py @@ -201,11 +201,11 @@ class Config: @model_validator(mode="wrap") def _run_root_validator2(cls, values: Any, handler: Callable, info: ValidationInfo) -> Any: - values_clone = copy.deepcopy(values) + # we skip pydantic before validation if it is an orm object if not (info and info.context and info.context.get("from_orm", False)): handler(values) - values_final = DjangoGetter(values_clone, cls, info.context) - return handler(values_final) + values = DjangoGetter(values, cls, info.context) + return handler(values) @classmethod def from_orm(cls: Type[S], obj: Any) -> S: From 7c295762ec838019a53ef5679a82bc4278fcd43d Mon Sep 17 00:00:00 2001 From: Ahmad Nofal Date: Mon, 11 Sep 2023 13:06:14 +0400 Subject: [PATCH 04/13] stuff is working --- ninja/operation.py | 27 ++++++++++----------------- ninja/schema.py | 35 +++++++++++++++++++++++++++++------ ninja/signature/details.py | 1 + 3 files changed, 40 insertions(+), 23 deletions(-) diff --git a/ninja/operation.py b/ninja/operation.py index 6ca3087c6..335026389 100644 --- a/ninja/operation.py +++ b/ninja/operation.py @@ -1,16 +1,5 @@ -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - Iterable, - List, - Optional, - Sequence, - Type, - Union, - cast, -) +from typing import (TYPE_CHECKING, Any, Callable, Dict, Iterable, List, + Optional, Sequence, Type, Union, cast) import django import pydantic @@ -204,13 +193,17 @@ def _result_to_response( return temporal_response resp_object = ResponseObject(result) - # ^ we need object because getter_dict seems work only with from_orm - result = response_model.from_orm(resp_object).model_dump( + # ^ we need object because getter_dict seems work only with model_validate + result = response_model.model_validate( + resp_object + ).model_dump( by_alias=self.by_alias, exclude_unset=self.exclude_unset, exclude_defaults=self.exclude_defaults, exclude_none=self.exclude_none, - )["response"] + )[ + "response" + ] return self.api.create_response( request, result, temporal_response=temporal_response ) @@ -419,7 +412,7 @@ def _not_allowed(self) -> HttpResponse: class ResponseObject: - "Basically this is just a helper to be able to pass response to pydantic's from_orm" + "Basically this is just a helper to be able to pass response to pydantic's model_validate" def __init__(self, response: HttpResponse) -> None: self.response = response diff --git a/ninja/schema.py b/ninja/schema.py index c795350e8..e8a7f3528 100644 --- a/ninja/schema.py +++ b/ninja/schema.py @@ -20,15 +20,16 @@ def resolve_initials(self, obj): return "".join(n[:1] for n in self.name.split()) """ -import copy import warnings from typing import Any, Callable, Dict, Type, TypeVar, Union, no_type_check import pydantic +from django.db import models from django.db.models import Manager, QuerySet from django.db.models.fields.files import FieldFile from django.template import Variable, VariableDoesNotExist -from pydantic import BaseModel, Field, ValidationInfo, model_validator, validator +from pydantic import (BaseModel, Field, ValidationInfo, model_validator, + validator) from pydantic._internal._model_construction import ModelMetaclass from pydantic.json_schema import GenerateJsonSchema, JsonSchemaValue @@ -200,16 +201,38 @@ class Config: from_attributes = True # aka orm_mode @model_validator(mode="wrap") - def _run_root_validator2(cls, values: Any, handler: Callable, info: ValidationInfo) -> Any: - # we skip pydantic before validation if it is an orm object - if not (info and info.context and info.context.get("from_orm", False)): + def _run_root_validator( + cls, values: Any, handler: Callable, info: ValidationInfo + ) -> Any: + print(info) + # we perform 'before' validations only if + is_dict = bool( + info and info.context and info.context.get("is_dict", False) + ) + if not is_dict : handler(values) values = DjangoGetter(values, cls, info.context) return handler(values) @classmethod def from_orm(cls: Type[S], obj: Any) -> S: - return cls.model_validate(obj, context={"from_orm":True}) + return cls.model_validate(obj) + + @classmethod + def model_validate( + cls: type[BaseModel], + obj: Any, + *args, + strict: bool | None = None, + from_attributes: bool | None = None, + context: dict[str, Any] | None = None, + ) -> BaseModel: + context = context or {} + if not isinstance(obj, dict): + context = {"is_dict": True} + return super().model_validate( + obj, *args, strict=strict, from_attributes=from_attributes, context=context + ) def dict(self, *a: Any, **kw: Any) -> DictStrAny: "Backward compatibility with pydantic 1.x" diff --git a/ninja/signature/details.py b/ninja/signature/details.py index 7576d56e8..08416e1eb 100644 --- a/ninja/signature/details.py +++ b/ninja/signature/details.py @@ -159,6 +159,7 @@ def _create_models(self) -> TModels: ) base_cls = param_cls._model + base_cls.model_config["from_attributes"] = True model_cls = type(cls_name, (base_cls,), attrs) # TODO: https://pydantic-docs.helpmanual.io/usage/models/#dynamic-model-creation - check if anything special in create_model method that I did not use result.append(model_cls) From ca0668a34d6cda2a667f67f2006c3e6073f55bf9 Mon Sep 17 00:00:00 2001 From: Ahmad Nofal Date: Fri, 8 Sep 2023 19:58:39 +0400 Subject: [PATCH 05/13] done all fixes --- ninja/schema.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/ninja/schema.py b/ninja/schema.py index e8a7f3528..c7f8b7ace 100644 --- a/ninja/schema.py +++ b/ninja/schema.py @@ -204,14 +204,16 @@ class Config: def _run_root_validator( cls, values: Any, handler: Callable, info: ValidationInfo ) -> Any: - print(info) - # we perform 'before' validations only if - is_dict = bool( - info and info.context and info.context.get("is_dict", False) - ) - if not is_dict : + + # We dont perform 'before' validations if an validating through 'model_validate' + through_model_validate = info and info.context and info.context.get("through_model_validate", False) + if not through_model_validate: handler(values) + + # We add our DjangoGetter for the Schema values = DjangoGetter(values, cls, info.context) + + # To update the schema with our DjangoGetter return handler(values) @classmethod @@ -228,8 +230,7 @@ def model_validate( context: dict[str, Any] | None = None, ) -> BaseModel: context = context or {} - if not isinstance(obj, dict): - context = {"is_dict": True} + context["through_model_validate"] = True return super().model_validate( obj, *args, strict=strict, from_attributes=from_attributes, context=context ) From f5bdab295721a7c04b0d50daddb723fb4ab32e02 Mon Sep 17 00:00:00 2001 From: Ahmad Nofal Date: Fri, 8 Sep 2023 20:04:50 +0400 Subject: [PATCH 06/13] format --- ninja/operation.py | 23 +++++++++++++++-------- ninja/schema.py | 9 ++++----- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/ninja/operation.py b/ninja/operation.py index 335026389..55321b6e0 100644 --- a/ninja/operation.py +++ b/ninja/operation.py @@ -1,5 +1,16 @@ -from typing import (TYPE_CHECKING, Any, Callable, Dict, Iterable, List, - Optional, Sequence, Type, Union, cast) +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Iterable, + List, + Optional, + Sequence, + Type, + Union, + cast, +) import django import pydantic @@ -194,16 +205,12 @@ def _result_to_response( resp_object = ResponseObject(result) # ^ we need object because getter_dict seems work only with model_validate - result = response_model.model_validate( - resp_object - ).model_dump( + result = response_model.model_validate(resp_object).model_dump( by_alias=self.by_alias, exclude_unset=self.exclude_unset, exclude_defaults=self.exclude_defaults, exclude_none=self.exclude_none, - )[ - "response" - ] + )["response"] return self.api.create_response( request, result, temporal_response=temporal_response ) diff --git a/ninja/schema.py b/ninja/schema.py index c7f8b7ace..83267614b 100644 --- a/ninja/schema.py +++ b/ninja/schema.py @@ -24,12 +24,10 @@ def resolve_initials(self, obj): from typing import Any, Callable, Dict, Type, TypeVar, Union, no_type_check import pydantic -from django.db import models from django.db.models import Manager, QuerySet from django.db.models.fields.files import FieldFile from django.template import Variable, VariableDoesNotExist -from pydantic import (BaseModel, Field, ValidationInfo, model_validator, - validator) +from pydantic import BaseModel, Field, ValidationInfo, model_validator, validator from pydantic._internal._model_construction import ModelMetaclass from pydantic.json_schema import GenerateJsonSchema, JsonSchemaValue @@ -204,9 +202,10 @@ class Config: def _run_root_validator( cls, values: Any, handler: Callable, info: ValidationInfo ) -> Any: - # We dont perform 'before' validations if an validating through 'model_validate' - through_model_validate = info and info.context and info.context.get("through_model_validate", False) + through_model_validate = ( + info and info.context and info.context.get("through_model_validate", False) + ) if not through_model_validate: handler(values) From 9f1923468dab5f08212654a4c4aef5babd88f8b9 Mon Sep 17 00:00:00 2001 From: Ahmad Nofal Date: Sat, 9 Sep 2023 17:59:16 +0400 Subject: [PATCH 07/13] lint mypy --- ninja/schema.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/ninja/schema.py b/ninja/schema.py index 83267614b..ab9c7423b 100644 --- a/ninja/schema.py +++ b/ninja/schema.py @@ -21,7 +21,7 @@ def resolve_initials(self, obj): """ import warnings -from typing import Any, Callable, Dict, Type, TypeVar, Union, no_type_check +from typing import Any, Callable, Dict, TypeVar, Union, no_type_check import pydantic from django.db.models import Manager, QuerySet @@ -29,6 +29,7 @@ def resolve_initials(self, obj): from django.template import Variable, VariableDoesNotExist from pydantic import BaseModel, Field, ValidationInfo, model_validator, validator from pydantic._internal._model_construction import ModelMetaclass +from pydantic.functional_validators import ModelWrapValidatorHandler from pydantic.json_schema import GenerateJsonSchema, JsonSchemaValue from ninja.signature.utils import get_args_names, has_kwargs @@ -45,7 +46,7 @@ def resolve_initials(self, obj): class DjangoGetter: __slots__ = ("_obj", "_schema_cls", "_context") - def __init__(self, obj: Any, schema_cls: "Schema", context: Any = None): + def __init__(self, obj: Any, schema_cls: type[S], context: Any = None): self._obj = obj self._schema_cls = schema_cls self._context = context @@ -199,9 +200,10 @@ class Config: from_attributes = True # aka orm_mode @model_validator(mode="wrap") + @classmethod def _run_root_validator( - cls, values: Any, handler: Callable, info: ValidationInfo - ) -> Any: + cls, values: Any, handler: ModelWrapValidatorHandler[S], info: ValidationInfo + ) -> S: # We dont perform 'before' validations if an validating through 'model_validate' through_model_validate = ( info and info.context and info.context.get("through_model_validate", False) @@ -216,22 +218,22 @@ def _run_root_validator( return handler(values) @classmethod - def from_orm(cls: Type[S], obj: Any) -> S: + def from_orm(cls: type[S], obj: Any) -> S: return cls.model_validate(obj) @classmethod def model_validate( - cls: type[BaseModel], + cls: type[S], obj: Any, - *args, + *, strict: bool | None = None, from_attributes: bool | None = None, context: dict[str, Any] | None = None, - ) -> BaseModel: + ) -> S: context = context or {} context["through_model_validate"] = True return super().model_validate( - obj, *args, strict=strict, from_attributes=from_attributes, context=context + obj, strict=strict, from_attributes=from_attributes, context=context ) def dict(self, *a: Any, **kw: Any) -> DictStrAny: From 1cd488a414d583261ef4b4dc0e59f1fb3f996646 Mon Sep 17 00:00:00 2001 From: Ahmad Nofal Date: Sat, 9 Sep 2023 18:03:03 +0400 Subject: [PATCH 08/13] lint mypy --- ninja/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ninja/schema.py b/ninja/schema.py index ab9c7423b..bb4ce8d6c 100644 --- a/ninja/schema.py +++ b/ninja/schema.py @@ -55,7 +55,7 @@ def __getattr__(self, key: str) -> Any: # if key.startswith("__pydantic"): # return getattr(self._obj, key) - resolver = self._schema_cls._ninja_resolvers.get(key) # type: ignore + resolver = self._schema_cls._ninja_resolvers.get(key) if resolver: value = resolver(getter=self) else: From 7b3bc2d8b6d9b3a3071dc3fdd786dbea33492690 Mon Sep 17 00:00:00 2001 From: Ahmad Nofal Date: Mon, 11 Sep 2023 13:13:17 +0400 Subject: [PATCH 09/13] added test cases --- tests/test_request.py | 81 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/tests/test_request.py b/tests/test_request.py index d7e4dfbe7..b17072e83 100644 --- a/tests/test_request.py +++ b/tests/test_request.py @@ -1,8 +1,23 @@ +from typing import Optional + import pytest +from pydantic import ConfigDict -from ninja import Cookie, Header, Router +from ninja import Body, Cookie, Header, Router, Schema from ninja.testing import TestClient + +class OptionalEmptySchema(Schema): + model_config = ConfigDict(extra="forbid") + name: Optional[str] = None + + +class ExtraForbidSchema(Schema): + model_config = ConfigDict(extra="forbid") + name: str + metadata: Optional[OptionalEmptySchema] = None + + router = Router() @@ -41,6 +56,11 @@ def cookies2(request, wpn: str = Cookie(..., alias="weapon")): return wpn +@router.post("/test-schema") +def test_schema(request, payload: ExtraForbidSchema = Body(...)): + return "ok" + + client = TestClient(router) @@ -77,3 +97,62 @@ def test_headers(path, expected_status, expected_response): assert response.status_code == expected_status, response.content print(response.json()) assert response.json() == expected_response + + +@pytest.mark.parametrize( + "path,json,expected_status,expected_response", + [ + ( + "/test-schema", + {"name": "test", "extra_name": "test2"}, + 422, + { + "detail": [ + { + "type": "extra_forbidden", + "loc": ["body", "payload", "extra_name"], + "msg": "Extra inputs are not permitted", + } + ] + }, + ), + ( + "/test-schema", + {"name": "test", "metadata": {"extra_name": "xxx"}}, + 422, + { + "detail": [ + { + "loc": ["body", "payload", "metadata", "extra_name"], + "msg": "Extra inputs are not permitted", + "type": "extra_forbidden", + } + ] + }, + ), + ( + "/test-schema", + {"name": "test", "metadata": "test2"}, + 422, + { + "detail": [ + { + "type": "model_attributes_type", + "loc": ["body", "payload", "metadata"], + "msg": "Input should be a valid dictionary or object to extract fields from", + } + ] + }, + ), + ], +) +def test_pydantic_config(path, json, expected_status, expected_response): + # test extra forbid + response = client.post(path, json=json) + assert response.json() == expected_response + assert response.status_code == expected_status + + # test extra forbid on nested schema + response = client.post( + path, json={"name": "test", "metadata": {"extra_name": "xxx"}} + ) From 5f1b1a4e36ae230a4e98d3c2324c83912f2cbf7d Mon Sep 17 00:00:00 2001 From: Ahmad Nofal Date: Mon, 11 Sep 2023 22:17:37 +0400 Subject: [PATCH 10/13] revert unneeded change --- ninja/signature/details.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ninja/signature/details.py b/ninja/signature/details.py index 08416e1eb..7576d56e8 100644 --- a/ninja/signature/details.py +++ b/ninja/signature/details.py @@ -159,7 +159,6 @@ def _create_models(self) -> TModels: ) base_cls = param_cls._model - base_cls.model_config["from_attributes"] = True model_cls = type(cls_name, (base_cls,), attrs) # TODO: https://pydantic-docs.helpmanual.io/usage/models/#dynamic-model-creation - check if anything special in create_model method that I did not use result.append(model_cls) From 68a7cbf8d4a5fe8cae982748798e135e390021f2 Mon Sep 17 00:00:00 2001 From: Ahmad Nofal Date: Mon, 11 Sep 2023 22:18:07 +0400 Subject: [PATCH 11/13] attempt to make lint work on older python versions --- ninja/schema.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ninja/schema.py b/ninja/schema.py index bb4ce8d6c..4ec1a5a59 100644 --- a/ninja/schema.py +++ b/ninja/schema.py @@ -21,7 +21,7 @@ def resolve_initials(self, obj): """ import warnings -from typing import Any, Callable, Dict, TypeVar, Union, no_type_check +from typing import Any, Callable, Dict, Type, TypeVar, Union, no_type_check import pydantic from django.db.models import Manager, QuerySet @@ -46,7 +46,7 @@ def resolve_initials(self, obj): class DjangoGetter: __slots__ = ("_obj", "_schema_cls", "_context") - def __init__(self, obj: Any, schema_cls: type[S], context: Any = None): + def __init__(self, obj: Any, schema_cls: Type[S], context: Any = None): self._obj = obj self._schema_cls = schema_cls self._context = context @@ -218,12 +218,12 @@ def _run_root_validator( return handler(values) @classmethod - def from_orm(cls: type[S], obj: Any) -> S: + def from_orm(cls: Type[S], obj: Any) -> S: return cls.model_validate(obj) @classmethod def model_validate( - cls: type[S], + cls: Type[S], obj: Any, *, strict: bool | None = None, From 5502e1b2e9d6ee90c4208e8dc613130a69ef06cd Mon Sep 17 00:00:00 2001 From: Ahmad Nofal Date: Mon, 11 Sep 2023 22:43:55 +0400 Subject: [PATCH 12/13] make tests run on python3.7 --- ninja/schema.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ninja/schema.py b/ninja/schema.py index 4ec1a5a59..a32d759bc 100644 --- a/ninja/schema.py +++ b/ninja/schema.py @@ -21,7 +21,7 @@ def resolve_initials(self, obj): """ import warnings -from typing import Any, Callable, Dict, Type, TypeVar, Union, no_type_check +from typing import Any, Callable, Dict, Optional, Type, TypeVar, Union, no_type_check import pydantic from django.db.models import Manager, QuerySet @@ -226,9 +226,9 @@ def model_validate( cls: Type[S], obj: Any, *, - strict: bool | None = None, - from_attributes: bool | None = None, - context: dict[str, Any] | None = None, + strict: Optional[bool] = None, + from_attributes: Optional[bool] = None, + context: Optional[Dict[str, Any]] = None, ) -> S: context = context or {} context["through_model_validate"] = True From 18f3ebcb539926dfeae690649edea2890b9fbce8 Mon Sep 17 00:00:00 2001 From: Ahmad Nofal Date: Tue, 12 Sep 2023 11:00:20 +0400 Subject: [PATCH 13/13] removed unused junk code in test --- tests/test_request.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/test_request.py b/tests/test_request.py index b17072e83..40d5dd5f3 100644 --- a/tests/test_request.py +++ b/tests/test_request.py @@ -151,8 +151,3 @@ def test_pydantic_config(path, json, expected_status, expected_response): response = client.post(path, json=json) assert response.json() == expected_response assert response.status_code == expected_status - - # test extra forbid on nested schema - response = client.post( - path, json={"name": "test", "metadata": {"extra_name": "xxx"}} - )