diff --git a/alembic/versions/dae93f1d9110_populate_tag_and_study_tag_tables_with_.py b/alembic/versions/dae93f1d9110_populate_tag_and_study_tag_tables_with_.py index 6fbb060115..7be22d1e24 100644 --- a/alembic/versions/dae93f1d9110_populate_tag_and_study_tag_tables_with_.py +++ b/alembic/versions/dae93f1d9110_populate_tag_and_study_tag_tables_with_.py @@ -7,7 +7,6 @@ """ import collections import itertools -import json import secrets import typing as t @@ -16,6 +15,7 @@ from sqlalchemy.engine import Connection # type: ignore from antarest.study.css4_colors import COLOR_NAMES +from antarest.core.serialization import from_json, to_json # revision identifiers, used by Alembic. revision = "dae93f1d9110" @@ -34,7 +34,7 @@ def _avoid_duplicates(tags: t.Iterable[str]) -> t.Sequence[str]: def _load_patch_obj(patch: t.Optional[str]) -> t.MutableMapping[str, t.Any]: """Load the patch object from the `patch` field in the `study_additional_data` table.""" - obj: t.MutableMapping[str, t.Any] = json.loads(patch or "{}") + obj: t.MutableMapping[str, t.Any] = from_json(patch or "{}") obj["study"] = obj.get("study") or {} obj["study"]["tags"] = _avoid_duplicates(obj["study"].get("tags") or []) return obj @@ -113,7 +113,7 @@ def downgrade() -> None: objects_by_ids[study_id] = obj # Updating objects in the `study_additional_data` table - bulk_patches = [{"study_id": id_, "patch": json.dumps(obj)} for id_, obj in objects_by_ids.items()] + bulk_patches = [{"study_id": id_, "patch": to_json(obj)} for id_, obj in objects_by_ids.items()] if bulk_patches: sql = sa.text("UPDATE study_additional_data SET patch = :patch WHERE study_id = :study_id") connexion.execute(sql, *bulk_patches) diff --git a/antarest/core/cache/business/redis_cache.py b/antarest/core/cache/business/redis_cache.py index 0234fd1666..7793f280c7 100644 --- a/antarest/core/cache/business/redis_cache.py +++ b/antarest/core/cache/business/redis_cache.py @@ -10,7 +10,6 @@ # # This file is part of the Antares project. -import json import logging from typing import List, Optional @@ -19,6 +18,7 @@ from antarest.core.interfaces.cache import ICache from antarest.core.model import JSON +from antarest.core.serialization import from_json logger = logging.getLogger(__name__) @@ -49,7 +49,7 @@ def get(self, id: str, refresh_timeout: Optional[int] = None) -> Optional[JSON]: logger.info(f"Trying to retrieve cache key {id}") if result is not None: logger.info(f"Cache key {id} found") - json_result = json.loads(result) + json_result = from_json(result) redis_element = RedisCacheElement(duration=json_result["duration"], data=json_result["data"]) self.redis.expire( redis_key, diff --git a/antarest/core/configdata/repository.py b/antarest/core/configdata/repository.py index 8439e028a6..3b7aea6ada 100644 --- a/antarest/core/configdata/repository.py +++ b/antarest/core/configdata/repository.py @@ -10,13 +10,13 @@ # # This file is part of the Antares project. -import json from operator import and_ from typing import Optional from antarest.core.configdata.model import ConfigData from antarest.core.jwt import DEFAULT_ADMIN_USER from antarest.core.model import JSON +from antarest.core.serialization import from_json, to_json_string from antarest.core.utils.fastapi_sqlalchemy import db @@ -43,14 +43,14 @@ def get(self, key: str, owner: Optional[int] = None) -> Optional[ConfigData]: def get_json(self, key: str, owner: Optional[int] = None) -> Optional[JSON]: configdata = self.get(key, owner) if configdata: - data: JSON = json.loads(configdata.value) + data: JSON = from_json(configdata.value) return data return None def put_json(self, key: str, data: JSON, owner: Optional[int] = None) -> None: configdata = ConfigData( key=key, - value=json.dumps(data), + value=to_json_string(data), owner=owner or DEFAULT_ADMIN_USER.id, ) configdata = db.session.merge(configdata) diff --git a/antarest/core/serialization/__init__.py b/antarest/core/serialization/__init__.py new file mode 100644 index 0000000000..6368c02f1e --- /dev/null +++ b/antarest/core/serialization/__init__.py @@ -0,0 +1,34 @@ +# Copyright (c) 2024, RTE (https://www.rte-france.com) +# +# See AUTHORS.txt +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# SPDX-License-Identifier: MPL-2.0 +# +# This file is part of the Antares project. +import typing as t + +import pydantic + +ADAPTER: pydantic.TypeAdapter[t.Any] = pydantic.TypeAdapter( + type=t.Any, config=pydantic.config.ConfigDict(ser_json_inf_nan="constants") +) # ser_json_inf_nan="constants" means infinity and NaN values will be serialized as `Infinity` and `NaN`. + + +# These utility functions allow to serialize with pydantic instead of using the built-in python "json" library. +# Since pydantic v2 is written in RUST it's way faster. + + +def from_json(data: t.Union[str, bytes, bytearray]) -> t.Dict[str, t.Any]: + return ADAPTER.validate_json(data) # type: ignore + + +def to_json(data: t.Any, indent: t.Optional[int] = None) -> bytes: + return ADAPTER.dump_json(data, indent=indent) + + +def to_json_string(data: t.Any, indent: t.Optional[int] = None) -> str: + return to_json(data, indent=indent).decode("utf-8") diff --git a/antarest/eventbus/web.py b/antarest/eventbus/web.py index ba363db6c4..a04a4571bb 100644 --- a/antarest/eventbus/web.py +++ b/antarest/eventbus/web.py @@ -11,13 +11,12 @@ # This file is part of the Antares project. import dataclasses -import json import logging from enum import Enum from http import HTTPStatus from typing import List, Optional -from fastapi import APIRouter, Depends, FastAPI, HTTPException, Query +from fastapi import Depends, HTTPException, Query from pydantic import BaseModel from starlette.websockets import WebSocket, WebSocketDisconnect @@ -27,6 +26,7 @@ from antarest.core.jwt import DEFAULT_ADMIN_USER, JWTUser from antarest.core.model import PermissionInfo, StudyPermissionType from antarest.core.permissions import check_permission +from antarest.core.serialization import to_json_string from antarest.fastapi_jwt_auth import AuthJWT from antarest.login.auth import Auth @@ -99,7 +99,7 @@ async def send_event_to_ws(event: Event) -> None: event_data = event.model_dump() del event_data["permissions"] del event_data["channel"] - await manager.broadcast(json.dumps(event_data), event.permissions, event.channel) + await manager.broadcast(to_json_string(event_data), event.permissions, event.channel) @app_ctxt.api_root.websocket("/ws") async def connect( diff --git a/antarest/launcher/model.py b/antarest/launcher/model.py index 9a330454dc..d80400a4ba 100644 --- a/antarest/launcher/model.py +++ b/antarest/launcher/model.py @@ -11,15 +11,15 @@ # This file is part of the Antares project. import enum -import json import typing as t from datetime import datetime -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, Field from sqlalchemy import Column, DateTime, Enum, ForeignKey, Integer, Sequence, String # type: ignore from sqlalchemy.orm import relationship # type: ignore from antarest.core.persistence import Base +from antarest.core.serialization import from_json from antarest.login.model import Identity, UserInfo from antarest.study.business.all_optional_meta import camel_case_model @@ -54,7 +54,7 @@ def from_launcher_params(cls, params: t.Optional[str]) -> "LauncherParametersDTO """ if params is None: return cls() - return cls.model_validate(json.loads(params)) + return cls.model_validate(from_json(params)) class LogType(str, enum.Enum): diff --git a/antarest/login/auth.py b/antarest/login/auth.py index ad4884111a..f86ca5903b 100644 --- a/antarest/login/auth.py +++ b/antarest/login/auth.py @@ -10,7 +10,6 @@ # # This file is part of the Antares project. -import json import logging from datetime import timedelta from typing import Any, Callable, Coroutine, Dict, Optional, Tuple, Union @@ -22,6 +21,7 @@ from antarest.core.config import Config from antarest.core.jwt import DEFAULT_ADMIN_USER, JWTUser +from antarest.core.serialization import from_json from antarest.fastapi_jwt_auth import AuthJWT logger = logging.getLogger(__name__) @@ -66,14 +66,14 @@ def get_current_user(self, auth_jwt: AuthJWT = Depends()) -> JWTUser: auth_jwt.jwt_required() - user = JWTUser.model_validate(json.loads(auth_jwt.get_jwt_subject())) + user = JWTUser.model_validate(from_json(auth_jwt.get_jwt_subject())) return user @staticmethod def get_user_from_token(token: str, jwt_manager: AuthJWT) -> Optional[JWTUser]: try: token_data = jwt_manager._verified_token(token) - return JWTUser.model_validate(json.loads(token_data["sub"])) + return JWTUser.model_validate(from_json(token_data["sub"])) except Exception as e: logger.debug("Failed to retrieve user from token", exc_info=e) return None diff --git a/antarest/login/main.py b/antarest/login/main.py index 09fac37ee5..ac6b2956c9 100644 --- a/antarest/login/main.py +++ b/antarest/login/main.py @@ -10,17 +10,16 @@ # # This file is part of the Antares project. -import json from http import HTTPStatus from typing import Any, Optional -from fastapi import APIRouter, FastAPI from starlette.requests import Request from starlette.responses import JSONResponse from antarest.core.application import AppBuildContext from antarest.core.config import Config from antarest.core.interfaces.eventbus import DummyEventBusService, IEventBus +from antarest.core.serialization import from_json from antarest.core.utils.fastapi_sqlalchemy import db from antarest.fastapi_jwt_auth import AuthJWT from antarest.fastapi_jwt_auth.exceptions import AuthJWTException @@ -78,7 +77,7 @@ def authjwt_exception_handler(request: Request, exc: AuthJWTException) -> Any: @AuthJWT.token_in_denylist_loader # type: ignore def check_if_token_is_revoked(decrypted_token: Any) -> bool: - subject = json.loads(decrypted_token["sub"]) + subject = from_json(decrypted_token["sub"]) user_id = subject["id"] token_type = subject["type"] with db(): diff --git a/antarest/login/web.py b/antarest/login/web.py index 6e2b20a19c..5bc85c62a1 100644 --- a/antarest/login/web.py +++ b/antarest/login/web.py @@ -10,7 +10,6 @@ # # This file is part of the Antares project. -import json import logging from datetime import timedelta from typing import Any, List, Optional, Union @@ -23,6 +22,7 @@ from antarest.core.jwt import JWTGroup, JWTUser from antarest.core.requests import RequestParameters, UserHasNotPermissionError from antarest.core.roles import RoleType +from antarest.core.serialization import from_json from antarest.core.utils.web import APITag from antarest.fastapi_jwt_auth import AuthJWT from antarest.login.auth import Auth @@ -103,7 +103,7 @@ def login( ) def refresh(jwt_manager: AuthJWT = Depends()) -> Any: jwt_manager.jwt_refresh_token_required() - identity = json.loads(jwt_manager.get_jwt_subject()) + identity = from_json(jwt_manager.get_jwt_subject()) logger.debug(f"Refreshing access token for {identity['id']}") user = service.get_jwt(identity["id"]) if user: diff --git a/antarest/main.py b/antarest/main.py index b1fd1480b6..e521124447 100644 --- a/antarest/main.py +++ b/antarest/main.py @@ -47,11 +47,11 @@ from antarest.login.auth import Auth, JwtSettings from antarest.login.model import init_admin_user from antarest.matrixstore.matrix_garbage_collector import MatrixGarbageCollector +from antarest.service_creator import SESSION_ARGS, Module, create_services, init_db_engine from antarest.singleton_services import start_all_services from antarest.study.storage.auto_archive_service import AutoArchiveService from antarest.study.storage.rawstudy.watcher import Watcher from antarest.tools.admin_lib import clean_locks -from antarest.utils import SESSION_ARGS, Module, create_services, init_db_engine logger = logging.getLogger(__name__) diff --git a/antarest/matrixstore/service.py b/antarest/matrixstore/service.py index 7f9b83cca2..7e9450fdec 100644 --- a/antarest/matrixstore/service.py +++ b/antarest/matrixstore/service.py @@ -12,7 +12,6 @@ import contextlib import io -import json import logging import tempfile import typing as t @@ -31,6 +30,7 @@ from antarest.core.filetransfer.service import FileTransferManager from antarest.core.jwt import JWTUser from antarest.core.requests import RequestParameters, UserHasNotPermissionError +from antarest.core.serialization import from_json from antarest.core.tasks.model import TaskResult, TaskType from antarest.core.tasks.service import ITaskService, TaskUpdateNotifier from antarest.core.utils.fastapi_sqlalchemy import db @@ -263,7 +263,7 @@ def _file_importation(self, file: bytes, *, is_json: bool = False) -> str: A SHA256 hash that identifies the imported matrix. """ if is_json: - obj = json.loads(file) + obj = from_json(file) content = MatrixContent(**obj) return self.create(content.data) # noinspection PyTypeChecker diff --git a/antarest/utils.py b/antarest/service_creator.py similarity index 89% rename from antarest/utils.py rename to antarest/service_creator.py index 1f7f4b7594..5859806a61 100644 --- a/antarest/utils.py +++ b/antarest/service_creator.py @@ -11,12 +11,11 @@ # This file is part of the Antares project. import logging +import typing as t from enum import Enum from pathlib import Path -from typing import Any, Dict, Mapping, Optional, Tuple import redis -from fastapi import APIRouter, FastAPI from ratelimit import RateLimitMiddleware # type: ignore from ratelimit.backends.redis import RedisBackend # type: ignore from ratelimit.backends.simple import MemoryBackend # type: ignore @@ -55,7 +54,7 @@ logger = logging.getLogger(__name__) -SESSION_ARGS: Mapping[str, bool] = { +SESSION_ARGS: t.Mapping[str, bool] = { "autocommit": False, "expire_on_commit": False, "autoflush": False, @@ -84,7 +83,7 @@ def init_db_engine( ) -> Engine: if auto_upgrade_db: upgrade_db(config_file) - connect_args: Dict[str, Any] = {} + connect_args: t.Dict[str, t.Any] = {} if config.db.db_url.startswith("sqlite"): connect_args["check_same_thread"] = False else: @@ -110,7 +109,7 @@ def init_db_engine( return engine -def create_event_bus(app_ctxt: Optional[AppBuildContext], config: Config) -> Tuple[IEventBus, Optional[redis.Redis]]: # type: ignore +def create_event_bus(app_ctxt: t.Optional[AppBuildContext], config: Config) -> t.Tuple[IEventBus, t.Optional[redis.Redis]]: # type: ignore redis_client = new_redis_instance(config.redis) if config.redis is not None else None return ( build_eventbus(app_ctxt, config, True, redis_client), @@ -119,8 +118,8 @@ def create_event_bus(app_ctxt: Optional[AppBuildContext], config: Config) -> Tup def create_core_services( - app_ctxt: Optional[AppBuildContext], config: Config -) -> Tuple[ICache, IEventBus, ITaskService, FileTransferManager, LoginService, MatrixService, StudyService,]: + app_ctxt: t.Optional[AppBuildContext], config: Config +) -> t.Tuple[ICache, IEventBus, ITaskService, FileTransferManager, LoginService, MatrixService, StudyService,]: event_bus, redis_client = create_event_bus(app_ctxt, config) cache = build_cache(config=config, redis_client=redis_client) filetransfer_service = build_filetransfer_service(app_ctxt, event_bus, config) @@ -157,8 +156,8 @@ def create_core_services( def create_watcher( config: Config, - app_ctxt: Optional[AppBuildContext], - study_service: Optional[StudyService] = None, + app_ctxt: t.Optional[AppBuildContext], + study_service: t.Optional[StudyService] = None, ) -> Watcher: if study_service: watcher = Watcher( @@ -182,9 +181,9 @@ def create_watcher( def create_matrix_gc( config: Config, - app_ctxt: Optional[AppBuildContext], - study_service: Optional[StudyService] = None, - matrix_service: Optional[MatrixService] = None, + app_ctxt: t.Optional[AppBuildContext], + study_service: t.Optional[StudyService] = None, + matrix_service: t.Optional[MatrixService] = None, ) -> MatrixGarbageCollector: if study_service and matrix_service: return MatrixGarbageCollector( @@ -205,7 +204,7 @@ def create_archive_worker( config: Config, workspace: str, local_root: Path = Path("/"), - event_bus: Optional[IEventBus] = None, + event_bus: t.Optional[IEventBus] = None, ) -> AbstractWorker: if not event_bus: event_bus, _ = create_event_bus(None, config) @@ -215,15 +214,17 @@ def create_archive_worker( def create_simulator_worker( config: Config, matrix_service: MatrixService, - event_bus: Optional[IEventBus] = None, + event_bus: t.Optional[IEventBus] = None, ) -> AbstractWorker: if not event_bus: event_bus, _ = create_event_bus(None, config) return SimulatorWorker(event_bus, matrix_service, config) -def create_services(config: Config, app_ctxt: Optional[AppBuildContext], create_all: bool = False) -> Dict[str, Any]: - services: Dict[str, Any] = {} +def create_services( + config: Config, app_ctxt: t.Optional[AppBuildContext], create_all: bool = False +) -> t.Dict[str, t.Any]: + services: t.Dict[str, t.Any] = {} ( cache, diff --git a/antarest/singleton_services.py b/antarest/singleton_services.py index 13395a439a..3b2373cc0c 100644 --- a/antarest/singleton_services.py +++ b/antarest/singleton_services.py @@ -19,8 +19,7 @@ from antarest.core.logging.utils import configure_logger from antarest.core.utils.fastapi_sqlalchemy import DBSessionMiddleware from antarest.core.utils.utils import get_local_path -from antarest.study.storage.auto_archive_service import AutoArchiveService -from antarest.utils import ( +from antarest.service_creator import ( SESSION_ARGS, Module, create_archive_worker, @@ -30,6 +29,7 @@ create_watcher, init_db_engine, ) +from antarest.study.storage.auto_archive_service import AutoArchiveService def _init(config_file: Path, services_list: List[Module]) -> Dict[Module, IService]: diff --git a/antarest/study/service.py b/antarest/study/service.py index 125c4e1531..c39576605f 100644 --- a/antarest/study/service.py +++ b/antarest/study/service.py @@ -15,7 +15,6 @@ import contextlib import http import io -import json import logging import os import time @@ -52,6 +51,7 @@ from antarest.core.jwt import DEFAULT_ADMIN_USER, JWTGroup, JWTUser from antarest.core.model import JSON, SUB_JSON, PermissionInfo, PublicMode, StudyPermissionType from antarest.core.requests import RequestParameters, UserHasNotPermissionError +from antarest.core.serialization import to_json from antarest.core.tasks.model import TaskListFilter, TaskResult, TaskStatus, TaskType from antarest.core.tasks.service import ITaskService, TaskUpdateNotifier, noop_notifier from antarest.core.utils.fastapi_sqlalchemy import db @@ -1329,13 +1329,7 @@ def export_task(_notifier: TaskUpdateNotifier) -> TaskResult: return FileResponse(tmp_export_file, headers=headers, media_type=filetype) else: - json_response = json.dumps( - matrix.model_dump(), - ensure_ascii=False, - allow_nan=True, - indent=None, - separators=(",", ":"), - ).encode("utf-8") + json_response = to_json(matrix.model_dump()) return Response(content=json_response, media_type="application/json") def get_study_sim_result(self, study_id: str, params: RequestParameters) -> t.List[StudySimResultDTO]: diff --git a/antarest/study/storage/abstract_storage_service.py b/antarest/study/storage/abstract_storage_service.py index 5f2c033ca3..967ece3ca6 100644 --- a/antarest/study/storage/abstract_storage_service.py +++ b/antarest/study/storage/abstract_storage_service.py @@ -10,7 +10,6 @@ # # This file is part of the Antares project. -import json import logging import shutil import tempfile @@ -23,6 +22,7 @@ from antarest.core.exceptions import BadOutputError, StudyOutputNotFoundError from antarest.core.interfaces.cache import CacheConstants, ICache from antarest.core.model import JSON, PublicMode +from antarest.core.serialization import from_json from antarest.core.utils.utils import StopWatch, extract_zip, unzip, zip_dir from antarest.login.model import GroupDTO from antarest.study.common.studystorage import IStudyStorageService, T @@ -87,8 +87,7 @@ def get_study_information( additional_data = study.additional_data or StudyAdditionalData() try: - patch_obj = json.loads(additional_data.patch or "{}") - patch = Patch.model_validate(patch_obj) + patch = Patch.model_validate(from_json(additional_data.patch or "{}")) except ValueError as e: # The conversion to JSON and the parsing can fail if the patch is not valid logger.warning(f"Failed to parse patch for study {study.id}", exc_info=e) diff --git a/antarest/study/storage/patch_service.py b/antarest/study/storage/patch_service.py index 0f3eb68c05..1c44e1ddc9 100644 --- a/antarest/study/storage/patch_service.py +++ b/antarest/study/storage/patch_service.py @@ -10,10 +10,10 @@ # # This file is part of the Antares project. -import json import typing as t from pathlib import Path +from antarest.core.serialization import from_json from antarest.study.model import Patch, PatchOutputs, RawStudy, StudyAdditionalData from antarest.study.repository import StudyMetadataRepository from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy @@ -34,8 +34,7 @@ def get(self, study: t.Union[RawStudy, VariantStudy], get_from_file: bool = Fals if not get_from_file and study.additional_data is not None: # the `study.additional_data.patch` field is optional if study.additional_data.patch: - patch_obj = json.loads(study.additional_data.patch or "{}") - return Patch.model_validate(patch_obj) + return Patch.model_validate(from_json(study.additional_data.patch)) patch = Patch() patch_path = Path(study.path) / PATCH_JSON diff --git a/antarest/study/storage/rawstudy/model/filesystem/config/files.py b/antarest/study/storage/rawstudy/model/filesystem/config/files.py index 15856dbf60..6cfed5b9bc 100644 --- a/antarest/study/storage/rawstudy/model/filesystem/config/files.py +++ b/antarest/study/storage/rawstudy/model/filesystem/config/files.py @@ -21,6 +21,7 @@ from pathlib import Path from antarest.core.model import JSON +from antarest.core.serialization import from_json from antarest.study.storage.rawstudy.ini_reader import IniReader from antarest.study.storage.rawstudy.model.filesystem.config.binding_constraint import ( DEFAULT_GROUP, @@ -330,7 +331,7 @@ def _parse_xpansion_version(path: Path) -> str: xpansion_json = path / "expansion" / "out.json" try: content = xpansion_json.read_text(encoding="utf-8") - obj = json.loads(content) + obj = from_json(content) return str(obj["antares_xpansion"]["version"]) except FileNotFoundError: return "" diff --git a/antarest/study/storage/rawstudy/model/filesystem/config/ini_properties.py b/antarest/study/storage/rawstudy/model/filesystem/config/ini_properties.py index 5464975bbf..e731ea203b 100644 --- a/antarest/study/storage/rawstudy/model/filesystem/config/ini_properties.py +++ b/antarest/study/storage/rawstudy/model/filesystem/config/ini_properties.py @@ -10,11 +10,12 @@ # # This file is part of the Antares project. -import json import typing as t from pydantic import BaseModel +from antarest.core.serialization import from_json, to_json + class IniProperties( BaseModel, @@ -49,7 +50,7 @@ def to_config(self) -> t.Dict[str, t.Any]: if isinstance(value, IniProperties): config[alias] = value.to_config() else: - config[alias] = json.loads(json.dumps(value)) + config[alias] = from_json(to_json(value)) return config @classmethod diff --git a/antarest/study/storage/rawstudy/model/filesystem/ini_file_node.py b/antarest/study/storage/rawstudy/model/filesystem/ini_file_node.py index e5443eb37f..bf9eecc574 100644 --- a/antarest/study/storage/rawstudy/model/filesystem/ini_file_node.py +++ b/antarest/study/storage/rawstudy/model/filesystem/ini_file_node.py @@ -13,18 +13,18 @@ import contextlib import functools import io -import json import logging import os import tempfile import typing as t import zipfile -from json import JSONDecodeError from pathlib import Path +import pydantic_core from filelock import FileLock from antarest.core.model import JSON, SUB_JSON +from antarest.core.serialization import from_json from antarest.study.storage.rawstudy.ini_reader import IniReader, IReader from antarest.study.storage.rawstudy.ini_writer import IniWriter from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig @@ -190,8 +190,8 @@ def save(self, data: SUB_JSON, url: t.Optional[t.List[str]] = None) -> None: info = self.reader.read(self.path) if self.path.exists() else {} obj = data if isinstance(data, str): - with contextlib.suppress(JSONDecodeError): - obj = json.loads(data) + with contextlib.suppress(pydantic_core.ValidationError): + obj = from_json(data) if len(url) == 2: if url[0] not in info: info[url[0]] = {} diff --git a/antarest/study/storage/rawstudy/model/filesystem/json_file_node.py b/antarest/study/storage/rawstudy/model/filesystem/json_file_node.py index ad95ee3187..54333e2c4b 100644 --- a/antarest/study/storage/rawstudy/model/filesystem/json_file_node.py +++ b/antarest/study/storage/rawstudy/model/filesystem/json_file_node.py @@ -15,6 +15,7 @@ from pathlib import Path from antarest.core.model import JSON +from antarest.core.serialization import from_json, to_json from antarest.study.storage.rawstudy.ini_reader import IReader from antarest.study.storage.rawstudy.ini_writer import IniWriter from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig @@ -47,7 +48,7 @@ def read(self, path: t.Any, **kwargs: t.Any) -> JSON: raise TypeError(repr(type(path))) try: - return t.cast(JSON, json.loads(content)) + return t.cast(JSON, from_json(content)) except json.JSONDecodeError as exc: err_msg = f"Failed to parse JSON file '{path}'" raise ValueError(err_msg) from exc @@ -59,8 +60,8 @@ class JsonWriter(IniWriter): """ def write(self, data: JSON, path: Path) -> None: - with open(path, "w") as fh: - json.dump(data, fh) + with open(path, "wb") as fh: + fh.write(to_json(data)) class JsonFileNode(IniFileNode): diff --git a/antarest/study/storage/study_download_utils.py b/antarest/study/storage/study_download_utils.py index 91b2b4f9af..9c5f6beed3 100644 --- a/antarest/study/storage/study_download_utils.py +++ b/antarest/study/storage/study_download_utils.py @@ -11,7 +11,6 @@ # This file is part of the Antares project. import csv -import json import logging import os import re @@ -26,6 +25,7 @@ from fastapi import HTTPException from antarest.core.exceptions import ChildNotFoundError +from antarest.core.serialization import to_json from antarest.study.model import ( ExportFormat, MatrixAggregationResult, @@ -343,15 +343,8 @@ def export( target_file: Path, ) -> None: if filetype == ExportFormat.JSON: - with open(target_file, "w") as fh: - json.dump( - matrix.model_dump(), - fh, - ensure_ascii=False, - allow_nan=True, - indent=None, - separators=(",", ":"), - ) + with open(target_file, "wb") as fh: + fh.write(to_json(matrix.model_dump())) else: StudyDownloader.write_inside_archive(target_file, filetype, matrix) diff --git a/antarest/study/storage/variantstudy/command_factory.py b/antarest/study/storage/variantstudy/command_factory.py index d178cfa79f..409cf81bb5 100644 --- a/antarest/study/storage/variantstudy/command_factory.py +++ b/antarest/study/storage/variantstudy/command_factory.py @@ -10,8 +10,8 @@ # # This file is part of the Antares project. -import typing as t import copy +import typing as t from antarest.core.model import JSON from antarest.matrixstore.service import ISimpleMatrixService diff --git a/antarest/study/storage/variantstudy/model/dbmodel.py b/antarest/study/storage/variantstudy/model/dbmodel.py index 061c586e03..421caf87d7 100644 --- a/antarest/study/storage/variantstudy/model/dbmodel.py +++ b/antarest/study/storage/variantstudy/model/dbmodel.py @@ -11,7 +11,6 @@ # This file is part of the Antares project. import datetime -import json import typing as t import uuid from pathlib import Path @@ -20,6 +19,7 @@ from sqlalchemy.orm import relationship # type: ignore from antarest.core.persistence import Base +from antarest.core.serialization import from_json from antarest.study.model import Study from antarest.study.storage.variantstudy.model.model import CommandDTO @@ -69,7 +69,7 @@ class CommandBlock(Base): # type: ignore def to_dto(self) -> CommandDTO: # Database may lack a version number, defaulting to 1 if so. version = self.version or 1 - return CommandDTO(id=self.id, action=self.command, args=json.loads(self.args), version=version) + return CommandDTO(id=self.id, action=self.command, args=from_json(self.args), version=version) def __str__(self) -> str: return ( diff --git a/antarest/study/storage/variantstudy/variant_study_service.py b/antarest/study/storage/variantstudy/variant_study_service.py index 34241f8552..4433a6ac07 100644 --- a/antarest/study/storage/variantstudy/variant_study_service.py +++ b/antarest/study/storage/variantstudy/variant_study_service.py @@ -11,7 +11,6 @@ # This file is part of the Antares project. import concurrent.futures -import json import logging import re import shutil @@ -43,6 +42,7 @@ from antarest.core.jwt import DEFAULT_ADMIN_USER from antarest.core.model import JSON, PermissionInfo, PublicMode, StudyPermissionType from antarest.core.requests import RequestParameters, UserHasNotPermissionError +from antarest.core.serialization import to_json_string from antarest.core.tasks.model import CustomTaskEventMessages, TaskDTO, TaskResult, TaskType from antarest.core.tasks.service import DEFAULT_AWAIT_MAX_TIMEOUT, ITaskService, TaskUpdateNotifier, noop_notifier from antarest.core.utils.utils import assert_this, suppress_exception @@ -188,7 +188,10 @@ def append_commands( # noinspection PyArgumentList new_commands = [ CommandBlock( - command=command.action, args=json.dumps(command.args), index=(first_index + i), version=command.version + command=command.action, + args=to_json_string(command.args), + index=(first_index + i), + version=command.version, ) for i, command in enumerate(validated_commands) ] @@ -223,7 +226,7 @@ def replace_commands( validated_commands = transform_command_to_dto(command_objs, commands) # noinspection PyArgumentList study.commands = [ - CommandBlock(command=command.action, args=json.dumps(command.args), index=i, version=command.version) + CommandBlock(command=command.action, args=to_json_string(command.args), index=i, version=command.version) for i, command in enumerate(validated_commands) ] self.invalidate_cache(study, invalidate_self_snapshot=True) @@ -314,7 +317,7 @@ def update_command( index = [command.id for command in study.commands].index(command_id) if index >= 0: study.commands[index].command = validated_commands[0].action - study.commands[index].args = json.dumps(validated_commands[0].args) + study.commands[index].args = to_json_string(validated_commands[0].args) self.invalidate_cache(study, invalidate_self_snapshot=True) def export_commands_matrices(self, study_id: str, params: RequestParameters) -> FileDownloadTaskDTO: diff --git a/antarest/study/web/raw_studies_blueprint.py b/antarest/study/web/raw_studies_blueprint.py index dc8554e55d..c76bce3e77 100644 --- a/antarest/study/web/raw_studies_blueprint.py +++ b/antarest/study/web/raw_studies_blueprint.py @@ -13,7 +13,6 @@ import collections import http import io -import json import logging import typing as t from pathlib import Path, PurePosixPath @@ -26,6 +25,7 @@ from antarest.core.jwt import JWTUser from antarest.core.model import SUB_JSON from antarest.core.requests import RequestParameters +from antarest.core.serialization import from_json, to_json from antarest.core.swagger import get_path_examples from antarest.core.utils.utils import sanitize_string, sanitize_uuid from antarest.core.utils.web import APITag @@ -148,7 +148,7 @@ def get_study( # Use `JSONResponse` to ensure to return a valid JSON response # that checks `NaN` and `Infinity` values. try: - output = json.loads(output) + output = from_json(output) return JSONResponse(content=output) except ValueError as exc: raise HTTPException( @@ -182,13 +182,7 @@ def get_study( # even though they are not standard JSON values because they are supported in JavaScript. # Additionally, we cannot use `orjson` because, despite its superior performance, it converts # `NaN` and other values to `null`, even when using a custom encoder. - json_response = json.dumps( - output, - ensure_ascii=False, - allow_nan=True, - indent=None, - separators=(",", ":"), - ).encode("utf-8") + json_response = to_json(output) return Response(content=json_response, media_type="application/json") @bp.get( diff --git a/antarest/study/web/xpansion_studies_blueprint.py b/antarest/study/web/xpansion_studies_blueprint.py index 93b60f01ff..775505baad 100644 --- a/antarest/study/web/xpansion_studies_blueprint.py +++ b/antarest/study/web/xpansion_studies_blueprint.py @@ -10,7 +10,6 @@ # # This file is part of the Antares project. -import json import logging import typing as t @@ -21,6 +20,7 @@ from antarest.core.jwt import JWTUser from antarest.core.model import JSON, StudyPermissionType from antarest.core.requests import RequestParameters +from antarest.core.serialization import to_json from antarest.core.utils.web import APITag from antarest.login.auth import Auth from antarest.study.business.xpansion_management import ( @@ -282,13 +282,7 @@ def get_resource_content( except (AttributeError, UnicodeDecodeError): pass - json_response = json.dumps( - output, - ensure_ascii=False, - allow_nan=True, - indent=None, - separators=(",", ":"), - ).encode("utf-8") + json_response = to_json(output) return Response(content=json_response, media_type="application/json") @bp.get( diff --git a/antarest/tools/lib.py b/antarest/tools/lib.py index 6bfd3df997..5f3aafc74a 100644 --- a/antarest/tools/lib.py +++ b/antarest/tools/lib.py @@ -24,6 +24,7 @@ from antarest.core.cache.business.local_chache import LocalCache from antarest.core.config import CacheConfig +from antarest.core.serialization import from_json, to_json_string from antarest.core.tasks.model import TaskDTO from antarest.core.utils.utils import StopWatch, get_local_path from antarest.matrixstore.repository import MatrixContentRepository @@ -117,7 +118,7 @@ def apply_commands( # This should not happen, but if it does, we return a failed result return GenerationResultInfoDTO(success=False, details=[]) - info = json.loads(task_result.result.return_value) + info = from_json(task_result.result.return_value) return GenerationResultInfoDTO(**info) def build_url(self, url: str) -> str: @@ -207,10 +208,7 @@ def extract_commands(study_path: Path, commands_output_dir: Path) -> None: command_list = extractor.extract(study) (commands_output_dir / COMMAND_FILE).write_text( - json.dumps( - [command.model_dump(exclude={"id"}) for command in command_list], - indent=2, - ) + to_json_string([command.model_dump(exclude={"id"}) for command in command_list], indent=2) ) @@ -299,10 +297,7 @@ def generate_diff( ) (output_dir / COMMAND_FILE).write_text( - json.dumps( - [command.to_dto().model_dump(exclude={"id"}) for command in diff_commands], - indent=2, - ) + to_json_string([command.to_dto().model_dump(exclude={"id"}) for command in diff_commands], indent=2) ) needed_matrices: Set[str] = set() diff --git a/antarest/worker/archive_worker_service.py b/antarest/worker/archive_worker_service.py index 587b6d15d6..b8e505fa8c 100644 --- a/antarest/worker/archive_worker_service.py +++ b/antarest/worker/archive_worker_service.py @@ -19,7 +19,7 @@ from antarest.core.config import Config from antarest.core.logging.utils import configure_logger from antarest.core.utils.utils import get_local_path -from antarest.utils import create_archive_worker +from antarest.service_creator import create_archive_worker # use the real module name instead of `__name__` (because `__name__ == "__main__"`) logger = logging.getLogger("antarest.worker.archive_worker_service") diff --git a/tests/cache/test_redis_cache.py b/tests/cache/test_redis_cache.py index 211de6c689..b3d9cabefd 100644 --- a/tests/cache/test_redis_cache.py +++ b/tests/cache/test_redis_cache.py @@ -10,11 +10,11 @@ # # This file is part of the Antares project. -import json from pathlib import Path from unittest.mock import Mock from antarest.core.cache.business.redis_cache import RedisCache, RedisCacheElement +from antarest.core.serialization import from_json from antarest.study.storage.rawstudy.model.filesystem.config.model import Area, FileStudyTreeConfigDTO @@ -44,7 +44,7 @@ def test_lifecycle(): # GET redis_client.get.return_value = cache_element - load = json.loads(cache_element) + load = from_json(cache_element) assert cache.get(id=id) == load["data"] redis_client.expire.assert_called_with(redis_key, duration) redis_client.get.assert_called_once_with(redis_key) diff --git a/tests/core/test_tasks.py b/tests/core/test_tasks.py index 89bc2ddfc2..cb2c635177 100644 --- a/tests/core/test_tasks.py +++ b/tests/core/test_tasks.py @@ -42,8 +42,8 @@ from antarest.eventbus.business.local_eventbus import LocalEventBus from antarest.eventbus.service import EventBusService from antarest.login.model import User +from antarest.service_creator import SESSION_ARGS from antarest.study.model import RawStudy -from antarest.utils import SESSION_ARGS from antarest.worker.worker import AbstractWorker, WorkerTaskCommand from tests.helpers import with_db_context diff --git a/tests/login/test_model.py b/tests/login/test_model.py index f8dde20c47..a37cdb7bd5 100644 --- a/tests/login/test_model.py +++ b/tests/login/test_model.py @@ -24,7 +24,7 @@ User, init_admin_user, ) -from antarest.utils import SESSION_ARGS +from antarest.service_creator import SESSION_ARGS TEST_ADMIN_PASS_WORD = "test"