build(python): bump main project dependencies (#1728)

bumps main projection dependencies: - `pydantic` from 1.9 to 2.8: huge breaking change but with large performance benefits expected on serialization - `fastapi` from 0.73 to 0.110 - `uvicorn` from to 0.15 to 0.30 - `mypy` from 1.4 to 1.11 It also brings few changes inside dependencies - Drop `requests` in favor of `httpx` - Drop `fastapi-jwt-auth` as they do not and will not support pydantic v2. We've decided to fork their code and adapt it as it's really light (see new folder `/antarest/fastapi_jwt_auth`) These changes also induced other minor dependencies bump: `jinja2`, `typing_extensions`, `PyJWT`, `python-multipart` Last, this work includes fixes on the API prefix addition when running in standalone mode (desktop version). We now distinguish properties root_path (used when behind a proxy) and api_prefix (which actually makes our server add a prefix). Co-authored-by: belthlemar <[email protected]> Co-authored-by: Sylvain Leclerc <[email protected]>
AntaresSimulatorTeam · Sep 26, 2024 · c3647ab · c3647ab
1 parent 5987cc3
commit c3647ab
Show file tree

Hide file tree

Showing 204 changed files with 3,415 additions and 3,797 deletions.
diff --git a/antarest/core/application.py b/antarest/core/application.py
@@ -0,0 +1,48 @@
+# Copyright (c) 2024, RTE (https://www.rte-france.com)
+#
+# See AUTHORS.txt
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This file is part of the Antares project.
+
+from dataclasses import dataclass
+from typing import Optional
+
+from fastapi import APIRouter, FastAPI
+
+
+@dataclass(frozen=True)
+class AppBuildContext:
+    """
+    Base elements of the application, for use at construction time:
+    - app: the actual fastapi application, where middlewares, exception handlers, etc. may be added
+    - api_root: the route under which all API and WS endpoints must be registered
+
+    API routes should not be added straight to app, but under api_root instead,
+    so that they are correctly prefixed if needed (/api for standalone mode).
+
+    Warning: the inclusion of api_root must happen AFTER all subroutes
+    have been registered, hence the build method.
+    """
+
+    app: FastAPI
+    api_root: APIRouter
+
+    def build(self) -> FastAPI:
+        """
+        Finalizes the app construction by including the API route.
+        Must be performed AFTER all subroutes have been added.
+        """
+        self.app.include_router(self.api_root)
+        return self.app
+
+
+def create_app_ctxt(app: FastAPI, api_root: Optional[APIRouter] = None) -> AppBuildContext:
+    if not api_root:
+        api_root = APIRouter()
+    return AppBuildContext(app, api_root)
diff --git a/antarest/core/cache/business/redis_cache.py b/antarest/core/cache/business/redis_cache.py
@@ -40,7 +40,7 @@ def put(self, id: str, data: JSON, duration: int = 3600) -> None:
         redis_element = RedisCacheElement(duration=duration, data=data)
         redis_key = f"cache:{id}"
         logger.info(f"Adding cache key {id}")
-        self.redis.set(redis_key, redis_element.json())
+        self.redis.set(redis_key, redis_element.model_dump_json())
         self.redis.expire(redis_key, duration)
 
     def get(self, id: str, refresh_timeout: Optional[int] = None) -> Optional[JSON]:

diff --git a/antarest/core/config.py b/antarest/core/config.py
@@ -13,6 +13,7 @@
 import multiprocessing
 import tempfile
 from dataclasses import asdict, dataclass, field
+from enum import Enum
 from pathlib import Path
 from typing import Dict, List, Optional
 
@@ -24,6 +25,12 @@
 DEFAULT_WORKSPACE_NAME = "default"
 
 
+class Launcher(str, Enum):
+    SLURM = "slurm"
+    LOCAL = "local"
+    DEFAULT = "default"
+
+
 @dataclass(frozen=True)
 class ExternalAuthConfig:
     """
@@ -399,7 +406,7 @@ def __post_init__(self) -> None:
         msg = f"Invalid configuration: {self.default=} must be one of {possible!r}"
         raise ValueError(msg)
 
-    def get_nb_cores(self, launcher: str) -> "NbCoresConfig":
+    def get_nb_cores(self, launcher: Launcher) -> "NbCoresConfig":
         """
         Retrieve the number of cores configuration for a given launcher: "local" or "slurm".
         If "default" is specified, retrieve the configuration of the default launcher.
@@ -416,12 +423,12 @@ def get_nb_cores(self, launcher: str) -> "NbCoresConfig":
         """
         config_map = {"local": self.local, "slurm": self.slurm}
         config_map["default"] = config_map[self.default]
-        launcher_config = config_map.get(launcher)
+        launcher_config = config_map.get(launcher.value)
         if launcher_config is None:
-            raise InvalidConfigurationError(launcher)
+            raise InvalidConfigurationError(launcher.value)
         return launcher_config.nb_cores
 
-    def get_time_limit(self, launcher: str) -> TimeLimitConfig:
+    def get_time_limit(self, launcher: Launcher) -> TimeLimitConfig:
         """
         Retrieve the time limit for a job of the given launcher: "local" or "slurm".
         If "default" is specified, retrieve the configuration of the default launcher.
@@ -438,7 +445,7 @@ def get_time_limit(self, launcher: str) -> TimeLimitConfig:
         """
         config_map = {"local": self.local, "slurm": self.slurm}
         config_map["default"] = config_map[self.default]
-        launcher_config = config_map.get(launcher)
+        launcher_config = config_map.get(launcher.value)
         if launcher_config is None:
             raise InvalidConfigurationError(launcher)
         return launcher_config.time_limit
@@ -586,6 +593,7 @@ class Config:
     cache: CacheConfig = CacheConfig()
     tasks: TaskConfig = TaskConfig()
     root_path: str = ""
+    api_prefix: str = ""
 
     @classmethod
     def from_dict(cls, data: JSON) -> "Config":
@@ -604,6 +612,7 @@ def from_dict(cls, data: JSON) -> "Config":
             cache=CacheConfig.from_dict(data["cache"]) if "cache" in data else defaults.cache,
             tasks=TaskConfig.from_dict(data["tasks"]) if "tasks" in data else defaults.tasks,
             root_path=data.get("root_path", defaults.root_path),
+            api_prefix=data.get("api_prefix", defaults.api_prefix),
         )
 
     @classmethod

diff --git a/antarest/core/core_blueprint.py b/antarest/core/core_blueprint.py
@@ -10,18 +10,14 @@
 #
 # This file is part of the Antares project.
 
-import logging
 from typing import Any
 
-from fastapi import APIRouter, Depends
+from fastapi import APIRouter
 from pydantic import BaseModel
 
 from antarest.core.config import Config
-from antarest.core.jwt import JWTUser
-from antarest.core.requests import UserHasNotPermissionError
 from antarest.core.utils.web import APITag
 from antarest.core.version_info import VersionInfoDTO, get_commit_id, get_dependencies
-from antarest.login.auth import Auth
 
 
 class StatusDTO(BaseModel):
@@ -36,7 +32,6 @@ def create_utils_routes(config: Config) -> APIRouter:
         config: main server configuration
     """
     bp = APIRouter()
-    auth = Auth(config)
 
     @bp.get("/health", tags=[APITag.misc], response_model=StatusDTO)
     def health() -> Any:
@@ -66,15 +61,4 @@ def version_info() -> Any:
             dependencies=get_dependencies(),
         )
 
-    @bp.get("/kill", include_in_schema=False)
-    def kill_worker(
-        current_user: JWTUser = Depends(auth.get_current_user),
-    ) -> Any:
-        if not current_user.is_site_admin():
-            raise UserHasNotPermissionError()
-        logging.getLogger(__name__).critical("Killing the worker")
-        # PyInstaller modifies the behavior of built-in functions, such as `exit`.
-        # It is advisable to use `sys.exit` or raise the `SystemExit` exception instead.
-        raise SystemExit(f"Worker killed by the user #{current_user.id}")
-
     return bp
diff --git a/antarest/core/filesystem_blueprint.py b/antarest/core/filesystem_blueprint.py
@@ -23,21 +23,21 @@
 
 import typing_extensions as te
 from fastapi import APIRouter, Depends, HTTPException
-from pydantic import BaseModel, Extra, Field
+from pydantic import BaseModel, Field
 from starlette.responses import PlainTextResponse, StreamingResponse
 
 from antarest.core.config import Config
 from antarest.core.utils.web import APITag
 from antarest.login.auth import Auth
 
-FilesystemName = te.Annotated[str, Field(regex=r"^\w+$", description="Filesystem name")]
-MountPointName = te.Annotated[str, Field(regex=r"^\w+$", description="Mount point name")]
+FilesystemName = te.Annotated[str, Field(pattern=r"^\w+$", description="Filesystem name")]
+MountPointName = te.Annotated[str, Field(pattern=r"^\w+$", description="Mount point name")]
 
 
 class FilesystemDTO(
     BaseModel,
-    extra=Extra.forbid,
-    schema_extra={
+    extra="forbid",
+    json_schema_extra={
         "example": {
             "name": "ws",
             "mount_dirs": {
@@ -62,8 +62,8 @@ class FilesystemDTO(
 
 class MountPointDTO(
     BaseModel,
-    extra=Extra.forbid,
-    schema_extra={
+    extra="forbid",
+    json_schema_extra={
         "example": {
             "name": "default",
             "path": "/path/to/workspaces/internal_studies",
@@ -89,10 +89,10 @@ class MountPointDTO(
 
     name: MountPointName
     path: Path = Field(description="Full path of the mount point in Antares Web Server")
-    total_bytes: int = Field(0, description="Total size of the mount point in bytes")
-    used_bytes: int = Field(0, description="Used size of the mount point in bytes")
-    free_bytes: int = Field(0, description="Free size of the mount point in bytes")
-    message: str = Field("", description="A message describing the status of the mount point")
+    total_bytes: int = Field(default=0, description="Total size of the mount point in bytes")
+    used_bytes: int = Field(default=0, description="Used size of the mount point in bytes")
+    free_bytes: int = Field(default=0, description="Free size of the mount point in bytes")
+    message: str = Field(default="", description="A message describing the status of the mount point")
 
     @classmethod
     async def from_path(cls, name: str, path: Path) -> "MountPointDTO":
@@ -110,8 +110,8 @@ async def from_path(cls, name: str, path: Path) -> "MountPointDTO":
 
 class FileInfoDTO(
     BaseModel,
-    extra=Extra.forbid,
-    schema_extra={
+    extra="forbid",
+    json_schema_extra={
         "example": {
             "path": "/path/to/workspaces/internal_studies/5a503c20-24a3-4734-9cf8-89565c9db5ec/study.antares",
             "file_type": "file",
@@ -142,12 +142,12 @@ class FileInfoDTO(
 
     path: Path = Field(description="Full path of the file or directory in Antares Web Server")
     file_type: str = Field(description="Type of the file or directory")
-    file_count: int = Field(1, description="Number of files and folders in the directory (1 for files)")
-    size_bytes: int = Field(0, description="Size of the file or total size of the directory in bytes")
+    file_count: int = Field(default=1, description="Number of files and folders in the directory (1 for files)")
+    size_bytes: int = Field(default=0, description="Size of the file or total size of the directory in bytes")
     created: datetime.datetime = Field(description="Creation date of the file or directory (local time)")
     modified: datetime.datetime = Field(description="Last modification date of the file or directory (local time)")
     accessed: datetime.datetime = Field(description="Last access date of the file or directory (local time)")
-    message: str = Field("OK", description="A message describing the status of the file")
+    message: str = Field(default="OK", description="A message describing the status of the file")
 
     @classmethod
     async def from_path(cls, full_path: Path, *, details: bool = False) -> "FileInfoDTO":
@@ -160,6 +160,7 @@ async def from_path(cls, full_path: Path, *, details: bool = False) -> "FileInfo
                 path=full_path,
                 file_type="unknown",
                 file_count=0,  # missing
+                size_bytes=0,  # missing
                 created=datetime.datetime.min,
                 modified=datetime.datetime.min,
                 accessed=datetime.datetime.min,
@@ -174,6 +175,7 @@ async def from_path(cls, full_path: Path, *, details: bool = False) -> "FileInfo
             created=datetime.datetime.fromtimestamp(file_stat.st_ctime),
             modified=datetime.datetime.fromtimestamp(file_stat.st_mtime),
             accessed=datetime.datetime.fromtimestamp(file_stat.st_atime),
+            message="OK",
         )
 
         if stat.S_ISDIR(file_stat.st_mode):

diff --git a/antarest/core/filetransfer/main.py b/antarest/core/filetransfer/main.py
@@ -12,8 +12,9 @@
 
 from typing import Optional
 
-from fastapi import FastAPI
+from fastapi import APIRouter, FastAPI
 
+from antarest.core.application import AppBuildContext
 from antarest.core.config import Config
 from antarest.core.filetransfer.repository import FileDownloadRepository
 from antarest.core.filetransfer.service import FileTransferManager
@@ -22,10 +23,10 @@
 
 
 def build_filetransfer_service(
-    application: Optional[FastAPI], event_bus: IEventBus, config: Config
+    app_ctxt: Optional[AppBuildContext], event_bus: IEventBus, config: Config
 ) -> FileTransferManager:
     ftm = FileTransferManager(repository=FileDownloadRepository(), event_bus=event_bus, config=config)
 
-    if application:
-        application.include_router(create_file_transfer_api(ftm, config))
+    if app_ctxt:
+        app_ctxt.api_root.include_router(create_file_transfer_api(ftm, config))
     return ftm
diff --git a/antarest/core/filetransfer/model.py b/antarest/core/filetransfer/model.py
@@ -41,7 +41,7 @@ class FileDownloadDTO(BaseModel):
     id: str
     name: str
     filename: str
-    expiration_date: Optional[str]
+    expiration_date: Optional[str] = None
     ready: bool
     failed: bool = False
     error_message: str = ""

diff --git a/antarest/core/maintenance/main.py b/antarest/core/maintenance/main.py
@@ -12,8 +12,9 @@
 
 from typing import Optional
 
-from fastapi import FastAPI
+from fastapi import APIRouter, FastAPI
 
+from antarest.core.application import AppBuildContext
 from antarest.core.config import Config
 from antarest.core.interfaces.cache import ICache
 from antarest.core.interfaces.eventbus import DummyEventBusService, IEventBus
@@ -23,15 +24,15 @@
 
 
 def build_maintenance_manager(
-    application: Optional[FastAPI],
+    app_ctxt: Optional[AppBuildContext],
     config: Config,
     cache: ICache,
     event_bus: IEventBus = DummyEventBusService(),
 ) -> MaintenanceService:
     repository = MaintenanceRepository()
     service = MaintenanceService(config, repository, event_bus, cache)
 
-    if application:
-        application.include_router(create_maintenance_api(service, config))
+    if app_ctxt:
+        app_ctxt.api_root.include_router(create_maintenance_api(service, config))
 
     return service
diff --git a/antarest/core/model.py b/antarest/core/model.py
@@ -21,7 +21,7 @@
 
 JSON = Dict[str, Any]
 ELEMENT = Union[str, int, float, bool, bytes]
-SUB_JSON = Union[ELEMENT, JSON, List, None]
+SUB_JSON = Union[ELEMENT, JSON, List[Any], None]
 
 
 class PublicMode(str, enum.Enum):

diff --git a/antarest/core/permissions.py b/antarest/core/permissions.py
@@ -11,6 +11,7 @@
 # This file is part of the Antares project.
 
 import logging
+import typing as t
 
 from antarest.core.jwt import JWTUser
 from antarest.core.model import PermissionInfo, PublicMode, StudyPermissionType
@@ -19,8 +20,8 @@
 logger = logging.getLogger(__name__)
 
 
-permission_matrix = {
-    StudyPermissionType.READ: {
+permission_matrix: t.Dict[str, t.Dict[str, t.Sequence[t.Union[RoleType, PublicMode]]]] = {
+    StudyPermissionType.READ.value: {
         "roles": [
             RoleType.ADMIN,
             RoleType.RUNNER,
@@ -34,15 +35,15 @@
             PublicMode.READ,
         ],
     },
-    StudyPermissionType.RUN: {
+    StudyPermissionType.RUN.value: {
         "roles": [RoleType.ADMIN, RoleType.RUNNER, RoleType.WRITER],
         "public_modes": [PublicMode.FULL, PublicMode.EDIT, PublicMode.EXECUTE],
     },
-    StudyPermissionType.WRITE: {
+    StudyPermissionType.WRITE.value: {
         "roles": [RoleType.ADMIN, RoleType.WRITER],
         "public_modes": [PublicMode.FULL, PublicMode.EDIT],
     },
-    StudyPermissionType.MANAGE_PERMISSIONS: {
+    StudyPermissionType.MANAGE_PERMISSIONS.value: {
         "roles": [RoleType.ADMIN],
         "public_modes": [],
     },
@@ -77,11 +78,11 @@ def check_permission(
 
     allowed_roles = permission_matrix[permission]["roles"]
     group_permission = any(
-        role in allowed_roles  # type: ignore
+        role in allowed_roles
         for role in [group.role for group in (user.groups or []) if group.id in permission_info.groups]
     )
     if group_permission:
         return True
 
     allowed_public_modes = permission_matrix[permission]["public_modes"]
-    return permission_info.public_mode in allowed_public_modes  # type: ignore
+    return permission_info.public_mode in allowed_public_modes