From c5aefd82bc633089419ebab5e0dbc8d706205001 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Tue, 21 Mar 2023 20:34:13 +0100 Subject: [PATCH 1/9] build: prepare next bugfix release v2.13.1 (unreleased) --- antarest/__init__.py | 4 ++-- docs/CHANGELOG.md | 6 ++++++ setup.py | 2 +- sonar-project.properties | 2 +- webapp/package.json | 2 +- 5 files changed, 11 insertions(+), 5 deletions(-) diff --git a/antarest/__init__.py b/antarest/__init__.py index 923dfadcf4..e25f88f074 100644 --- a/antarest/__init__.py +++ b/antarest/__init__.py @@ -7,9 +7,9 @@ # Standard project metadata -__version__ = "2.13.0" +__version__ = "2.13.1" __author__ = "RTE, Antares Web Team" -__date__ = "2023-03-09" +__date__ = "(unreleased)" # noinspection SpellCheckingInspection __credits__ = "(c) Réseau de Transport de l’Électricité (RTE)" diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 3a3924f0a9..1a5f2dfce0 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -4,6 +4,12 @@ Antares Web Changelog v2.13.0 (unreleased) -------------------- +### Bug Fixes + + +v2.13.0 (2023-03-09) +-------------------- + ### Features * **ui-common:** add doc link on subsections (#1241) ([1331232](https://github.com/AntaresSimulatorTeam/AntaREST/commit/1331232e418ebfbf3cc1a82725b95cb11cf8b9bc)) diff --git a/setup.py b/setup.py index 9f329df65e..e37c23a030 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="AntaREST", - version="2.13.0", + version="2.13.1", description="Antares Server", long_description=long_description, long_description_content_type="text/markdown", diff --git a/sonar-project.properties b/sonar-project.properties index 1f6f0c0f83..0b5c59021b 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -6,5 +6,5 @@ sonar.exclusions=antarest/gui.py,antarest/main.py sonar.python.coverage.reportPaths=coverage.xml sonar.python.version=3.8 sonar.javascript.lcov.reportPaths=webapp/coverage/lcov.info -sonar.projectVersion=2.13.0 +sonar.projectVersion=2.13.1 sonar.coverage.exclusions=antarest/gui.py,antarest/main.py,antarest/singleton_services.py,antarest/worker/archive_worker_service.py,webapp/**/* \ No newline at end of file diff --git a/webapp/package.json b/webapp/package.json index acb70e12b3..ebd23c387e 100644 --- a/webapp/package.json +++ b/webapp/package.json @@ -1,6 +1,6 @@ { "name": "antares-web", - "version": "2.13.0", + "version": "2.13.1", "private": true, "dependencies": { "@emotion/react": "11.10.6", From 6979e871dac39a34e76fe6a72b2ccf4502e8a288 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE <43534797+laurent-laporte-pro@users.noreply.github.com> Date: Wed, 22 Mar 2023 14:34:55 +0100 Subject: [PATCH 2/9] fix(desktop): use Antares Solver v8.5 for Antares Web Desktop version (#1414) --- scripts/package_antares_web.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/package_antares_web.sh b/scripts/package_antares_web.sh index b80a947bc9..59e6c63fb9 100755 --- a/scripts/package_antares_web.sh +++ b/scripts/package_antares_web.sh @@ -1,6 +1,6 @@ #!/bin/bash -ANTARES_SOLVER_VERSION="8.4" +ANTARES_SOLVER_VERSION="8.5" ANTARES_SOLVER_FULL_VERSION="$ANTARES_SOLVER_VERSION.1" ANTARES_SOLVER_FULL_VERSION_INT=$(echo $ANTARES_SOLVER_FULL_VERSION | sed 's/\.//g') From 101dd8c2a149c5112669d557d0851a9b1659d683 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE <43534797+laurent-laporte-pro@users.noreply.github.com> Date: Wed, 22 Mar 2023 14:44:17 +0100 Subject: [PATCH 3/9] fix(launcher): improved reliability of task state retrieval sent to SLUM (#1417) --- .../adapters/slurm_launcher/slurm_launcher.py | 47 +++++++++---------- requirements.txt | 2 +- 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/antarest/launcher/adapters/slurm_launcher/slurm_launcher.py b/antarest/launcher/adapters/slurm_launcher/slurm_launcher.py index ae247009f8..96609c6ded 100644 --- a/antarest/launcher/adapters/slurm_launcher/slurm_launcher.py +++ b/antarest/launcher/adapters/slurm_launcher/slurm_launcher.py @@ -334,35 +334,34 @@ def _import_xpansion_result(self, job_id: str, xpansion_mode: str) -> None: logger.warning("Output path in xpansion result not found") def _check_studies_state(self) -> None: - try: - with self.antares_launcher_lock: + with self.antares_launcher_lock: + try: self._call_launcher( arguments=self.launcher_args, parameters=self.launcher_params, ) - except Exception as e: - logger.info("Could not get data on remote server", exc_info=e) - - study_list = self.data_repo_tinydb.get_list_of_studies() - for study in study_list: - log_path = SlurmLauncher._get_log_path(study) - if study.with_error: - self.log_tail_manager.stop_tracking(log_path) - self._handle_failure(study) - elif study.done: - self.log_tail_manager.stop_tracking(log_path) - self._handle_success(study) - else: - # study.started => still running - # study.finished => waiting for ZIP + logs retrieval (or failure) - self.log_tail_manager.track( - log_path, self.create_update_log(study.name) - ) + except Exception as e: + logger.info("Could not get data on remote server", exc_info=e) + + study_list = self.data_repo_tinydb.get_list_of_studies() + for study in study_list: + log_path = SlurmLauncher._get_log_path(study) + if study.with_error: + self.log_tail_manager.stop_tracking(log_path) + self._handle_failure(study) + elif study.done: + self.log_tail_manager.stop_tracking(log_path) + self._handle_success(study) + else: + # study.started => still running + # study.finished => waiting for ZIP + logs retrieval (or failure) + self.log_tail_manager.track( + log_path, self.create_update_log(study.name) + ) - # Re-fetching the study list is necessary as new studies may have been added - # during the `import_output` process. Afterward, we clean up the list to ensure - # that any removed studies are removed from the database. - with self.antares_launcher_lock: + # Re-fetching the study list is necessary as new studies may have been added + # during the `import_output` process. Afterward, we clean up the list to ensure + # that any removed studies are removed from the database. # fmt: off cleanup_list = [s for s in study_list if s.with_error or s.done] for study in cleanup_list: diff --git a/requirements.txt b/requirements.txt index 06f220802c..500878c52b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Antares-Launcher~=1.2.2 +Antares-Launcher~=1.2.4 aiofiles~=0.8.0 alembic~=1.7.5 From 12bfa849e2232ea275851ad11407faf70bb91d2c Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE <43534797+laurent-laporte-pro@users.noreply.github.com> Date: Wed, 22 Mar 2023 14:46:40 +0100 Subject: [PATCH 4/9] fix(api): show Antares Launcher version in the `/version` end point (#1415) --- antares-launcher | 2 +- antarest/core/core_blueprint.py | 42 +++++++++++----------- antarest/core/utils/utils.py | 46 ++---------------------- antarest/core/version_info.py | 45 +++++++++++++++++++++++ tests/integration/test_core_blueprint.py | 20 +++++++++++ 5 files changed, 89 insertions(+), 66 deletions(-) create mode 100644 antarest/core/version_info.py create mode 100644 tests/integration/test_core_blueprint.py diff --git a/antares-launcher b/antares-launcher index 142879530e..ba92020341 160000 --- a/antares-launcher +++ b/antares-launcher @@ -1 +1 @@ -Subproject commit 142879530e1b181e9b20e26e2343b71ed3b604c2 +Subproject commit ba92020341f85c526f067aa0b6658b1ec4cef893 diff --git a/antarest/core/core_blueprint.py b/antarest/core/core_blueprint.py index 7b2fc425d6..40a2e4dba4 100644 --- a/antarest/core/core_blueprint.py +++ b/antarest/core/core_blueprint.py @@ -1,39 +1,26 @@ import logging -import subprocess -from pathlib import Path -from typing import Any, Optional +from typing import Any -from fastapi import APIRouter, Depends -from pydantic import BaseModel - -from antarest import __version__ from antarest.core.config import Config from antarest.core.jwt import JWTUser from antarest.core.requests import UserHasNotPermissionError -from antarest.core.utils.utils import get_commit_id from antarest.core.utils.web import APITag +from antarest.core.version_info import VersionInfoDTO, get_commit_id from antarest.login.auth import Auth +from fastapi import APIRouter, Depends +from pydantic import BaseModel class StatusDTO(BaseModel): status: str -class VersionDTO(BaseModel): - version: str - gitcommit: Optional[str] - - def create_utils_routes(config: Config) -> APIRouter: """ Utility endpoints Args: - storage_service: study service facade to handle request config: main server configuration - - Returns: - """ bp = APIRouter() auth = Auth(config) @@ -46,11 +33,24 @@ def health() -> Any: "/version", tags=[APITag.misc], summary="Get application version", - response_model=VersionDTO, + response_model=VersionInfoDTO, ) - def version() -> Any: - return VersionDTO( - version=__version__, gitcommit=get_commit_id(config.resources_path) + def version_info() -> Any: + """ + Returns the current version of the application, along with relevant dependency information. + + - `version`: The current version of the application. + - `gitcommit`: The commit ID of the current version's Git repository. + - `dependencies`: A dictionary of dependencies, where the key is + the dependency name and the value is its version number. + """ + from antareslauncher import __version__ as antares_launcher_version + from antarest import __version__ as antarest_version + + return VersionInfoDTO( + version=antarest_version, + gitcommit=get_commit_id(config.resources_path), + dependencies={"Antares_Launcher": antares_launcher_version}, ) @bp.get("/kill", include_in_schema=False) diff --git a/antarest/core/utils/utils.py b/antarest/core/utils/utils.py index ae56f8a874..0466052eec 100644 --- a/antarest/core/utils/utils.py +++ b/antarest/core/utils/utils.py @@ -1,37 +1,17 @@ import logging import os import shutil -import subprocess import tempfile import time from glob import escape from pathlib import Path -from typing import ( - IO, - Any, - Optional, - Callable, - TypeVar, - List, - Union, - Awaitable, - Tuple, -) -from zipfile import ( - ZipFile, - BadZipFile, - ZIP_STORED, - ZIP_DEFLATED, - ZIP_BZIP2, - ZIP_LZMA, -) +from typing import IO, Any, Callable, List, Optional, Tuple, TypeVar +from zipfile import ZIP_DEFLATED, BadZipFile, ZipFile import redis - from antarest.core.config import RedisConfig from antarest.core.exceptions import BadZipBinary, ShouldNotHappenException - logger = logging.getLogger(__name__) @@ -105,28 +85,6 @@ def get_local_path() -> Path: return filepath -def get_commit_id(path_resources: Path) -> Optional[str]: - commit_id = None - - path_commit_id = path_resources / "commit_id" - if path_commit_id.exists(): - commit_id = path_commit_id.read_text()[:-1] - else: - command = "git log -1 HEAD --format=%H" - process = subprocess.run(command, stdout=subprocess.PIPE, shell=True) - if process.returncode == 0: - commit_id = process.stdout.decode("utf-8") - - if commit_id is not None: - - def remove_carriage_return(value: str) -> str: - return value[:-1] - - commit_id = remove_carriage_return(commit_id) - - return commit_id - - def new_redis_instance(config: RedisConfig) -> redis.Redis: # type: ignore redis_client = redis.Redis( host=config.host, diff --git a/antarest/core/version_info.py b/antarest/core/version_info.py new file mode 100644 index 0000000000..88572704a3 --- /dev/null +++ b/antarest/core/version_info.py @@ -0,0 +1,45 @@ +""" +Python module that is dedicated to printing application version and dependencies information +""" +import subprocess +from pathlib import Path +from typing import Dict, Optional + +from pydantic import BaseModel + + +class VersionInfoDTO(BaseModel): + name: str = "AntaREST" + version: str + gitcommit: str + dependencies: Dict[str, str] + + +def get_commit_id(resources_dir: Path) -> str: + """ + Returns the contents of the file :file:`resources/commit_id` + if it exists and is not empty, or the commit ID of the current Git HEAD, if available. + If neither the commit ID nor the file is available, returns "". + + Note: + The :file:`commit_id` is generated during the "deploy" stage + in the :file:`.github/workflows/deploy.yml` GitHub workflow. + + Args: + resources_dir: The path to the ``resources`` directory. + + Returns: + The contents of the file :file:`resources/commit_id`, + the commit ID of the current Git HEAD, or "". + """ + path_commit_id = resources_dir.joinpath("commit_id") + try: + return path_commit_id.read_text(encoding="utf-8").strip() + except FileNotFoundError: + command = "git log -1 HEAD --format=%H" + try: + return subprocess.check_output( + command, encoding="utf-8", shell=True + ).strip() + except (subprocess.CalledProcessError, FileNotFoundError): + return "" diff --git a/tests/integration/test_core_blueprint.py b/tests/integration/test_core_blueprint.py new file mode 100644 index 0000000000..aaa6cb83e9 --- /dev/null +++ b/tests/integration/test_core_blueprint.py @@ -0,0 +1,20 @@ +from unittest import mock + +from fastapi import FastAPI +from http import HTTPStatus +from starlette.testclient import TestClient + + +class TestVersionInfo: + def test_version_info(self, app: FastAPI): + client = TestClient(app, raise_server_exceptions=False) + res = client.get("/version") + assert res.status_code == HTTPStatus.OK + actual = res.json() + expected = { + "name": "AntaREST", + "version": mock.ANY, + "gitcommit": mock.ANY, + "dependencies": {"Antares_Launcher": mock.ANY}, + } + assert actual == expected From 8f55667b52eea39a7d0e646811f16ef024afbbe0 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE <43534797+laurent-laporte-pro@users.noreply.github.com> Date: Wed, 22 Mar 2023 15:50:42 +0100 Subject: [PATCH 5/9] fix(desktop): use Antares Solver v8.5.0 for Antares Web Desktop version (#1419) --- scripts/package_antares_web.sh | 75 ++++++++++++++++------------------ 1 file changed, 35 insertions(+), 40 deletions(-) diff --git a/scripts/package_antares_web.sh b/scripts/package_antares_web.sh index 59e6c63fb9..54f6d14c9a 100755 --- a/scripts/package_antares_web.sh +++ b/scripts/package_antares_web.sh @@ -1,65 +1,60 @@ #!/bin/bash ANTARES_SOLVER_VERSION="8.5" -ANTARES_SOLVER_FULL_VERSION="$ANTARES_SOLVER_VERSION.1" +ANTARES_SOLVER_FULL_VERSION="8.5.1" ANTARES_SOLVER_FULL_VERSION_INT=$(echo $ANTARES_SOLVER_FULL_VERSION | sed 's/\.//g') - if [[ "$OSTYPE" == "msys"* ]]; then - ANTARES_SOLVER_FOLDER_NAME="rte-antares-$ANTARES_SOLVER_FULL_VERSION-installer-64bits" - ANTARES_SOLVER_ZIPFILE_NAME="$ANTARES_SOLVER_FOLDER_NAME.zip" + ANTARES_SOLVER_FOLDER_NAME="rte-antares-$ANTARES_SOLVER_FULL_VERSION-installer-64bits" + ANTARES_SOLVER_ZIPFILE_NAME="$ANTARES_SOLVER_FOLDER_NAME.zip" else - ANTARES_SOLVER_FOLDER_NAME="antares-$ANTARES_SOLVER_FULL_VERSION-Ubuntu-20.04" - ANTARES_SOLVER_ZIPFILE_NAME="$ANTARES_SOLVER_FOLDER_NAME.tar.gz" + ANTARES_SOLVER_FOLDER_NAME="antares-$ANTARES_SOLVER_FULL_VERSION-Ubuntu-20.04" + ANTARES_SOLVER_ZIPFILE_NAME="$ANTARES_SOLVER_FOLDER_NAME.tar.gz" fi LINK="https://github.com/AntaresSimulatorTeam/Antares_Simulator/releases/download/v$ANTARES_SOLVER_FULL_VERSION/$ANTARES_SOLVER_ZIPFILE_NAME" DESTINATION="../dist/AntaresWeb/antares_solver" echo "Downloading AntaresSimulator from $LINK" - wget $LINK +wget $LINK echo "Unzipping $ANTARES_SOLVER_ZIPFILE_NAME and move Antares solver to $DESTINATION" - 7z x $ANTARES_SOLVER_ZIPFILE_NAME - if [[ "$OSTYPE" != "msys"* ]]; then - 7z x "$ANTARES_SOLVER_FOLDER_NAME.tar" - fi - - mkdir $DESTINATION +7z x $ANTARES_SOLVER_ZIPFILE_NAME +if [[ "$OSTYPE" != "msys"* ]]; then + 7z x "$ANTARES_SOLVER_FOLDER_NAME.tar" +fi - if [[ "$OSTYPE" == "msys"* ]]; then - mv "$ANTARES_SOLVER_FOLDER_NAME/bin/antares-$ANTARES_SOLVER_VERSION-solver.exe" $DESTINATION - mv $ANTARES_SOLVER_FOLDER_NAME/bin/sirius_solver.dll $DESTINATION - mv $ANTARES_SOLVER_FOLDER_NAME/bin/zlib1.dll $DESTINATION - else - mv "$ANTARES_SOLVER_FOLDER_NAME/bin/antares-$ANTARES_SOLVER_VERSION-solver" $DESTINATION - mv "$ANTARES_SOLVER_FOLDER_NAME/bin/libsirius_solver.so" $DESTINATION - fi +mkdir $DESTINATION +if [[ "$OSTYPE" == "msys"* ]]; then + mv "$ANTARES_SOLVER_FOLDER_NAME/bin/antares-$ANTARES_SOLVER_VERSION-solver.exe" $DESTINATION + mv $ANTARES_SOLVER_FOLDER_NAME/bin/sirius_solver.dll $DESTINATION + mv $ANTARES_SOLVER_FOLDER_NAME/bin/zlib1.dll $DESTINATION +else + mv "$ANTARES_SOLVER_FOLDER_NAME/bin/antares-$ANTARES_SOLVER_VERSION-solver" $DESTINATION + mv "$ANTARES_SOLVER_FOLDER_NAME/bin/libsirius_solver.so" $DESTINATION +fi echo "Copy basic configuration files" - cp -r ../resources/deploy/* ../dist/ - if [[ "$OSTYPE" == "msys"* ]]; then - sed -i "s/700: path\/to\/700/$ANTARES_SOLVER_FULL_VERSION_INT: .\/AntaresWeb\/antares_solver\/antares-$ANTARES_SOLVER_VERSION-solver.exe/g" ../dist/config.yaml - else - sed -i "s/700: path\/to\/700/$ANTARES_SOLVER_FULL_VERSION_INT: .\/AntaresWeb\/antares_solver\/antares-$ANTARES_SOLVER_VERSION-solver/g" ../dist/config.yaml - fi - - +cp -r ../resources/deploy/* ../dist/ +if [[ "$OSTYPE" == "msys"* ]]; then + sed -i "s/700: path\/to\/700/$ANTARES_SOLVER_FULL_VERSION_INT: .\/AntaresWeb\/antares_solver\/antares-$ANTARES_SOLVER_VERSION-solver.exe/g" ../dist/config.yaml +else + sed -i "s/700: path\/to\/700/$ANTARES_SOLVER_FULL_VERSION_INT: .\/AntaresWeb\/antares_solver\/antares-$ANTARES_SOLVER_VERSION-solver/g" ../dist/config.yaml +fi echo "Creating shortcuts" - if [[ "$OSTYPE" == "msys"* ]]; then - cp ../resources/AntaresWebServerShortcut.lnk ../dist/ - else - ln -s ../dist/AntaresWeb/AntaresWebServer ../dist/AntaresWebServer - fi +if [[ "$OSTYPE" == "msys"* ]]; then + cp ../resources/AntaresWebServerShortcut.lnk ../dist/ +else + ln -s ../dist/AntaresWeb/AntaresWebServer ../dist/AntaresWebServer +fi echo "Unzipping example study" - cd ../dist/examples/studies - 7z x example_study.zip - rm example_study.zip +cd ../dist/examples/studies || exit +7z x example_study.zip +rm example_study.zip echo "Cleaning up" - rm $ANTARES_SOLVER_ZIPFILE_NAME - rm -rf $ANTARES_SOLVER_FOLDER_NAME - +rm $ANTARES_SOLVER_ZIPFILE_NAME +rm -rf $ANTARES_SOLVER_FOLDER_NAME From a2d0de073582070282131b3bcd346e6fbe7315ab Mon Sep 17 00:00:00 2001 From: MartinBelthle <102529366+MartinBelthle@users.noreply.github.com> Date: Thu, 23 Mar 2023 15:36:30 +0100 Subject: [PATCH 6/9] fix(api): better handling of exception to catch the stacktrace (#1422) --- antarest/eventbus/business/redis_eventbus.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/antarest/eventbus/business/redis_eventbus.py b/antarest/eventbus/business/redis_eventbus.py index 3a4f5871bb..40740d6f7d 100644 --- a/antarest/eventbus/business/redis_eventbus.py +++ b/antarest/eventbus/business/redis_eventbus.py @@ -35,8 +35,8 @@ def get_events(self) -> List[Event]: event = self.pubsub.get_message(ignore_subscribe_messages=True) if event is not None: return [Event.parse_raw(event["data"])] - except Exception as e: - logger.error("Failed to retrieve or parse event !", exc_info=e) + except Exception: + logger.error("Failed to retrieve or parse event !", exc_info=True) return [] From 35c4fa768999d49a228670bcceab1faaf5a648b6 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE <43534797+laurent-laporte-pro@users.noreply.github.com> Date: Wed, 29 Mar 2023 16:25:20 +0200 Subject: [PATCH 7/9] feat: standardize log messsage formats (#1431) --- antarest/core/logging/utils.py | 34 ++++-- antarest/gui.py | 68 ++++------- antarest/main.py | 210 ++++++++++++++++++++++----------- antarest/singleton_services.py | 8 +- antarest/utils.py | 17 +-- 5 files changed, 195 insertions(+), 142 deletions(-) diff --git a/antarest/core/logging/utils.py b/antarest/core/logging/utils.py index b0feb93ad2..56a6e00fc3 100644 --- a/antarest/core/logging/utils.py +++ b/antarest/core/logging/utils.py @@ -1,12 +1,11 @@ import logging import logging.config -import os import re import uuid -from pythonjsonlogger.jsonlogger import JsonFormatter from contextvars import ContextVar, Token -from typing import Optional, Type, Any, Dict +from typing import Any, Dict, Optional, Type +from antarest.core.config import Config from starlette.middleware.base import ( BaseHTTPMiddleware, RequestResponseEndpoint, @@ -14,8 +13,6 @@ from starlette.requests import Request from starlette.responses import Response -from antarest.core.config import Config - _request: ContextVar[Optional[Request]] = ContextVar("_request", default=None) _request_id: ContextVar[Optional[str]] = ContextVar( "_request_id", default=None @@ -41,11 +38,29 @@ def configure_logger(config: Config) -> None: "formatters": { "console": { "class": "antarest.core.logging.utils.CustomDefaultFormatter", - "format": "%(asctime)s - %(trace_id)s - %(threadName)s - %(name)s - %(ip)s - %(user)s - %(pid)s - %(levelname)s - %(message)s", + "format": ( + "[%(asctime)s] [%(process)s] [%(name)s]" + " - %(trace_id)s" + " - %(threadName)s" + " - %(ip)s" + " - %(user)s" + " - %(levelname)s" + " - %(message)s" + ), }, "json": { - "class": f"{JsonFormatter.__module__}.{JsonFormatter.__name__}", - "format": "%(asctime)s - %(trace_id)s - %(threadName)s - %(name)s - %(ip)s - %(user)s - %(pid)s - %(levelname)s - %(message)s", + "class": "pythonjsonlogger.jsonlogger.JsonFormatter", + "format": ( + "%(asctime)s" + " - %(process)s" + " - %(name)s" + " - %(trace_id)s" + " - %(threadName)s" + " - %(ip)s" + " - %(user)s" + " - %(levelname)s" + " - %(message)s" + ), }, }, "handlers": { @@ -110,9 +125,8 @@ def filter(self, record: logging.LogRecord) -> bool: request: Optional[Request] = _request.get() request_id: Optional[str] = _request_id.get() if request is not None: - record.ip = request.scope.get("client", ("undefined"))[0] + record.ip = request.scope.get("client", "undefined")[0] record.trace_id = request_id - record.pid = os.getpid() return True diff --git a/antarest/gui.py b/antarest/gui.py index d95ae37b39..56497e9ceb 100644 --- a/antarest/gui.py +++ b/antarest/gui.py @@ -1,6 +1,6 @@ +import contextlib import multiprocessing import platform -import sys import time import webbrowser from multiprocessing import Process @@ -8,21 +8,22 @@ import requests import uvicorn # type: ignore -from antarest import __version__ -from antarest.core.utils.utils import get_local_path -from antarest.main import fastapi_app, get_arguments from PyQt5.QtGui import QIcon from PyQt5.QtWidgets import QAction, QApplication, QMenu, QSystemTrayIcon +from antarest.core.utils.utils import get_local_path +from antarest.main import fastapi_app, parse_arguments + RESOURCE_PATH = get_local_path() / "resources" def run_server(config_file: Path) -> None: - app, _ = fastapi_app( + app = fastapi_app( config_file, mount_front=True, auto_upgrade_db=True, - ) + )[0] + # noinspection PyTypeChecker uvicorn.run(app, host="127.0.0.1", port=8080) @@ -30,21 +31,11 @@ def open_app() -> None: webbrowser.open("http://localhost:8080") -if __name__ == "__main__": +def main() -> None: multiprocessing.freeze_support() - ( - config_file, - display_version, - no_front, - auto_upgrade_db, - module, - ) = get_arguments() - - if display_version: - print(__version__) - sys.exit() - + arguments = parse_arguments() if platform.system() == "Windows": + # noinspection PyPackageRequirements from win10toast import ToastNotifier # type: ignore toaster = ToastNotifier() @@ -64,54 +55,41 @@ def open_app() -> None: app_icon=RESOURCE_PATH / "webapp" / "favicon.ico", timeout=600, ) - app = QApplication([]) app.setQuitOnLastWindowClosed(False) - # Adding an icon icon = QIcon(str(RESOURCE_PATH / "webapp" / "logo16.png")) - # Adding item on the menu bar tray = QSystemTrayIcon() tray.setIcon(icon) tray.setVisible(True) - # Creating the options menu = QMenu() - openapp = QAction("Open application") - menu.addAction(openapp) - openapp.triggered.connect(open_app) - + open_app_action = QAction("Open application") + menu.addAction(open_app_action) + open_app_action.triggered.connect(open_app) # To quit the app - quit = QAction("Quit") - quit.triggered.connect(app.quit) - menu.addAction(quit) - + quit_action = QAction("Quit") + quit_action.triggered.connect(app.quit) + menu.addAction(quit_action) # Adding options to the System Tray tray.setContextMenu(menu) app.processEvents() - tray.setToolTip("AntaresWebServer") - server = Process( target=run_server, - args=(config_file,), + args=(arguments.config_file,), ) server.start() - - countdown = 30 - server_started = False - while countdown > 0: - try: + for _ in range(30, 0, -1): + with contextlib.suppress(requests.ConnectionError): res = requests.get("http://localhost:8080") if res.status_code == 200: - server_started = True break - except requests.ConnectionError: - pass time.sleep(1) - countdown -= 1 - app.exec_() - server.kill() + + +if __name__ == "__main__": + main() diff --git a/antarest/main.py b/antarest/main.py index 217022f162..10d452c59a 100644 --- a/antarest/main.py +++ b/antarest/main.py @@ -1,11 +1,12 @@ import argparse +import copy import logging -import sys from pathlib import Path -from typing import Tuple, Any, Optional, Dict, cast +from typing import Any, Dict, Optional, Tuple, cast import sqlalchemy.ext.baked # type: ignore import uvicorn # type: ignore +import uvicorn.config # type: ignore from fastapi import FastAPI, HTTPException from fastapi_jwt_auth import AuthJWT # type: ignore from ratelimit import RateLimitMiddleware # type: ignore @@ -20,7 +21,7 @@ from antarest import __version__ from antarest.core.config import Config from antarest.core.core_blueprint import create_utils_routes -from antarest.core.logging.utils import configure_logger, LoggingMiddleware +from antarest.core.logging.utils import LoggingMiddleware, configure_logger from antarest.core.requests import RATE_LIMIT_CONFIG from antarest.core.swagger import customize_openapi from antarest.core.utils.utils import get_local_path @@ -33,83 +34,144 @@ from antarest.study.storage.auto_archive_service import AutoArchiveService from antarest.study.storage.rawstudy.watcher import Watcher from antarest.tools.admin_lib import clean_locks -from antarest.utils import ( - Module, - get_default_config_path_or_raise, - init_db, - create_services, -) +from antarest.utils import Module, create_services, init_db logger = logging.getLogger(__name__) +class PathType: + """file or directory path type for `argparse` parser + + The `PathType` class represents a type of argument that can be used + with the `argparse` library. + This class takes three boolean arguments, `exists`, `file_ok`, and `dir_ok`, + which specify whether the path argument must exist, whether it can be a file, + and whether it can be a directory, respectively. + + Example Usage: + + ```python + import argparse + from antarest.main import PathType + + parser = argparse.ArgumentParser() + parser.add_argument('--input', type=PathType(file_ok=True, exists=True)) + args = parser.parse_args() + + print(args.input) + ``` + + In the above example, `PathType` is used to specify the type of the `--input` + argument for the `argparse` parser. The argument must be an existing file path. + If the given path is not an existing file, the argparse library raises an error. + The Path object representing the given path is then printed to the console. + """ + + def __init__( + self, + exists: bool = False, + file_ok: bool = False, + dir_ok: bool = False, + ) -> None: + if not (file_ok or dir_ok): + msg = "Either `file_ok` or `dir_ok` must be set at a minimum." + raise ValueError(msg) + self.exists = exists + self.file_ok = file_ok + self.dir_ok = dir_ok + + def __call__(self, string: str) -> Path: + """ + Check whether the given string represents a valid path. + + If `exists` is `False`, the method simply returns the given path. + If `exists` is True, it checks whether the path exists and whether it is + a file or a directory, depending on the values of `file_ok` and `dir_ok`. + If the path exists and is of the correct type, the method returns the path; + otherwise, it raises an :class:`argparse.ArgumentTypeError` with an + appropriate error message. + + Args: + string: file or directory path + + Returns: + the file or directory path + + Raises + argparse.ArgumentTypeError: if the path is invalid + """ + file_path = Path(string).expanduser() + if not self.exists: + return file_path + if self.file_ok and self.dir_ok: + if file_path.exists(): + return file_path + msg = f"The file or directory path does not exist: '{file_path}'" + raise argparse.ArgumentTypeError(msg) + elif self.file_ok: + if file_path.is_file(): + return file_path + elif file_path.exists(): + msg = f"The path is not a regular file: '{file_path}'" + else: + msg = f"The file path does not exist: '{file_path}'" + raise argparse.ArgumentTypeError(msg) + elif self.dir_ok: + if file_path.is_dir(): + return file_path + elif file_path.exists(): + msg = f"The path is not a directory: '{file_path}'" + else: + msg = f"The directory path does not exist: '{file_path}'" + raise argparse.ArgumentTypeError(msg) + else: # pragma: no cover + raise NotImplementedError((self.file_ok, self.dir_ok)) + + def parse_arguments() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument( "-c", "--config", + type=PathType(exists=True, file_ok=True), dest="config_file", - help="path to the config file", + help="path to the config file [default: '%(default)s']", + default="./config.yaml", ) parser.add_argument( "-v", "--version", - dest="version", - help="Server version", - action="store_true", - required=False, + action="version", + help="Display the server version and exit", + version=__version__, ) parser.add_argument( "--no-front", dest="no_front", - help="Not embed the front build", + help="Exclude the embedding of the front-end build", action="store_true", - required=False, + default=False, ) parser.add_argument( "--auto-upgrade-db", dest="auto_upgrade_db", help="Automatically upgrade db", action="store_true", - required=False, + default=False, ) + # noinspection PyUnresolvedReferences parser.add_argument( "--module", + type=Module, dest="module", - help="Select a module to run (default is the application server)", + help="Select a module to run [default: application server]", choices=[mod.value for mod in Module], action="store", default=Module.APP.value, - required=False, ) return parser.parse_args() -def get_arguments() -> Tuple[Path, bool, bool, bool, Module]: - arguments = parse_arguments() - - display_version = arguments.version or False - if display_version: - return ( - Path("."), - display_version, - arguments.no_front, - arguments.auto_upgrade_db, - Module.APP, - ) - - config_file = Path( - arguments.config_file or get_default_config_path_or_raise() - ) - return ( - config_file, - display_version, - arguments.no_front, - arguments.auto_upgrade_db, - Module(arguments.module), - ) - - def fastapi_app( config_file: Path, resource_path: Optional[Path] = None, @@ -179,8 +241,8 @@ def home(request: Request) -> Any: @application.on_event("startup") def set_default_executor() -> None: - from concurrent.futures import ThreadPoolExecutor import asyncio + from concurrent.futures import ThreadPoolExecutor loop = asyncio.get_running_loop() loop.set_default_executor( @@ -273,27 +335,39 @@ def handle_all_exception(request: Request, exc: Exception) -> Any: return application, services -if __name__ == "__main__": - ( - config_file, - display_version, - no_front, - auto_upgrade_db, - module, - ) = get_arguments() - - if display_version: - print(__version__) - sys.exit() +LOGGING_CONFIG = copy.deepcopy(uvicorn.config.LOGGING_CONFIG) +LOGGING_CONFIG["formatters"]["default"]["fmt"] = ( + # fmt: off + "[%(asctime)s] [%(process)s]" + " %(levelprefix)s" + " %(message)s" + # fmt: on +) +LOGGING_CONFIG["formatters"]["access"]["fmt"] = ( + # fmt: off + "[%(asctime)s] [%(process)s] [%(name)s]" + " %(levelprefix)s" + " %(client_addr)s - \"%(request_line)s\"" + " %(status_code)s" + # fmt: on +) + + +def main() -> None: + arguments = parse_arguments() + if arguments.module == Module.APP: + clean_locks(arguments.config_file) + app = fastapi_app( + arguments.config_file, + mount_front=not arguments.no_front, + auto_upgrade_db=arguments.auto_upgrade_db, + )[0] + # noinspection PyTypeChecker + uvicorn.run(app, host="0.0.0.0", port=8080, log_config=LOGGING_CONFIG) else: - if module == Module.APP: - clean_locks(config_file) - app, _ = fastapi_app( - config_file, - mount_front=not no_front, - auto_upgrade_db=auto_upgrade_db, - ) - uvicorn.run(app, host="0.0.0.0", port=8080) - else: - services = SingletonServices(config_file, [module]) - services.start() + services = SingletonServices(arguments.config_file, [arguments.module]) + services.start() + + +if __name__ == "__main__": + main() diff --git a/antarest/singleton_services.py b/antarest/singleton_services.py index 6cf72bdc16..5805cffdf0 100644 --- a/antarest/singleton_services.py +++ b/antarest/singleton_services.py @@ -1,7 +1,7 @@ import logging import time from pathlib import Path -from typing import List, Dict, Any +from typing import Dict, List from antarest.core.config import Config from antarest.core.interfaces.service import IService @@ -10,12 +10,12 @@ from antarest.study.storage.auto_archive_service import AutoArchiveService from antarest.utils import ( Module, - init_db, - create_watcher, - create_matrix_gc, create_archive_worker, create_core_services, + create_matrix_gc, create_simulator_worker, + create_watcher, + init_db, ) logger = logging.getLogger(__name__) diff --git a/antarest/utils.py b/antarest/utils.py index 68319d9cf2..4256e2d453 100644 --- a/antarest/utils.py +++ b/antarest/utils.py @@ -1,7 +1,7 @@ import logging from enum import Enum from pathlib import Path -from typing import Tuple, Any, Optional, Dict +from typing import Any, Dict, Optional, Tuple import redis import sqlalchemy.ext.baked # type: ignore @@ -26,11 +26,7 @@ from antarest.core.tasks.main import build_taskjob_manager from antarest.core.tasks.service import ITaskService from antarest.core.utils.fastapi_sqlalchemy import DBSessionMiddleware -from antarest.core.utils.utils import ( - get_default_config_path, - get_local_path, - new_redis_instance, -) +from antarest.core.utils.utils import get_local_path, new_redis_instance from antarest.eventbus.main import build_eventbus from antarest.launcher.main import build_launcher from antarest.login.main import build_login @@ -61,15 +57,6 @@ class Module(str, Enum): SIMULATOR_WORKER = "simulator_worker" -def get_default_config_path_or_raise() -> Path: - config_path = get_default_config_path() - if not config_path: - raise ValueError( - "Config file not found. Set it by '-c' with command line or place it at ./config.yaml, ../config.yaml or ~/.antares/config.yaml" - ) - return config_path - - def init_db( config_file: Path, config: Config, From 425c5f082e15e1cc94c221494b8c8ccb13632cee Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Wed, 5 Apr 2023 16:03:15 +0200 Subject: [PATCH 8/9] docs: update change log to prepare the release --- docs/CHANGELOG.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 1a5f2dfce0..fd3b9b3539 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,11 +1,23 @@ Antares Web Changelog ===================== -v2.13.0 (unreleased) +v2.13.1 (unreleased) -------------------- ### Bug Fixes +* **desktop:** use Antares Solver v8.5 for Antares Web Desktop version (#1414) ([6979e87](https://github.com/AntaresSimulatorTeam/AntaREST/commit/6979e871dac39a34e76fe6a72b2ccf4502e8a288)) +* **launcher:** improved reliability of task state retrieval sent to SLUM (#1417) ([101dd8c](https://github.com/AntaresSimulatorTeam/AntaREST/commit/101dd8c2a149c5112669d557d0851a9b1659d683)) +* **api:** show Antares Launcher version in the `/version` end point (#1415) ([12bfa84](https://github.com/AntaresSimulatorTeam/AntaREST/commit/12bfa849e2232ea275851ad11407faf70bb91d2c)) +* **desktop:** use Antares Solver v8.5.0 for Antares Web Desktop version (#1419) ([8f55667](https://github.com/AntaresSimulatorTeam/AntaREST/commit/8f55667b52eea39a7d0e646811f16ef024afbbe0)) +* **api:** better handling of exception to catch the stacktrace (#1422) ([a2d0de0](https://github.com/AntaresSimulatorTeam/AntaREST/commit/a2d0de073582070282131b3bcd346e6fbe7315ab)) + + +### Contributors + +Laurent LAPORTE, and +MartinBelthle + v2.13.0 (2023-03-09) -------------------- @@ -68,7 +80,7 @@ v2.13.0 (2023-03-09) ### Contributors hdinia, -laurent-laporte-pro, +Laurent LAPORTE, pl-buiquang, skamril, MartinBelthle From e19cae82e50894c9eddf39971fb8a7faf1190ab1 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Tue, 11 Apr 2023 11:10:02 +0200 Subject: [PATCH 9/9] build: new hotfix release v2.13.1 (2023-04-11) --- antarest/__init__.py | 2 +- docs/CHANGELOG.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/antarest/__init__.py b/antarest/__init__.py index e25f88f074..7ae1001d6a 100644 --- a/antarest/__init__.py +++ b/antarest/__init__.py @@ -9,7 +9,7 @@ __version__ = "2.13.1" __author__ = "RTE, Antares Web Team" -__date__ = "(unreleased)" +__date__ = "2023-04-11" # noinspection SpellCheckingInspection __credits__ = "(c) Réseau de Transport de l’Électricité (RTE)" diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index fd3b9b3539..f97a85131e 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,7 +1,7 @@ Antares Web Changelog ===================== -v2.13.1 (unreleased) +v2.13.1 (2023-04-11) -------------------- ### Bug Fixes