Skip to content

Commit

Permalink
v2.13.1 (#1418)
Browse files Browse the repository at this point in the history
  • Loading branch information
laurent-laporte-pro authored and skamril committed Apr 11, 2023
2 parents e7c9ceb + e19cae8 commit f0757df
Show file tree
Hide file tree
Showing 19 changed files with 371 additions and 283 deletions.
2 changes: 1 addition & 1 deletion antares-launcher
Submodule antares-launcher updated 82 files
+40 −0 .github/workflows/binary.yml
+33 −0 .github/workflows/python-package.yml
+42 −0 .github/workflows/python-publish.yml
+0 −8 .gitignore
+101 −0 CHANGES.md
+23 −2 MANIFEST.in
+6 −12 Makefile
+114 −48 README.md
+0 −0 __init__.py
+47 −6 antareslauncher/__init__.py
+12 −15 antareslauncher/advanced_launch.py
+12 −16 antareslauncher/basic_launch.py
+313 −0 antareslauncher/config.py
+15 −32 antareslauncher/data_repo/data_repo_tinydb.py
+0 −82 antareslauncher/definitions.py
+74 −0 antareslauncher/exceptions.py
+10 −8 antareslauncher/main.py
+1 −1 antareslauncher/main_option_parser.py
+16 −19 antareslauncher/parameters_reader.py
+0 −140 antareslauncher/remote_environnement/iconnection.py
+0 −74 antareslauncher/remote_environnement/iremote_environment.py
+332 −204 antareslauncher/remote_environnement/remote_environment_with_slurm.py
+1 −38 antareslauncher/remote_environnement/slurm_script_features.py
+310 −99 antareslauncher/remote_environnement/ssh_connection.py
+1 −1 antareslauncher/study_dto.py
+1 −3 antareslauncher/use_cases/check_remote_queue/check_queue_controller.py
+4 −4 antareslauncher/use_cases/check_remote_queue/slurm_queue_show.py
+11 −25 antareslauncher/use_cases/create_list/study_list_composer.py
+3 −3 antareslauncher/use_cases/kill_job/job_kill_controller.py
+4 −6 antareslauncher/use_cases/launch/launch_controller.py
+3 −3 antareslauncher/use_cases/launch/study_submitter.py
+3 −3 antareslauncher/use_cases/launch/study_zip_uploader.py
+1 −1 antareslauncher/use_cases/launch/study_zipper.py
+6 −4 antareslauncher/use_cases/retrieve/clean_remote_server.py
+58 −38 antareslauncher/use_cases/retrieve/download_final_zip.py
+13 −18 antareslauncher/use_cases/retrieve/log_downloader.py
+6 −10 antareslauncher/use_cases/retrieve/retrieve_controller.py
+4 −3 antareslauncher/use_cases/retrieve/state_updater.py
+3 −9 antareslauncher/use_cases/retrieve/study_retriever.py
+10 −0 data/.gitignore
+130 −0 data/README.md
+0 −18 data/configuration.yaml
+0 −8 data/default_ssh_connection_template.json
+0 −9 doc/Makefile
+16 −12 doc/README.md
+1 −1 doc/source/_templates/layout.html
+1 −0 doc/source/api/.gitignore
+11 −2 doc/source/binary_generation.md
+1 −12 doc/source/changelog.rst
+2 −2 doc/source/conf.py
+55 −0 doc/source/development_guide.rst
+0 −55 doc/source/developper_documentation.rst
+5 −18 doc/source/index.rst
+0 −35 doc/source/introduction.rst
+36 −34 doc/source/user_guide.rst
+0 −1 pytest.ini
+32 −0 requirements-docs.txt
+13 −0 requirements-test.txt
+10 −18 requirements.txt
+89 −13 setup.py
+0 −31 standards.md
+7 −0 tests/data/__init__.py
+0 −0 tests/end-to-end/__init__.py
+0 −102 tests/end-to-end/test_end_to_end.py
+1 −1 tests/integration/test_integration_check_queue_controller.py
+2 −1 tests/integration/test_integration_job_kill_controller.py
+34 −12 tests/integration/test_integration_launch_controller.py
+1 −10 tests/integration/test_integration_study_list_composer.py
+7 −9 tests/unit/launcher/test_launch_controller.py
+3 −3 tests/unit/launcher/test_submitter.py
+4 −4 tests/unit/launcher/test_zip_uploader.py
+15 −13 tests/unit/retriever/test_download_final_zip.py
+10 −5 tests/unit/retriever/test_log_downloader.py
+6 −5 tests/unit/retriever/test_retrieve_controller.py
+5 −2 tests/unit/retriever/test_server_cleaner.py
+6 −12 tests/unit/retriever/test_study_retriever.py
+401 −0 tests/unit/test_config.py
+27 −23 tests/unit/test_file_manager.py
+17 −13 tests/unit/test_parameters_reader.py
+364 −562 tests/unit/test_remote_environment_with_slurm.py
+149 −0 tests/unit/test_ssh_connection.py
+13 −17 tests/unit/test_study_list_composer.py
4 changes: 2 additions & 2 deletions antarest/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@

# Standard project metadata

__version__ = "2.13.0"
__version__ = "2.13.1"
__author__ = "RTE, Antares Web Team"
__date__ = "2023-03-09"
__date__ = "2023-04-11"
# noinspection SpellCheckingInspection
__credits__ = "(c) Réseau de Transport de l’Électricité (RTE)"

Expand Down
42 changes: 21 additions & 21 deletions antarest/core/core_blueprint.py
Original file line number Diff line number Diff line change
@@ -1,39 +1,26 @@
import logging
import subprocess
from pathlib import Path
from typing import Any, Optional
from typing import Any

from fastapi import APIRouter, Depends
from pydantic import BaseModel

from antarest import __version__
from antarest.core.config import Config
from antarest.core.jwt import JWTUser
from antarest.core.requests import UserHasNotPermissionError
from antarest.core.utils.utils import get_commit_id
from antarest.core.utils.web import APITag
from antarest.core.version_info import VersionInfoDTO, get_commit_id
from antarest.login.auth import Auth
from fastapi import APIRouter, Depends
from pydantic import BaseModel


class StatusDTO(BaseModel):
status: str


class VersionDTO(BaseModel):
version: str
gitcommit: Optional[str]


def create_utils_routes(config: Config) -> APIRouter:
"""
Utility endpoints
Args:
storage_service: study service facade to handle request
config: main server configuration
Returns:
"""
bp = APIRouter()
auth = Auth(config)
Expand All @@ -46,11 +33,24 @@ def health() -> Any:
"/version",
tags=[APITag.misc],
summary="Get application version",
response_model=VersionDTO,
response_model=VersionInfoDTO,
)
def version() -> Any:
return VersionDTO(
version=__version__, gitcommit=get_commit_id(config.resources_path)
def version_info() -> Any:
"""
Returns the current version of the application, along with relevant dependency information.
- `version`: The current version of the application.
- `gitcommit`: The commit ID of the current version's Git repository.
- `dependencies`: A dictionary of dependencies, where the key is
the dependency name and the value is its version number.
"""
from antareslauncher import __version__ as antares_launcher_version
from antarest import __version__ as antarest_version

return VersionInfoDTO(
version=antarest_version,
gitcommit=get_commit_id(config.resources_path),
dependencies={"Antares_Launcher": antares_launcher_version},
)

@bp.get("/kill", include_in_schema=False)
Expand Down
34 changes: 24 additions & 10 deletions antarest/core/logging/utils.py
Original file line number Diff line number Diff line change
@@ -1,21 +1,18 @@
import logging
import logging.config
import os
import re
import uuid
from pythonjsonlogger.jsonlogger import JsonFormatter
from contextvars import ContextVar, Token
from typing import Optional, Type, Any, Dict
from typing import Any, Dict, Optional, Type

from antarest.core.config import Config
from starlette.middleware.base import (
BaseHTTPMiddleware,
RequestResponseEndpoint,
)
from starlette.requests import Request
from starlette.responses import Response

from antarest.core.config import Config

_request: ContextVar[Optional[Request]] = ContextVar("_request", default=None)
_request_id: ContextVar[Optional[str]] = ContextVar(
"_request_id", default=None
Expand All @@ -41,11 +38,29 @@ def configure_logger(config: Config) -> None:
"formatters": {
"console": {
"class": "antarest.core.logging.utils.CustomDefaultFormatter",
"format": "%(asctime)s - %(trace_id)s - %(threadName)s - %(name)s - %(ip)s - %(user)s - %(pid)s - %(levelname)s - %(message)s",
"format": (
"[%(asctime)s] [%(process)s] [%(name)s]"
" - %(trace_id)s"
" - %(threadName)s"
" - %(ip)s"
" - %(user)s"
" - %(levelname)s"
" - %(message)s"
),
},
"json": {
"class": f"{JsonFormatter.__module__}.{JsonFormatter.__name__}",
"format": "%(asctime)s - %(trace_id)s - %(threadName)s - %(name)s - %(ip)s - %(user)s - %(pid)s - %(levelname)s - %(message)s",
"class": "pythonjsonlogger.jsonlogger.JsonFormatter",
"format": (
"%(asctime)s"
" - %(process)s"
" - %(name)s"
" - %(trace_id)s"
" - %(threadName)s"
" - %(ip)s"
" - %(user)s"
" - %(levelname)s"
" - %(message)s"
),
},
},
"handlers": {
Expand Down Expand Up @@ -110,9 +125,8 @@ def filter(self, record: logging.LogRecord) -> bool:
request: Optional[Request] = _request.get()
request_id: Optional[str] = _request_id.get()
if request is not None:
record.ip = request.scope.get("client", ("undefined"))[0]
record.ip = request.scope.get("client", "undefined")[0]
record.trace_id = request_id
record.pid = os.getpid()
return True


Expand Down
46 changes: 2 additions & 44 deletions antarest/core/utils/utils.py
Original file line number Diff line number Diff line change
@@ -1,37 +1,17 @@
import logging
import os
import shutil
import subprocess
import tempfile
import time
from glob import escape
from pathlib import Path
from typing import (
IO,
Any,
Optional,
Callable,
TypeVar,
List,
Union,
Awaitable,
Tuple,
)
from zipfile import (
ZipFile,
BadZipFile,
ZIP_STORED,
ZIP_DEFLATED,
ZIP_BZIP2,
ZIP_LZMA,
)
from typing import IO, Any, Callable, List, Optional, Tuple, TypeVar
from zipfile import ZIP_DEFLATED, BadZipFile, ZipFile

import redis

from antarest.core.config import RedisConfig
from antarest.core.exceptions import BadZipBinary, ShouldNotHappenException


logger = logging.getLogger(__name__)


Expand Down Expand Up @@ -105,28 +85,6 @@ def get_local_path() -> Path:
return filepath


def get_commit_id(path_resources: Path) -> Optional[str]:
commit_id = None

path_commit_id = path_resources / "commit_id"
if path_commit_id.exists():
commit_id = path_commit_id.read_text()[:-1]
else:
command = "git log -1 HEAD --format=%H"
process = subprocess.run(command, stdout=subprocess.PIPE, shell=True)
if process.returncode == 0:
commit_id = process.stdout.decode("utf-8")

if commit_id is not None:

def remove_carriage_return(value: str) -> str:
return value[:-1]

commit_id = remove_carriage_return(commit_id)

return commit_id


def new_redis_instance(config: RedisConfig) -> redis.Redis: # type: ignore
redis_client = redis.Redis(
host=config.host,
Expand Down
45 changes: 45 additions & 0 deletions antarest/core/version_info.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
"""
Python module that is dedicated to printing application version and dependencies information
"""
import subprocess
from pathlib import Path
from typing import Dict, Optional

from pydantic import BaseModel


class VersionInfoDTO(BaseModel):
name: str = "AntaREST"
version: str
gitcommit: str
dependencies: Dict[str, str]


def get_commit_id(resources_dir: Path) -> str:
"""
Returns the contents of the file :file:`resources/commit_id`
if it exists and is not empty, or the commit ID of the current Git HEAD, if available.
If neither the commit ID nor the file is available, returns "".
Note:
The :file:`commit_id` is generated during the "deploy" stage
in the :file:`.github/workflows/deploy.yml` GitHub workflow.
Args:
resources_dir: The path to the ``resources`` directory.
Returns:
The contents of the file :file:`resources/commit_id`,
the commit ID of the current Git HEAD, or "".
"""
path_commit_id = resources_dir.joinpath("commit_id")
try:
return path_commit_id.read_text(encoding="utf-8").strip()
except FileNotFoundError:
command = "git log -1 HEAD --format=%H"
try:
return subprocess.check_output(
command, encoding="utf-8", shell=True
).strip()
except (subprocess.CalledProcessError, FileNotFoundError):
return ""
4 changes: 2 additions & 2 deletions antarest/eventbus/business/redis_eventbus.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,8 @@ def get_events(self) -> List[Event]:
event = self.pubsub.get_message(ignore_subscribe_messages=True)
if event is not None:
return [Event.parse_raw(event["data"])]
except Exception as e:
logger.error("Failed to retrieve or parse event !", exc_info=e)
except Exception:
logger.error("Failed to retrieve or parse event !", exc_info=True)

return []

Expand Down
Loading

0 comments on commit f0757df

Please sign in to comment.