diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 51e26d2b89..dce98f7665 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -25,7 +25,7 @@ jobs: with: options: --check --diff - name: Check Typing (mypy) - continue-on-error: true + #continue-on-error: true run: | mypy --install-types --non-interactive mypy diff --git a/antarest/__init__.py b/antarest/__init__.py index 19d92fe27e..2f31bd9d5d 100644 --- a/antarest/__init__.py +++ b/antarest/__init__.py @@ -1,4 +1,4 @@ -__version__ = "2.1.5" +__version__ = "2.2.0" from pathlib import Path diff --git a/antarest/core/cache/business/redis_cache.py b/antarest/core/cache/business/redis_cache.py index 3faddd4d3f..8234ee5c76 100644 --- a/antarest/core/cache/business/redis_cache.py +++ b/antarest/core/cache/business/redis_cache.py @@ -17,7 +17,7 @@ class RedisCacheElement(BaseModel): class RedisCache(ICache): - def __init__(self, redis_client: Redis): + def __init__(self, redis_client: Redis): # type: ignore self.redis = redis_client def start(self) -> None: diff --git a/antarest/core/cache/main.py b/antarest/core/cache/main.py index a6836cf30b..6fbc518a77 100644 --- a/antarest/core/cache/main.py +++ b/antarest/core/cache/main.py @@ -12,7 +12,7 @@ def build_cache( - config: Config, redis_client: Optional[Redis] = None + config: Config, redis_client: Optional[Redis] = None # type: ignore ) -> ICache: cache = ( RedisCache(redis_client) diff --git a/antarest/core/config.py b/antarest/core/config.py index 92a37f89da..6a29ad2ecf 100644 --- a/antarest/core/config.py +++ b/antarest/core/config.py @@ -131,7 +131,7 @@ class SlurmConfig: password: str = "" default_wait_time: int = 0 default_time_limit: int = 0 - default_n_cpu: int = 0 + default_n_cpu: int = 1 default_json_db_name: str = "" slurm_script_path: str = "" antares_versions_on_remote_server: List[str] = field( diff --git a/antarest/core/exceptions.py b/antarest/core/exceptions.py index 8a9835581c..28cc3ac60d 100644 --- a/antarest/core/exceptions.py +++ b/antarest/core/exceptions.py @@ -32,6 +32,11 @@ def __init__(self, message: str) -> None: super().__init__(HTTPStatus.EXPECTATION_FAILED, message) +class CommandApplicationError(HTTPException): + def __init__(self, message: str) -> None: + super().__init__(HTTPStatus.INTERNAL_SERVER_ERROR, message) + + class CommandUpdateAuthorizationError(HTTPException): def __init__(self, message: str) -> None: super().__init__(HTTPStatus.LOCKED, message) diff --git a/antarest/core/persistence.py b/antarest/core/persistence.py index bdb49f0739..777a0de0b0 100644 --- a/antarest/core/persistence.py +++ b/antarest/core/persistence.py @@ -3,9 +3,9 @@ from io import StringIO from pathlib import Path -from alembic import command # type: ignore -from alembic.config import Config # type: ignore -from alembic.util import CommandError # type: ignore +from alembic import command +from alembic.config import Config +from alembic.util import CommandError from sqlalchemy.ext.declarative import declarative_base # type: ignore from antarest.core.utils.utils import get_local_path @@ -18,7 +18,7 @@ def upgrade_db(config_file: Path) -> None: os.environ.setdefault("ANTAREST_CONF", str(config_file)) - alembic_cfg = Config(get_local_path() / "alembic.ini") + alembic_cfg = Config(str(get_local_path() / "alembic.ini")) alembic_cfg.stdout = StringIO() alembic_cfg.set_main_option( "script_location", str(get_local_path() / "alembic") @@ -35,7 +35,7 @@ def upgrade_db(config_file: Path) -> None: raise e alembic_cfg.stdout = StringIO() - command.heads(alembic_cfg) + command.heads(alembic_cfg) # type: ignore head_output = alembic_cfg.stdout.getvalue() head = head_output.split(" ")[0].strip() if current_version != head: diff --git a/antarest/core/requests.py b/antarest/core/requests.py index b224b4de4a..6ba7b879c5 100644 --- a/antarest/core/requests.py +++ b/antarest/core/requests.py @@ -2,7 +2,7 @@ from typing import Optional from fastapi import HTTPException -from markupsafe import escape +from markupsafe import escape # type: ignore from antarest.core.jwt import JWTUser diff --git a/antarest/core/tasks/model.py b/antarest/core/tasks/model.py index 311feba38c..7c8c520c15 100644 --- a/antarest/core/tasks/model.py +++ b/antarest/core/tasks/model.py @@ -132,7 +132,7 @@ def to_dto(self, with_logs: bool = False) -> TaskDTO: if self.completion_date else None, logs=sorted( - [log.to_dto() for log in self.logs], key=lambda l: l.id + [log.to_dto() for log in self.logs], key=lambda l: l.id # type: ignore ) if with_logs else None, diff --git a/antarest/core/utils/utils.py b/antarest/core/utils/utils.py index a185d8d545..1a66f69964 100644 --- a/antarest/core/utils/utils.py +++ b/antarest/core/utils/utils.py @@ -76,7 +76,7 @@ def get_local_path() -> Path: return filepath -def new_redis_instance(config: RedisConfig) -> redis.Redis: +def new_redis_instance(config: RedisConfig) -> redis.Redis: # type: ignore return redis.Redis(host=config.host, port=config.port, db=0) diff --git a/antarest/eventbus/business/redis_eventbus.py b/antarest/eventbus/business/redis_eventbus.py index c5b692cc6c..bdbfc8b317 100644 --- a/antarest/eventbus/business/redis_eventbus.py +++ b/antarest/eventbus/business/redis_eventbus.py @@ -14,7 +14,7 @@ class RedisEventBus(IEventBusBackend): - def __init__(self, redis_client: Redis) -> None: + def __init__(self, redis_client: Redis) -> None: # type: ignore self.redis = redis_client self.pubsub = self.redis.pubsub() self.pubsub.subscribe(REDIS_STORE_KEY) diff --git a/antarest/eventbus/main.py b/antarest/eventbus/main.py index 1578aa9984..cff096fee3 100644 --- a/antarest/eventbus/main.py +++ b/antarest/eventbus/main.py @@ -15,7 +15,7 @@ def build_eventbus( application: FastAPI, config: Config, autostart: bool = True, - redis_client: Optional[Redis] = None, + redis_client: Optional[Redis] = None, # type: ignore ) -> IEventBus: eventbus = EventBusService( diff --git a/antarest/eventbus/service.py b/antarest/eventbus/service.py index 5bec4ca9c1..b65dae5a1c 100644 --- a/antarest/eventbus/service.py +++ b/antarest/eventbus/service.py @@ -22,7 +22,6 @@ def __init__( self.start() def push(self, event: Event) -> None: - # TODO add arg permissions with group/role, user, public self.backend.push_event(event) def add_listener( diff --git a/antarest/eventbus/web.py b/antarest/eventbus/web.py index 40e055591f..e4a1eae185 100644 --- a/antarest/eventbus/web.py +++ b/antarest/eventbus/web.py @@ -13,7 +13,7 @@ from antarest.core.config import Config from antarest.core.interfaces.eventbus import IEventBus, Event from antarest.core.jwt import JWTUser, DEFAULT_ADMIN_USER -from antarest.core.model import PermissionInfo, StudyPermissionType +from antarest.core.model import PermissionInfo, StudyPermissionType, PublicMode from antarest.core.permissions import check_permission from antarest.login.auth import Auth @@ -82,7 +82,7 @@ async def broadcast( self, message: str, permissions: PermissionInfo, channel: Optional[str] ) -> None: for connection in self.active_connections: - if check_permission( + if channel is not None or check_permission( connection.user, permissions, StudyPermissionType.READ ): if ( @@ -119,7 +119,6 @@ async def connect( raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED) user = Auth.get_user_from_token(token, jwt_manager) if user is None: - # TODO check auth and subscribe to rooms raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED) except Exception as e: logger.error( diff --git a/antarest/gui.py b/antarest/gui.py index ca52563d98..8fede8b9d3 100644 --- a/antarest/gui.py +++ b/antarest/gui.py @@ -64,11 +64,11 @@ def open_app() -> None: menu = QMenu() openapp = QAction("Open application") menu.addAction(openapp) - openapp.triggered.connect(open_app) + openapp.triggered.connect(open_app) # type: ignore # To quit the app quit = QAction("Quit") - quit.triggered.connect(app.quit) + quit.triggered.connect(app.quit) # type: ignore menu.addAction(quit) # Adding options to the System Tray diff --git a/antarest/launcher/adapters/abstractlauncher.py b/antarest/launcher/adapters/abstractlauncher.py index a313ea1094..df3f48b8ba 100644 --- a/antarest/launcher/adapters/abstractlauncher.py +++ b/antarest/launcher/adapters/abstractlauncher.py @@ -1,8 +1,9 @@ from abc import ABC, abstractmethod -from typing import Callable, NamedTuple, Optional +from typing import Callable, NamedTuple, Optional, Any from uuid import UUID from antarest.core.config import Config +from antarest.core.model import JSON from antarest.core.requests import RequestParameters from antarest.launcher.model import JobStatus, LogType from antarest.study.service import StudyService @@ -31,7 +32,11 @@ def __init__( @abstractmethod def run_study( - self, study_uuid: str, version: str, params: RequestParameters + self, + study_uuid: str, + version: str, + launcher_parameters: Optional[JSON], + params: RequestParameters, ) -> UUID: raise NotImplementedError() diff --git a/antarest/launcher/adapters/local_launcher/local_launcher.py b/antarest/launcher/adapters/local_launcher/local_launcher.py index f291e43811..b91b3f10f7 100644 --- a/antarest/launcher/adapters/local_launcher/local_launcher.py +++ b/antarest/launcher/adapters/local_launcher/local_launcher.py @@ -5,6 +5,7 @@ from uuid import UUID, uuid4 from antarest.core.config import Config +from antarest.core.model import JSON from antarest.core.requests import RequestParameters from antarest.core.utils.fastapi_sqlalchemy import db from antarest.launcher.adapters.abstractlauncher import ( @@ -31,7 +32,11 @@ def __init__( self.job_id_to_study_id: Dict[str, str] = {} def run_study( - self, study_uuid: str, version: str, params: RequestParameters + self, + study_uuid: str, + version: str, + launcher_parameters: Optional[JSON], + params: RequestParameters, ) -> UUID: if self.config.launcher.local is None: raise LauncherInitException() diff --git a/antarest/launcher/adapters/slurm_launcher/slurm_launcher.py b/antarest/launcher/adapters/slurm_launcher/slurm_launcher.py index bcda5b4093..3db5a56259 100644 --- a/antarest/launcher/adapters/slurm_launcher/slurm_launcher.py +++ b/antarest/launcher/adapters/slurm_launcher/slurm_launcher.py @@ -24,6 +24,7 @@ EventChannelDirectory, ) from antarest.core.jwt import DEFAULT_ADMIN_USER +from antarest.core.model import JSON from antarest.core.requests import RequestParameters from antarest.core.utils.fastapi_sqlalchemy import db from antarest.launcher.adapters.abstractlauncher import ( @@ -39,6 +40,11 @@ logging.getLogger("paramiko").setLevel("WARN") +MAX_NB_CPU = 24 +MAX_TIME_LIMIT = 604800 +MIN_TIME_LIMIT = 3600 + + class VersionNotSupportedError(Exception): pass @@ -155,14 +161,45 @@ def _delete_study(self, study_path: Path) -> None: if study_path.exists(): shutil.rmtree(study_path) - def _import_study_output(self, job_id: str) -> Optional[str]: + def _import_study_output( + self, job_id: str, xpansion_mode: bool = False + ) -> Optional[str]: study_id = self.job_id_to_study_id[job_id] + if xpansion_mode: + self._import_xpansion_result(job_id, study_id) return self.storage_service.import_output( study_id, self.slurm_config.local_workspace / "OUTPUT" / job_id / "output", params=RequestParameters(DEFAULT_ADMIN_USER), ) + def _import_xpansion_result(self, job_id: str, study_id: str) -> None: + output_path = ( + self.slurm_config.local_workspace / "OUTPUT" / job_id / "output" + ) + if output_path.exists() and len(os.listdir(output_path)) == 1: + output_path = output_path / os.listdir(output_path)[0] + shutil.copytree( + self.slurm_config.local_workspace + / "OUTPUT" + / job_id + / "input" + / "links", + output_path / "updated_links", + ) + study = self.storage_service.get_study(study_id) + if int(study.version) < 800: + shutil.copytree( + self.slurm_config.local_workspace + / "OUTPUT" + / job_id + / "user" + / "expansion", + output_path / "results", + ) + else: + logger.warning("Output path in xpansion result not found") + def _check_studies_state(self) -> None: try: run_with( @@ -191,11 +228,13 @@ def _check_studies_state(self) -> None: with db(): output_id: Optional[str] = None if not study.with_error: - output_id = self._import_study_output(study.name) + output_id = self._import_study_output( + study.name, study.xpansion_study + ) self.callbacks.update_status( study.name, JobStatus.FAILED - if study.with_error + if study.with_error or output_id is None else JobStatus.SUCCESS, None, output_id, @@ -283,7 +322,11 @@ def _clean_up_study(self, launch_id: str) -> None: del self.job_id_to_study_id[launch_id] def _run_study( - self, study_uuid: str, launch_uuid: str, params: RequestParameters + self, + study_uuid: str, + launch_uuid: str, + launcher_params: Optional[JSON], + params: RequestParameters, ) -> None: with db(): study_path = Path(self.launcher_args.studies_in) / str(launch_uuid) @@ -301,8 +344,11 @@ def _run_study( self._assert_study_version_is_supported(study_uuid, params) + launcher_args = self._check_and_apply_launcher_params( + launcher_params + ) run_with( - self.launcher_args, self.launcher_params, show_banner=False + launcher_args, self.launcher_params, show_banner=False ) self.callbacks.update_status( str(launch_uuid), JobStatus.RUNNING, None, None @@ -322,13 +368,47 @@ def _run_study( self._delete_study(study_path) + def _check_and_apply_launcher_params( + self, launcher_params: Optional[JSON] + ) -> argparse.Namespace: + if launcher_params: + launcher_args = deepcopy(self.launcher_args) + if launcher_params.get("xpansion", False): + launcher_args.xpansion_mode = True + time_limit = launcher_params.get("time_limit", None) + if time_limit and isinstance(time_limit, int): + if MIN_TIME_LIMIT < time_limit < MAX_TIME_LIMIT: + launcher_args.time_limit = time_limit + else: + logger.warning( + f"Invalid slurm launcher time limit ({time_limit}), should be between {MIN_TIME_LIMIT} and {MAX_TIME_LIMIT}" + ) + post_processing = launcher_params.get("post_processing", False) + if isinstance(post_processing, bool): + launcher_args.post_processing = post_processing + nb_cpu = launcher_params.get("nb_cpu", None) + if nb_cpu and isinstance(nb_cpu, int): + if 0 < nb_cpu <= MAX_NB_CPU: + launcher_args.n_cpu = nb_cpu + else: + logger.warning( + f"Invalid slurm launcher nb_cpu ({nb_cpu}), should be between 1 and 24" + ) + return launcher_args + return self.launcher_args + def run_study( - self, study_uuid: str, version: str, params: RequestParameters + self, + study_uuid: str, + version: str, + launcher_parameters: Optional[JSON], + params: RequestParameters, ) -> UUID: # TODO: version ? launch_uuid = uuid4() thread = threading.Thread( - target=self._run_study, args=(study_uuid, launch_uuid, params) + target=self._run_study, + args=(study_uuid, launch_uuid, launcher_parameters, params), ) thread.start() @@ -351,5 +431,12 @@ def kill_job(self, job_id: str) -> None: launcher_args, self.launcher_params, show_banner=False ) return - - raise JobIdNotFound() + logger.warning( + "Failed to retrieve job id in antares launcher database" + ) + self.callbacks.update_status( + job_id, + JobStatus.FAILED, + None, + None, + ) diff --git a/antarest/launcher/service.py b/antarest/launcher/service.py index f019189bd4..8409cfb1f6 100644 --- a/antarest/launcher/service.py +++ b/antarest/launcher/service.py @@ -25,6 +25,7 @@ from antarest.core.model import ( StudyPermissionType, PermissionInfo, + JSON, ) from antarest.study.storage.utils import ( assert_permission, @@ -102,7 +103,11 @@ def _assert_launcher_is_initialized(self, launcher: str) -> None: raise LauncherServiceNotAvailableException(launcher) def run_study( - self, study_uuid: str, params: RequestParameters, launcher: str + self, + study_uuid: str, + launcher: str, + launcher_parameters: Optional[JSON], + params: RequestParameters, ) -> UUID: study_info = self.study_service.get_study_information( uuid=study_uuid, params=params @@ -112,7 +117,7 @@ def run_study( self._assert_launcher_is_initialized(launcher) job_uuid: UUID = self.launchers[launcher].run_study( - study_uuid, str(study_version), params + study_uuid, str(study_version), launcher_parameters, params ) job_status = JobResult( diff --git a/antarest/launcher/web.py b/antarest/launcher/web.py index 35c63ee2bd..cb9ff40059 100644 --- a/antarest/launcher/web.py +++ b/antarest/launcher/web.py @@ -6,6 +6,7 @@ from antarest.core.config import Config from antarest.core.jwt import JWTUser +from antarest.core.model import JSON from antarest.core.requests import RequestParameters from antarest.core.utils.web import APITag from antarest.launcher.model import ( @@ -35,10 +36,12 @@ def create_launcher_api(service: LauncherService, config: Config) -> APIRouter: def run( study_id: str, engine: Optional[str] = None, + engine_parameters: Optional[JSON] = None, current_user: JWTUser = Depends(auth.get_current_user), ) -> Any: logger.info( - f"Launching study {study_id}", extra={"user": current_user.id} + f"Launching study {study_id} with options {engine_parameters}", + extra={"user": current_user.id}, ) selected_engine = ( engine if engine is not None else config.launcher.default @@ -46,7 +49,11 @@ def run( params = RequestParameters(user=current_user) return JobCreationDTO( - job_id=str(service.run_study(study_id, params, selected_engine)) + job_id=str( + service.run_study( + study_id, selected_engine, engine_parameters, params + ) + ) ) @bp.get( diff --git a/antarest/login/web.py b/antarest/login/web.py index 6e1571c2d0..e9a92cad6a 100644 --- a/antarest/login/web.py +++ b/antarest/login/web.py @@ -5,9 +5,8 @@ from fastapi import Depends, APIRouter, HTTPException from fastapi_jwt_auth import AuthJWT # type: ignore -from markupsafe import escape +from markupsafe import escape # type: ignore from pydantic import BaseModel -from starlette.responses import JSONResponse from antarest.core.config import Config from antarest.core.jwt import JWTUser, JWTGroup diff --git a/antarest/matrixstore/model.py b/antarest/matrixstore/model.py index 8af183f604..de8bb06c75 100644 --- a/antarest/matrixstore/model.py +++ b/antarest/matrixstore/model.py @@ -150,7 +150,7 @@ def __eq__(self, other: Any) -> bool: # https://github.com/samuelcolvin/pydantic/issues/1423 # https://github.com/samuelcolvin/pydantic/issues/1599 # https://github.com/samuelcolvin/pydantic/issues/1930 -# TODO maybe we should reverting to only float because Any cause problem retrieving data from a node will have pandas forcing all to float anyway... +# Reverting to only float because Any cause problem retrieving data from a node will have pandas forcing all to float anyway... # this cause matrix dump on disk (and then hash id) to be different for basically the same matrices MatrixData = float diff --git a/antarest/study/common/studystorage.py b/antarest/study/common/studystorage.py index 908cf24c54..375e61d595 100644 --- a/antarest/study/common/studystorage.py +++ b/antarest/study/common/studystorage.py @@ -4,12 +4,16 @@ from antarest.core.exceptions import StudyNotFoundError from antarest.core.model import JSON +from antarest.core.requests import RequestParameters from antarest.study.model import ( Study, StudySimResultDTO, StudyMetadataDTO, StudyMetadataPatchDTO, ) +from antarest.study.storage.rawstudy.model.filesystem.config.model import ( + FileStudyTreeConfigDTO, +) from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy T = TypeVar("T", bound=Study) @@ -112,12 +116,12 @@ def get_study_information( raise NotImplementedError() @abstractmethod - def get_raw(self, metadata: T) -> FileStudy: + def get_raw(self, metadata: T, use_cache: bool = True) -> FileStudy: """ Fetch a study raw tree object and its config Args: metadata: study - + use_cache: use cache Returns: the config and study tree object """ @@ -247,3 +251,17 @@ def export_study_flat( """ raise NotImplementedError() + + @abstractmethod + def get_synthesis( + self, metadata: T, params: Optional[RequestParameters] = None + ) -> FileStudyTreeConfigDTO: + """ + Return study synthesis + Args: + metadata: study + params: RequestParameters + Returns: FileStudyTreeConfigDTO + + """ + raise NotImplementedError() diff --git a/antarest/study/model.py b/antarest/study/model.py index 621abc99a8..68c64fa249 100644 --- a/antarest/study/model.py +++ b/antarest/study/model.py @@ -24,13 +24,12 @@ STUDY_REFERENCE_TEMPLATES: Dict[str, str] = { "600": "empty_study_613.zip", - "613": "empty_study_613.zip", + "610": "empty_study_613.zip", "640": "empty_study_613.zip", "700": "empty_study_700.zip", "710": "empty_study_710.zip", "720": "empty_study_720.zip", "800": "empty_study_803.zip", - "803": "empty_study_803.zip", "810": "empty_study_810.zip", } diff --git a/antarest/study/service.py b/antarest/study/service.py index 7920e553e4..3373334cbf 100644 --- a/antarest/study/service.py +++ b/antarest/study/service.py @@ -1,3 +1,4 @@ +import base64 import io import logging import os @@ -9,7 +10,7 @@ from uuid import uuid4 from fastapi import HTTPException -from markupsafe import escape +from markupsafe import escape # type: ignore from antarest.core.config import Config from antarest.core.filetransfer.model import ( @@ -21,6 +22,7 @@ StudyTypeUnsupported, UnsupportedOperationOnArchivedStudy, NotAManagedStudyException, + CommandApplicationError, ) from antarest.core.interfaces.cache import ICache, CacheConstants from antarest.core.interfaces.eventbus import IEventBus, Event, EventType @@ -62,6 +64,9 @@ PatchStudy, ) from antarest.study.repository import StudyMetadataRepository +from antarest.study.storage.rawstudy.model.filesystem.config.model import ( + FileStudyTreeConfigDTO, +) from antarest.study.storage.rawstudy.model.filesystem.ini_file_node import ( IniFileNode, ) @@ -93,6 +98,9 @@ from antarest.study.storage.variantstudy.model.command.update_config import ( UpdateConfig, ) +from antarest.study.storage.variantstudy.model.command.update_raw_file import ( + UpdateRawFile, +) from antarest.study.storage.variantstudy.model.dbmodel import VariantStudy from antarest.study.storage.variantstudy.variant_study_service import ( VariantStudyService, @@ -453,6 +461,24 @@ def create_study( ) return str(raw.id) + def get_study_synthesis( + self, study_id: str, params: RequestParameters + ) -> FileStudyTreeConfigDTO: + """ + Return study synthesis + Args: + study_id: study id + params: request parameters + + Returns: study synthesis + + """ + study = self.get_study(study_id) + assert_permission(params.user, study, StudyPermissionType.READ) + return self._get_study_storage_service(study).get_synthesis( + study, params + ) + def remove_duplicates(self) -> None: study_paths: Dict[str, List[str]] = {} for study in self.repository.get_all(): @@ -998,15 +1024,19 @@ def _create_edit_study_command( else data, command_context=self.variant_study_service.command_factory.command_context, ) - elif ( - isinstance(tree_node, RawFileNode) - and url.split("/")[-1] == "comments" - ): - return UpdateComments( - target=url, - comments=data, - command_context=self.variant_study_service.command_factory.command_context, - ) + elif isinstance(tree_node, RawFileNode): + if url.split("/")[-1] == "comments": + return UpdateComments( + target=url, + comments=data, + command_context=self.variant_study_service.command_factory.command_context, + ) + elif isinstance(data, bytes): + return UpdateRawFile( + target=url, + b64Data=base64.b64encode(data).decode("utf-8"), + command_context=self.variant_study_service.command_factory.command_context, + ) raise NotImplementedError() def _edit_study_using_command( @@ -1028,9 +1058,11 @@ def _edit_study_using_command( tree_node=tree_node, url=url, data=data ) if isinstance(study_service, RawStudyService): - command.apply(study_data=file_study) + res = command.apply(study_data=file_study) if not is_managed(study): tree_node.denormalize() + if not res.status: + raise CommandApplicationError(res.message) lastsave_url = "study/antares/lastsave" lastsave_node = file_study.tree.get_node(lastsave_url.split("/")) diff --git a/antarest/study/storage/abstract_storage_service.py b/antarest/study/storage/abstract_storage_service.py index c53391fab1..a96b4c7216 100644 --- a/antarest/study/storage/abstract_storage_service.py +++ b/antarest/study/storage/abstract_storage_service.py @@ -2,7 +2,7 @@ import os import shutil import tempfile -from abc import abstractmethod +from abc import abstractmethod, ABC from datetime import datetime from pathlib import Path from typing import List, Union, Optional, IO @@ -12,6 +12,7 @@ from antarest.core.model import JSON, PublicMode from antarest.core.exceptions import BadOutputError, StudyOutputNotFoundError from antarest.core.interfaces.cache import CacheConstants, ICache +from antarest.core.requests import RequestParameters from antarest.core.utils.utils import extract_zip, StopWatch from antarest.login.model import GroupDTO from antarest.study.common.studystorage import IStudyStorageService, T @@ -30,6 +31,7 @@ from antarest.study.storage.rawstudy.model.filesystem.config.model import ( Simulation, FileStudyTreeConfig, + FileStudyTreeConfigDTO, ) from antarest.study.storage.rawstudy.model.filesystem.factory import ( StudyFactory, @@ -40,7 +42,7 @@ logger = logging.getLogger(__name__) -class AbstractStorageService(IStudyStorageService[T]): +class AbstractStorageService(IStudyStorageService[T], ABC): def __init__( self, config: Config, @@ -96,6 +98,14 @@ def get_study_information( ) raw_study = self.study_factory.create_from_config(config) file_metadata = raw_study.get(url=["study", "antares"]) + study_version = str( + file_metadata.get("version", study.version) + ) + if study_version != study.version: + logger.warning( + f"Study version in file ({study_version}) is different from the one stored in db ({study.version}), returning file version" + ) + study.version = study_version file_settings = raw_study.get( url=["settings", "generaldata", "general"] ) @@ -348,107 +358,3 @@ def export_output(self, metadata: T, output_id: str, target: Path) -> Path: ) ) return target.parent / filename - - @abstractmethod - def export_study_flat( - self, - metadata: T, - dest: Path, - outputs: bool = True, - denormalize: bool = True, - ) -> None: - raise NotImplementedError() - - @abstractmethod - def create(self, metadata: T) -> T: - """ - Create empty new study - Args: - metadata: study information - - Returns: new study information - - """ - raise NotImplementedError() - - @abstractmethod - def exists(self, metadata: T) -> bool: - """ - Check study exist. - Args: - metadata: study - - Returns: true if study presents in disk, false else. - - """ - raise NotImplementedError() - - @abstractmethod - def copy( - self, src_meta: T, dest_name: str, with_outputs: bool = False - ) -> T: - """ - Copy study to a new destination - Args: - src_meta: source study - dest_meta: destination study - with_outputs: indicate either to copy the output or not - - Returns: destination study - - """ - raise NotImplementedError() - - @abstractmethod - def get_raw(self, metadata: T, use_cache: bool = True) -> FileStudy: - """ - Fetch a study raw tree object and its config - Args: - metadata: study - use_cache: indicate if the cache should be used - - Returns: the config and study tree object - - """ - raise NotImplementedError() - - @abstractmethod - def set_reference_output( - self, metadata: T, output_id: str, status: bool - ) -> None: - """ - Set an output to the reference output of a study - Args: - metadata: study - output_id: the id of output to set the reference status - status: true to set it as reference, false to unset it - - Returns: - - """ - raise NotImplementedError() - - @abstractmethod - def delete(self, metadata: T) -> None: - """ - Delete study - Args: - metadata: study - - Returns: - - """ - raise NotImplementedError() - - @abstractmethod - def delete_output(self, metadata: T, output_id: str) -> None: - """ - Delete a simulation output - Args: - metadata: study - output_id: output simulation - - Returns: - - """ - raise NotImplementedError() diff --git a/antarest/study/storage/rawstudy/model/filesystem/bucket_node.py b/antarest/study/storage/rawstudy/model/filesystem/bucket_node.py index ac9a190bd2..f9b6db0071 100644 --- a/antarest/study/storage/rawstudy/model/filesystem/bucket_node.py +++ b/antarest/study/storage/rawstudy/model/filesystem/bucket_node.py @@ -23,16 +23,26 @@ def save( data: Union[str, int, bool, float, bytes, JSON], url: Optional[List[str]] = None, ) -> None: - assert isinstance(data, Dict) - for key, value in data.items(): - if isinstance(value, (str, bytes)): - RawFileNode(self.context, self.config.next_file(key)).save( - value - ) - elif isinstance(value, dict): + if url is None or len(url) == 0: + assert isinstance(data, Dict) + for key, value in data.items(): + self._save(value, key) + else: + key = url[0] + if len(url) > 1: BucketNode(self.context, self.config.next_file(key)).save( - value + data, url[1:] ) + else: + self._save(data, key) + + def _save( + self, data: Union[str, int, bool, float, bytes, JSON], key: str + ) -> None: + if isinstance(data, (str, bytes)): + RawFileNode(self.context, self.config.next_file(key)).save(data) + elif isinstance(data, dict): + BucketNode(self.context, self.config.next_file(key)).save(data) def build(self) -> TREE: if not self.config.path.exists(): diff --git a/antarest/study/storage/rawstudy/model/filesystem/common/prepro.py b/antarest/study/storage/rawstudy/model/filesystem/common/prepro.py index df060deffa..5c7ad5f3b3 100644 --- a/antarest/study/storage/rawstudy/model/filesystem/common/prepro.py +++ b/antarest/study/storage/rawstudy/model/filesystem/common/prepro.py @@ -14,9 +14,6 @@ from antarest.study.storage.rawstudy.model.filesystem.matrix.input_series_matrix import ( InputSeriesMatrix, ) -from antarest.study.storage.rawstudy.model.filesystem.raw_file_node import ( - RawFileNode, -) class PreproCorrelation(IniFileNode): diff --git a/antarest/study/storage/rawstudy/model/filesystem/raw_file_node.py b/antarest/study/storage/rawstudy/model/filesystem/raw_file_node.py index dae60c1bdf..111ff3bc76 100644 --- a/antarest/study/storage/rawstudy/model/filesystem/raw_file_node.py +++ b/antarest/study/storage/rawstudy/model/filesystem/raw_file_node.py @@ -7,7 +7,6 @@ from antarest.study.storage.rawstudy.model.filesystem.context import ( ContextServer, ) -from antarest.study.storage.rawstudy.model.filesystem.inode import TREE from antarest.study.storage.rawstudy.model.filesystem.lazy_node import ( LazyNode, ) diff --git a/antarest/study/storage/rawstudy/raw_study_service.py b/antarest/study/storage/rawstudy/raw_study_service.py index 716e8e6a12..ccb76377c5 100644 --- a/antarest/study/storage/rawstudy/raw_study_service.py +++ b/antarest/study/storage/rawstudy/raw_study_service.py @@ -11,6 +11,8 @@ StudyDeletionNotAllowed, ) from antarest.core.interfaces.cache import ICache +from antarest.core.model import JSON +from antarest.core.requests import RequestParameters from antarest.core.utils.utils import extract_zip from antarest.study.model import ( RawStudy, @@ -21,6 +23,9 @@ AbstractStorageService, ) from antarest.study.storage.patch_service import PatchService +from antarest.study.storage.rawstudy.model.filesystem.config.model import ( + FileStudyTreeConfigDTO, +) from antarest.study.storage.rawstudy.model.filesystem.factory import ( StudyFactory, FileStudy, @@ -111,7 +116,7 @@ def get_raw(self, metadata: RawStudy, use_cache: bool = True) -> FileStudy: Fetch a study object and its config Args: metadata: study - + use_cache: use cache Returns: the config and study tree object """ @@ -122,6 +127,16 @@ def get_raw(self, metadata: RawStudy, use_cache: bool = True) -> FileStudy: ) return FileStudy(config=study_config, tree=study_tree) + def get_synthesis( + self, metadata: RawStudy, params: Optional[RequestParameters] = None + ) -> FileStudyTreeConfigDTO: + self._check_study_exists(metadata) + study_path = self.get_study_path(metadata) + study_config, _ = self.study_factory.create_from_fs( + study_path, metadata.id + ) + return FileStudyTreeConfigDTO.from_build_config(study_config) + def create(self, metadata: RawStudy) -> RawStudy: """ Create empty new study diff --git a/antarest/study/storage/rawstudy/watcher.py b/antarest/study/storage/rawstudy/watcher.py index 4cb9c2e014..c5c5b95b44 100644 --- a/antarest/study/storage/rawstudy/watcher.py +++ b/antarest/study/storage/rawstudy/watcher.py @@ -6,7 +6,7 @@ from time import time, sleep from typing import List -from filelock import FileLock # type: ignore +from filelock import FileLock from antarest.core.config import Config from antarest.core.utils.fastapi_sqlalchemy import db diff --git a/antarest/study/storage/variantstudy/business/matrix_constants_generator.py b/antarest/study/storage/variantstudy/business/matrix_constants_generator.py index 9c47e6c4ec..4e26d895be 100644 --- a/antarest/study/storage/variantstudy/business/matrix_constants_generator.py +++ b/antarest/study/storage/variantstudy/business/matrix_constants_generator.py @@ -1,6 +1,6 @@ from typing import Dict -from filelock import FileLock # type: ignore +from filelock import FileLock from antarest.matrixstore.service import ISimpleMatrixService from antarest.study.storage.variantstudy.business import matrix_constants diff --git a/antarest/study/storage/variantstudy/command_factory.py b/antarest/study/storage/variantstudy/command_factory.py index dc05b9a784..caa5a9910c 100644 --- a/antarest/study/storage/variantstudy/command_factory.py +++ b/antarest/study/storage/variantstudy/command_factory.py @@ -52,6 +52,9 @@ from antarest.study.storage.variantstudy.model.command.update_config import ( UpdateConfig, ) +from antarest.study.storage.variantstudy.model.command.update_raw_file import ( + UpdateRawFile, +) from antarest.study.storage.variantstudy.model.command_context import ( CommandContext, ) @@ -160,6 +163,11 @@ def _to_single_icommand(self, action: str, args: JSON) -> ICommand: **args, command_context=self.command_context, ) + elif action == CommandName.UPDATE_FILE.value: + return UpdateRawFile( + **args, + command_context=self.command_context, + ) raise NotImplementedError() def to_icommand(self, command_dto: CommandDTO) -> List[ICommand]: diff --git a/antarest/study/storage/variantstudy/model/command/common.py b/antarest/study/storage/variantstudy/model/command/common.py index 448a733944..4b0b68a23c 100644 --- a/antarest/study/storage/variantstudy/model/command/common.py +++ b/antarest/study/storage/variantstudy/model/command/common.py @@ -41,3 +41,4 @@ class CommandName(Enum): REPLACE_MATRIX = "replace_matrix" UPDATE_CONFIG = "update_config" UPDATE_COMMENTS = "update_comments" + UPDATE_FILE = "update_file" diff --git a/antarest/study/storage/variantstudy/model/command/create_area.py b/antarest/study/storage/variantstudy/model/command/create_area.py index ea694d45e5..9e417261f2 100644 --- a/antarest/study/storage/variantstudy/model/command/create_area.py +++ b/antarest/study/storage/variantstudy/model/command/create_area.py @@ -1,9 +1,10 @@ -from typing import Any, Optional, List +from typing import Any, Optional, List, Tuple, Dict from antarest.core.model import JSON from antarest.study.storage.rawstudy.model.filesystem.config.model import ( Area, transform_name_to_id, + FileStudyTreeConfig, ) from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy from antarest.study.storage.variantstudy.business.default_values import ( @@ -51,21 +52,24 @@ def _generate_new_thermal_areas_ini( return new_areas - def _apply(self, study_data: FileStudy) -> CommandOutput: + def _apply_config( + self, study_data: FileStudyTreeConfig + ) -> Tuple[CommandOutput, Dict[str, Any]]: if self.command_context.generator_matrix_constants is None: raise ValueError() area_id = transform_name_to_id(self.area_name) - if area_id in study_data.config.areas.keys(): - return CommandOutput( - status=False, - message=f"Area '{self.area_name}' already exists and could not be created", + if area_id in study_data.areas.keys(): + return ( + CommandOutput( + status=False, + message=f"Area '{self.area_name}' already exists and could not be created", + ), + dict(), ) - version = study_data.config.version - - study_data.config.areas[area_id] = Area( + study_data.areas[area_id] = Area( name=self.area_name, links={}, thermals=[], @@ -73,6 +77,19 @@ def _apply(self, study_data: FileStudy) -> CommandOutput: filters_synthesis=[], filters_year=[], ) + return ( + CommandOutput( + status=True, message=f"Area '{self.area_name}' created" + ), + {"area_id": area_id}, + ) + + def _apply(self, study_data: FileStudy) -> CommandOutput: + output, data = self._apply_config(study_data.config) + if not output.status: + return output + area_id = data["area_id"] + version = study_data.config.version hydro_config = study_data.tree.get(["input", "hydro", "hydro"]) get_or_create_section(hydro_config, "inter-daily-breakdown")[ @@ -241,9 +258,7 @@ def _apply(self, study_data: FileStudy) -> CommandOutput: study_data.tree.save(new_area_data) - return CommandOutput( - status=True, message=f"Area '{self.area_name}' created" - ) + return output def to_dto(self) -> CommandDTO: return CommandDTO( diff --git a/antarest/study/storage/variantstudy/model/command/create_binding_constraint.py b/antarest/study/storage/variantstudy/model/command/create_binding_constraint.py index 2cb7e4209f..92323002be 100644 --- a/antarest/study/storage/variantstudy/model/command/create_binding_constraint.py +++ b/antarest/study/storage/variantstudy/model/command/create_binding_constraint.py @@ -1,10 +1,11 @@ -from typing import Dict, List, Union, Any, Optional, cast +from typing import Dict, List, Union, Any, Optional, cast, Tuple from pydantic import validator from antarest.matrixstore.model import MatrixData from antarest.study.storage.rawstudy.model.filesystem.config.model import ( transform_name_to_id, + FileStudyTreeConfig, ) from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy from antarest.study.storage.variantstudy.model.command.common import ( @@ -55,12 +56,19 @@ def validate_series( else: return validate_matrix(v, values) + def _apply_config( + self, study_data: FileStudyTreeConfig + ) -> Tuple[CommandOutput, Dict[str, Any]]: + bd_id = transform_name_to_id(self.name) + if bd_id not in study_data.bindings: + study_data.bindings.append(bd_id) + return CommandOutput(status=True), {} + def _apply(self, study_data: FileStudy) -> CommandOutput: assert isinstance(self.values, str) binding_constraints = study_data.tree.get( ["input", "bindingconstraints", "bindingconstraints"] ) - new_key = len(binding_constraints.keys()) bd_id = transform_name_to_id(self.name) return apply_binding_constraint( diff --git a/antarest/study/storage/variantstudy/model/command/create_cluster.py b/antarest/study/storage/variantstudy/model/command/create_cluster.py index 8c2915c132..bef358bbe7 100644 --- a/antarest/study/storage/variantstudy/model/command/create_cluster.py +++ b/antarest/study/storage/variantstudy/model/command/create_cluster.py @@ -1,4 +1,4 @@ -from typing import Dict, Union, List, Any, Optional, cast +from typing import Dict, Union, List, Any, Optional, cast, Tuple from pydantic import validator @@ -7,6 +7,7 @@ from antarest.study.storage.rawstudy.model.filesystem.config.model import ( Cluster, transform_name_to_id, + FileStudyTreeConfig, ) from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy from antarest.study.storage.variantstudy.model.command.common import ( @@ -72,24 +73,44 @@ def validate_modulation( else: return validate_matrix(v, values) - def _apply(self, study_data: FileStudy) -> CommandOutput: - if self.area_id not in study_data.config.areas: - return CommandOutput( - status=False, - message=f"Area '{self.area_id}' does not exist", + def _apply_config( + self, study_data: FileStudyTreeConfig + ) -> Tuple[CommandOutput, Dict[str, Any]]: + if self.area_id not in study_data.areas: + return ( + CommandOutput( + status=False, + message=f"Area '{self.area_id}' does not exist", + ), + dict(), ) - cluster_id = transform_name_to_id(self.cluster_name) - for cluster in study_data.config.areas[self.area_id].thermals: + for cluster in study_data.areas[self.area_id].thermals: if cluster.id == cluster_id: - return CommandOutput( - status=False, - message=f"Cluster '{self.cluster_name}' already exist", + return ( + CommandOutput( + status=False, + message=f"Cluster '{self.cluster_name}' already exist", + ), + dict(), ) - - study_data.config.areas[self.area_id].thermals.append( + study_data.areas[self.area_id].thermals.append( Cluster(id=cluster_id, name=self.cluster_name) ) + return ( + CommandOutput( + status=True, + message=f"Cluster '{self.cluster_name}' added to area '{self.area_id}'", + ), + {"cluster_id": cluster_id}, + ) + + def _apply(self, study_data: FileStudy) -> CommandOutput: + output, data = self._apply_config(study_data.config) + if not output.status: + return output + + cluster_id = data["cluster_id"] cluster_list_config = study_data.tree.get( ["input", "thermal", "clusters", self.area_id, "list"] @@ -121,10 +142,7 @@ def _apply(self, study_data: FileStudy) -> CommandOutput: } study_data.tree.save(new_cluster_data) - return CommandOutput( - status=True, - message=f"Cluster '{self.cluster_name}' added to area '{self.area_id}'", - ) + return output def to_dto(self) -> CommandDTO: return CommandDTO( diff --git a/antarest/study/storage/variantstudy/model/command/create_district.py b/antarest/study/storage/variantstudy/model/command/create_district.py index 2f9fd31580..69bebdea44 100644 --- a/antarest/study/storage/variantstudy/model/command/create_district.py +++ b/antarest/study/storage/variantstudy/model/command/create_district.py @@ -1,11 +1,12 @@ from enum import Enum -from typing import Any, Optional, List, cast +from typing import Any, Optional, List, cast, Tuple, Dict from pydantic import validator from antarest.study.storage.rawstudy.model.filesystem.config.model import ( transform_name_to_id, Set, + FileStudyTreeConfig, ) from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy from antarest.study.storage.variantstudy.model.command.common import ( @@ -45,24 +46,39 @@ def validate_district_name(cls, val: str) -> str: ) return val - def _apply(self, study_data: FileStudy) -> CommandOutput: + def _apply_config( + self, study_data: FileStudyTreeConfig + ) -> Tuple[CommandOutput, Dict[str, Any]]: district_id = transform_name_to_id(self.name) - if district_id in study_data.config.sets: - return CommandOutput( - status=False, - message=f"District '{self.name}' already exists and could not be created", + if district_id in study_data.sets: + return ( + CommandOutput( + status=False, + message=f"District '{self.name}' already exists and could not be created", + ), + dict(), ) base_filter = self.base_filter or DistrictBaseFilter.remove_all inverted_set = base_filter == DistrictBaseFilter.add_all - study_data.config.sets[district_id] = Set( + study_data.sets[district_id] = Set( name=self.name, areas=self.filter_items or [], output=self.output if self.output is not None else True, inverted_set=inverted_set, ) - item_key = "-" if inverted_set else "+" + return CommandOutput(status=True, message=district_id), { + "district_id": district_id, + "item_key": item_key, + } + + def _apply(self, study_data: FileStudy) -> CommandOutput: + output, data = self._apply_config(study_data.config) + if not output.status: + return output + district_id = data["district_id"] + item_key = data["item_key"] study_data.tree.save( { "caption": self.name, @@ -76,7 +92,7 @@ def _apply(self, study_data: FileStudy) -> CommandOutput: ["input", "areas", "sets", district_id], ) - return CommandOutput(status=True, message=district_id) + return output def to_dto(self) -> CommandDTO: return CommandDTO( diff --git a/antarest/study/storage/variantstudy/model/command/create_link.py b/antarest/study/storage/variantstudy/model/command/create_link.py index 534eef84bd..a225584614 100644 --- a/antarest/study/storage/variantstudy/model/command/create_link.py +++ b/antarest/study/storage/variantstudy/model/command/create_link.py @@ -1,10 +1,13 @@ -from typing import Dict, List, Union, Any, Optional, cast +from typing import Dict, List, Union, Any, Optional, cast, Tuple from pydantic import validator from antarest.core.model import JSON from antarest.matrixstore.model import MatrixData -from antarest.study.storage.rawstudy.model.filesystem.config.model import Link +from antarest.study.storage.rawstudy.model.filesystem.config.model import ( + Link, + FileStudyTreeConfig, +) from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy from antarest.study.storage.variantstudy.business.default_values import ( LinkProperties, @@ -51,10 +54,10 @@ def validate_series( return validate_matrix(v, values) def _create_link_in_config( - self, area_from: str, area_to: str, study_data: FileStudy + self, area_from: str, area_to: str, study_data: FileStudyTreeConfig ) -> None: self.parameters = self.parameters or {} - study_data.config.areas[area_from].links[area_to] = Link( + study_data.areas[area_from].links[area_to] = Link( filters_synthesis=[ step.strip() for step in self.parameters.get( @@ -118,37 +121,64 @@ def generate_link_properties(parameters: JSON) -> JSON: ), } - def _apply(self, study_data: FileStudy) -> CommandOutput: - if self.area1 not in study_data.config.areas: - return CommandOutput( - status=False, message=f"The area '{self.area1}' does not exist" + def _apply_config( + self, study_data: FileStudyTreeConfig + ) -> Tuple[CommandOutput, Dict[str, Any]]: + if self.area1 not in study_data.areas: + return ( + CommandOutput( + status=False, + message=f"The area '{self.area1}' does not exist", + ), + dict(), ) - if self.area2 not in study_data.config.areas: - return CommandOutput( - status=False, message=f"The area '{self.area2}' does not exist" + if self.area2 not in study_data.areas: + return ( + CommandOutput( + status=False, + message=f"The area '{self.area2}' does not exist", + ), + dict(), ) area_from, area_to = sorted([self.area1, self.area2]) - if area_to in study_data.config.areas[area_from].links: - return CommandOutput( - status=False, - message=f"The link between {self.area1} and {self.area2} already exist.", + if area_to in study_data.areas[area_from].links: + return ( + CommandOutput( + status=False, + message=f"The link between {self.area1} and {self.area2} already exist.", + ), + dict(), ) self._create_link_in_config(area_from, area_to, study_data) if ( - study_data.config.path - / "input" - / "links" - / area_from - / f"{area_to}.txt" + study_data.path / "input" / "links" / area_from / f"{area_to}.txt" ).exists(): - return CommandOutput( - status=False, - message=f"The link between {self.area1} and {self.area2} already exist", + return ( + CommandOutput( + status=False, + message=f"The link between {self.area1} and {self.area2} already exist", + ), + dict(), ) + return ( + CommandOutput( + status=True, + message=f"Link between '{self.area1}' and '{self.area2}' created", + ), + {"area_from": area_from, "area_to": area_to}, + ) + + def _apply(self, study_data: FileStudy) -> CommandOutput: + output, data = self._apply_config(study_data.config) + if not output.status: + return output + area_from = data["area_from"] + area_to = data["area_to"] + self.parameters = self.parameters or {} link_property = CreateLink.generate_link_properties(self.parameters) @@ -160,11 +190,7 @@ def _apply(self, study_data: FileStudy) -> CommandOutput: study_data.tree.save( self.series, ["input", "links", area_from, area_to] ) - - return CommandOutput( - status=True, - message=f"Link between '{self.area1}' and '{self.area2}' created", - ) + return output def to_dto(self) -> CommandDTO: return CommandDTO( diff --git a/antarest/study/storage/variantstudy/model/command/icommand.py b/antarest/study/storage/variantstudy/model/command/icommand.py index e64ef7ba54..696a9fadba 100644 --- a/antarest/study/storage/variantstudy/model/command/icommand.py +++ b/antarest/study/storage/variantstudy/model/command/icommand.py @@ -1,11 +1,13 @@ import logging from abc import ABC, abstractmethod -from typing import Optional, List +from typing import List, Tuple, Dict, Any from pydantic import BaseModel +from antarest.study.storage.rawstudy.model.filesystem.config.model import ( + FileStudyTreeConfig, +) from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy -from antarest.study.storage.variantstudy.model.model import CommandDTO from antarest.study.storage.variantstudy.model.command.common import ( CommandOutput, CommandName, @@ -13,6 +15,7 @@ from antarest.study.storage.variantstudy.model.command_context import ( CommandContext, ) +from antarest.study.storage.variantstudy.model.model import CommandDTO MATCH_SIGNATURE_SEPARATOR = "%" logger = logging.getLogger(__name__) @@ -27,6 +30,16 @@ class ICommand(ABC, BaseModel): def _apply(self, study_data: FileStudy) -> CommandOutput: raise NotImplementedError() + @abstractmethod + def _apply_config( + self, study_data: FileStudyTreeConfig + ) -> Tuple[CommandOutput, Dict[str, Any]]: + raise NotImplementedError() + + def apply_config(self, study_data: FileStudyTreeConfig) -> CommandOutput: + output, _ = self._apply_config(study_data) + return output + def apply(self, study_data: FileStudy) -> CommandOutput: try: return self._apply(study_data) diff --git a/antarest/study/storage/variantstudy/model/command/remove_area.py b/antarest/study/storage/variantstudy/model/command/remove_area.py index d3fc6f96d7..6237318c42 100644 --- a/antarest/study/storage/variantstudy/model/command/remove_area.py +++ b/antarest/study/storage/variantstudy/model/command/remove_area.py @@ -1,10 +1,15 @@ -from typing import Any, List, Optional +import logging +from typing import Any, List, Optional, Tuple, Dict from antarest.core.model import JSON from antarest.study.storage.rawstudy.model.filesystem.config.model import ( transform_name_to_id, + FileStudyTreeConfig, ) from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy +from antarest.study.storage.rawstudy.model.filesystem.folder_node import ( + ChildNotFoundError, +) from antarest.study.storage.variantstudy.model.command.common import ( CommandOutput, CommandName, @@ -24,6 +29,15 @@ def __init__(self, **data: Any) -> None: command_name=CommandName.REMOVE_AREA, version=1, **data ) + def _apply_config( + self, study_data: FileStudyTreeConfig + ) -> Tuple[CommandOutput, Dict[str, Any]]: + del study_data.areas[self.id] + return ( + CommandOutput(status=True, message=f"Area '{self.id}' deleted"), + dict(), + ) + def _apply(self, study_data: FileStudy) -> CommandOutput: study_data.tree.delete(["input", "areas", self.id]) @@ -130,7 +144,7 @@ def _apply(self, study_data: FileStudy) -> CommandOutput: ] ) - del study_data.config.areas[self.id] + output, _ = self._apply_config(study_data.config) for area_name, area in study_data.config.areas.items(): for link in area.links.keys(): if link == self.id: @@ -151,7 +165,7 @@ def _apply(self, study_data: FileStudy) -> CommandOutput: # todo remove bindinconstraint using this area ? # todo remove area from districts - return CommandOutput(status=True, message=f"Area '{self.id}' deleted") + return output def to_dto(self) -> CommandDTO: return CommandDTO( @@ -189,12 +203,19 @@ def revert( # todo revert binding constraints that has the area in constraint and also search in base for one return [command] - area_commands, links_commands = ( - self.command_context.command_extractor - or CommandExtraction(self.command_context.matrix_service) - ).extract_area(base, self.id) - return area_commands + links_commands - # todo revert binding constraints that has the area in constraint + try: + area_commands, links_commands = ( + self.command_context.command_extractor + or CommandExtraction(self.command_context.matrix_service) + ).extract_area(base, self.id) + # todo revert binding constraints that has the area in constraint + return area_commands + links_commands + except ChildNotFoundError as e: + logging.getLogger(__name__).warning( + f"Failed to extract revert command for remove_area {self.id}", + exc_info=e, + ) + return [] def _create_diff(self, other: "ICommand") -> List["ICommand"]: return [] diff --git a/antarest/study/storage/variantstudy/model/command/remove_binding_constraint.py b/antarest/study/storage/variantstudy/model/command/remove_binding_constraint.py index 1814716154..765069b0fc 100644 --- a/antarest/study/storage/variantstudy/model/command/remove_binding_constraint.py +++ b/antarest/study/storage/variantstudy/model/command/remove_binding_constraint.py @@ -1,8 +1,10 @@ -from typing import Any, List, Optional +import logging +from typing import Any, List, Optional, Tuple, Dict from antarest.core.model import JSON from antarest.study.storage.rawstudy.model.filesystem.config.model import ( transform_name_to_id, + FileStudyTreeConfig, ) from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy from antarest.study.storage.variantstudy.model.command.common import ( @@ -26,12 +28,24 @@ def __init__(self, **data: Any) -> None: **data, ) + def _apply_config( + self, study_data: FileStudyTreeConfig + ) -> Tuple[CommandOutput, Dict[str, Any]]: + if self.id not in study_data.bindings: + return ( + CommandOutput( + status=False, message="Binding constraint not found" + ), + dict(), + ) + study_data.bindings.remove(self.id) + return CommandOutput(status=True), dict() + def _apply(self, study_data: FileStudy) -> CommandOutput: if self.id not in study_data.config.bindings: return CommandOutput( status=False, message="Binding constraint not found" ) - binding_constraints = study_data.tree.get( ["input", "bindingconstraints", "bindingconstraints"] ) @@ -42,14 +56,13 @@ def _apply(self, study_data: FileStudy) -> CommandOutput: continue new_binding_constraints[str(index)] = binding_constraints[bd] index += 1 - study_data.tree.save( new_binding_constraints, ["input", "bindingconstraints", "bindingconstraints"], ) study_data.tree.delete(["input", "bindingconstraints", self.id]) - study_data.config.bindings.remove(self.id) - return CommandOutput(status=True) + output, _ = self._apply_config(study_data.config) + return output def to_dto(self) -> CommandDTO: return CommandDTO( @@ -86,10 +99,17 @@ def revert( ): return [command] - return ( - self.command_context.command_extractor - or CommandExtraction(self.command_context.matrix_service) - ).extract_binding_constraint(base, self.id) + try: + return ( + self.command_context.command_extractor + or CommandExtraction(self.command_context.matrix_service) + ).extract_binding_constraint(base, self.id) + except Exception as e: + logging.getLogger(__name__).warning( + f"Failed to extract revert command for remove_binding_constraint {self.id}", + exc_info=e, + ) + return [] def _create_diff(self, other: "ICommand") -> List["ICommand"]: return [] diff --git a/antarest/study/storage/variantstudy/model/command/remove_cluster.py b/antarest/study/storage/variantstudy/model/command/remove_cluster.py index c7a2782962..e63d64152b 100644 --- a/antarest/study/storage/variantstudy/model/command/remove_cluster.py +++ b/antarest/study/storage/variantstudy/model/command/remove_cluster.py @@ -1,9 +1,14 @@ -from typing import Any, List +import logging +from typing import Any, List, Tuple, Dict from antarest.study.storage.rawstudy.model.filesystem.config.model import ( transform_name_to_id, + FileStudyTreeConfig, ) from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy +from antarest.study.storage.rawstudy.model.filesystem.folder_node import ( + ChildNotFoundError, +) from antarest.study.storage.variantstudy.model.command.common import ( CommandOutput, CommandName, @@ -24,6 +29,53 @@ def __init__(self, **data: Any) -> None: command_name=CommandName.REMOVE_CLUSTER, version=1, **data ) + def _remove_cluster(self, study_data: FileStudyTreeConfig) -> None: + study_data.areas[self.area_id].thermals = [ + cluster + for cluster in study_data.areas[self.area_id].thermals + if cluster.id != self.cluster_id.lower() + ] + + def _apply_config( + self, study_data: FileStudyTreeConfig + ) -> Tuple[CommandOutput, Dict[str, Any]]: + if self.area_id not in study_data.areas: + return ( + CommandOutput( + status=False, + message=f"Area '{self.area_id}' does not exist", + ), + dict(), + ) + + if ( + len( + [ + cluster + for cluster in study_data.areas[self.area_id].thermals + if cluster.id == self.cluster_id + ] + ) + == 0 + ): + return ( + CommandOutput( + status=False, + message=f"Cluster '{self.cluster_id}' does not exist", + ), + dict(), + ) + self._remove_cluster(study_data) + # todo remove binding constraint using this cluster ? + + return ( + CommandOutput( + status=True, + message=f"Cluster '{self.cluster_id}' removed from area '{self.area_id}'", + ), + dict(), + ) + def _apply(self, study_data: FileStudy) -> CommandOutput: if self.area_id not in study_data.config.areas: return CommandOutput( @@ -47,6 +99,7 @@ def _apply(self, study_data: FileStudy) -> CommandOutput: status=False, message=f"Cluster '{self.cluster_id}' does not exist", ) + study_data.tree.delete( [ "input", @@ -76,12 +129,7 @@ def _apply(self, study_data: FileStudy) -> CommandOutput: ] ) - study_data.config.areas[self.area_id].thermals = [ - cluster - for cluster in study_data.config.areas[self.area_id].thermals - if cluster.id != self.cluster_id.lower() - ] - # todo remove binding constraint using this cluster ? + self._remove_cluster(study_data.config) return CommandOutput( status=True, @@ -131,11 +179,18 @@ def revert( # todo revert binding constraints that has the cluster in constraint and also search in base for one return [command] - return ( - self.command_context.command_extractor - or CommandExtraction(self.command_context.matrix_service) - ).extract_cluster(base, self.area_id, self.cluster_id) - # todo revert binding constraints that has the cluster in constraint + try: + return ( + self.command_context.command_extractor + or CommandExtraction(self.command_context.matrix_service) + ).extract_cluster(base, self.area_id, self.cluster_id) + # todo revert binding constraints that has the cluster in constraint + except ChildNotFoundError as e: + logging.getLogger(__name__).warning( + f"Failed to extract revert command for remove_cluster {self.area_id}#{self.cluster_id}", + exc_info=e, + ) + return [] def _create_diff(self, other: "ICommand") -> List["ICommand"]: return [] diff --git a/antarest/study/storage/variantstudy/model/command/remove_district.py b/antarest/study/storage/variantstudy/model/command/remove_district.py index f1d87df2c6..4d8d7d98a5 100644 --- a/antarest/study/storage/variantstudy/model/command/remove_district.py +++ b/antarest/study/storage/variantstudy/model/command/remove_district.py @@ -1,9 +1,14 @@ -from typing import Any, List, Optional +import logging +from typing import Any, List, Optional, Tuple, Dict from antarest.study.storage.rawstudy.model.filesystem.config.model import ( transform_name_to_id, + FileStudyTreeConfig, ) from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy +from antarest.study.storage.rawstudy.model.filesystem.folder_node import ( + ChildNotFoundError, +) from antarest.study.storage.variantstudy.model.command.common import ( CommandOutput, CommandName, @@ -23,10 +28,16 @@ def __init__(self, **data: Any) -> None: command_name=CommandName.REMOVE_DISTRICT, version=1, **data ) + def _apply_config( + self, study_data: FileStudyTreeConfig + ) -> Tuple[CommandOutput, Dict[str, Any]]: + del study_data.sets[self.id] + return CommandOutput(status=True, message=self.id), dict() + def _apply(self, study_data: FileStudy) -> CommandOutput: - del study_data.config.sets[self.id] + output, _ = self._apply_config(study_data.config) study_data.tree.delete(["input", "areas", "sets", self.id]) - return CommandOutput(status=True, message=self.id) + return output def to_dto(self) -> CommandDTO: return CommandDTO( @@ -62,10 +73,17 @@ def revert( and transform_name_to_id(command.name) == self.id ): return [command] - return ( - self.command_context.command_extractor - or CommandExtraction(self.command_context.matrix_service) - ).extract_district(base, self.id) + try: + return ( + self.command_context.command_extractor + or CommandExtraction(self.command_context.matrix_service) + ).extract_district(base, self.id) + except Exception as e: + logging.getLogger(__name__).warning( + f"Failed to extract revert command for remove_district {self.id}", + exc_info=e, + ) + return [] def _create_diff(self, other: "ICommand") -> List["ICommand"]: return [] diff --git a/antarest/study/storage/variantstudy/model/command/remove_link.py b/antarest/study/storage/variantstudy/model/command/remove_link.py index 4a57049a68..bde9421ad6 100644 --- a/antarest/study/storage/variantstudy/model/command/remove_link.py +++ b/antarest/study/storage/variantstudy/model/command/remove_link.py @@ -1,6 +1,13 @@ -from typing import Any, List, Optional +import logging +from typing import Any, List, Optional, Tuple, Dict +from antarest.study.storage.rawstudy.model.filesystem.config.model import ( + FileStudyTreeConfig, +) from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy +from antarest.study.storage.rawstudy.model.filesystem.folder_node import ( + ChildNotFoundError, +) from antarest.study.storage.variantstudy.model.command.common import ( CommandOutput, CommandName, @@ -21,34 +28,55 @@ def __init__(self, **data: Any) -> None: command_name=CommandName.REMOVE_LINK, version=1, **data ) - def _apply(self, study_data: FileStudy) -> CommandOutput: - if self.area1 not in study_data.config.areas: - return CommandOutput( - status=False, - message=f"The area '{self.area1}' does not exist.", + def _apply_config( + self, study_data: FileStudyTreeConfig + ) -> Tuple[CommandOutput, Dict[str, Any]]: + if self.area1 not in study_data.areas: + return ( + CommandOutput( + status=False, + message=f"The area '{self.area1}' does not exist.", + ), + dict(), ) - if self.area2 not in study_data.config.areas: - return CommandOutput( - status=False, - message=f"The area '{self.area2}' does not exist.", + if self.area2 not in study_data.areas: + return ( + CommandOutput( + status=False, + message=f"The area '{self.area2}' does not exist.", + ), + dict(), ) area_from, area_to = sorted([self.area1, self.area2]) - if area_to not in study_data.config.areas[area_from].links: - return CommandOutput( - status=False, - message=f"The link between {self.area1} and {self.area2} does not exist.", + if area_to not in study_data.areas[area_from].links: + return ( + CommandOutput( + status=False, + message=f"The link between {self.area1} and {self.area2} does not exist.", + ), + dict(), ) + return ( + CommandOutput( + status=True, + message=f"Link between {self.area1} and {self.area2} removed", + ), + {"area_from": area_from, "area_to": area_to}, + ) + def _apply(self, study_data: FileStudy) -> CommandOutput: + output, data = self._apply_config(study_data.config) + if not output.status: + return output + area_from = data["area_from"] + area_to = data["area_to"] study_data.tree.delete(["input", "links", area_from, area_to]) study_data.tree.delete( ["input", "links", area_from, "properties", area_to] ) - return CommandOutput( - status=True, - message=f"Link between {self.area1} and {self.area2} removed", - ) + return output def to_dto(self) -> CommandDTO: return CommandDTO( @@ -91,10 +119,17 @@ def revert( ): return [command] area_from, area_to = sorted([self.area1, self.area2]) - return ( - self.command_context.command_extractor - or CommandExtraction(self.command_context.matrix_service) - ).extract_link(base, area_from, area_to) + try: + return ( + self.command_context.command_extractor + or CommandExtraction(self.command_context.matrix_service) + ).extract_link(base, area_from, area_to) + except ChildNotFoundError as e: + logging.getLogger(__name__).warning( + f"Failed to extract revert command for remove_link {self.area1}/{self.area2}", + exc_info=e, + ) + return [] def _create_diff(self, other: "ICommand") -> List["ICommand"]: return [] diff --git a/antarest/study/storage/variantstudy/model/command/replace_matrix.py b/antarest/study/storage/variantstudy/model/command/replace_matrix.py index 3da4a5e842..1f1650cca2 100644 --- a/antarest/study/storage/variantstudy/model/command/replace_matrix.py +++ b/antarest/study/storage/variantstudy/model/command/replace_matrix.py @@ -1,9 +1,12 @@ -from typing import Union, List, Any, Optional +from typing import Union, List, Any, Optional, Tuple, Dict from pydantic import validator from antarest.core.model import JSON from antarest.matrixstore.model import MatrixData +from antarest.study.storage.rawstudy.model.filesystem.config.model import ( + FileStudyTreeConfig, +) from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy from antarest.study.storage.rawstudy.model.filesystem.folder_node import ( ChildNotFoundError, @@ -39,6 +42,17 @@ def __init__(self, **data: Any) -> None: command_name=CommandName.REPLACE_MATRIX, version=1, **data ) + def _apply_config( + self, study_data: FileStudyTreeConfig + ) -> Tuple[CommandOutput, Dict[str, Any]]: + return ( + CommandOutput( + status=True, + message=f"Matrix '{self.target}' has been successfully replaced.", + ), + dict(), + ) + def _apply(self, study_data: FileStudy) -> CommandOutput: replace_matrix_data: JSON = {} target_matrix = replace_matrix_data @@ -64,11 +78,8 @@ def _apply(self, study_data: FileStudy) -> CommandOutput: ) study_data.tree.save(replace_matrix_data) - - return CommandOutput( - status=True, - message=f"Matrix '{self.target}' has been successfully replaced.", - ) + output, _ = self._apply_config(study_data.config) + return output def to_dto(self) -> CommandDTO: return CommandDTO( diff --git a/antarest/study/storage/variantstudy/model/command/update_binding_constraint.py b/antarest/study/storage/variantstudy/model/command/update_binding_constraint.py index 5cd0043dbc..67df370693 100644 --- a/antarest/study/storage/variantstudy/model/command/update_binding_constraint.py +++ b/antarest/study/storage/variantstudy/model/command/update_binding_constraint.py @@ -1,4 +1,4 @@ -from typing import List, Optional, Dict, Union, Any +from typing import List, Optional, Dict, Union, Any, Tuple from pydantic import validator @@ -6,6 +6,7 @@ from antarest.matrixstore.model import MatrixData from antarest.study.storage.rawstudy.model.filesystem.config.model import ( transform_name_to_id, + FileStudyTreeConfig, ) from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy from antarest.study.storage.variantstudy.model.command.common import ( @@ -52,6 +53,11 @@ def validate_series( return validate_matrix(v, values) return None + def _apply_config( + self, study_data: FileStudyTreeConfig + ) -> Tuple[CommandOutput, Dict[str, Any]]: + return CommandOutput(status=True), {} + def _apply(self, study_data: FileStudy) -> CommandOutput: binding_constraints = study_data.tree.get( ["input", "bindingconstraints", "bindingconstraints"] diff --git a/antarest/study/storage/variantstudy/model/command/update_comments.py b/antarest/study/storage/variantstudy/model/command/update_comments.py index bd56f214f4..00d1168fa0 100644 --- a/antarest/study/storage/variantstudy/model/command/update_comments.py +++ b/antarest/study/storage/variantstudy/model/command/update_comments.py @@ -1,6 +1,9 @@ -from typing import Any, List, Optional +from typing import Any, List, Optional, Tuple, Dict from antarest.core.model import JSON +from antarest.study.storage.rawstudy.model.filesystem.config.model import ( + FileStudyTreeConfig, +) from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy from antarest.study.storage.rawstudy.model.filesystem.folder_node import ( ChildNotFoundError, @@ -25,6 +28,17 @@ def __init__(self, **data: Any) -> None: command_name=CommandName.UPDATE_COMMENTS, version=1, **data ) + def _apply_config( + self, study_data: FileStudyTreeConfig + ) -> Tuple[CommandOutput, Dict[str, Any]]: + return ( + CommandOutput( + status=True, + message=f"Comment '{self.comments}' has been successfully replaced.", + ), + dict(), + ) + def _apply(self, study_data: FileStudy) -> CommandOutput: replace_comment_data: JSON = { "settings": {"comments": self.comments.encode("utf-8")} @@ -32,10 +46,8 @@ def _apply(self, study_data: FileStudy) -> CommandOutput: study_data.tree.save(replace_comment_data) - return CommandOutput( - status=True, - message=f"Comment '{self.comments}' has been successfully replaced.", - ) + output, _ = self._apply_config(study_data.config) + return output def to_dto(self) -> CommandDTO: return CommandDTO( diff --git a/antarest/study/storage/variantstudy/model/command/update_config.py b/antarest/study/storage/variantstudy/model/command/update_config.py index 867fe628b4..546202e274 100644 --- a/antarest/study/storage/variantstudy/model/command/update_config.py +++ b/antarest/study/storage/variantstudy/model/command/update_config.py @@ -1,7 +1,10 @@ import logging -from typing import Any, Union, List, Optional +from typing import Any, Union, List, Tuple, Dict from antarest.core.model import JSON +from antarest.study.storage.rawstudy.model.filesystem.config.model import ( + FileStudyTreeConfig, +) from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy from antarest.study.storage.rawstudy.model.filesystem.folder_node import ( ChildNotFoundError, @@ -29,6 +32,11 @@ def __init__(self, **data: Any) -> None: command_name=CommandName.UPDATE_CONFIG, version=1, **data ) + def _apply_config( + self, study_data: FileStudyTreeConfig + ) -> Tuple[CommandOutput, Dict[str, Any]]: + return CommandOutput(status=True, message="ok"), dict() + def _apply(self, study_data: FileStudy) -> CommandOutput: url = self.target.split("/") tree_node = study_data.tree.get_node(url) @@ -39,7 +47,8 @@ def _apply(self, study_data: FileStudy) -> CommandOutput: ) study_data.tree.save(self.data, url) - return CommandOutput(status=True, message="ok") + output, _ = self._apply_config(study_data.config) + return output def to_dto(self) -> CommandDTO: return CommandDTO( diff --git a/antarest/study/storage/variantstudy/model/command/update_raw_file.py b/antarest/study/storage/variantstudy/model/command/update_raw_file.py new file mode 100644 index 0000000000..40d45b8974 --- /dev/null +++ b/antarest/study/storage/variantstudy/model/command/update_raw_file.py @@ -0,0 +1,93 @@ +import base64 +from typing import List, Any, Tuple, Dict + +from antarest.study.storage.rawstudy.model.filesystem.config.model import ( + FileStudyTreeConfig, +) +from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy +from antarest.study.storage.rawstudy.model.filesystem.raw_file_node import ( + RawFileNode, +) +from antarest.study.storage.variantstudy.model.command.common import ( + CommandOutput, + CommandName, +) +from antarest.study.storage.variantstudy.model.command.icommand import ( + ICommand, + MATCH_SIGNATURE_SEPARATOR, +) +from antarest.study.storage.variantstudy.model.model import CommandDTO + + +class UpdateRawFile(ICommand): + target: str + b64Data: str + + def __init__(self, **data: Any) -> None: + super().__init__( + command_name=CommandName.UPDATE_FILE, version=1, **data + ) + + def _apply_config( + self, study_data: FileStudyTreeConfig + ) -> Tuple[CommandOutput, Dict[str, Any]]: + return CommandOutput(status=True, message="ok"), {} + + def _apply(self, study_data: FileStudy) -> CommandOutput: + url = self.target.split("/") + tree_node = study_data.tree.get_node(url) + if not isinstance(tree_node, RawFileNode): + return CommandOutput( + status=False, + message=f"Study node at path {self.target} is invalid", + ) + + study_data.tree.save( + base64.decodebytes(self.b64Data.encode("utf-8")), url + ) + return CommandOutput(status=True, message="ok") + + def to_dto(self) -> CommandDTO: + return CommandDTO( + action=self.command_name.value, + args={"target": self.target, "b64Data": self.b64Data}, + ) + + def match_signature(self) -> str: + return str( + self.command_name.value + MATCH_SIGNATURE_SEPARATOR + self.target + ) + + def match(self, other: ICommand, equal: bool = False) -> bool: + if not isinstance(other, UpdateRawFile): + return False + simple_match = self.target == other.target + if not equal: + return simple_match + return simple_match and self.b64Data == other.b64Data + + def revert( + self, history: List["ICommand"], base: FileStudy + ) -> List["ICommand"]: + for command in reversed(history): + if ( + isinstance(command, UpdateRawFile) + and command.target == self.target + ): + return [command] + from antarest.study.storage.variantstudy.model.command.utils_extractor import ( + CommandExtraction, + ) + + return [ + ( + self.command_context.command_extractor + or CommandExtraction(self.command_context.matrix_service) + ).generate_update_rawfile(base.tree, self.target.split("/")) + ] + + def _create_diff(self, other: "ICommand") -> List["ICommand"]: + return [other] + + def get_inner_matrices(self) -> List[str]: + return [] diff --git a/antarest/study/storage/variantstudy/model/command/utils_binding_constraint.py b/antarest/study/storage/variantstudy/model/command/utils_binding_constraint.py index 3dce7c7589..0c306222c4 100644 --- a/antarest/study/storage/variantstudy/model/command/utils_binding_constraint.py +++ b/antarest/study/storage/variantstudy/model/command/utils_binding_constraint.py @@ -63,7 +63,8 @@ def apply_binding_constraint( [str(coeff_val) for coeff_val in coeffs[link_or_thermal]] ) - study_data.config.bindings.append(bd_id) + if bd_id not in study_data.config.bindings: + study_data.config.bindings.append(bd_id) study_data.tree.save( binding_constraints, ["input", "bindingconstraints", "bindingconstraints"], diff --git a/antarest/study/storage/variantstudy/model/command/utils_extractor.py b/antarest/study/storage/variantstudy/model/command/utils_extractor.py index eb7c41bd2f..5038858ac2 100644 --- a/antarest/study/storage/variantstudy/model/command/utils_extractor.py +++ b/antarest/study/storage/variantstudy/model/command/utils_extractor.py @@ -1,5 +1,6 @@ +import base64 import logging -from typing import Optional, List, Tuple, Union +from typing import Optional, List, Tuple, Union, cast from antarest.core.model import JSON from antarest.core.utils.utils import StopWatch @@ -43,6 +44,9 @@ from antarest.study.storage.variantstudy.model.command.update_config import ( UpdateConfig, ) +from antarest.study.storage.variantstudy.model.command.update_raw_file import ( + UpdateRawFile, +) from antarest.study.storage.variantstudy.model.command.utils import ( strip_matrix_protocol, ) @@ -460,6 +464,16 @@ def generate_update_config( command_context=self.command_context, ) + def generate_update_rawfile( + self, study_tree: FileStudyTree, url: List[str] + ) -> ICommand: + data = study_tree.get(url) + return UpdateRawFile( + target="/".join(url), + b64Data=base64.b64encode(cast(bytes, data)).decode("utf-8"), + command_context=self.command_context, + ) + def generate_update_comments( self, study_tree: FileStudyTree, diff --git a/antarest/study/storage/variantstudy/model/interfaces.py b/antarest/study/storage/variantstudy/model/interfaces.py index c6b393f28c..1bf53885ba 100644 --- a/antarest/study/storage/variantstudy/model/interfaces.py +++ b/antarest/study/storage/variantstudy/model/interfaces.py @@ -65,6 +65,14 @@ def generate_update_config( ) -> "ICommand": # type: ignore raise NotImplementedError() + @abstractmethod + def generate_update_rawfile( + self, + study_tree: FileStudyTree, + url: List[str], + ) -> "ICommand": # type: ignore + raise NotImplementedError() + @abstractmethod def generate_update_comments( self, diff --git a/antarest/study/storage/variantstudy/variant_command_extractor.py b/antarest/study/storage/variantstudy/variant_command_extractor.py index 1155b959be..581516aa86 100644 --- a/antarest/study/storage/variantstudy/variant_command_extractor.py +++ b/antarest/study/storage/variantstudy/variant_command_extractor.py @@ -50,12 +50,6 @@ def extract(self, study: FileStudy) -> List[CommandDTO]: study_tree, ["layers", "layers"] ) ) - # todo create something out of variant manager commands to replace single rawnode files ? - # study_commands.append( - # self._generate_update_config( - # study_tree, ["settings", "comments"] - # ) - # ) stopwatch.log_elapsed( lambda x: logger.info(f"General command extraction done in {x}s") ) diff --git a/antarest/study/storage/variantstudy/variant_command_generator.py b/antarest/study/storage/variantstudy/variant_command_generator.py index e523a98751..d207f80aa5 100644 --- a/antarest/study/storage/variantstudy/variant_command_generator.py +++ b/antarest/study/storage/variantstudy/variant_command_generator.py @@ -1,14 +1,20 @@ import logging import shutil from pathlib import Path -from typing import List, Optional, Callable +from typing import List, Optional, Callable, Tuple, Union, cast from antarest.core.utils.utils import StopWatch +from antarest.study.storage.rawstudy.model.filesystem.config.model import ( + FileStudyTreeConfig, +) from antarest.study.storage.rawstudy.model.filesystem.factory import ( FileStudy, StudyFactory, ) from antarest.study.storage.utils import update_antares_info +from antarest.study.storage.variantstudy.model.command.common import ( + CommandOutput, +) from antarest.study.storage.variantstudy.model.command.icommand import ICommand from antarest.study.storage.variantstudy.model.dbmodel import VariantStudy from antarest.study.storage.variantstudy.model.model import ( @@ -17,30 +23,24 @@ logger = logging.getLogger(__name__) +APPLY_CALLBACK = Callable[ + [ICommand, Union[FileStudyTreeConfig, FileStudy]], CommandOutput +] + class VariantCommandGenerator: def __init__(self, study_factory: StudyFactory) -> None: self.study_factory = study_factory - def generate( - self, + @staticmethod + def _generate( commands: List[List[ICommand]], - dest_path: Path, + data: Union[FileStudy, FileStudyTreeConfig], + applier: APPLY_CALLBACK, metadata: Optional[VariantStudy] = None, - delete_on_failure: bool = True, notifier: Optional[Callable[[int, bool, str], None]] = None, ) -> GenerationResultInfoDTO: stopwatch = StopWatch() - - # Build file study - logger.info("Building study tree") - study_config, study_tree = self.study_factory.create_from_fs( - dest_path, "", use_cache=False - ) - if metadata: - update_antares_info(metadata, study_tree) - file_study = FileStudy(config=study_config, tree=study_tree) - # Apply commands results: GenerationResultInfoDTO = GenerationResultInfoDTO( success=True, details=[] @@ -63,7 +63,7 @@ def generate( command_index += 1 command_output_messages: List[str] = [] for command in command_batch: - output = command.apply(file_study) + output = applier(command, data) command_output_messages.append(output.message) command_output_status = ( command_output_status and output.status @@ -101,11 +101,61 @@ def generate( if not results.success: break - - if not results.success and delete_on_failure: - shutil.rmtree(dest_path) + data_type = isinstance(data, FileStudy) stopwatch.log_elapsed( - lambda x: logger.info(f"Variant generation done in {x}s"), + lambda x: logger.info( + f"Variant generation done in {x}s" + if data_type + else f"Variant light generation done in {x}s" + ), since_start=True, ) return results + + def generate( + self, + commands: List[List[ICommand]], + dest_path: Path, + metadata: Optional[VariantStudy] = None, + delete_on_failure: bool = True, + notifier: Optional[Callable[[int, bool, str], None]] = None, + ) -> GenerationResultInfoDTO: + # Build file study + logger.info("Building study tree") + study_config, study_tree = self.study_factory.create_from_fs( + dest_path, "", use_cache=False + ) + if metadata: + update_antares_info(metadata, study_tree) + file_study = FileStudy(config=study_config, tree=study_tree) + + results = VariantCommandGenerator._generate( + commands, + file_study, + lambda command, data: command.apply(cast(FileStudy, data)), + metadata, + notifier, + ) + + if not results.success and delete_on_failure: + shutil.rmtree(dest_path) + return results + + def generate_config( + self, + commands: List[List[ICommand]], + config: FileStudyTreeConfig, + metadata: Optional[VariantStudy] = None, + notifier: Optional[Callable[[int, bool, str], None]] = None, + ) -> Tuple[GenerationResultInfoDTO, FileStudyTreeConfig]: + logger.info("Building config (light generation)") + results = VariantCommandGenerator._generate( + commands, + config, + lambda command, data: command.apply_config( + cast(FileStudyTreeConfig, data) + ), + metadata, + notifier, + ) + return results, config diff --git a/antarest/study/storage/variantstudy/variant_study_service.py b/antarest/study/storage/variantstudy/variant_study_service.py index a771d42cf4..29a9ad23e4 100644 --- a/antarest/study/storage/variantstudy/variant_study_service.py +++ b/antarest/study/storage/variantstudy/variant_study_service.py @@ -4,11 +4,11 @@ import time from datetime import datetime from pathlib import Path -from typing import List, Optional, cast +from typing import List, Optional, cast, Tuple, Callable from uuid import uuid4 from fastapi import HTTPException -from filelock import FileLock # type: ignore +from filelock import FileLock from antarest.core.config import Config from antarest.core.exceptions import ( @@ -30,7 +30,7 @@ ) from antarest.core.jwt import DEFAULT_ADMIN_USER from antarest.core.model import JSON, StudyPermissionType -from antarest.core.requests import RequestParameters +from antarest.core.requests import RequestParameters, UserHasNotPermissionError from antarest.core.tasks.model import ( TaskResult, TaskDTO, @@ -45,11 +45,16 @@ Study, StudyMetadataDTO, StudySimResultDTO, + RawStudy, ) from antarest.study.storage.abstract_storage_service import ( AbstractStorageService, ) from antarest.study.storage.patch_service import PatchService +from antarest.study.storage.rawstudy.model.filesystem.config.model import ( + FileStudyTreeConfigDTO, + FileStudyTreeConfig, +) from antarest.study.storage.rawstudy.model.filesystem.factory import ( FileStudy, StudyFactory, @@ -391,23 +396,24 @@ def get_all_variants_children( study = self._get_variant_study( parent_id, params, raw_study_accepted=True ) + children_tree = VariantTreeDTO( node=self.get_study_information(study, summary=True), children=[] ) - children = self._get_variants_children(parent_id, params) + children = self._get_variants_children(parent_id) for child in children: - children_tree.children.append( - self.get_all_variants_children(child.id, params) - ) + try: + children_tree.children.append( + self.get_all_variants_children(child.id, params) + ) + except UserHasNotPermissionError: + logger.info( + f"Filtering children {child.id} in variant tree since user has not permission on this study" + ) return children_tree - def _get_variants_children( - self, parent_id: str, params: RequestParameters - ) -> List[StudyMetadataDTO]: - self._get_variant_study( - parent_id, params, raw_study_accepted=True - ) # check permissions + def _get_variants_children(self, parent_id: str) -> List[StudyMetadataDTO]: children = self.repository.get_children(parent_id=parent_id) output_list: List[StudyMetadataDTO] = [] for child in children: @@ -541,7 +547,9 @@ def create_variant_study( return str(variant_study.id) def generate_task( - self, metadata: VariantStudy, denormalize: bool = False + self, + metadata: VariantStudy, + denormalize: bool = False, ) -> str: with FileLock( str( @@ -573,10 +581,10 @@ def generate_task( def callback(notifier: TaskUpdateNotifier) -> TaskResult: generate_result = self._generate( - study_id, - denormalize, - RequestParameters(DEFAULT_ADMIN_USER), - notifier, + variant_study_id=study_id, + denormalize=denormalize, + params=RequestParameters(DEFAULT_ADMIN_USER), + notifier=notifier, ) return TaskResult( success=generate_result.success, @@ -612,11 +620,25 @@ def generate( return self.generate_task(variant_study, denormalize) + def generate_study_config( + self, + variant_study_id: str, + params: RequestParameters, + ) -> Tuple[GenerationResultInfoDTO, FileStudyTreeConfig]: + # Get variant study + variant_study = self._get_variant_study(variant_study_id, params) + + # Get parent study + if variant_study.parent_id is None: + raise NoParentStudyError(variant_study_id) + + return self._generate_study_config(variant_study, None) + def _generate( self, variant_study_id: str, - denormalize: bool, params: RequestParameters, + denormalize: bool = True, notifier: TaskUpdateNotifier = noop_notifier, ) -> GenerationResultInfoDTO: logger.info(f"Generating variant study {variant_study_id}") @@ -650,7 +672,10 @@ def _generate( if isinstance(parent_study, VariantStudy): self._safe_generation(parent_study) self.export_study_flat( - metadata=parent_study, dest=dest_path, outputs=False + metadata=parent_study, + dest=dest_path, + outputs=False, + denormalize=False, ) else: self.raw_study_service.export_study_flat( @@ -660,8 +685,10 @@ def _generate( denormalize=False, ) - results = self._generate_snapshot(variant_study, notifier) - + # Copy parent study to dest + results = self._generate_snapshot( + variant_study=variant_study, dest_path=dest_path, notifier=notifier + ) if results.success: variant_study.snapshot = VariantStudySnapshot( id=variant_study.id, @@ -678,21 +705,35 @@ def _generate( study_tree.denormalize() return results - def _generate_snapshot( - self, - variant_study: VariantStudy, - notifier: TaskUpdateNotifier = noop_notifier, - ) -> GenerationResultInfoDTO: + def _generate_study_config( + self, metadata: VariantStudy, config: Optional[FileStudyTreeConfig] + ) -> Tuple[GenerationResultInfoDTO, FileStudyTreeConfig]: + parent_study = self.repository.get(metadata.parent_id) + if parent_study is None: + raise StudyNotFoundError(metadata.parent_id) + + if isinstance(parent_study, RawStudy): + parent_config, _ = self.study_factory.create_from_fs( + Path(parent_study.path), + parent_study.id, + Path(parent_study.path) / "output", + use_cache=True, + ) + else: + res, parent_config = self._generate_study_config( + parent_study, config + ) + if res is not None and not res.success: + return res, parent_config - # Copy parent study to dest - dest_path = Path(variant_study.path) / SNAPSHOT_RELATIVE_PATH + # Generate + return self._generate_config(metadata, parent_config) + def _get_commands_and_notifier( + self, variant_study: VariantStudy, notifier: TaskUpdateNotifier + ) -> Tuple[List[List[ICommand]], Callable[[int, bool, str], None]]: # Generate - commands: List[List[ICommand]] = [] - for command_block in variant_study.commands: - commands.append( - self.command_factory.to_icommand(command_block.to_dto()) - ) + commands: List[List[ICommand]] = self._to_icommand(variant_study) def notify( command_index: int, command_result: bool, command_message: str @@ -719,6 +760,40 @@ def notify( exc_info=e, ) + return commands, notify + + def _to_icommand(self, metadata: VariantStudy) -> List[List[ICommand]]: + commands: List[List[ICommand]] = [] + for command_block in metadata.commands: + commands.append( + self.command_factory.to_icommand(command_block.to_dto()) + ) + return commands + + def _generate_config( + self, + variant_study: VariantStudy, + config: FileStudyTreeConfig, + notifier: TaskUpdateNotifier = noop_notifier, + ) -> Tuple[GenerationResultInfoDTO, FileStudyTreeConfig]: + + commands, notify = self._get_commands_and_notifier( + variant_study=variant_study, notifier=notifier + ) + return self.generator.generate_config( + commands, config, variant_study, notifier=notify + ) + + def _generate_snapshot( + self, + variant_study: VariantStudy, + dest_path: Path, + notifier: TaskUpdateNotifier = noop_notifier, + ) -> GenerationResultInfoDTO: + + commands, notify = self._get_commands_and_notifier( + variant_study=variant_study, notifier=notifier + ) return self.generator.generate( commands, dest_path, variant_study, notifier=notify ) @@ -764,7 +839,7 @@ def copy( Copy study to a new destination Args: src_meta: source study - dest_meta: destination study + dest_name: destination study with_outputs: indicate either to copy the output or not Returns: destination study """ @@ -828,6 +903,7 @@ def get_raw( Fetch a study raw tree object and its config Args: metadata: study + use_cache: use cache Returns: the config and study tree object """ self._safe_generation(metadata) @@ -933,3 +1009,27 @@ def export_study_flat( study.denormalize() duration = "{:.3f}".format(time.time() - stop_time) logger.info(f"Study {path_study} denormalized in {duration}s") + + def get_synthesis( + self, + metadata: VariantStudy, + params: Optional[RequestParameters] = None, + ) -> FileStudyTreeConfigDTO: + """ + Return study synthesis + Args: + metadata: study + params: RequestParameters + Returns: FileStudyTreeConfigDTO + + """ + if params is None: + raise UserHasNotPermissionError() + + results, config = self.generate_study_config(metadata.id, params) + if results.success: + return FileStudyTreeConfigDTO.from_build_config(config) + + raise VariantGenerationError( + f"Error during light generation of {metadata.id}" + ) diff --git a/antarest/study/web/studies_blueprint.py b/antarest/study/web/studies_blueprint.py index 397fa81550..941c97d709 100644 --- a/antarest/study/web/studies_blueprint.py +++ b/antarest/study/web/studies_blueprint.py @@ -1,17 +1,15 @@ import io import logging -import time from http import HTTPStatus from pathlib import Path from typing import Any, Optional, List, Dict -from fastapi import APIRouter, File, Depends, Request, HTTPException, Body -from markupsafe import escape +from fastapi import APIRouter, File, Depends, Request, HTTPException +from markupsafe import escape # type: ignore from starlette.responses import FileResponse from antarest.core.config import Config from antarest.core.filetransfer.model import ( - FileDownloadDTO, FileDownloadTaskDTO, ) from antarest.core.jwt import JWTUser @@ -32,6 +30,9 @@ StudyDownloadDTO, ) from antarest.study.service import StudyService +from antarest.study.storage.rawstudy.model.filesystem.config.model import ( + FileStudyTreeConfigDTO, +) from antarest.study.storage.study_download_utils import StudyDownloader logger = logging.getLogger(__name__) @@ -195,6 +196,24 @@ def create_study( return uuid + @bp.get( + "/studies/{uuid}/synthesis", + tags=[APITag.study_management], + summary="Return study synthesis", + response_model=FileStudyTreeConfigDTO, + ) + def get_study_synthesis( + uuid: str, + current_user: JWTUser = Depends(auth.get_current_user), + ) -> Any: + study_id = sanitize_uuid(uuid) + logger.info( + f"Return a synthesis for study '{study_id}'", + extra={"user": current_user.id}, + ) + params = RequestParameters(user=current_user) + return study_service.get_study_synthesis(study_id, params) + @bp.get( "/studies/{uuid}/export", tags=[APITag.study_management], diff --git a/antarest/tools/cli.py b/antarest/tools/cli.py index 45eed88f29..bba11b4824 100644 --- a/antarest/tools/cli.py +++ b/antarest/tools/cli.py @@ -2,26 +2,25 @@ from pathlib import Path from typing import Optional -import click +import click # type: ignore from antarest.study.model import NEW_DEFAULT_STUDY_VERSION from antarest.tools.lib import ( generate_diff, extract_commands, - apply_commands_from_dir, generate_study, ) logging.basicConfig(level=logging.INFO) -@click.group() +@click.group() # type: ignore def commands() -> None: pass -@commands.command() -@click.option( +@commands.command() # type: ignore +@click.option( # type: ignore "--host", "-h", nargs=1, @@ -29,7 +28,7 @@ def commands() -> None: type=str, help="Host URL of the antares web instance", ) -@click.option( +@click.option( # type: ignore "--auth_token", nargs=1, required=False, @@ -37,7 +36,7 @@ def commands() -> None: type=str, help="Authentication token if server needs one", ) -@click.option( +@click.option( # type: ignore "--output", "-o", nargs=1, @@ -45,7 +44,7 @@ def commands() -> None: type=str, help="Output study path", ) -@click.option( +@click.option( # type: ignore "--input", "-i", nargs=1, @@ -53,7 +52,7 @@ def commands() -> None: type=click.Path(exists=True), help="Variant script source path", ) -@click.option( +@click.option( # type: ignore "--study_id", "-s", nargs=1, @@ -83,8 +82,8 @@ def apply_script( print(res) -@commands.command() -@click.option( +@commands.command() # type: ignore +@click.option( # type: ignore "--input", "-i", nargs=1, @@ -92,7 +91,7 @@ def apply_script( type=click.Path(exists=True), help="Study path", ) -@click.option( +@click.option( # type: ignore "--output", "-o", nargs=1, @@ -105,22 +104,22 @@ def generate_script(input: str, output: str) -> None: extract_commands(Path(input), Path(output)) -@commands.command() -@click.option( +@commands.command() # type: ignore +@click.option( # type: ignore "--base", nargs=1, required=True, type=click.Path(exists=True), help="Base study path", ) -@click.option( +@click.option( # type: ignore "--variant", nargs=1, required=True, type=click.Path(exists=True), help="Variant study path", ) -@click.option( +@click.option( # type: ignore "--output", "-o", nargs=1, @@ -128,7 +127,7 @@ def generate_script(input: str, output: str) -> None: type=click.Path(exists=False), help="Script output path", ) -@click.option( +@click.option( # type: ignore "--version", "-v", nargs=1, diff --git a/setup.py b/setup.py index d3a4ecf83d..6bc41162a5 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="AntaREST", - version="2.1.5", + version="2.2.0", description="Antares Server", long_description=long_description, long_description_content_type="text/markdown", diff --git a/sonar-project.properties b/sonar-project.properties index cb15f8f5f3..f46f3b9a03 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -4,4 +4,4 @@ sonar.sources=antarest sonar.language=python sonar.exclusions=antarest/gui.py,antarest/main.py sonar.python.coverage.reportPaths=coverage.xml -sonar.projectVersion=2.1.5 \ No newline at end of file +sonar.projectVersion=2.2.0 \ No newline at end of file diff --git a/tests/eventbus/test_websocket_manager.py b/tests/eventbus/test_websocket_manager.py new file mode 100644 index 0000000000..5ae1044058 --- /dev/null +++ b/tests/eventbus/test_websocket_manager.py @@ -0,0 +1,53 @@ +import asyncio +from unittest import IsolatedAsyncioTestCase +from unittest.mock import Mock, MagicMock + +from starlette.websockets import WebSocket + +from antarest.core.jwt import JWTUser +from antarest.core.model import PermissionInfo, PublicMode +from antarest.eventbus.web import ( + ConnectionManager, + WebsocketMessage, + WebsocketMessageAction, +) + + +class AsyncMock(MagicMock): + async def __call__(self, *args, **kwargs): + return super(AsyncMock, self).__call__(*args, **kwargs) + + +class ConnectionManagerTest(IsolatedAsyncioTestCase): + async def test_subscriptions(self): + ws_manager = ConnectionManager() + + user = JWTUser(id=1, type="user", impersonator=1, groups=[]) + subscription_message = WebsocketMessage( + action=WebsocketMessageAction.SUBSCRIBE, payload="foo" + ) + unsubscription_message = WebsocketMessage( + action=WebsocketMessageAction.UNSUBSCRIBE, payload="foo" + ) + mock_connection = AsyncMock(spec=WebSocket) + await ws_manager.connect(mock_connection, user) + assert len(ws_manager.active_connections) == 1 + + ws_manager.process_message( + subscription_message.json(), mock_connection, user + ) + assert len(ws_manager.active_connections[0].channel_subscriptions) == 1 + assert ( + ws_manager.active_connections[0].channel_subscriptions[0] == "foo" + ) + + await ws_manager.broadcast("hello", PermissionInfo(), channel="foo") + mock_connection.send_text.assert_called_with("hello") + + ws_manager.process_message( + unsubscription_message.json(), mock_connection, user + ) + assert len(ws_manager.active_connections[0].channel_subscriptions) == 0 + + ws_manager.disconnect(mock_connection) + assert len(ws_manager.active_connections) == 0 diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 02d9afcbdb..fab179049d 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -131,6 +131,15 @@ def test_main(app: FastAPI): ) assert res.json() == comments + # study synthesis + res = client.get( + f"/v1/studies/{study_id}/synthesis", + headers={ + "Authorization": f'Bearer {george_credentials["access_token"]}' + }, + ) + assert res.status_code == 200 + # study creation created = client.post( "/v1/studies?name=foo", diff --git a/tests/launcher/test_service.py b/tests/launcher/test_service.py index 179c11bbbc..699112f956 100644 --- a/tests/launcher/test_service.py +++ b/tests/launcher/test_service.py @@ -69,6 +69,8 @@ def test_service_run_study(get_current_user_mock): job_id = launcher_service.run_study( "study_uuid", + "local", + None, RequestParameters( user=JWTUser( id=0, @@ -76,7 +78,6 @@ def test_service_run_study(get_current_user_mock): type="users", ) ), - "local", ) assert job_id == uuid diff --git a/tests/launcher/test_slurm_launcher.py b/tests/launcher/test_slurm_launcher.py index ee1065d8dd..97fa80a1f1 100644 --- a/tests/launcher/test_slurm_launcher.py +++ b/tests/launcher/test_slurm_launcher.py @@ -1,4 +1,5 @@ import os +import shutil import uuid from argparse import Namespace from pathlib import Path @@ -16,7 +17,7 @@ SlurmLauncher, ) from antarest.launcher.model import JobStatus -from antarest.study.model import StudyMetadataDTO +from antarest.study.model import StudyMetadataDTO, RawStudy @pytest.fixture @@ -158,6 +159,50 @@ def test_slurm_launcher_delete_function(tmp_path: str): assert not directory_path.exists() +def test_extra_parameters(launcher_config: Config): + slurm_launcher = SlurmLauncher( + config=launcher_config, + study_service=Mock(), + callbacks=Mock(), + event_bus=Mock(), + ) + launcher_params = slurm_launcher._check_and_apply_launcher_params({}) + assert launcher_params.n_cpu == 1 + assert launcher_params.time_limit == 0 + assert not launcher_params.xpansion_mode + assert not launcher_params.post_processing + + launcher_params = slurm_launcher._check_and_apply_launcher_params( + {"nb_cpu": 12} + ) + assert launcher_params.n_cpu == 12 + + launcher_params = slurm_launcher._check_and_apply_launcher_params( + {"nb_cpu": 48} + ) + assert launcher_params.n_cpu == 1 + + launcher_params = slurm_launcher._check_and_apply_launcher_params( + {"time_limit": 999999999} + ) + assert launcher_params.time_limit == 0 + + launcher_params = slurm_launcher._check_and_apply_launcher_params( + {"time_limit": 99999} + ) + assert launcher_params.time_limit == 99999 + + launcher_params = slurm_launcher._check_and_apply_launcher_params( + {"xpansion": True} + ) + assert launcher_params.xpansion_mode + + launcher_params = slurm_launcher._check_and_apply_launcher_params( + {"post_processing": True} + ) + assert launcher_params.post_processing + + @pytest.mark.parametrize( "version,job_status", [(42, JobStatus.RUNNING), (99, JobStatus.FAILED)] ) @@ -198,7 +243,9 @@ def test_run_study( slurm_launcher.start = Mock() slurm_launcher._delete_study = Mock() - slurm_launcher._run_study(study_uuid, str(uuid.uuid4()), params=params) + slurm_launcher._run_study( + study_uuid, str(uuid.uuid4()), None, params=params + ) slurm_launcher._clean_local_workspace.assert_called_once() storage_service.export_study_flat.assert_called_once() @@ -309,6 +356,50 @@ def test_import_study_output(launcher_config): ) assert res == "output" + link_dir = ( + launcher_config.launcher.slurm.local_workspace + / "OUTPUT" + / "1" + / "input" + / "links" + ) + link_dir.mkdir(parents=True) + link_file = link_dir / "something" + link_file.write_text("hello") + xpansion_dir = Path( + launcher_config.launcher.slurm.local_workspace + / "OUTPUT" + / "1" + / "user" + / "expansion" + ) + xpansion_dir.mkdir(parents=True) + xpansion_test_file = xpansion_dir / "something_else" + xpansion_test_file.write_text("world") + output_dir = ( + launcher_config.launcher.slurm.local_workspace + / "OUTPUT" + / "1" + / "output" + / "output_name" + ) + output_dir.mkdir(parents=True) + slurm_launcher.storage_service.get_study.side_effect = [ + Mock(spec=RawStudy, version="800"), + Mock(spec=RawStudy, version="700"), + ] + assert not (output_dir / "updated_links" / "something").exists() + assert not (output_dir / "updated_links" / "something").exists() + + slurm_launcher._import_study_output("1", True) + assert (output_dir / "updated_links" / "something").exists() + assert (output_dir / "updated_links" / "something").read_text() == "hello" + shutil.rmtree(output_dir / "updated_links") + + slurm_launcher._import_study_output("1", True) + assert (output_dir / "results" / "something_else").exists() + assert (output_dir / "results" / "something_else").read_text() == "world" + @patch("antarest.launcher.adapters.slurm_launcher.slurm_launcher.run_with") @pytest.mark.unit_test @@ -340,7 +431,7 @@ def test_kill_job( job_id_to_kill=42, json_ssh_config=None, log_dir=str(tmp_path / "LOGS"), - n_cpu=0, + n_cpu=1, output_dir=str(tmp_path / "OUTPUT"), post_processing=False, studies_in=str(tmp_path / "STUDIES_IN"), diff --git a/tests/launcher/test_web.py b/tests/launcher/test_web.py index 78f0ccdb01..8b298a3953 100644 --- a/tests/launcher/test_web.py +++ b/tests/launcher/test_web.py @@ -47,7 +47,7 @@ def test_run() -> None: assert res.status_code == 200 assert res.json() == {"job_id": str(job)} service.run_study.assert_called_once_with( - study, RequestParameters(ADMIN), "local" + study, "local", None, RequestParameters(ADMIN) ) diff --git a/tests/storage/business/test_raw_study_service.py b/tests/storage/business/test_raw_study_service.py index 76dc296619..1e4c5671c5 100644 --- a/tests/storage/business/test_raw_study_service.py +++ b/tests/storage/business/test_raw_study_service.py @@ -274,11 +274,12 @@ def create_study(version: str): ) return study_service.create(metadata) - md613 = create_study("613") + md613 = create_study("610") md700 = create_study("700") md710 = create_study("710") md720 = create_study("720") - md803 = create_study("803") + md803 = create_study("800") + md810 = create_study("810") path_study = path_studies / md613.id general_data_file = path_study / "settings" / "generaldata.ini" @@ -362,6 +363,18 @@ def create_study(version: str): is not None ) + path_study = path_studies / md810.id + general_data_file = path_study / "settings" / "generaldata.ini" + general_data = general_data_file.read_text() + assert ( + re.search( + "^renewable-generation-modelling = false", + general_data, + flags=re.MULTILINE, + ) + is None + ) + @pytest.mark.unit_test def test_copy_study( diff --git a/tests/storage/business/test_variant_study_service.py b/tests/storage/business/test_variant_study_service.py index bde8889cf8..59b1b70b66 100644 --- a/tests/storage/business/test_variant_study_service.py +++ b/tests/storage/business/test_variant_study_service.py @@ -11,12 +11,19 @@ VariantGenerationError, ) from antarest.core.interfaces.cache import CacheConstants +from antarest.core.jwt import JWTUser +from antarest.core.model import PublicMode +from antarest.core.requests import RequestParameters from antarest.core.tasks.model import TaskDTO, TaskStatus, TaskResult +from antarest.login.model import User from antarest.study.model import DEFAULT_WORKSPACE_NAME from antarest.study.storage.variantstudy.model.dbmodel import ( VariantStudy, CommandBlock, ) +from antarest.study.storage.variantstudy.repository import ( + VariantStudyRepository, +) from antarest.study.storage.variantstudy.variant_study_service import ( VariantStudyService, ) @@ -297,3 +304,69 @@ def test_delete_study(tmp_path: Path) -> None: ] ) assert not study_path.exists() + + +@pytest.mark.unit_test +def test_get_variant_children(tmp_path: Path) -> None: + name = "my-study" + study_path = tmp_path / name + study_path.mkdir() + (study_path / "study.antares").touch() + + cache = Mock() + repo_mock = Mock(spec=VariantStudyRepository) + study_service = VariantStudyService( + raw_study_service=Mock(), + cache=cache, + task_service=Mock(), + command_factory=Mock(), + study_factory=Mock(), + config=build_config(tmp_path), + repository=repo_mock, + event_bus=Mock(), + patch_service=Mock(), + ) + + parent = VariantStudy( + id="parent", + name="parent", + type="variant", + archived=False, + path=str(study_path), + version="700", + owner=User(id=2, name="me"), + groups=[], + public_mode=PublicMode.NONE, + ) + children = [ + VariantStudy( + id="child1", + name="child1", + type="variant", + archived=False, + path=str(study_path), + version="700", + owner=User(id=2, name="me"), + groups=[], + public_mode=PublicMode.NONE, + ), + VariantStudy( + id="child2", + name="child2", + type="variant", + archived=False, + path=str(study_path), + version="700", + owner=User(id=3, name="not me"), + groups=[], + public_mode=PublicMode.NONE, + ), + ] + repo_mock.get.side_effect = [parent] + children + repo_mock.get_children.side_effect = [children, [], []] + + tree = study_service.get_all_variants_children( + "parent", + RequestParameters(user=JWTUser(id=2, type="user", impersonator=2)), + ) + assert len(tree.children) == 1 diff --git a/tests/storage/repository/filesystem/test_bucket_node.py b/tests/storage/repository/filesystem/test_bucket_node.py index 38a661d7b7..f87d451ee6 100644 --- a/tests/storage/repository/filesystem/test_bucket_node.py +++ b/tests/storage/repository/filesystem/test_bucket_node.py @@ -61,3 +61,9 @@ def test_save_bucket(tmp_path: Path): node.save(data={"fileA.txt": b"Hello, World"}) assert (file / "fileA.txt").read_text() == "Hello, World" + + node.save(data={"fileC.txt": b"test"}, url=["folder"]) + assert (file / "folder" / "fileC.txt").read_text() == "test" + + node.save(data=b"test2", url=["folder", "fileC.txt"]) + assert (file / "folder" / "fileC.txt").read_text() == "test2" diff --git a/tests/variantstudy/model/command/conftest.py b/tests/variantstudy/model/command/conftest.py index 30b83b4b33..a077432ab1 100644 --- a/tests/variantstudy/model/command/conftest.py +++ b/tests/variantstudy/model/command/conftest.py @@ -62,6 +62,11 @@ def command_context(matrix_service: MatrixService) -> CommandContext: command_extractor, x ) ) + command_extractor.generate_update_rawfile.side_effect = ( + lambda x, u: CommandExtraction.generate_update_rawfile( + command_extractor, x, u + ) + ) command_context = CommandContext( generator_matrix_constants=GeneratorMatrixConstants( matrix_service=matrix_service diff --git a/tests/variantstudy/model/command/data.png b/tests/variantstudy/model/command/data.png new file mode 100644 index 0000000000..c36f9c7ba4 Binary files /dev/null and b/tests/variantstudy/model/command/data.png differ diff --git a/tests/variantstudy/model/command/test_manage_binding_constraints.py b/tests/variantstudy/model/command/test_manage_binding_constraints.py index dd99397682..9a47de6a81 100644 --- a/tests/variantstudy/model/command/test_manage_binding_constraints.py +++ b/tests/variantstudy/model/command/test_manage_binding_constraints.py @@ -3,6 +3,9 @@ from antarest.matrixstore.service import MatrixService from antarest.study.storage.rawstudy.io.reader import IniReader from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy +from antarest.study.storage.rawstudy.model.filesystem.folder_node import ( + ChildNotFoundError, +) from antarest.study.storage.variantstudy.business.matrix_constants_generator import ( GeneratorMatrixConstants, ) @@ -382,6 +385,9 @@ def test_revert(command_context: CommandContext): command_context=command_context, ) ] + base.command_context.command_extractor.extract_binding_constraint.side_effect = ( + ChildNotFoundError() + ) base.revert([], study) base.command_context.command_extractor.extract_binding_constraint.assert_called_with( study, "foo" diff --git a/tests/variantstudy/model/command/test_manage_district.py b/tests/variantstudy/model/command/test_manage_district.py index e413d52cc0..d6aba041dd 100644 --- a/tests/variantstudy/model/command/test_manage_district.py +++ b/tests/variantstudy/model/command/test_manage_district.py @@ -9,6 +9,9 @@ transform_name_to_id, ) from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy +from antarest.study.storage.rawstudy.model.filesystem.folder_node import ( + ChildNotFoundError, +) from antarest.study.storage.variantstudy.business.matrix_constants_generator import ( GeneratorMatrixConstants, ) @@ -193,6 +196,9 @@ def test_revert(command_context: CommandContext): base = RemoveDistrict(id="id", command_context=command_context) study = FileStudy(config=Mock(), tree=Mock()) + base.command_context.command_extractor.extract_district.side_effect = ( + ChildNotFoundError() + ) base.revert([], study) base.command_context.command_extractor.extract_district.assert_called_with( study, "id" diff --git a/tests/variantstudy/model/command/test_remove_area.py b/tests/variantstudy/model/command/test_remove_area.py index 6cf2ecf582..909e76fa0f 100644 --- a/tests/variantstudy/model/command/test_remove_area.py +++ b/tests/variantstudy/model/command/test_remove_area.py @@ -7,6 +7,9 @@ transform_name_to_id, ) from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy +from antarest.study.storage.rawstudy.model.filesystem.folder_node import ( + ChildNotFoundError, +) from antarest.study.storage.variantstudy.business.matrix_constants_generator import ( GeneratorMatrixConstants, ) @@ -107,6 +110,9 @@ def test_revert(command_context: CommandContext): [Mock()], [Mock()], ) + base.command_context.command_extractor.extract_area.side_effect = ( + ChildNotFoundError() + ) base.revert([], study) base.command_context.command_extractor.extract_area.assert_called_with( study, "foo" diff --git a/tests/variantstudy/model/command/test_remove_cluster.py b/tests/variantstudy/model/command/test_remove_cluster.py index e27455890e..48a489dd1d 100644 --- a/tests/variantstudy/model/command/test_remove_cluster.py +++ b/tests/variantstudy/model/command/test_remove_cluster.py @@ -7,6 +7,9 @@ transform_name_to_id, ) from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy +from antarest.study.storage.rawstudy.model.filesystem.folder_node import ( + ChildNotFoundError, +) from antarest.study.storage.variantstudy.business.matrix_constants_generator import ( GeneratorMatrixConstants, ) @@ -135,6 +138,9 @@ def test_revert(command_context: CommandContext): ) ] study = FileStudy(config=Mock(), tree=Mock()) + base.command_context.command_extractor.extract_cluster.side_effect = ( + ChildNotFoundError() + ) base.revert([], study) base.command_context.command_extractor.extract_cluster.assert_called_with( study, "foo", "bar" diff --git a/tests/variantstudy/model/command/test_remove_link.py b/tests/variantstudy/model/command/test_remove_link.py index f149ba1c83..e8fa19c6b3 100644 --- a/tests/variantstudy/model/command/test_remove_link.py +++ b/tests/variantstudy/model/command/test_remove_link.py @@ -8,6 +8,9 @@ transform_name_to_id, ) from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy +from antarest.study.storage.rawstudy.model.filesystem.folder_node import ( + ChildNotFoundError, +) from antarest.study.storage.variantstudy.business.default_values import ( FilteringOptions, LinkProperties, @@ -104,6 +107,9 @@ def test_revert(command_context: CommandContext): area1="foo", area2="bar", command_context=command_context ) study = FileStudy(config=Mock(), tree=Mock()) + base.command_context.command_extractor.extract_link.side_effect = ( + ChildNotFoundError() + ) base.revert([], study) base.command_context.command_extractor.extract_link.assert_called_with( study, "bar", "foo" diff --git a/tests/variantstudy/model/command/test_update_rawfile.py b/tests/variantstudy/model/command/test_update_rawfile.py new file mode 100644 index 0000000000..3f75613cd8 --- /dev/null +++ b/tests/variantstudy/model/command/test_update_rawfile.py @@ -0,0 +1,55 @@ +import base64 +import os.path +from pathlib import Path +from typing import cast + +from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy +from antarest.study.storage.variantstudy.model.command.common import ( + CommandName, +) +from antarest.study.storage.variantstudy.model.command.update_raw_file import ( + UpdateRawFile, +) +from antarest.study.storage.variantstudy.model.command_context import ( + CommandContext, +) +from antarest.study.storage.variantstudy.model.model import CommandDTO + + +def test_update_rawfile( + empty_study: FileStudy, command_context: CommandContext +) -> None: + data_path = Path(os.path.dirname(__file__)) / "data.png" + data = base64.b64encode(data_path.read_bytes()).decode("utf-8") + + original_data = empty_study.tree.get(["settings", "resources", "study"]) + + command = UpdateRawFile( + target="settings/resources/study", + b64Data=data, + command_context=command_context, + ) + + reverted_commands = command.revert([], empty_study) + assert cast( + UpdateRawFile, reverted_commands[0] + ).b64Data == base64.b64encode(original_data).decode("utf-8") + + alt_command = UpdateRawFile( + target="settings/resources/study", + b64Data="", + command_context=command_context, + ) + reverted_commands = command.revert([alt_command], empty_study) + assert cast(UpdateRawFile, reverted_commands[0]).b64Data == "" + + assert command.match(alt_command) + assert not command.match(alt_command, True) + assert len(command.get_inner_matrices()) == 0 + assert [alt_command] == command.create_diff(alt_command) + + res = command.apply(empty_study) + assert res.status + new_data = empty_study.tree.get(["settings", "resources", "study"]) + assert original_data != new_data + assert cast(bytes, new_data).startswith(b"\x89PNG") diff --git a/tests/variantstudy/model/test_variant_model.py b/tests/variantstudy/model/test_variant_model.py index f32d9697e3..7a87a46443 100644 --- a/tests/variantstudy/model/test_variant_model.py +++ b/tests/variantstudy/model/test_variant_model.py @@ -120,6 +120,6 @@ def test_commands_service() -> VariantStudyService: service._generate_snapshot = Mock() expected_result = GenerationResultInfoDTO(success=True, details=[]) service._generate_snapshot.return_value = expected_result - results = service._generate(saved_id, False, SADMIN) + results = service._generate(saved_id, SADMIN, False) assert results == expected_result assert study.snapshot.id == study.id diff --git a/tests/variantstudy/test_command_factory.py b/tests/variantstudy/test_command_factory.py index 7300b3354b..0c3985b678 100644 --- a/tests/variantstudy/test_command_factory.py +++ b/tests/variantstudy/test_command_factory.py @@ -211,6 +211,13 @@ action=CommandName.UPDATE_COMMENTS.value, args=[{"comments": "comments"}], ), + CommandDTO( + action=CommandName.UPDATE_FILE.value, + args={ + "target": "settings/resources/study", + "b64Data": "", + }, + ), ], ) @pytest.mark.unit_test diff --git a/webapp/package.json b/webapp/package.json index f2a75be817..1874d5833b 100644 --- a/webapp/package.json +++ b/webapp/package.json @@ -1,6 +1,6 @@ { "name": "antares-web", - "version": "2.1.5", + "version": "2.2.0", "private": true, "dependencies": { "@fortawesome/fontawesome-svg-core": "^1.2.36", diff --git a/webapp/public/locales/en/singlestudy.json b/webapp/public/locales/en/singlestudy.json index 4f09ad7557..0ba8dfc322 100644 --- a/webapp/public/locales/en/singlestudy.json +++ b/webapp/public/locales/en/singlestudy.json @@ -2,6 +2,12 @@ "informations": "Information", "treeView": "Tree view", "variants": "Variant management", + "runStudy": "Run study", + "xpansionMode": "Xpansion mode", + "postProcessing": "Post processing", + "timeLimit": "Time limit", + "timeLimitHelper": "Time limit (in hours)", + "nbCpu": "Number of core", "currentTask": "Current tasks", "failtoloadjobs": "Failed to load jobs", "taskId": "Id", diff --git a/webapp/public/locales/en/variants.json b/webapp/public/locales/en/variants.json index b0feb2f5cf..d2b19e32bb 100644 --- a/webapp/public/locales/en/variants.json +++ b/webapp/public/locales/en/variants.json @@ -23,6 +23,7 @@ "addError": "Command not added", "moveError": "Command not moved", "fetchCommandError": "Error while retrieving commands", + "fetchSynthesisError": "Error while fetching study synthesis", "importSuccess": "File imported successfully", "importError": "Error during file importation", "jsonParsingError": "Error while parsing json command file", diff --git a/webapp/public/locales/fr/singlestudy.json b/webapp/public/locales/fr/singlestudy.json index 3bed9b4527..41f3f36b66 100644 --- a/webapp/public/locales/fr/singlestudy.json +++ b/webapp/public/locales/fr/singlestudy.json @@ -2,6 +2,12 @@ "informations": "Informations", "treeView": "Vue détaillée", "variants": "Gestion des variantes", + "runStudy": "Lancer l'étude", + "xpansionMode": "Mode Xpansion", + "postProcessing": "Post processing", + "timeLimit": "Limite de temps", + "timeLimitHelper": "Limite de temps (en heures)", + "nbCpu": "Nombre de coeurs", "currentTask": "Tâches", "failtoloadjobs": "Echec du chargement des jobs", "taskId": "Id", diff --git a/webapp/public/locales/fr/variants.json b/webapp/public/locales/fr/variants.json index 0be6ff5351..edb9366f14 100644 --- a/webapp/public/locales/fr/variants.json +++ b/webapp/public/locales/fr/variants.json @@ -22,6 +22,7 @@ "addError": "Erreur lors de l'ajout de votre commande", "moveError": "Erreur lors du changement de position", "fetchCommandError": "Erreur lors de la récupération des commandes", + "fetchSynthesisError": "Erreur lors de la récupération de la synthèse de l'étude", "importSuccess": "Fichier importé avec succès", "importError": "Erreur lors de l'importation du fichier", "jsonParsingError": "Erreur lors de la lecture du fichier de commande", diff --git a/webapp/src/App/Pages/StudyManagement.tsx b/webapp/src/App/Pages/StudyManagement.tsx index d0af77e367..3ac8455ae1 100644 --- a/webapp/src/App/Pages/StudyManagement.tsx +++ b/webapp/src/App/Pages/StudyManagement.tsx @@ -13,7 +13,7 @@ import { AxiosError } from 'axios'; import { AppState } from '../reducers'; import StudyCreationTools from '../../components/StudyCreationTools'; import StudyListing from '../../components/StudyListing'; -import { initStudies } from '../../ducks/study'; +import { initStudies, initStudiesVersion } from '../../ducks/study'; import { getStudies } from '../../services/api/study'; import MainContentLoader from '../../components/ui/loaders/MainContentLoader'; import SortView from '../../components/ui/SortView'; @@ -25,6 +25,7 @@ import theme from '../theme'; import { getGroups, getUsers } from '../../services/api/user'; import { loadState, saveState } from '../../services/utils/localStorage'; import enqueueErrorSnackbar from '../../components/ui/ErrorSnackBar'; +import { convertVersions } from '../../services/utils'; const DEFAULT_LIST_MODE_KEY = 'studylisting.listmode'; const DEFAULT_FILTER_USER = 'studylisting.filter.user'; @@ -63,10 +64,12 @@ const useStyles = makeStyles(() => createStyles({ const mapState = (state: AppState) => ({ studies: state.study.studies, + versions: state.study.versionList, }); const mapDispatch = ({ loadStudies: initStudies, + loadVersions: initStudiesVersion, }); const connector = connect(mapState, mapDispatch); @@ -74,7 +77,7 @@ type ReduxProps = ConnectedProps; type PropTypes = ReduxProps; const StudyManagement = (props: PropTypes) => { - const { studies, loadStudies } = props; + const { studies, loadStudies, loadVersions, versions } = props; const classes = useStyles(); const [t] = useTranslation(); const { enqueueSnackbar } = useSnackbar(); @@ -107,11 +110,7 @@ const StudyManagement = (props: PropTypes) => { const [currentGroup, setCurrentGroup] = useState(loadState(DEFAULT_FILTER_GROUP)); const [currentVersion, setCurrentVersion] = useState(loadState(DEFAULT_FILTER_VERSION)); - const versionList = [{ id: '640', name: '6.4.0' }, - { id: '700', name: '7.0.0' }, - { id: '710', name: '7.1.0' }, - { id: '720', name: '7.2.0' }, - { id: '800', name: '8.0.0' }]; + const versionList = convertVersions(versions || []); const init = async () => { try { @@ -128,6 +127,9 @@ const StudyManagement = (props: PropTypes) => { useEffect(() => { init(); getAllStudies(false); + if (!versions) { + loadVersions(); + } }, []); useEffect(() => { diff --git a/webapp/src/common/types.ts b/webapp/src/common/types.ts index 424c65ae98..4426c70f69 100644 --- a/webapp/src/common/types.ts +++ b/webapp/src/common/types.ts @@ -317,4 +317,57 @@ export interface TaskEventPayload { message: string; } +export interface Cluster { + id: string; + name: string; + enabled: boolean; +} + +export interface Link{ + filters_synthesis: Array; + filters_year: Array; +} + +export interface Area { + name: string; + links: {[elm: string]: Link}; + thermals: Array; + renewables: Array; + filters_synthesis: Array; + filters_year: Array; +} +export interface Set { + name?: string; + inverted_set: boolean; + areas?: Array; + output: boolean; + filters_synthesis: Array; + filters_year: Array; +} + +export interface Simulation { + name: string; + date: string; + mode: string; + nbyears: number; + synthesis: boolean; + by_year: boolean; + error: boolean; +} + +export interface FileStudyTreeConfigDTO { + study_path: string; + path: string; + study_id: string; + version: number; + output_path?: string; + areas: {[elm: string]: Area}; + sets: {[elm: string]: Set}; + outputs: {[elm: string]: Simulation}; + bindings: Array; + store_new_set: boolean; + archive_input_series: Array; + enr_modelling: string; +} + export default {}; diff --git a/webapp/src/components/Data/DataView.tsx b/webapp/src/components/Data/DataView.tsx index f632ad3fcb..dcc226535e 100644 --- a/webapp/src/components/Data/DataView.tsx +++ b/webapp/src/components/Data/DataView.tsx @@ -136,7 +136,7 @@ const DataView = (props: PropTypes) => { } }; - const matchFilter = (input: string): boolean => input.search(filter) >= 0; + const matchFilter = (input: MatrixDataSetDTO): boolean => input.name.search(filter) >= 0 || !!input.matrices.find((matrix: MatrixInfoDTO) => matrix.id.search(filter) >= 0); useEffect(() => { const initToogleList: Array = []; @@ -150,7 +150,7 @@ const DataView = (props: PropTypes) => { {data.map( (dataset, index) => - matchFilter(dataset.name) && ( + matchFilter(dataset) && ( { const theme = useTheme(); const { enqueueSnackbar } = useSnackbar(); const history = useHistory(); + const [studyToLaunch, setStudyToLaunch] = useState(); const [openConfirmationModal, setOpenConfirmationModal] = useState(false); const [openPermissionModal, setOpenPermissionModal] = useState(false); const [openRenameModal, setOpenRenameModal] = useState(false); const [outputList, setOutputList] = useState>(); const [outputExportButtonAnchor, setOutputExportButtonAnchor] = React.useState(null); - const launchStudy = async () => { + const openStudyLauncher = (): void => { if (study) { - try { - await callLaunchStudy(study.id); - enqueueSnackbar(t('studymanager:studylaunched', { studyname: study.name }), { - variant: 'success', - }); - } catch (e) { - enqueueErrorSnackbar(enqueueSnackbar, t('studymanager:failtorunstudy'), e as AxiosError); - logError('Failed to launch study', study, e); - } + setStudyToLaunch(study); } }; @@ -448,7 +441,7 @@ const InformationView = (props: PropTypes) => { ) : ( <> - exportStudy(study.id, false)} fakeDelay={500}> @@ -518,6 +511,7 @@ const InformationView = (props: PropTypes) => { onClose={() => setOpenRenameModal(false)} /> )} + { setStudyToLaunch(undefined); }} /> ) : null; }; diff --git a/webapp/src/components/SingleStudy/TaskView.tsx b/webapp/src/components/SingleStudy/TaskView.tsx index 6f5d3652d4..b74067fba2 100644 --- a/webapp/src/components/SingleStudy/TaskView.tsx +++ b/webapp/src/components/SingleStudy/TaskView.tsx @@ -148,16 +148,20 @@ const TaskView = (props: PropTypes) => { const [jobIdDetail, setJobIdDetail] = useState(); const [jobIdKill, setJobIdKill] = useState(); const [logModalContent, setLogModalContent] = useState(); + const [logModalContentLoading, setLogModalContentLoading] = useState(false); const [openConfirmationModal, setOpenConfirmationModal] = useState(false); const openLogView = (jobId: string) => { + setJobIdDetail(jobId); + setLogModalContentLoading(true); (async () => { try { const logData = await getStudyJobLog(jobId); setLogModalContent(logData); - setJobIdDetail(jobId); } catch (e) { enqueueErrorSnackbar(enqueueSnackbar, t('singlestudy:failtofetchlogs'), e as AxiosError); + } finally { + setLogModalContentLoading(false); } })(); }; @@ -202,6 +206,7 @@ const TaskView = (props: PropTypes) => { title={t('singlestudy:taskLog')} jobId={jobIdDetail} content={logModalContent} + loading={logModalContentLoading} close={() => setJobIdDetail(undefined)} />
diff --git a/webapp/src/components/StudyCreationTools/CreateStudyForm.tsx b/webapp/src/components/StudyCreationTools/CreateStudyForm.tsx index 397b5554b9..5352af8bc5 100644 --- a/webapp/src/components/StudyCreationTools/CreateStudyForm.tsx +++ b/webapp/src/components/StudyCreationTools/CreateStudyForm.tsx @@ -1,33 +1,29 @@ -import React from 'react'; +import React, { useEffect } from 'react'; import { Controller, useForm } from 'react-hook-form'; import { useTranslation } from 'react-i18next'; import { Button, Input, MenuItem, Select } from '@material-ui/core'; import debug from 'debug'; import { connect, ConnectedProps } from 'react-redux'; import { createStudy, getStudyMetadata } from '../../services/api/study'; -import { addStudies } from '../../ducks/study'; +import { addStudies, initStudiesVersion } from '../../ducks/study'; import { StudyMetadata } from '../../common/types'; +import { AppState } from '../../App/reducers'; +import { displayVersionName } from '../../services/utils'; const logErr = debug('antares:createstudyform:error'); -const AVAILABLE_VERSIONS: Record = { - '8.0.3': 803, - '7.2.0': 720, - '7.1.0': 710, - '7.0.0': 700, - '6.1.3': 613, -}; -const DEFAULT_VERSION = 803; - interface Inputs { studyname: string; version: number; } -const mapState = () => ({ /* noop */ }); +const mapState = (state: AppState) => ({ + versions: state.study.versionList, +}); const mapDispatch = ({ addStudy: (study: StudyMetadata) => addStudies([study]), + loadVersions: initStudiesVersion, }); const connector = connect(mapState, mapDispatch); @@ -39,7 +35,7 @@ type PropTypes = PropsFromRedux & OwnProps; const CreateStudyForm = (props: PropTypes) => { const [t] = useTranslation(); - const { useStyles, addStudy } = props; + const { useStyles, addStudy, loadVersions, versions } = props; const classes = useStyles(); const { control, register, handleSubmit } = useForm(); @@ -55,30 +51,40 @@ const CreateStudyForm = (props: PropTypes) => { } }; + const defaultVersion = versions && versions[versions.length - 1]; + + useEffect(() => { + if (!versions) { + loadVersions(); + } + }); + return (
- ( - - )} - /> + { defaultVersion && ( + ( + + )} + /> + )} ); }; diff --git a/webapp/src/components/StudyListing/StudyListingItemView/StudyListSummaryView.tsx b/webapp/src/components/StudyListing/StudyListingItemView/StudyListSummaryView.tsx index 0891e8cc0e..21380980c4 100644 --- a/webapp/src/components/StudyListing/StudyListingItemView/StudyListSummaryView.tsx +++ b/webapp/src/components/StudyListing/StudyListingItemView/StudyListSummaryView.tsx @@ -129,14 +129,16 @@ const StudyListSummaryView = (props: StudyListingItemPropTypes) => {
{convertUTCToLocalTime(study.modificationDate)} -
- - {study.id} - - - copyId(study.id)} /> - -
+ {study.managed && ( +
+ + {study.id} + + + copyId(study.id)} /> + +
+ )}
{study.archived ? ( @@ -149,14 +151,13 @@ const StudyListSummaryView = (props: StudyListingItemPropTypes) => { ) : ( <> - launchStudy(study)} - fakeDelay={1000} > {t('main:launch')} - + { const classes = useStyles(); const { studies, removeStudy, isList, scrollPosition, updateScroll } = props; const { enqueueSnackbar } = useSnackbar(); + const [studyToLaunch, setStudyToLaunch] = useState(); const [t] = useTranslation(); - const launchStudy = async (study: StudyMetadata) => { - try { - await callLaunchStudy(study.id); - enqueueSnackbar(t('studymanager:studylaunched', { studyname: study.name }), { variant: 'success' }); - } catch (e) { - enqueueSnackbar(t('studymanager:failtorunstudy'), { variant: 'error' }); - logError('Failed to launch study', study, e); - } + const openStudyLauncher = (study: StudyMetadata): void => { + setStudyToLaunch(study); }; - const debouncedLaunchStudy = _.debounce(launchStudy, 5000, { leading: true, trailing: false }); - const importStudy = async (study: StudyMetadata, withOutputs = false) => { try { await callCopyStudy(study.id, `${study.name} (${t('main:copy')})`, withOutputs); @@ -174,15 +167,16 @@ const StudyListing = (props: PropTypes) => { innerElementType={innerElementType} itemCount={studies.length} itemSize={66} - itemData={{ studies, importStudy, launchStudy: debouncedLaunchStudy, deleteStudy, archiveStudy, unarchiveStudy }} + itemData={{ studies, importStudy, launchStudy: openStudyLauncher, deleteStudy, archiveStudy, unarchiveStudy }} > {Row} ) } - ) : + ) : } + { setStudyToLaunch(undefined); }} />
); }; diff --git a/webapp/src/components/StudySearchTool/index.tsx b/webapp/src/components/StudySearchTool/index.tsx index 08d8291090..90e8d2376f 100644 --- a/webapp/src/components/StudySearchTool/index.tsx +++ b/webapp/src/components/StudySearchTool/index.tsx @@ -113,7 +113,7 @@ const StudySearchTool = (props: PropTypes) => { }; const filter = (currentName: string): StudyMetadata[] => sortStudies() - .filter((s) => !currentName || s.name.search(new RegExp(currentName, 'i')) !== -1) + .filter((s) => !currentName || (s.name.search(new RegExp(currentName, 'i')) !== -1) || (s.id.search(new RegExp(currentName, 'i')) !== -1)) .filter((s) => !versionFilter || versionFilter.id === s.version) .filter((s) => (userFilter ? (s.owner.id && userFilter.id === s.owner.id) : true)) .filter((s) => (groupFilter ? s.groups.findIndex((elm) => elm.id === groupFilter.id) >= 0 : true)) diff --git a/webapp/src/components/ui/LauncherModal.tsx b/webapp/src/components/ui/LauncherModal.tsx new file mode 100644 index 0000000000..20d1c5508d --- /dev/null +++ b/webapp/src/components/ui/LauncherModal.tsx @@ -0,0 +1,207 @@ +import React, { useState } from 'react'; +import { createStyles, makeStyles, Theme, Button, Paper, FormGroup, FormControlLabel, Checkbox, Typography, FormControl, TextField } from '@material-ui/core'; +import { useSnackbar } from 'notistack'; +import Modal from '@material-ui/core/Modal'; +import Backdrop from '@material-ui/core/Backdrop'; +import Fade from '@material-ui/core/Fade'; +import { useTranslation } from 'react-i18next'; +import { AxiosError } from 'axios'; +import clsx from 'clsx'; +import enqueueErrorSnackbar from './ErrorSnackBar'; +import { LaunchOptions, launchStudy } from '../../services/api/study'; +import { StudyMetadata } from '../../common/types'; + +const useStyles = makeStyles((theme: Theme) => createStyles({ + root: { + display: 'flex', + justifyContent: 'center', + alignItems: 'center', + overflowY: 'auto', + boxSizing: 'border-box', + padding: theme.spacing(2), + }, + main: { + backgroundColor: 'white', + display: 'flex', + flexFlow: 'column nowrap', + alignItems: 'center', + width: '400px', + height: '450px', + }, + titlebox: { + height: '40px', + width: '100%', + display: 'flex', + flexFlow: 'row nowrap', + alignItems: 'center', + backgroundColor: theme.palette.primary.main, + }, + title: { + fontWeight: 'bold', + color: 'white', + marginLeft: theme.spacing(2), + overflow: 'hidden', + }, + content: { + flex: '1', + minWidth: '100px', + width: '100%', + display: 'flex', + flexFlow: 'column nowrap', + justifyContent: 'flex-start', + alignItems: 'flex-start', + overflow: 'hidden', + boxSizing: 'border-box', + padding: theme.spacing(2), + }, + footer: { + height: '60px', + width: '100%', + display: 'flex', + flexFlow: 'row nowrap', + justifyContent: 'center', + alignItems: 'center', + overflow: 'hidden', + }, + optionTitle: { + fontSize: '1.2em', + fontWeight: 'bold', + marginBottom: theme.spacing(3), + }, + fieldSection: { + marginTop: theme.spacing(1), + width: '100%', + }, + button: { + margin: theme.spacing(2), + }, + +})); + +interface PropTypes { + open: boolean; + study?: StudyMetadata; + close: () => void; +} + +const LauncherModal = (props: PropTypes) => { + const { open, study, close } = props; + const [t] = useTranslation(); + const { enqueueSnackbar } = useSnackbar(); + const [options, setOptions] = useState({}); + const classes = useStyles(); + + const launch = async () => { + if (!study) { + enqueueSnackbar(t('studymanager:failtorunstudy'), { variant: 'error' }); + return; + } + try { + await launchStudy(study.id, options); + enqueueSnackbar(t('studymanager:studylaunched', { studyname: study.name }), { variant: 'success' }); + close(); + } catch (e) { + enqueueErrorSnackbar(enqueueSnackbar, t('studymanager:failtorunstudy'), e as AxiosError); + } + }; + + const handleChange = (field: string, value: number | string | boolean) => { + setOptions({ + ...options, + [field]: value, + }); + }; + + const timeLimitParse = (value: any): number => { + try { + return parseInt(value, 10) * 3600; + } catch { + return 48 * 3600; + } + }; + + return ( + + + +
+
+ {t('singlestudy:runStudy')} +
+
+
+ + Options + + {/* + {`${t('singlestudy:nbCpu')} : ${options.nb_cpu || 12}`} + + + handleChange('nb_cpu', value as number)} + valueLabelDisplay="auto" + aria-labelledby="continuous-slider" + /> + */} + + handleChange('time_limit', timeLimitParse(e.target.value))} + InputLabelProps={{ + shrink: true, + }} + helperText={t('singlestudy:timeLimitHelper')} + /> + + {/* + handleChange('post_processing', checked)} />} label={t('singlestudy:postProcessing')} /> + */} + + handleChange('xpansion', checked)} />} label={t('singlestudy:xpansionMode')} /> + +
+
+ + +
+ + + + ); +}; + +LauncherModal.defaultProps = { + study: undefined, +}; + +export default LauncherModal; diff --git a/webapp/src/components/ui/LogModal.tsx b/webapp/src/components/ui/LogModal.tsx index c4605b6190..c1012eff3e 100644 --- a/webapp/src/components/ui/LogModal.tsx +++ b/webapp/src/components/ui/LogModal.tsx @@ -10,6 +10,7 @@ import { connect, ConnectedProps } from 'react-redux'; import { exportText } from '../../services/utils/index'; import { addListener, removeListener } from '../../ducks/websockets'; import { WSEvent, WSLogMessage, WSMessage } from '../../common/types'; +import SimpleLoader from './loaders/SimpleLoader'; const useStyles = makeStyles((theme: Theme) => createStyles({ @@ -45,6 +46,7 @@ const useStyles = makeStyles((theme: Theme) => flex: '1', width: '100%', overflow: 'auto', + position: 'relative', }, content: { padding: theme.spacing(3), @@ -83,6 +85,7 @@ interface OwnTypes { close: () => void; // eslint-disable-next-line react/require-default-props style?: CSSProperties; + loading?: boolean; } const mapState = () => ({ @@ -98,7 +101,7 @@ type ReduxProps = ConnectedProps; type PropTypes = ReduxProps & OwnTypes; const LogModal = (props: PropTypes) => { - const { title, style, jobId, isOpen, content, close, addWsListener, removeWsListener } = props; + const { title, style, jobId, loading, isOpen, content, close, addWsListener, removeWsListener } = props; const [logDetail, setLogDetail] = useState(content); const divRef = useRef(null); const logRef = useRef(null); @@ -187,9 +190,12 @@ const LogModal = (props: PropTypes) => {
-
- {logDetail} -
+ {loading ? : + ( +
+ {logDetail} +
+ )}