Merge pull request #893 from AntaresSimulatorTeam/dev

v2.4.0

.github/workflows/deploy.yml

@@ -33,8 +33,6 @@ jobs:
         working-directory: scripts
         env:
           NODE_OPTIONS: --max-old-space-size=8192
-      - name: Copy build to app resources
-        run: cp -r webapp/build resources/webapp
       - name: Set up Python
         uses: actions/setup-python@v1
         with:

.github/workflows/main.yml

@@ -90,10 +90,6 @@ jobs:
         working-directory: webapp
         env:
           NODE_OPTIONS: --max-old-space-size=8192
-      - name: Check extracted deps
-        if: matrix.os == 'ubuntu-latest'
-        run: npm run check
-        working-directory: webapp
       - name: Lint
         if: matrix.os == 'ubuntu-latest'
         run: npm run lint
@@ -102,31 +98,3 @@ jobs:
         run: npm run test
         working-directory: webapp
 
-  npm-test2:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      max-parallel: 9
-      matrix:
-        os: [ ubuntu-latest ]
-    steps:
-      - name: Checkout github repo
-        uses: actions/checkout@v1
-      - name: Set up Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v1
-        with:
-          node-version: 14.x
-      - name: Install dependencies
-        run: npm install
-        working-directory: webapp_v2
-      - name: Build
-        run: npm run build
-        working-directory: webapp_v2
-        env:
-          NODE_OPTIONS: --max-old-space-size=8192
-      - name: Lint
-        run: npm run lint
-        working-directory: webapp_v2
-      - name: Run tests
-        run: npm run test
-        working-directory: webapp_v2
-

alembic/versions/e6bb88859884_add_study_additional_data.py

@@ -0,0 +1,34 @@
+"""add_study_additional_data
+
+Revision ID: e6bb88859884
+Revises: ef72a8a1c9cf
+Create Date: 2022-04-20 14:30:54.685433
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = 'e6bb88859884'
+down_revision = 'f83043f34a3f'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('study_additional_data',
+    sa.Column('study_id', sa.String(length=36), nullable=False),
+    sa.Column('author', sa.String(length=255), nullable=True),
+    sa.Column('horizon', sa.String(), nullable=True),
+    sa.Column('patch', sa.String(), nullable=True),
+    sa.ForeignKeyConstraint(['study_id'], ['study.id'], ),
+    sa.PrimaryKeyConstraint('study_id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('study_additional_data')
+    # ### end Alembic commands ###

alembic/versions/f83043f34a3f_add_job_launcher_params.py

@@ -0,0 +1,32 @@
+"""add_job_launcher_params
+
+Revision ID: f83043f34a3f
+Revises: ef72a8a1c9cf
+Create Date: 2022-03-22 17:10:10.888752
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'f83043f34a3f'
+down_revision = 'ef72a8a1c9cf'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('job_result', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('launcher_params', sa.String(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('job_result', schema=None) as batch_op:
+        batch_op.drop_column('launcher_params')
+
+    # ### end Alembic commands ###

antarest/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "2.3.2"
+__version__ = "2.4.0"
 
 from pathlib import Path
 

antarest/core/config.py

@@ -197,6 +197,7 @@ class LauncherConfig:
     default: str = "local"
     local: Optional[LocalConfig] = LocalConfig()
     slurm: Optional[SlurmConfig] = SlurmConfig()
+    batch_size: int = 9999
 
     @staticmethod
     def from_dict(data: JSON) -> "LauncherConfig":
@@ -213,6 +214,7 @@ def from_dict(data: JSON) -> "LauncherConfig":
             default=data.get("default", "local"),
             local=local,
             slurm=slurm,
+            batch_size=data.get("batch_size", 9999),
         )
 
 

antarest/core/configdata/repository.py

@@ -1,7 +1,10 @@
+import json
 from operator import and_
 from typing import Optional
 
 from antarest.core.configdata.model import ConfigData
+from antarest.core.jwt import DEFAULT_ADMIN_USER
+from antarest.core.model import JSON
 from antarest.core.utils.fastapi_sqlalchemy import db
 
 
@@ -12,10 +15,38 @@ def save(self, configdata: ConfigData) -> ConfigData:
         db.session.commit()
         return configdata
 
-    def get(self, key: str, owner: int) -> Optional[ConfigData]:
+    def get(
+        self, key: str, owner: Optional[int] = None
+    ) -> Optional[ConfigData]:
         configdata: ConfigData = (
             db.session.query(ConfigData)
-            .filter(and_(ConfigData.owner == owner, ConfigData.key == key))
+            .filter(
+                and_(
+                    ConfigData.owner == (owner or DEFAULT_ADMIN_USER.id),
+                    ConfigData.key == key,
+                )
+            )
             .first()
         )
         return configdata
+
+    def get_json(
+        self, key: str, owner: Optional[int] = None
+    ) -> Optional[JSON]:
+        configdata = self.get(key, owner)
+        if configdata:
+            data: JSON = json.loads(configdata.value)
+            return data
+        return None
+
+    def put_json(
+        self, key: str, data: JSON, owner: Optional[int] = None
+    ) -> None:
+        configdata = ConfigData(
+            key=key,
+            value=json.dumps(data),
+            owner=owner or DEFAULT_ADMIN_USER.id,
+        )
+        configdata = db.session.merge(configdata)
+        db.session.add(configdata)
+        db.session.commit()

antarest/core/interfaces/cache.py

@@ -8,9 +8,7 @@
 class CacheConstants(Enum):
     RAW_STUDY = "RAW_STUDY"
     STUDY_FACTORY = "STUDY_FACTORY"
-    STUDY_LISTING_SUMMARY = "STUDY_LISTING_SUMMARY"
     STUDY_LISTING = "STUDY_LISTING"
-    STUDY_LISTING_SUMMARY_MANAGED = "STUDY_LISTING_SUMMARY_MANAGED"
     STUDY_LISTING_MANAGED = "STUDY_LISTING_MANAGED"
 
 

antarest/core/jwt.py

@@ -1,12 +1,12 @@
-from dataclasses import dataclass, field
 from typing import List, Union
 
 from pydantic import BaseModel
 
-from antarest.core.model import JSON
 from antarest.core.roles import RoleType
 from antarest.login.model import Group, Identity
 
+ADMIN_ID = 1
+
 
 class JWTGroup(BaseModel):
     """
@@ -28,6 +28,13 @@ class JWTUser(BaseModel):
     impersonator: int
     groups: List[JWTGroup] = []
 
+    def is_admin_token(self) -> bool:
+        """
+        Returns: true if the user is a bot of admin
+
+        """
+        return self.impersonator == ADMIN_ID
+
     def is_site_admin(self) -> bool:
         """
         Returns: true if connected user is admin
@@ -85,8 +92,8 @@ def is_or_impersonate(self, user_id: int) -> bool:
 
 
 DEFAULT_ADMIN_USER = JWTUser(
-    id=1,
-    impersonator=1,
+    id=ADMIN_ID,
+    impersonator=ADMIN_ID,
     type="users",
     groups=[JWTGroup(id="admin", name="admin", role=RoleType.ADMIN)],
 )

antarest/core/logging/utils.py

@@ -68,6 +68,11 @@ def configure_logger(config: Config) -> None:
                 "level": "INFO",
                 "propagate": True,
             },
+            "antarest.study.storage.rawstudy.watcher": {
+                "handlers": ["default"],
+                "level": "WARN",
+                "propagate": True,
+            },
         },
     }
     if config.logging.logfile is not None:

antarest/core/permissions.py

@@ -61,7 +61,7 @@ def check_permission(
     Returns: true if user match permission requirements, false else.
 
     """
-    if user.is_site_admin():
+    if user.is_site_admin() or user.is_admin_token():
         return True
 
     if (

antarest/core/utils/utils.py

@@ -1,7 +1,7 @@
 import time
 from glob import escape
 from pathlib import Path
-from typing import IO, Any, Optional, Callable, TypeVar
+from typing import IO, Any, Optional, Callable, TypeVar, List
 from zipfile import ZipFile, BadZipFile
 
 import redis
@@ -123,3 +123,11 @@ def retry(
 def assert_this(b: Any) -> None:
     if not b:
         raise AssertionError
+
+
+def concat_files(files: List[Path], target: Path) -> None:
+    with open(target, "w") as fh:
+        for item in files:
+            with open(item, "r") as infile:
+                for line in infile:
+                    fh.write(line)

antarest/launcher/adapters/abstractlauncher.py

@@ -1,6 +1,6 @@
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import Callable, NamedTuple, Optional, Dict
+from typing import Callable, NamedTuple, Optional, Dict, List
 
 from antarest.core.config import Config
 from antarest.core.interfaces.eventbus import (
@@ -28,7 +28,7 @@ class LauncherCallbacks(NamedTuple):
     append_before_log: Callable[[str, str], None]
     append_after_log: Callable[[str, str], None]
     # args: job_id, output_path, additional_logs
-    import_output: Callable[[str, Path, Dict[str, Path]], Optional[str]]
+    import_output: Callable[[str, Path, Dict[str, List[Path]]], Optional[str]]
 
 
 class AbstractLauncher(ABC):

antarest/launcher/adapters/log_manager.py

@@ -15,6 +15,9 @@ def __init__(self, log_base_dir: Path) -> None:
         self.log_base_dir = log_base_dir
         self.tracked_logs: Dict[str, Thread] = {}
 
+    def is_tracking(self, log_path: Optional[Path]) -> bool:
+        return str(log_path) in self.tracked_logs if log_path else False
+
     def track(
         self, log_path: Optional[Path], handler: Callable[[str], None]
     ) -> None:

antarest/launcher/adapters/slurm_launcher/slurm_launcher.py

@@ -235,17 +235,29 @@ def _import_study_output(
         if xpansion_mode is not None:
             self._import_xpansion_result(job_id, xpansion_mode)
 
-        launcher_logs: Dict[str, Path] = {}
+        launcher_logs: Dict[str, List[Path]] = {}
         if log_dir is not None:
             launcher_logs = {
                 log_name: log_path
                 for log_name, log_path in {
-                    "antares-out.log": SlurmLauncher._get_log_path_from_log_dir(
-                        Path(log_dir), LogType.STDOUT
-                    ),
-                    "antares-err.log": SlurmLauncher._get_log_path_from_log_dir(
-                        Path(log_dir), LogType.STDERR
-                    ),
+                    "antares-out.log": [
+                        p
+                        for p in [
+                            SlurmLauncher._get_log_path_from_log_dir(
+                                Path(log_dir), LogType.STDOUT
+                            )
+                        ]
+                        if p is not None
+                    ],
+                    "antares-err.log": [
+                        p
+                        for p in [
+                            SlurmLauncher._get_log_path_from_log_dir(
+                                Path(log_dir), LogType.STDERR
+                            )
+                        ]
+                        if p is not None
+                    ],
                 }.items()
                 if log_path
             }