From 96fb9c98d5fa90cc761705c6663584914385fc11 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Thu, 8 Feb 2024 14:55:05 +0100 Subject: [PATCH 01/31] feature(raw): add endpoint for matrices download --- antarest/core/filetransfer/service.py | 42 +-- antarest/study/service.py | 92 +++++- antarest/study/storage/utils.py | 124 +++++++- antarest/study/web/raw_studies_blueprint.py | 81 ++++- requirements.txt | 1 + .../test_download_matrices.py | 281 ++++++++++++++++++ .../test_study_matrix_index.py | 10 +- .../business/test_study_service_utils.py | 14 +- tests/storage/test_service.py | 6 +- 9 files changed, 615 insertions(+), 36 deletions(-) create mode 100644 tests/integration/raw_studies_blueprint/test_download_matrices.py diff --git a/antarest/core/filetransfer/service.py b/antarest/core/filetransfer/service.py index 760573a42f..80a81e6927 100644 --- a/antarest/core/filetransfer/service.py +++ b/antarest/core/filetransfer/service.py @@ -43,6 +43,8 @@ def request_download( filename: str, name: Optional[str] = None, owner: Optional[JWTUser] = None, + use_notification: bool = True, + expiration_time_in_minutes: int = 0, ) -> FileDownload: fh, path = tempfile.mkstemp(dir=self.tmp_dir, suffix=filename) os.close(fh) @@ -55,36 +57,40 @@ def request_download( path=str(tmpfile), owner=owner.impersonator if owner is not None else None, expiration_date=datetime.datetime.utcnow() - + datetime.timedelta(minutes=self.download_default_expiration_timeout_minutes), + + datetime.timedelta( + minutes=expiration_time_in_minutes or self.download_default_expiration_timeout_minutes + ), ) self.repository.add(download) - self.event_bus.push( - Event( - type=EventType.DOWNLOAD_CREATED, - payload=download.to_dto(), - permissions=PermissionInfo(owner=owner.impersonator) - if owner - else PermissionInfo(public_mode=PublicMode.READ), + if use_notification: + self.event_bus.push( + Event( + type=EventType.DOWNLOAD_CREATED, + payload=download.to_dto(), + permissions=PermissionInfo(owner=owner.impersonator) + if owner + else PermissionInfo(public_mode=PublicMode.READ), + ) ) - ) return download - def set_ready(self, download_id: str) -> None: + def set_ready(self, download_id: str, use_notification: bool = True) -> None: download = self.repository.get(download_id) if not download: raise FileDownloadNotFound() download.ready = True self.repository.save(download) - self.event_bus.push( - Event( - type=EventType.DOWNLOAD_READY, - payload=download.to_dto(), - permissions=PermissionInfo(owner=download.owner) - if download.owner - else PermissionInfo(public_mode=PublicMode.READ), + if use_notification: + self.event_bus.push( + Event( + type=EventType.DOWNLOAD_READY, + payload=download.to_dto(), + permissions=PermissionInfo(owner=download.owner) + if download.owner + else PermissionInfo(public_mode=PublicMode.READ), + ) ) - ) def fail(self, download_id: str, reason: str = "") -> None: download = self.repository.get(download_id) diff --git a/antarest/study/service.py b/antarest/study/service.py index ae86fe62ae..6c97a16d66 100644 --- a/antarest/study/service.py +++ b/antarest/study/service.py @@ -4,6 +4,7 @@ import json import logging import os +import re import time import typing as t from datetime import datetime, timedelta @@ -12,6 +13,7 @@ from uuid import uuid4 import numpy as np +import pandas as pd from fastapi import HTTPException, UploadFile from markupsafe import escape from starlette.responses import FileResponse, Response @@ -20,6 +22,7 @@ from antarest.core.exceptions import ( BadEditInstructionException, CommandApplicationError, + IncorrectPathError, NotAManagedStudyException, StudyDeletionNotAllowed, StudyNotFoundError, @@ -54,6 +57,7 @@ from antarest.study.business.areas.thermal_management import ThermalManager from antarest.study.business.binding_constraint_management import BindingConstraintManager from antarest.study.business.config_management import ConfigManager +from antarest.study.business.correlation_management import CorrelationManager from antarest.study.business.district_manager import DistrictManager from antarest.study.business.general_management import GeneralManager from antarest.study.business.link_management import LinkInfoDTO, LinkManager @@ -109,7 +113,14 @@ should_study_be_denormalized, upgrade_study, ) -from antarest.study.storage.utils import assert_permission, get_start_date, is_managed, remove_from_cache +from antarest.study.storage.utils import ( + MatrixProfile, + assert_permission, + get_specific_matrices_according_to_version, + get_start_date, + is_managed, + remove_from_cache, +) from antarest.study.storage.variantstudy.model.command.icommand import ICommand from antarest.study.storage.variantstudy.model.command.replace_matrix import ReplaceMatrix from antarest.study.storage.variantstudy.model.command.update_comments import UpdateComments @@ -141,6 +152,40 @@ def get_disk_usage(path: t.Union[str, Path]) -> int: return total_size +def _handle_specific_matrices( + df: pd.DataFrame, + matrix_profile: MatrixProfile, + matrix_path: str, + *, + with_index: bool, + with_columns: bool, +) -> pd.DataFrame: + if with_columns: + if Path(matrix_path).parts[1] == "links": + cols = _handle_links_columns(matrix_path, matrix_profile) + else: + cols = matrix_profile.cols + if cols: + df.columns = pd.Index(cols) + rows = matrix_profile.rows + if with_index and rows: + df.index = rows # type: ignore + return df + + +def _handle_links_columns(matrix_path: str, matrix_profile: MatrixProfile) -> t.List[str]: + path_parts = Path(matrix_path).parts + area_id_1 = path_parts[2] + area_id_2 = path_parts[3] + result = matrix_profile.cols + for k, col in enumerate(result): + if col == "Hurdle costs direct": + result[k] = f"{col} ({area_id_1}->{area_id_2})" + elif col == "Hurdle costs indirect": + result[k] = f"{col} ({area_id_2}->{area_id_1})" + return result + + class StudyUpgraderTask: """ Task to perform a study upgrade. @@ -268,6 +313,7 @@ def __init__( self.xpansion_manager = XpansionManager(self.storage_service) self.matrix_manager = MatrixManager(self.storage_service) self.binding_constraint_manager = BindingConstraintManager(self.storage_service) + self.correlation_manager = CorrelationManager(self.storage_service) self.cache_service = cache_service self.config = config self.on_deletion_callbacks: t.List[t.Callable[[str], None]] = [] @@ -2379,3 +2425,47 @@ def get_disk_usage(self, uuid: str, params: RequestParameters) -> int: study_path = self.storage_service.raw_study_service.get_study_path(study) # If the study is a variant, it's possible that it only exists in DB and not on disk. If so, we return 0. return get_disk_usage(study_path) if study_path.exists() else 0 + + def get_matrix_with_index_and_header( + self, *, study_id: str, path: str, with_index: bool, with_columns: bool, parameters: RequestParameters + ) -> pd.DataFrame: + matrix_path = Path(path) + study = self.get_study(study_id) + for aggregate in ["allocation", "correlation"]: + if matrix_path == Path("input") / "hydro" / aggregate: + all_areas = t.cast( + t.List[AreaInfoDTO], + self.get_all_areas(study_id, area_type=AreaType.AREA, ui=False, params=parameters), + ) + if aggregate == "allocation": + hydro_matrix = self.allocation_manager.get_allocation_matrix(study, all_areas) + else: + hydro_matrix = self.correlation_manager.get_correlation_matrix(all_areas, study, []) # type: ignore + return pd.DataFrame(data=hydro_matrix.data, columns=hydro_matrix.columns, index=hydro_matrix.index) + + json_matrix = self.get(study_id, path, depth=3, formatted=True, params=parameters) + for key in ["data", "index", "columns"]: + if key not in json_matrix: + raise IncorrectPathError(f"The path filled does not correspond to a matrix : {path}") + if not json_matrix["data"]: + return pd.DataFrame() + df_matrix = pd.DataFrame(data=json_matrix["data"], columns=json_matrix["columns"], index=json_matrix["index"]) + + if with_index: + matrix_index = self.get_input_matrix_startdate(study_id, path, parameters) + time_column = pd.date_range( + start=matrix_index.start_date, periods=len(df_matrix), freq=matrix_index.level.value[0] + ) + df_matrix.index = time_column + + specific_matrices = get_specific_matrices_according_to_version(int(study.version)) + for specific_matrix in specific_matrices: + if re.match(specific_matrix, path): + return _handle_specific_matrices( + df_matrix, + specific_matrices[specific_matrix], + path, + with_index=with_index, + with_columns=with_columns, + ) + return df_matrix diff --git a/antarest/study/storage/utils.py b/antarest/study/storage/utils.py index a0fc7a02fe..81e7b5dfca 100644 --- a/antarest/study/storage/utils.py +++ b/antarest/study/storage/utils.py @@ -1,4 +1,5 @@ import calendar +import copy import logging import math import os @@ -269,6 +270,127 @@ def assert_permission( ) +def _generate_columns(column_suffix: str) -> t.List[str]: + return [f"{i}{column_suffix}" for i in range(101)] + + +class MatrixProfile(t.NamedTuple): + """ + Matrix profile for time series or classic tables. + """ + + cols: t.List[str] + rows: t.List[str] + stats: bool + + +SPECIFIC_MATRICES = { + "input/hydro/common/capacity/creditmodulations_*": MatrixProfile( + cols=_generate_columns(""), + rows=["Generating Power", "Pumping Power"], + stats=False, + ), + "input/hydro/common/capacity/maxpower_*": MatrixProfile( + cols=[ + "Generating Max Power (MW)", + "Generating Max Energy (Hours at Pmax)", + "Pumping Max Power (MW)", + "Pumping Max Energy (Hours at Pmax)", + ], + rows=[], + stats=False, + ), + "input/hydro/common/capacity/reservoir_*": MatrixProfile( + cols=["Lev Low (p.u)", "Lev Avg (p.u)", "Lev High (p.u)"], + rows=[], + stats=False, + ), + "input/hydro/common/capacity/waterValues_*": MatrixProfile(cols=_generate_columns("%"), rows=[], stats=False), + "input/hydro/series/*/mod": MatrixProfile(cols=[], rows=[], stats=True), + "input/hydro/series/*/ror": MatrixProfile(cols=[], rows=[], stats=True), + "input/hydro/common/capacity/inflowPattern_*": MatrixProfile(cols=["Inflow Pattern (X)"], rows=[], stats=False), + "input/hydro/prepro/*/energy": MatrixProfile( + cols=["Expectation (MWh)", "Std Deviation (MWh)", "Min. (MWh)", "Max. (MWh)", "ROR Share"], + rows=[ + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", + ], + stats=False, + ), + "input/thermal/prepro/*/*/modulation": MatrixProfile( + cols=["Marginal cost modulation", "Market bid modulation", "Capacity modulation", "Min gen modulation"], + rows=[], + stats=False, + ), + "input/thermal/prepro/*/*/data": MatrixProfile( + cols=["FO Duration", "PO Duration", "FO Rate", "PO Rate", "NPO Min", "NPO Max"], + rows=[], + stats=False, + ), + "input/reserves/*": MatrixProfile( + cols=["Primary Res. (draft)", "Strategic Res. (draft)", "DSM", "Day Ahead"], + rows=[], + stats=False, + ), + "input/misc-gen/miscgen-*": MatrixProfile( + cols=["CHP", "Bio Mass", "Bio Gaz", "Waste", "GeoThermal", "Other", "PSP", "ROW Balance"], + rows=[], + stats=False, + ), + "input/bindingconstraints/*": MatrixProfile(cols=["<", ">", "="], rows=[], stats=False), + "input/links/*/*": MatrixProfile( + cols=[ + "Capacités de transmission directes", + "Capacités de transmission indirectes", + "Hurdle costs direct", + "Hurdle costs indirect", + "Impedances", + "Loop flow", + "P.Shift Min", + "P.Shift Max", + ], + rows=[], + stats=False, + ), +} + + +SPECIFIC_MATRICES_820 = copy.deepcopy(SPECIFIC_MATRICES) +SPECIFIC_MATRICES_820["input/links/*/*"] = MatrixProfile( + cols=[ + "Hurdle costs direct", + "Hurdle costs indirect", + "Impedances", + "Loop flow", + "P.Shift Min", + "P.Shift Max", + ], + rows=[], + stats=False, +) + +SPECIFIC_MATRICES_870 = copy.deepcopy(SPECIFIC_MATRICES_820) +SPECIFIC_MATRICES_870["input/bindingconstraints/*"] = MatrixProfile(cols=[], rows=[], stats=False) + + +def get_specific_matrices_according_to_version(study_version: int) -> t.Dict[str, MatrixProfile]: + if study_version < 820: + return SPECIFIC_MATRICES + elif study_version < 870: + return SPECIFIC_MATRICES_820 + return SPECIFIC_MATRICES_870 + + def get_start_date( file_study: FileStudy, output_id: t.Optional[str] = None, @@ -293,7 +415,7 @@ def get_start_date( starting_month_index = MONTHS.index(starting_month.title()) + 1 starting_day_index = DAY_NAMES.index(starting_day.title()) - target_year = 2000 + target_year = 2018 while True: if leapyear == calendar.isleap(target_year): first_day = datetime(target_year, starting_month_index, 1) diff --git a/antarest/study/web/raw_studies_blueprint.py b/antarest/study/web/raw_studies_blueprint.py index 41e214d1ad..6454ea3175 100644 --- a/antarest/study/web/raw_studies_blueprint.py +++ b/antarest/study/web/raw_studies_blueprint.py @@ -4,12 +4,15 @@ import logging import pathlib import typing as t +from enum import Enum +import pandas as pd from fastapi import APIRouter, Body, Depends, File, HTTPException from fastapi.params import Param, Query -from starlette.responses import JSONResponse, PlainTextResponse, Response, StreamingResponse +from starlette.responses import FileResponse, JSONResponse, PlainTextResponse, Response, StreamingResponse from antarest.core.config import Config +from antarest.core.filetransfer.model import FileDownloadNotFound from antarest.core.jwt import JWTUser from antarest.core.model import SUB_JSON from antarest.core.requests import RequestParameters @@ -49,6 +52,11 @@ } +class ExpectedFormatTypes(Enum): + XLSX = "xlsx" + CSV = "csv" + + def create_raw_study_routes( study_service: StudyService, config: Config, @@ -243,4 +251,75 @@ def validate( ) return study_service.check_errors(uuid) + @bp.get( + "/studies/{uuid}/raw/download", + summary="Download a matrix in a given format", + tags=[APITag.study_raw_data], + response_class=FileResponse, + ) + def get_matrix( + uuid: str, + path: str, + format: ExpectedFormatTypes, + header: bool = True, + index: bool = True, + current_user: JWTUser = Depends(auth.get_current_user), + ) -> FileResponse: + parameters = RequestParameters(user=current_user) + df_matrix = study_service.get_matrix_with_index_and_header( + study_id=uuid, path=path, with_index=index, with_columns=header, parameters=parameters + ) + + export_file_download = study_service.file_transfer_manager.request_download( + f"{pathlib.Path(path).stem}.{format.value}", + f"Exporting matrix {pathlib.Path(path).stem} to format {format.value} for study {uuid}", + current_user, + use_notification=False, + expiration_time_in_minutes=10, + ) + export_path = pathlib.Path(export_file_download.path) + export_id = export_file_download.id + + try: + _create_matrix_files(df_matrix, header, index, format, export_path) + study_service.file_transfer_manager.set_ready(export_id, use_notification=False) + except ValueError as e: + study_service.file_transfer_manager.fail(export_id, str(e)) + raise HTTPException( + status_code=http.HTTPStatus.UNPROCESSABLE_ENTITY, + detail=f"The Excel file {export_path} already exists and cannot be replaced due to Excel policy :{str(e)}", + ) from e + except FileDownloadNotFound as e: + study_service.file_transfer_manager.fail(export_id, str(e)) + raise HTTPException( + status_code=http.HTTPStatus.UNPROCESSABLE_ENTITY, + detail=f"The file download does not exist in database :{str(e)}", + ) from e + + return FileResponse( + export_path, + headers={"Content-Disposition": f'attachment; filename="{export_file_download.filename}"'}, + media_type="application/octet-stream", + ) + return bp + + +def _create_matrix_files( + df_matrix: pd.DataFrame, header: bool, index: bool, format: ExpectedFormatTypes, export_path: pathlib.Path +) -> None: + if format == ExpectedFormatTypes.CSV: + df_matrix.to_csv( + export_path, + sep="\t", + header=header, + index=index, + float_format="%.6f", + ) + else: + df_matrix.to_excel( + export_path, + header=header, + index=index, + float_format="%.6f", + ) diff --git a/requirements.txt b/requirements.txt index 4e12840d32..2d8f4be828 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,7 @@ Jinja2~=3.0.3 jsonref~=0.2 MarkupSafe~=2.0.1 numpy~=1.22.1 +openpyxl~=3.1.2 pandas~=1.4.0 paramiko~=2.12.0 plyer~=2.0.0 diff --git a/tests/integration/raw_studies_blueprint/test_download_matrices.py b/tests/integration/raw_studies_blueprint/test_download_matrices.py new file mode 100644 index 0000000000..849a0e2675 --- /dev/null +++ b/tests/integration/raw_studies_blueprint/test_download_matrices.py @@ -0,0 +1,281 @@ +import io + +import numpy as np +import pandas as pd +import pytest +from starlette.testclient import TestClient + +from antarest.core.tasks.model import TaskStatus +from tests.integration.utils import wait_task_completion + + +@pytest.mark.integration_test +class TestDownloadMatrices: + """ + Checks the retrieval of matrices with the endpoint GET studies/uuid/raw/download + """ + + def test_download_matrices(self, client: TestClient, admin_access_token: str, study_id: str) -> None: + admin_headers = {"Authorization": f"Bearer {admin_access_token}"} + + # ============================= + # STUDIES PREPARATION + # ============================= + + # Manage parent study and upgrades it to v8.2 + # This is done to test matrix headers according to different versions + copied = client.post( + f"/v1/studies/{study_id}/copy", params={"dest": "copied", "use_task": False}, headers=admin_headers + ) + parent_id = copied.json() + res = client.put(f"/v1/studies/{parent_id}/upgrade", params={"target_version": 820}, headers=admin_headers) + assert res.status_code == 200 + task_id = res.json() + assert task_id + task = wait_task_completion(client, admin_access_token, task_id, timeout=20) + assert task.status == TaskStatus.COMPLETED + + # Create Variant + res = client.post( + f"/v1/studies/{parent_id}/variants", + headers=admin_headers, + params={"name": "variant_1"}, + ) + assert res.status_code == 200 + variant_id = res.json() + + # Create a new area to implicitly create normalized matrices + area_name = "new_area" + res = client.post( + f"/v1/studies/{variant_id}/areas", + headers=admin_headers, + json={"name": area_name, "type": "AREA", "metadata": {"country": "FR"}}, + ) + assert res.status_code in {200, 201} + + # Change study start_date + res = client.put( + f"/v1/studies/{variant_id}/config/general/form", json={"firstMonth": "july"}, headers=admin_headers + ) + assert res.status_code == 200 + + # Really generates the snapshot + client.get(f"/v1/studies/{variant_id}/areas", headers=admin_headers) + assert res.status_code == 200 + + # ============================= + # TESTS NOMINAL CASE ON RAW AND VARIANT STUDY + # ============================= + + raw_matrix_path = r"input/load/series/load_de" + variant_matrix_path = f"input/load/series/load_{area_name}" + + for uuid, path in zip([parent_id, variant_id], [raw_matrix_path, variant_matrix_path]): + # get downloaded bytes + res = client.get( + f"/v1/studies/{uuid}/raw/download", params={"path": path, "format": "xlsx"}, headers=admin_headers + ) + assert res.status_code == 200 + + # load into dataframe + dataframe = pd.read_excel(io.BytesIO(res.content), index_col=0) + + # check time coherence + generated_index = dataframe.index + first_date = generated_index[0].to_pydatetime() + second_date = generated_index[1].to_pydatetime() + assert first_date.month == second_date.month == 1 if uuid == parent_id else 7 + assert first_date.day == second_date.day == 1 + assert first_date.hour == 0 + assert second_date.hour == 1 + + # reformat into a json to help comparison + new_cols = [int(col) for col in dataframe.columns] + dataframe.columns = new_cols + dataframe.index = range(len(dataframe)) + actual_matrix = dataframe.to_dict(orient="split") + + # asserts that the result is the same as the one we get with the classic get /raw endpoint + res = client.get(f"/v1/studies/{uuid}/raw", params={"path": path, "formatted": True}, headers=admin_headers) + expected_matrix = res.json() + assert actual_matrix == expected_matrix + + # ============================= + # TESTS INDEX AND HEADER PARAMETERS + # ============================= + + # test only few possibilities as each API call is quite long + for header in [True, False]: + index = not header + res = client.get( + f"/v1/studies/{parent_id}/raw/download", + params={"path": raw_matrix_path, "format": "csv", "header": header, "index": index}, + headers=admin_headers, + ) + assert res.status_code == 200 + content = io.BytesIO(res.content) + dataframe = pd.read_csv( + content, index_col=0 if index else None, header="infer" if header else None, sep="\t" + ) + first_index = dataframe.index[0] + assert first_index == "2018-01-01 00:00:00" if index else first_index == 0 + assert isinstance(dataframe.columns[0], str) if header else isinstance(dataframe.columns[0], np.int64) + + # ============================= + # TEST SPECIFIC MATRICES + # ============================= + + # tests links headers before v8.2 + res = client.get( + f"/v1/studies/{study_id}/raw/download", + params={"path": "input/links/de/fr", "format": "csv", "index": False}, + headers=admin_headers, + ) + assert res.status_code == 200 + content = io.BytesIO(res.content) + dataframe = pd.read_csv(content, sep="\t") + assert list(dataframe.columns) == [ + "Capacités de transmission directes", + "Capacités de transmission indirectes", + "Hurdle costs direct (de->fr)", + "Hurdle costs indirect (fr->de)", + "Impedances", + "Loop flow", + "P.Shift Min", + "P.Shift Max", + ] + + # tests links headers after v8.2 + res = client.get( + f"/v1/studies/{parent_id}/raw/download", + params={"path": "input/links/de/fr_parameters", "format": "csv"}, + headers=admin_headers, + ) + assert res.status_code == 200 + content = io.BytesIO(res.content) + dataframe = pd.read_csv(content, index_col=0, sep="\t") + assert list(dataframe.columns) == [ + "Hurdle costs direct (de->fr_parameters)", + "Hurdle costs indirect (fr_parameters->de)", + "Impedances", + "Loop flow", + "P.Shift Min", + "P.Shift Max", + ] + + # allocation and correlation matrices + for path in ["input/hydro/allocation", "input/hydro/correlation"]: + res = client.get( + f"/v1/studies/{parent_id}/raw/download", params={"path": path, "format": "csv"}, headers=admin_headers + ) + assert res.status_code == 200 + content = io.BytesIO(res.content) + dataframe = pd.read_csv(content, index_col=0, sep="\t") + assert list(dataframe.index) == list(dataframe.columns) == ["de", "es", "fr", "it"] + for i in range((len(dataframe))): + assert dataframe.iloc[i, i] == 1.0 + + # test for empty matrix + res = client.get( + f"/v1/studies/{study_id}/raw/download", + params={"path": "input/hydro/common/capacity/waterValues_de", "format": "csv"}, + headers=admin_headers, + ) + assert res.status_code == 200 + content = io.BytesIO(res.content) + dataframe = pd.read_csv(content, index_col=0, sep="\t") + assert dataframe.empty + + # modulation matrix + res = client.get( + f"/v1/studies/{parent_id}/raw/download", + params={"path": "input/thermal/prepro/de/01_solar/modulation", "format": "csv"}, + headers=admin_headers, + ) + assert res.status_code == 200 + content = io.BytesIO(res.content) + dataframe = pd.read_csv(content, index_col=0, sep="\t") + assert dataframe.index[0] == "2018-01-01 00:00:00" + dataframe.index = range(len(dataframe)) + liste_transposee = list(zip(*[8760 * [1.0], 8760 * [1.0], 8760 * [1.0], 8760 * [0.0]])) + expected_df = pd.DataFrame(columns=["0", "1", "2", "3"], index=range(8760), data=liste_transposee) + assert dataframe.equals(expected_df) + + # asserts endpoint returns the right columns for output matrix + res = client.get( + f"/v1/studies/{study_id}/raw/download", + params={ + "path": "output/20201014-1422eco-hello/economy/mc-ind/00001/links/de/fr/values-hourly", + "format": "csv", + }, + headers=admin_headers, + ) + assert res.status_code == 200 + content = io.BytesIO(res.content) + dataframe = pd.read_csv(content, index_col=0, sep="\t") + assert list(dataframe.columns) == [ + "('FLOW LIN.', 'MWh', '')", + "('UCAP LIN.', 'MWh', '')", + "('LOOP FLOW', 'MWh', '')", + "('FLOW QUAD.', 'MWh', '')", + "('CONG. FEE (ALG.)', 'Euro', '')", + "('CONG. FEE (ABS.)', 'Euro', '')", + "('MARG. COST', 'Euro/MW', '')", + "('CONG. PROB +', '%', '')", + "('CONG. PROB -', '%', '')", + "('HURDLE COST', 'Euro', '')", + ] + + # test energy matrix to test the regex + res = client.get( + f"/v1/studies/{study_id}/raw/download", + params={"path": "input/hydro/prepro/de/energy", "format": "csv"}, + headers=admin_headers, + ) + assert res.status_code == 200 + content = io.BytesIO(res.content) + dataframe = pd.read_csv(content, index_col=0, sep="\t") + assert dataframe.empty + + # ============================= + # ERRORS + # ============================= + + fake_str = "fake_str" + + # fake study_id + res = client.get( + f"/v1/studies/{fake_str}/raw/download", + params={"path": raw_matrix_path, "format": "csv"}, + headers=admin_headers, + ) + assert res.status_code == 404 + assert res.json()["exception"] == "StudyNotFoundError" + + # fake path + res = client.get( + f"/v1/studies/{parent_id}/raw/download", + params={"path": f"input/links/de/{fake_str}", "format": "csv"}, + headers=admin_headers, + ) + assert res.status_code == 404 + assert res.json()["exception"] == "ChildNotFoundError" + + # path that does not lead to a matrix + res = client.get( + f"/v1/studies/{parent_id}/raw/download", + params={"path": "settings/generaldata", "format": "csv"}, + headers=admin_headers, + ) + assert res.status_code == 404 + assert res.json()["exception"] == "IncorrectPathError" + assert res.json()["description"] == "The path filled does not correspond to a matrix : settings/generaldata" + + # wrong format + res = client.get( + f"/v1/studies/{parent_id}/raw/download", + params={"path": raw_matrix_path, "format": fake_str}, + headers=admin_headers, + ) + assert res.status_code == 422 + assert res.json()["exception"] == "RequestValidationError" diff --git a/tests/integration/studies_blueprint/test_study_matrix_index.py b/tests/integration/studies_blueprint/test_study_matrix_index.py index 69880cb357..4aeacceff4 100644 --- a/tests/integration/studies_blueprint/test_study_matrix_index.py +++ b/tests/integration/studies_blueprint/test_study_matrix_index.py @@ -33,7 +33,7 @@ def test_get_study_matrix_index( expected = { "first_week_size": 7, "level": "hourly", - "start_date": "2001-01-01 00:00:00", + "start_date": "2018-01-01 00:00:00", "steps": 8760, } assert actual == expected @@ -50,7 +50,7 @@ def test_get_study_matrix_index( expected = { "first_week_size": 7, "level": "daily", - "start_date": "2001-01-01 00:00:00", + "start_date": "2018-01-01 00:00:00", "steps": 365, } assert actual == expected @@ -67,7 +67,7 @@ def test_get_study_matrix_index( expected = { "first_week_size": 7, "level": "hourly", - "start_date": "2001-01-01 00:00:00", + "start_date": "2018-01-01 00:00:00", "steps": 8760, } assert actual == expected @@ -80,7 +80,7 @@ def test_get_study_matrix_index( actual = res.json() expected = { "first_week_size": 7, - "start_date": "2001-01-01 00:00:00", + "start_date": "2018-01-01 00:00:00", "steps": 8760, "level": "hourly", } @@ -96,5 +96,5 @@ def test_get_study_matrix_index( ) assert res.status_code == 200 actual = res.json() - expected = {"first_week_size": 7, "start_date": "2001-01-01 00:00:00", "steps": 7, "level": "daily"} + expected = {"first_week_size": 7, "start_date": "2018-01-01 00:00:00", "steps": 7, "level": "daily"} assert actual == expected diff --git a/tests/storage/business/test_study_service_utils.py b/tests/storage/business/test_study_service_utils.py index 623f17a55e..dcd674e0e3 100644 --- a/tests/storage/business/test_study_service_utils.py +++ b/tests/storage/business/test_study_service_utils.py @@ -104,7 +104,7 @@ def test_output_downloads_export(tmp_path: Path): }, StudyDownloadLevelDTO.WEEKLY, MatrixIndex( - start_date=str(datetime.datetime(2001, 1, 1)), + start_date=str(datetime.datetime(2018, 1, 1)), steps=51, first_week_size=7, level=StudyDownloadLevelDTO.WEEKLY, @@ -121,7 +121,7 @@ def test_output_downloads_export(tmp_path: Path): }, StudyDownloadLevelDTO.WEEKLY, MatrixIndex( - start_date=str(datetime.datetime(2002, 7, 5)), + start_date=str(datetime.datetime(2019, 7, 5)), steps=48, first_week_size=5, level=StudyDownloadLevelDTO.WEEKLY, @@ -138,7 +138,7 @@ def test_output_downloads_export(tmp_path: Path): }, StudyDownloadLevelDTO.MONTHLY, MatrixIndex( - start_date=str(datetime.datetime(2002, 7, 1)), + start_date=str(datetime.datetime(2019, 7, 1)), steps=7, first_week_size=7, level=StudyDownloadLevelDTO.MONTHLY, @@ -155,7 +155,7 @@ def test_output_downloads_export(tmp_path: Path): }, StudyDownloadLevelDTO.MONTHLY, MatrixIndex( - start_date=str(datetime.datetime(2002, 7, 1)), + start_date=str(datetime.datetime(2019, 7, 1)), steps=4, first_week_size=7, level=StudyDownloadLevelDTO.MONTHLY, @@ -172,7 +172,7 @@ def test_output_downloads_export(tmp_path: Path): }, StudyDownloadLevelDTO.HOURLY, MatrixIndex( - start_date=str(datetime.datetime(2010, 3, 5)), + start_date=str(datetime.datetime(2021, 3, 5)), steps=2304, first_week_size=3, level=StudyDownloadLevelDTO.HOURLY, @@ -189,7 +189,7 @@ def test_output_downloads_export(tmp_path: Path): }, StudyDownloadLevelDTO.ANNUAL, MatrixIndex( - start_date=str(datetime.datetime(2010, 3, 5)), + start_date=str(datetime.datetime(2021, 3, 5)), steps=1, first_week_size=3, level=StudyDownloadLevelDTO.ANNUAL, @@ -206,7 +206,7 @@ def test_output_downloads_export(tmp_path: Path): }, StudyDownloadLevelDTO.DAILY, MatrixIndex( - start_date=str(datetime.datetime(2009, 3, 3)), + start_date=str(datetime.datetime(2026, 3, 3)), steps=98, first_week_size=3, level=StudyDownloadLevelDTO.DAILY, diff --git a/tests/storage/test_service.py b/tests/storage/test_service.py index e7e8662394..12f61e6489 100644 --- a/tests/storage/test_service.py +++ b/tests/storage/test_service.py @@ -571,7 +571,7 @@ def test_download_output() -> None: # AREA TYPE res_matrix = MatrixAggregationResultDTO( index=MatrixIndex( - start_date="2001-01-01 00:00:00", + start_date="2018-01-01 00:00:00", steps=1, first_week_size=7, level=StudyDownloadLevelDTO.ANNUAL, @@ -631,7 +631,7 @@ def test_download_output() -> None: input_data.filter = ["east>west"] res_matrix = MatrixAggregationResultDTO( index=MatrixIndex( - start_date="2001-01-01 00:00:00", + start_date="2018-01-01 00:00:00", steps=1, first_week_size=7, level=StudyDownloadLevelDTO.ANNUAL, @@ -661,7 +661,7 @@ def test_download_output() -> None: input_data.filterIn = "n" res_matrix = MatrixAggregationResultDTO( index=MatrixIndex( - start_date="2001-01-01 00:00:00", + start_date="2018-01-01 00:00:00", steps=1, first_week_size=7, level=StudyDownloadLevelDTO.ANNUAL, From 34cc269e185f8809d357ef563bdcf5ac8e4b5dd0 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Thu, 8 Feb 2024 23:14:38 +0100 Subject: [PATCH 02/31] refactor(api-download): rename `ExpectedFormatTypes` into `TableExportFormat` --- antarest/study/web/raw_studies_blueprint.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/antarest/study/web/raw_studies_blueprint.py b/antarest/study/web/raw_studies_blueprint.py index 6454ea3175..1707ba8de3 100644 --- a/antarest/study/web/raw_studies_blueprint.py +++ b/antarest/study/web/raw_studies_blueprint.py @@ -1,10 +1,10 @@ +import enum import http import io import json import logging import pathlib import typing as t -from enum import Enum import pandas as pd from fastapi import APIRouter, Body, Depends, File, HTTPException @@ -52,7 +52,7 @@ } -class ExpectedFormatTypes(Enum): +class TableExportFormat(enum.Enum): XLSX = "xlsx" CSV = "csv" @@ -260,7 +260,7 @@ def validate( def get_matrix( uuid: str, path: str, - format: ExpectedFormatTypes, + format: TableExportFormat, header: bool = True, index: bool = True, current_user: JWTUser = Depends(auth.get_current_user), @@ -306,9 +306,9 @@ def get_matrix( def _create_matrix_files( - df_matrix: pd.DataFrame, header: bool, index: bool, format: ExpectedFormatTypes, export_path: pathlib.Path + df_matrix: pd.DataFrame, header: bool, index: bool, format: TableExportFormat, export_path: pathlib.Path ) -> None: - if format == ExpectedFormatTypes.CSV: + if format == TableExportFormat.CSV: df_matrix.to_csv( export_path, sep="\t", From e58c9171e71ac373169c6aac61be66850686f622 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Thu, 8 Feb 2024 23:19:14 +0100 Subject: [PATCH 03/31] refactor(api-download): turn `TableExportFormat` enum into case-insensitive enum --- antarest/study/web/raw_studies_blueprint.py | 6 ++++-- .../raw_studies_blueprint/test_download_matrices.py | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/antarest/study/web/raw_studies_blueprint.py b/antarest/study/web/raw_studies_blueprint.py index 1707ba8de3..fe61cbe6d8 100644 --- a/antarest/study/web/raw_studies_blueprint.py +++ b/antarest/study/web/raw_studies_blueprint.py @@ -1,4 +1,3 @@ -import enum import http import io import json @@ -20,6 +19,7 @@ from antarest.core.utils.utils import sanitize_uuid from antarest.core.utils.web import APITag from antarest.login.auth import Auth +from antarest.study.business.enum_ignore_case import EnumIgnoreCase from antarest.study.service import StudyService logger = logging.getLogger(__name__) @@ -52,7 +52,9 @@ } -class TableExportFormat(enum.Enum): +class TableExportFormat(EnumIgnoreCase): + """Export format for tables.""" + XLSX = "xlsx" CSV = "csv" diff --git a/tests/integration/raw_studies_blueprint/test_download_matrices.py b/tests/integration/raw_studies_blueprint/test_download_matrices.py index 849a0e2675..e1f354f823 100644 --- a/tests/integration/raw_studies_blueprint/test_download_matrices.py +++ b/tests/integration/raw_studies_blueprint/test_download_matrices.py @@ -105,11 +105,12 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st # ============================= # test only few possibilities as each API call is quite long + # (also check that the format is case-insensitive) for header in [True, False]: index = not header res = client.get( f"/v1/studies/{parent_id}/raw/download", - params={"path": raw_matrix_path, "format": "csv", "header": header, "index": index}, + params={"path": raw_matrix_path, "format": "CSV", "header": header, "index": index}, headers=admin_headers, ) assert res.status_code == 200 From 75b3a203e3e0f28975691db416e25210e2659056 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Thu, 8 Feb 2024 23:22:51 +0100 Subject: [PATCH 04/31] refactor(api-download): remplace "csv" by "tsv" in `TableExportFormat` enum --- antarest/study/web/raw_studies_blueprint.py | 6 ++--- .../test_download_matrices.py | 22 +++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/antarest/study/web/raw_studies_blueprint.py b/antarest/study/web/raw_studies_blueprint.py index fe61cbe6d8..19f3ff4b50 100644 --- a/antarest/study/web/raw_studies_blueprint.py +++ b/antarest/study/web/raw_studies_blueprint.py @@ -56,7 +56,7 @@ class TableExportFormat(EnumIgnoreCase): """Export format for tables.""" XLSX = "xlsx" - CSV = "csv" + TSV = "tsv" def create_raw_study_routes( @@ -98,7 +98,7 @@ def get_study( - `formatted`: A flag specifying whether the data should be returned in a formatted manner. Returns the fetched data: a JSON object (in most cases), a plain text file - or a file attachment (Microsoft Office document, CSV/TSV file...). + or a file attachment (Microsoft Office document, TSV/TSV file...). """ logger.info( f"📘 Fetching data at {path} (depth={depth}) from study {uuid}", @@ -310,7 +310,7 @@ def get_matrix( def _create_matrix_files( df_matrix: pd.DataFrame, header: bool, index: bool, format: TableExportFormat, export_path: pathlib.Path ) -> None: - if format == TableExportFormat.CSV: + if format == TableExportFormat.TSV: df_matrix.to_csv( export_path, sep="\t", diff --git a/tests/integration/raw_studies_blueprint/test_download_matrices.py b/tests/integration/raw_studies_blueprint/test_download_matrices.py index e1f354f823..3fb7af4164 100644 --- a/tests/integration/raw_studies_blueprint/test_download_matrices.py +++ b/tests/integration/raw_studies_blueprint/test_download_matrices.py @@ -110,7 +110,7 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st index = not header res = client.get( f"/v1/studies/{parent_id}/raw/download", - params={"path": raw_matrix_path, "format": "CSV", "header": header, "index": index}, + params={"path": raw_matrix_path, "format": "TSV", "header": header, "index": index}, headers=admin_headers, ) assert res.status_code == 200 @@ -129,7 +129,7 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st # tests links headers before v8.2 res = client.get( f"/v1/studies/{study_id}/raw/download", - params={"path": "input/links/de/fr", "format": "csv", "index": False}, + params={"path": "input/links/de/fr", "format": "tsv", "index": False}, headers=admin_headers, ) assert res.status_code == 200 @@ -149,7 +149,7 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st # tests links headers after v8.2 res = client.get( f"/v1/studies/{parent_id}/raw/download", - params={"path": "input/links/de/fr_parameters", "format": "csv"}, + params={"path": "input/links/de/fr_parameters", "format": "tsv"}, headers=admin_headers, ) assert res.status_code == 200 @@ -167,7 +167,7 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st # allocation and correlation matrices for path in ["input/hydro/allocation", "input/hydro/correlation"]: res = client.get( - f"/v1/studies/{parent_id}/raw/download", params={"path": path, "format": "csv"}, headers=admin_headers + f"/v1/studies/{parent_id}/raw/download", params={"path": path, "format": "tsv"}, headers=admin_headers ) assert res.status_code == 200 content = io.BytesIO(res.content) @@ -179,7 +179,7 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st # test for empty matrix res = client.get( f"/v1/studies/{study_id}/raw/download", - params={"path": "input/hydro/common/capacity/waterValues_de", "format": "csv"}, + params={"path": "input/hydro/common/capacity/waterValues_de", "format": "tsv"}, headers=admin_headers, ) assert res.status_code == 200 @@ -190,7 +190,7 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st # modulation matrix res = client.get( f"/v1/studies/{parent_id}/raw/download", - params={"path": "input/thermal/prepro/de/01_solar/modulation", "format": "csv"}, + params={"path": "input/thermal/prepro/de/01_solar/modulation", "format": "tsv"}, headers=admin_headers, ) assert res.status_code == 200 @@ -207,7 +207,7 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st f"/v1/studies/{study_id}/raw/download", params={ "path": "output/20201014-1422eco-hello/economy/mc-ind/00001/links/de/fr/values-hourly", - "format": "csv", + "format": "tsv", }, headers=admin_headers, ) @@ -230,7 +230,7 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st # test energy matrix to test the regex res = client.get( f"/v1/studies/{study_id}/raw/download", - params={"path": "input/hydro/prepro/de/energy", "format": "csv"}, + params={"path": "input/hydro/prepro/de/energy", "format": "tsv"}, headers=admin_headers, ) assert res.status_code == 200 @@ -247,7 +247,7 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st # fake study_id res = client.get( f"/v1/studies/{fake_str}/raw/download", - params={"path": raw_matrix_path, "format": "csv"}, + params={"path": raw_matrix_path, "format": "tsv"}, headers=admin_headers, ) assert res.status_code == 404 @@ -256,7 +256,7 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st # fake path res = client.get( f"/v1/studies/{parent_id}/raw/download", - params={"path": f"input/links/de/{fake_str}", "format": "csv"}, + params={"path": f"input/links/de/{fake_str}", "format": "tsv"}, headers=admin_headers, ) assert res.status_code == 404 @@ -265,7 +265,7 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st # path that does not lead to a matrix res = client.get( f"/v1/studies/{parent_id}/raw/download", - params={"path": "settings/generaldata", "format": "csv"}, + params={"path": "settings/generaldata", "format": "tsv"}, headers=admin_headers, ) assert res.status_code == 404 From 62d24b00cb99c7d0d7512ebd47326fce23bc0a10 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Thu, 8 Feb 2024 23:28:10 +0100 Subject: [PATCH 05/31] refactor(api-download): add `suffix` and `media_type` properties to `TableExportFormat` enum --- antarest/study/web/raw_studies_blueprint.py | 32 +++++++++++++++++-- .../test_download_matrices.py | 10 ++++-- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/antarest/study/web/raw_studies_blueprint.py b/antarest/study/web/raw_studies_blueprint.py index 19f3ff4b50..c296e875d5 100644 --- a/antarest/study/web/raw_studies_blueprint.py +++ b/antarest/study/web/raw_studies_blueprint.py @@ -58,6 +58,31 @@ class TableExportFormat(EnumIgnoreCase): XLSX = "xlsx" TSV = "tsv" + def __str__(self) -> str: + """Return the format as a string for display.""" + return self.value.title() + + @property + def media_type(self) -> str: + """Return the media type used for the HTTP response.""" + if self == TableExportFormat.XLSX: + # noinspection SpellCheckingInspection + return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + elif self == TableExportFormat.TSV: + return "text/tab-separated-values" + else: # pragma: no cover + raise NotImplementedError(f"Export format '{self}' is not implemented") + + @property + def suffix(self) -> str: + """Return the file suffix for the format.""" + if self == TableExportFormat.XLSX: + return ".xlsx" + elif self == TableExportFormat.TSV: + return ".tsv" + else: # pragma: no cover + raise NotImplementedError(f"Export format '{self}' is not implemented") + def create_raw_study_routes( study_service: StudyService, @@ -272,9 +297,10 @@ def get_matrix( study_id=uuid, path=path, with_index=index, with_columns=header, parameters=parameters ) + matrix_name = pathlib.Path(path).stem export_file_download = study_service.file_transfer_manager.request_download( - f"{pathlib.Path(path).stem}.{format.value}", - f"Exporting matrix {pathlib.Path(path).stem} to format {format.value} for study {uuid}", + f"{matrix_name}{format.suffix}", + f"Exporting matrix '{matrix_name}' to {format} format for study '{uuid}'", current_user, use_notification=False, expiration_time_in_minutes=10, @@ -301,7 +327,7 @@ def get_matrix( return FileResponse( export_path, headers={"Content-Disposition": f'attachment; filename="{export_file_download.filename}"'}, - media_type="application/octet-stream", + media_type=format.media_type, ) return bp diff --git a/tests/integration/raw_studies_blueprint/test_download_matrices.py b/tests/integration/raw_studies_blueprint/test_download_matrices.py index 3fb7af4164..2411f30f07 100644 --- a/tests/integration/raw_studies_blueprint/test_download_matrices.py +++ b/tests/integration/raw_studies_blueprint/test_download_matrices.py @@ -70,12 +70,16 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st raw_matrix_path = r"input/load/series/load_de" variant_matrix_path = f"input/load/series/load_{area_name}" - for uuid, path in zip([parent_id, variant_id], [raw_matrix_path, variant_matrix_path]): + for uuid, path in [(parent_id, raw_matrix_path), (variant_id, variant_matrix_path)]: # get downloaded bytes res = client.get( - f"/v1/studies/{uuid}/raw/download", params={"path": path, "format": "xlsx"}, headers=admin_headers + f"/v1/studies/{uuid}/raw/download", + params={"path": path, "format": "xlsx"}, + headers=admin_headers, ) assert res.status_code == 200 + # noinspection SpellCheckingInspection + assert res.headers["content-type"] == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" # load into dataframe dataframe = pd.read_excel(io.BytesIO(res.content), index_col=0) @@ -114,6 +118,8 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st headers=admin_headers, ) assert res.status_code == 200 + assert res.headers["content-type"] == "text/tab-separated-values; charset=utf-8" + content = io.BytesIO(res.content) dataframe = pd.read_csv( content, index_col=0 if index else None, header="infer" if header else None, sep="\t" From 793c17c56cc39ddc398b0a7826b8a5e842c7206c Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Thu, 8 Feb 2024 23:37:50 +0100 Subject: [PATCH 06/31] refactor(api-download): add `export_table` method (to replace `_create_matrix_files`) --- antarest/study/web/raw_studies_blueprint.py | 50 ++++++++++----------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/antarest/study/web/raw_studies_blueprint.py b/antarest/study/web/raw_studies_blueprint.py index c296e875d5..f47691efd0 100644 --- a/antarest/study/web/raw_studies_blueprint.py +++ b/antarest/study/web/raw_studies_blueprint.py @@ -2,8 +2,8 @@ import io import json import logging -import pathlib import typing as t +from pathlib import Path, PurePosixPath import pandas as pd from fastapi import APIRouter, Body, Depends, File, HTTPException @@ -83,6 +83,22 @@ def suffix(self) -> str: else: # pragma: no cover raise NotImplementedError(f"Export format '{self}' is not implemented") + def export_table( + self, + df: pd.DataFrame, + export_path: t.Union[str, Path], + *, + with_index: bool = True, + with_header: bool = True, + ) -> None: + """Export a table to a file in the given format.""" + if self == TableExportFormat.XLSX: + return df.to_excel(export_path, index=with_index, header=with_header, engine="openpyxl") + elif self == TableExportFormat.TSV: + return df.to_csv(export_path, sep="\t", index=with_index, header=with_header, float_format="%.6f") + else: # pragma: no cover + raise NotImplementedError(f"Export format '{self}' is not implemented") + def create_raw_study_routes( study_service: StudyService, @@ -134,10 +150,10 @@ def get_study( if isinstance(output, bytes): # Guess the suffix form the target data - resource_path = pathlib.PurePosixPath(path) + resource_path = PurePosixPath(path) parent_cfg = study_service.get(uuid, str(resource_path.parent), depth=2, formatted=True, params=parameters) child = parent_cfg[resource_path.name] - suffix = pathlib.PurePosixPath(child).suffix + suffix = PurePosixPath(child).suffix content_type, encoding = CONTENT_TYPES.get(suffix, (None, None)) if content_type == "application/json": @@ -297,7 +313,7 @@ def get_matrix( study_id=uuid, path=path, with_index=index, with_columns=header, parameters=parameters ) - matrix_name = pathlib.Path(path).stem + matrix_name = Path(path).stem export_file_download = study_service.file_transfer_manager.request_download( f"{matrix_name}{format.suffix}", f"Exporting matrix '{matrix_name}' to {format} format for study '{uuid}'", @@ -305,17 +321,17 @@ def get_matrix( use_notification=False, expiration_time_in_minutes=10, ) - export_path = pathlib.Path(export_file_download.path) + export_path = Path(export_file_download.path) export_id = export_file_download.id try: - _create_matrix_files(df_matrix, header, index, format, export_path) + format.export_table(df_matrix, export_path, with_index=index, with_header=header) study_service.file_transfer_manager.set_ready(export_id, use_notification=False) except ValueError as e: study_service.file_transfer_manager.fail(export_id, str(e)) raise HTTPException( status_code=http.HTTPStatus.UNPROCESSABLE_ENTITY, - detail=f"The Excel file {export_path} already exists and cannot be replaced due to Excel policy :{str(e)}", + detail=f"Cannot replace '{export_path}' due to Excel policy: {e}", ) from e except FileDownloadNotFound as e: study_service.file_transfer_manager.fail(export_id, str(e)) @@ -331,23 +347,3 @@ def get_matrix( ) return bp - - -def _create_matrix_files( - df_matrix: pd.DataFrame, header: bool, index: bool, format: TableExportFormat, export_path: pathlib.Path -) -> None: - if format == TableExportFormat.TSV: - df_matrix.to_csv( - export_path, - sep="\t", - header=header, - index=index, - float_format="%.6f", - ) - else: - df_matrix.to_excel( - export_path, - header=header, - index=index, - float_format="%.6f", - ) From 9ee16bcf71c758d946ae4de557962370ed92b520 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Thu, 8 Feb 2024 23:51:20 +0100 Subject: [PATCH 07/31] refactor(api-download): rename endpoint parameters and use `alias`, `description` and `title`. NOTE: `description` and `title` are displayed in the Swagger --- antarest/study/web/raw_studies_blueprint.py | 29 ++++++++++++------- .../test_download_matrices.py | 6 ++-- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/antarest/study/web/raw_studies_blueprint.py b/antarest/study/web/raw_studies_blueprint.py index f47691efd0..bd1b254472 100644 --- a/antarest/study/web/raw_studies_blueprint.py +++ b/antarest/study/web/raw_studies_blueprint.py @@ -298,25 +298,32 @@ def validate( "/studies/{uuid}/raw/download", summary="Download a matrix in a given format", tags=[APITag.study_raw_data], - response_class=FileResponse, ) def get_matrix( uuid: str, - path: str, - format: TableExportFormat, - header: bool = True, - index: bool = True, + matrix_path: str = Query( # type: ignore + ..., alias="path", description="Relative path of the matrix to download", title="Matrix Path" + ), + export_format: TableExportFormat = Query( # type: ignore + TableExportFormat.XLSX, alias="format", description="Export format", title="Export Format" + ), + with_header: bool = Query( # type: ignore + True, alias="header", description="Whether to include the header or not", title="With Header" + ), + with_index: bool = Query( # type: ignore + True, alias="index", description="Whether to include the index or not", title="With Index" + ), current_user: JWTUser = Depends(auth.get_current_user), ) -> FileResponse: parameters = RequestParameters(user=current_user) df_matrix = study_service.get_matrix_with_index_and_header( - study_id=uuid, path=path, with_index=index, with_columns=header, parameters=parameters + study_id=uuid, path=matrix_path, with_index=with_index, with_columns=with_header, parameters=parameters ) - matrix_name = Path(path).stem + matrix_name = Path(matrix_path).stem export_file_download = study_service.file_transfer_manager.request_download( - f"{matrix_name}{format.suffix}", - f"Exporting matrix '{matrix_name}' to {format} format for study '{uuid}'", + f"{matrix_name}{export_format.suffix}", + f"Exporting matrix '{matrix_name}' to {export_format} format for study '{uuid}'", current_user, use_notification=False, expiration_time_in_minutes=10, @@ -325,7 +332,7 @@ def get_matrix( export_id = export_file_download.id try: - format.export_table(df_matrix, export_path, with_index=index, with_header=header) + export_format.export_table(df_matrix, export_path, with_index=with_index, with_header=with_header) study_service.file_transfer_manager.set_ready(export_id, use_notification=False) except ValueError as e: study_service.file_transfer_manager.fail(export_id, str(e)) @@ -343,7 +350,7 @@ def get_matrix( return FileResponse( export_path, headers={"Content-Disposition": f'attachment; filename="{export_file_download.filename}"'}, - media_type=format.media_type, + media_type=export_format.media_type, ) return bp diff --git a/tests/integration/raw_studies_blueprint/test_download_matrices.py b/tests/integration/raw_studies_blueprint/test_download_matrices.py index 2411f30f07..23147dbf05 100644 --- a/tests/integration/raw_studies_blueprint/test_download_matrices.py +++ b/tests/integration/raw_studies_blueprint/test_download_matrices.py @@ -71,10 +71,11 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st variant_matrix_path = f"input/load/series/load_{area_name}" for uuid, path in [(parent_id, raw_matrix_path), (variant_id, variant_matrix_path)]: - # get downloaded bytes + # Export the matrix in xlsx format (which is the default format) + # and retrieve it as binary content (a ZIP-like file). res = client.get( f"/v1/studies/{uuid}/raw/download", - params={"path": path, "format": "xlsx"}, + params={"path": path}, headers=admin_headers, ) assert res.status_code == 200 @@ -82,6 +83,7 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st assert res.headers["content-type"] == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" # load into dataframe + # noinspection PyTypeChecker dataframe = pd.read_excel(io.BytesIO(res.content), index_col=0) # check time coherence From 2e46cbfefe019ac0cca916a3d354172963eece36 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Fri, 9 Feb 2024 00:01:32 +0100 Subject: [PATCH 08/31] refactor(api-download): rename the parameters and variables from `with_columns` to `with_header` It is clearer for DataFrames. --- antarest/study/service.py | 8 ++++---- antarest/study/web/raw_studies_blueprint.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/antarest/study/service.py b/antarest/study/service.py index 6c97a16d66..df673a5a0e 100644 --- a/antarest/study/service.py +++ b/antarest/study/service.py @@ -158,9 +158,9 @@ def _handle_specific_matrices( matrix_path: str, *, with_index: bool, - with_columns: bool, + with_header: bool, ) -> pd.DataFrame: - if with_columns: + if with_header: if Path(matrix_path).parts[1] == "links": cols = _handle_links_columns(matrix_path, matrix_profile) else: @@ -2427,7 +2427,7 @@ def get_disk_usage(self, uuid: str, params: RequestParameters) -> int: return get_disk_usage(study_path) if study_path.exists() else 0 def get_matrix_with_index_and_header( - self, *, study_id: str, path: str, with_index: bool, with_columns: bool, parameters: RequestParameters + self, *, study_id: str, path: str, with_index: bool, with_header: bool, parameters: RequestParameters ) -> pd.DataFrame: matrix_path = Path(path) study = self.get_study(study_id) @@ -2466,6 +2466,6 @@ def get_matrix_with_index_and_header( specific_matrices[specific_matrix], path, with_index=with_index, - with_columns=with_columns, + with_header=with_header, ) return df_matrix diff --git a/antarest/study/web/raw_studies_blueprint.py b/antarest/study/web/raw_studies_blueprint.py index bd1b254472..9a401d7135 100644 --- a/antarest/study/web/raw_studies_blueprint.py +++ b/antarest/study/web/raw_studies_blueprint.py @@ -317,7 +317,7 @@ def get_matrix( ) -> FileResponse: parameters = RequestParameters(user=current_user) df_matrix = study_service.get_matrix_with_index_and_header( - study_id=uuid, path=matrix_path, with_index=with_index, with_columns=with_header, parameters=parameters + study_id=uuid, path=matrix_path, with_index=with_index, with_header=with_header, parameters=parameters ) matrix_name = Path(matrix_path).stem From f3925c64f496080cd91f4c590e96c3661239095a Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Fri, 9 Feb 2024 00:04:02 +0100 Subject: [PATCH 09/31] refactor(api-download): correct spelling in unit tests --- .../raw_studies_blueprint/test_download_matrices.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/integration/raw_studies_blueprint/test_download_matrices.py b/tests/integration/raw_studies_blueprint/test_download_matrices.py index 23147dbf05..9b9a269702 100644 --- a/tests/integration/raw_studies_blueprint/test_download_matrices.py +++ b/tests/integration/raw_studies_blueprint/test_download_matrices.py @@ -88,7 +88,9 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st # check time coherence generated_index = dataframe.index + # noinspection PyUnresolvedReferences first_date = generated_index[0].to_pydatetime() + # noinspection PyUnresolvedReferences second_date = generated_index[1].to_pydatetime() assert first_date.month == second_date.month == 1 if uuid == parent_id else 7 assert first_date.day == second_date.day == 1 @@ -206,8 +208,8 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st dataframe = pd.read_csv(content, index_col=0, sep="\t") assert dataframe.index[0] == "2018-01-01 00:00:00" dataframe.index = range(len(dataframe)) - liste_transposee = list(zip(*[8760 * [1.0], 8760 * [1.0], 8760 * [1.0], 8760 * [0.0]])) - expected_df = pd.DataFrame(columns=["0", "1", "2", "3"], index=range(8760), data=liste_transposee) + transposed_matrix = list(zip(*[8760 * [1.0], 8760 * [1.0], 8760 * [1.0], 8760 * [0.0]])) + expected_df = pd.DataFrame(columns=["0", "1", "2", "3"], index=range(8760), data=transposed_matrix) assert dataframe.equals(expected_df) # asserts endpoint returns the right columns for output matrix @@ -222,6 +224,7 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st assert res.status_code == 200 content = io.BytesIO(res.content) dataframe = pd.read_csv(content, index_col=0, sep="\t") + # noinspection SpellCheckingInspection assert list(dataframe.columns) == [ "('FLOW LIN.', 'MWh', '')", "('UCAP LIN.', 'MWh', '')", From 3cf81d7bf2e4e4f7b083faba824b8891662fae62 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Fri, 9 Feb 2024 00:42:56 +0100 Subject: [PATCH 10/31] refactor(api-download): correct implementation of `get_matrix_with_index_and_header` --- antarest/study/service.py | 48 +++++++++---------- antarest/study/storage/utils.py | 30 ++---------- antarest/study/web/raw_studies_blueprint.py | 6 ++- .../test_download_matrices.py | 8 +++- 4 files changed, 40 insertions(+), 52 deletions(-) diff --git a/antarest/study/service.py b/antarest/study/service.py index df673a5a0e..2a6113d5bc 100644 --- a/antarest/study/service.py +++ b/antarest/study/service.py @@ -1,10 +1,10 @@ import base64 import contextlib +import fnmatch import io import json import logging import os -import re import time import typing as t from datetime import datetime, timedelta @@ -116,7 +116,7 @@ from antarest.study.storage.utils import ( MatrixProfile, assert_permission, - get_specific_matrices_according_to_version, + get_matrix_profile_by_version, get_start_date, is_managed, remove_from_cache, @@ -2431,26 +2431,25 @@ def get_matrix_with_index_and_header( ) -> pd.DataFrame: matrix_path = Path(path) study = self.get_study(study_id) - for aggregate in ["allocation", "correlation"]: - if matrix_path == Path("input") / "hydro" / aggregate: - all_areas = t.cast( - t.List[AreaInfoDTO], - self.get_all_areas(study_id, area_type=AreaType.AREA, ui=False, params=parameters), - ) - if aggregate == "allocation": - hydro_matrix = self.allocation_manager.get_allocation_matrix(study, all_areas) - else: - hydro_matrix = self.correlation_manager.get_correlation_matrix(all_areas, study, []) # type: ignore - return pd.DataFrame(data=hydro_matrix.data, columns=hydro_matrix.columns, index=hydro_matrix.index) - - json_matrix = self.get(study_id, path, depth=3, formatted=True, params=parameters) - for key in ["data", "index", "columns"]: - if key not in json_matrix: - raise IncorrectPathError(f"The path filled does not correspond to a matrix : {path}") - if not json_matrix["data"]: + + if matrix_path.parts in [("input", "hydro", "allocation"), ("input", "hydro", "correlation")]: + all_areas = t.cast( + t.List[AreaInfoDTO], + self.get_all_areas(study_id, area_type=AreaType.AREA, ui=False, params=parameters), + ) + if matrix_path.parts[-1] == "allocation": + hydro_matrix = self.allocation_manager.get_allocation_matrix(study, all_areas) + else: + hydro_matrix = self.correlation_manager.get_correlation_matrix(all_areas, study, []) # type: ignore + return pd.DataFrame(data=hydro_matrix.data, columns=hydro_matrix.columns, index=hydro_matrix.index) + + matrix_obj = self.get(study_id, path, depth=3, formatted=True, params=parameters) + if set(matrix_obj) != {"data", "index", "columns"}: + raise IncorrectPathError(f"The provided path does not point to a valid matrix: '{path}'") + if not matrix_obj["data"]: return pd.DataFrame() - df_matrix = pd.DataFrame(data=json_matrix["data"], columns=json_matrix["columns"], index=json_matrix["index"]) + df_matrix = pd.DataFrame(**matrix_obj) if with_index: matrix_index = self.get_input_matrix_startdate(study_id, path, parameters) time_column = pd.date_range( @@ -2458,14 +2457,15 @@ def get_matrix_with_index_and_header( ) df_matrix.index = time_column - specific_matrices = get_specific_matrices_according_to_version(int(study.version)) - for specific_matrix in specific_matrices: - if re.match(specific_matrix, path): + matrix_profiles = get_matrix_profile_by_version(int(study.version)) + for pattern, matrix_profile in matrix_profiles.items(): + if fnmatch.fnmatch(path, pattern): return _handle_specific_matrices( df_matrix, - specific_matrices[specific_matrix], + matrix_profile, path, with_index=with_index, with_header=with_header, ) + return df_matrix diff --git a/antarest/study/storage/utils.py b/antarest/study/storage/utils.py index 81e7b5dfca..65a33ac1f0 100644 --- a/antarest/study/storage/utils.py +++ b/antarest/study/storage/utils.py @@ -244,30 +244,9 @@ def assert_permission( MATRIX_INPUT_DAYS_COUNT = 365 -MONTHS = ( - "January", - "February", - "March", - "April", - "May", - "June", - "July", - "August", - "September", - "October", - "November", - "December", -) +MONTHS = calendar.month_name[1:] -DAY_NAMES = ( - "Monday", - "Tuesday", - "Wednesday", - "Thursday", - "Friday", - "Saturday", - "Sunday", -) +DAY_NAMES = calendar.day_name[:] def _generate_columns(column_suffix: str) -> t.List[str]: @@ -383,12 +362,13 @@ class MatrixProfile(t.NamedTuple): SPECIFIC_MATRICES_870["input/bindingconstraints/*"] = MatrixProfile(cols=[], rows=[], stats=False) -def get_specific_matrices_according_to_version(study_version: int) -> t.Dict[str, MatrixProfile]: +def get_matrix_profile_by_version(study_version: int) -> t.Dict[str, MatrixProfile]: if study_version < 820: return SPECIFIC_MATRICES elif study_version < 870: return SPECIFIC_MATRICES_820 - return SPECIFIC_MATRICES_870 + else: + return SPECIFIC_MATRICES_870 def get_start_date( diff --git a/antarest/study/web/raw_studies_blueprint.py b/antarest/study/web/raw_studies_blueprint.py index 9a401d7135..d452a53e9e 100644 --- a/antarest/study/web/raw_studies_blueprint.py +++ b/antarest/study/web/raw_studies_blueprint.py @@ -317,7 +317,11 @@ def get_matrix( ) -> FileResponse: parameters = RequestParameters(user=current_user) df_matrix = study_service.get_matrix_with_index_and_header( - study_id=uuid, path=matrix_path, with_index=with_index, with_header=with_header, parameters=parameters + study_id=uuid, + path=matrix_path, + with_index=with_index, + with_header=with_header, + parameters=parameters, ) matrix_name = Path(matrix_path).stem diff --git a/tests/integration/raw_studies_blueprint/test_download_matrices.py b/tests/integration/raw_studies_blueprint/test_download_matrices.py index 9b9a269702..14b8aa9a58 100644 --- a/tests/integration/raw_studies_blueprint/test_download_matrices.py +++ b/tests/integration/raw_studies_blueprint/test_download_matrices.py @@ -209,7 +209,11 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st assert dataframe.index[0] == "2018-01-01 00:00:00" dataframe.index = range(len(dataframe)) transposed_matrix = list(zip(*[8760 * [1.0], 8760 * [1.0], 8760 * [1.0], 8760 * [0.0]])) - expected_df = pd.DataFrame(columns=["0", "1", "2", "3"], index=range(8760), data=transposed_matrix) + expected_df = pd.DataFrame( + columns=["Marginal cost modulation", "Market bid modulation", "Capacity modulation", "Min gen modulation"], + index=range(8760), + data=transposed_matrix, + ) assert dataframe.equals(expected_df) # asserts endpoint returns the right columns for output matrix @@ -281,7 +285,7 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st ) assert res.status_code == 404 assert res.json()["exception"] == "IncorrectPathError" - assert res.json()["description"] == "The path filled does not correspond to a matrix : settings/generaldata" + assert res.json()["description"] == "The provided path does not point to a valid matrix: 'settings/generaldata'" # wrong format res = client.get( From 05e69f5d31774bad2dbfa48bbe8fdfbd55f8e1bf Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Fri, 9 Feb 2024 01:01:36 +0100 Subject: [PATCH 11/31] refactor(api-download): move `MatrixProfile` and specifics matrix in `matrix_profile` module --- antarest/study/service.py | 49 +------ antarest/study/storage/matrix_profile.py | 160 +++++++++++++++++++++++ antarest/study/storage/utils.py | 123 ----------------- 3 files changed, 164 insertions(+), 168 deletions(-) create mode 100644 antarest/study/storage/matrix_profile.py diff --git a/antarest/study/service.py b/antarest/study/service.py index 2a6113d5bc..05c7b24595 100644 --- a/antarest/study/service.py +++ b/antarest/study/service.py @@ -97,6 +97,7 @@ StudySimResultDTO, ) from antarest.study.repository import StudyFilter, StudyMetadataRepository, StudyPagination, StudySortBy +from antarest.study.storage.matrix_profile import get_matrix_profiles_by_version from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfigDTO from antarest.study.storage.rawstudy.model.filesystem.folder_node import ChildNotFoundError from antarest.study.storage.rawstudy.model.filesystem.ini_file_node import IniFileNode @@ -113,14 +114,7 @@ should_study_be_denormalized, upgrade_study, ) -from antarest.study.storage.utils import ( - MatrixProfile, - assert_permission, - get_matrix_profile_by_version, - get_start_date, - is_managed, - remove_from_cache, -) +from antarest.study.storage.utils import assert_permission, get_start_date, is_managed, remove_from_cache from antarest.study.storage.variantstudy.model.command.icommand import ICommand from antarest.study.storage.variantstudy.model.command.replace_matrix import ReplaceMatrix from antarest.study.storage.variantstudy.model.command.update_comments import UpdateComments @@ -152,40 +146,6 @@ def get_disk_usage(path: t.Union[str, Path]) -> int: return total_size -def _handle_specific_matrices( - df: pd.DataFrame, - matrix_profile: MatrixProfile, - matrix_path: str, - *, - with_index: bool, - with_header: bool, -) -> pd.DataFrame: - if with_header: - if Path(matrix_path).parts[1] == "links": - cols = _handle_links_columns(matrix_path, matrix_profile) - else: - cols = matrix_profile.cols - if cols: - df.columns = pd.Index(cols) - rows = matrix_profile.rows - if with_index and rows: - df.index = rows # type: ignore - return df - - -def _handle_links_columns(matrix_path: str, matrix_profile: MatrixProfile) -> t.List[str]: - path_parts = Path(matrix_path).parts - area_id_1 = path_parts[2] - area_id_2 = path_parts[3] - result = matrix_profile.cols - for k, col in enumerate(result): - if col == "Hurdle costs direct": - result[k] = f"{col} ({area_id_1}->{area_id_2})" - elif col == "Hurdle costs indirect": - result[k] = f"{col} ({area_id_2}->{area_id_1})" - return result - - class StudyUpgraderTask: """ Task to perform a study upgrade. @@ -2457,12 +2417,11 @@ def get_matrix_with_index_and_header( ) df_matrix.index = time_column - matrix_profiles = get_matrix_profile_by_version(int(study.version)) + matrix_profiles = get_matrix_profiles_by_version(int(study.version)) for pattern, matrix_profile in matrix_profiles.items(): if fnmatch.fnmatch(path, pattern): - return _handle_specific_matrices( + return matrix_profile.handle_specific_matrices( df_matrix, - matrix_profile, path, with_index=with_index, with_header=with_header, diff --git a/antarest/study/storage/matrix_profile.py b/antarest/study/storage/matrix_profile.py new file mode 100644 index 0000000000..55833d0382 --- /dev/null +++ b/antarest/study/storage/matrix_profile.py @@ -0,0 +1,160 @@ +import copy +import typing as t +from pathlib import Path + +import pandas as pd + + +class MatrixProfile(t.NamedTuple): + """ + Matrix profile for time series or classic tables. + """ + + cols: t.Sequence[str] + rows: t.Sequence[str] + stats: bool + + def handle_specific_matrices( + self, + df: pd.DataFrame, + matrix_path: str, + *, + with_index: bool, + with_header: bool, + ) -> pd.DataFrame: + if with_header: + if Path(matrix_path).parts[1] == "links": + cols = self.handle_links_columns(matrix_path) + else: + cols = self.cols + if cols: + df.columns = pd.Index(cols) + rows = self.rows + if with_index and rows: + df.index = rows # type: ignore + return df + + def handle_links_columns(self, matrix_path: str) -> t.Sequence[str]: + path_parts = Path(matrix_path).parts + area_id_1 = path_parts[2] + area_id_2 = path_parts[3] + result = list(self.cols) + for k, col in enumerate(result): + if col == "Hurdle costs direct": + result[k] = f"{col} ({area_id_1}->{area_id_2})" + elif col == "Hurdle costs indirect": + result[k] = f"{col} ({area_id_2}->{area_id_1})" + return result + + +# noinspection SpellCheckingInspection +SPECIFIC_MATRICES = { + "input/hydro/common/capacity/creditmodulations_*": MatrixProfile( + cols=[str(i) for i in range(101)], + rows=["Generating Power", "Pumping Power"], + stats=False, + ), + "input/hydro/common/capacity/maxpower_*": MatrixProfile( + cols=[ + "Generating Max Power (MW)", + "Generating Max Energy (Hours at Pmax)", + "Pumping Max Power (MW)", + "Pumping Max Energy (Hours at Pmax)", + ], + rows=[], + stats=False, + ), + "input/hydro/common/capacity/reservoir_*": MatrixProfile( + cols=["Lev Low (p.u)", "Lev Avg (p.u)", "Lev High (p.u)"], + rows=[], + stats=False, + ), + "input/hydro/common/capacity/waterValues_*": MatrixProfile( + cols=[f"{i}%" for i in range(101)], + rows=[], + stats=False, + ), + "input/hydro/series/*/mod": MatrixProfile(cols=[], rows=[], stats=True), + "input/hydro/series/*/ror": MatrixProfile(cols=[], rows=[], stats=True), + "input/hydro/common/capacity/inflowPattern_*": MatrixProfile(cols=["Inflow Pattern (X)"], rows=[], stats=False), + "input/hydro/prepro/*/energy": MatrixProfile( + cols=["Expectation (MWh)", "Std Deviation (MWh)", "Min. (MWh)", "Max. (MWh)", "ROR Share"], + rows=[ + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", + ], + stats=False, + ), + "input/thermal/prepro/*/*/modulation": MatrixProfile( + cols=["Marginal cost modulation", "Market bid modulation", "Capacity modulation", "Min gen modulation"], + rows=[], + stats=False, + ), + "input/thermal/prepro/*/*/data": MatrixProfile( + cols=["FO Duration", "PO Duration", "FO Rate", "PO Rate", "NPO Min", "NPO Max"], + rows=[], + stats=False, + ), + "input/reserves/*": MatrixProfile( + cols=["Primary Res. (draft)", "Strategic Res. (draft)", "DSM", "Day Ahead"], + rows=[], + stats=False, + ), + "input/misc-gen/miscgen-*": MatrixProfile( + cols=["CHP", "Bio Mass", "Bio Gaz", "Waste", "GeoThermal", "Other", "PSP", "ROW Balance"], + rows=[], + stats=False, + ), + "input/bindingconstraints/*": MatrixProfile(cols=["<", ">", "="], rows=[], stats=False), + "input/links/*/*": MatrixProfile( + cols=[ + "Capacités de transmission directes", + "Capacités de transmission indirectes", + "Hurdle costs direct", + "Hurdle costs indirect", + "Impedances", + "Loop flow", + "P.Shift Min", + "P.Shift Max", + ], + rows=[], + stats=False, + ), +} + +SPECIFIC_MATRICES_820 = copy.deepcopy(SPECIFIC_MATRICES) +SPECIFIC_MATRICES_820["input/links/*/*"] = MatrixProfile( + cols=[ + "Hurdle costs direct", + "Hurdle costs indirect", + "Impedances", + "Loop flow", + "P.Shift Min", + "P.Shift Max", + ], + rows=[], + stats=False, +) + +SPECIFIC_MATRICES_870 = copy.deepcopy(SPECIFIC_MATRICES_820) +# noinspection SpellCheckingInspection +SPECIFIC_MATRICES_870["input/bindingconstraints/*"] = MatrixProfile(cols=[], rows=[], stats=False) + + +def get_matrix_profiles_by_version(study_version: int) -> t.Dict[str, MatrixProfile]: + if study_version < 820: + return SPECIFIC_MATRICES + elif study_version < 870: + return SPECIFIC_MATRICES_820 + else: + return SPECIFIC_MATRICES_870 diff --git a/antarest/study/storage/utils.py b/antarest/study/storage/utils.py index 65a33ac1f0..365eb1f370 100644 --- a/antarest/study/storage/utils.py +++ b/antarest/study/storage/utils.py @@ -1,5 +1,4 @@ import calendar -import copy import logging import math import os @@ -249,128 +248,6 @@ def assert_permission( DAY_NAMES = calendar.day_name[:] -def _generate_columns(column_suffix: str) -> t.List[str]: - return [f"{i}{column_suffix}" for i in range(101)] - - -class MatrixProfile(t.NamedTuple): - """ - Matrix profile for time series or classic tables. - """ - - cols: t.List[str] - rows: t.List[str] - stats: bool - - -SPECIFIC_MATRICES = { - "input/hydro/common/capacity/creditmodulations_*": MatrixProfile( - cols=_generate_columns(""), - rows=["Generating Power", "Pumping Power"], - stats=False, - ), - "input/hydro/common/capacity/maxpower_*": MatrixProfile( - cols=[ - "Generating Max Power (MW)", - "Generating Max Energy (Hours at Pmax)", - "Pumping Max Power (MW)", - "Pumping Max Energy (Hours at Pmax)", - ], - rows=[], - stats=False, - ), - "input/hydro/common/capacity/reservoir_*": MatrixProfile( - cols=["Lev Low (p.u)", "Lev Avg (p.u)", "Lev High (p.u)"], - rows=[], - stats=False, - ), - "input/hydro/common/capacity/waterValues_*": MatrixProfile(cols=_generate_columns("%"), rows=[], stats=False), - "input/hydro/series/*/mod": MatrixProfile(cols=[], rows=[], stats=True), - "input/hydro/series/*/ror": MatrixProfile(cols=[], rows=[], stats=True), - "input/hydro/common/capacity/inflowPattern_*": MatrixProfile(cols=["Inflow Pattern (X)"], rows=[], stats=False), - "input/hydro/prepro/*/energy": MatrixProfile( - cols=["Expectation (MWh)", "Std Deviation (MWh)", "Min. (MWh)", "Max. (MWh)", "ROR Share"], - rows=[ - "January", - "February", - "March", - "April", - "May", - "June", - "July", - "August", - "September", - "October", - "November", - "December", - ], - stats=False, - ), - "input/thermal/prepro/*/*/modulation": MatrixProfile( - cols=["Marginal cost modulation", "Market bid modulation", "Capacity modulation", "Min gen modulation"], - rows=[], - stats=False, - ), - "input/thermal/prepro/*/*/data": MatrixProfile( - cols=["FO Duration", "PO Duration", "FO Rate", "PO Rate", "NPO Min", "NPO Max"], - rows=[], - stats=False, - ), - "input/reserves/*": MatrixProfile( - cols=["Primary Res. (draft)", "Strategic Res. (draft)", "DSM", "Day Ahead"], - rows=[], - stats=False, - ), - "input/misc-gen/miscgen-*": MatrixProfile( - cols=["CHP", "Bio Mass", "Bio Gaz", "Waste", "GeoThermal", "Other", "PSP", "ROW Balance"], - rows=[], - stats=False, - ), - "input/bindingconstraints/*": MatrixProfile(cols=["<", ">", "="], rows=[], stats=False), - "input/links/*/*": MatrixProfile( - cols=[ - "Capacités de transmission directes", - "Capacités de transmission indirectes", - "Hurdle costs direct", - "Hurdle costs indirect", - "Impedances", - "Loop flow", - "P.Shift Min", - "P.Shift Max", - ], - rows=[], - stats=False, - ), -} - - -SPECIFIC_MATRICES_820 = copy.deepcopy(SPECIFIC_MATRICES) -SPECIFIC_MATRICES_820["input/links/*/*"] = MatrixProfile( - cols=[ - "Hurdle costs direct", - "Hurdle costs indirect", - "Impedances", - "Loop flow", - "P.Shift Min", - "P.Shift Max", - ], - rows=[], - stats=False, -) - -SPECIFIC_MATRICES_870 = copy.deepcopy(SPECIFIC_MATRICES_820) -SPECIFIC_MATRICES_870["input/bindingconstraints/*"] = MatrixProfile(cols=[], rows=[], stats=False) - - -def get_matrix_profile_by_version(study_version: int) -> t.Dict[str, MatrixProfile]: - if study_version < 820: - return SPECIFIC_MATRICES - elif study_version < 870: - return SPECIFIC_MATRICES_820 - else: - return SPECIFIC_MATRICES_870 - - def get_start_date( file_study: FileStudy, output_id: t.Optional[str] = None, From da198d825620dbfbb14d24458eb5b46bbe1ab3e1 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Fri, 9 Feb 2024 01:06:59 +0100 Subject: [PATCH 12/31] refactor(api-download): simplify implementation of `SPECIFIC_MATRICES` --- antarest/study/storage/matrix_profile.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/antarest/study/storage/matrix_profile.py b/antarest/study/storage/matrix_profile.py index 55833d0382..24efebfeb2 100644 --- a/antarest/study/storage/matrix_profile.py +++ b/antarest/study/storage/matrix_profile.py @@ -1,3 +1,4 @@ +import calendar import copy import typing as t from pathlib import Path @@ -79,20 +80,7 @@ def handle_links_columns(self, matrix_path: str) -> t.Sequence[str]: "input/hydro/common/capacity/inflowPattern_*": MatrixProfile(cols=["Inflow Pattern (X)"], rows=[], stats=False), "input/hydro/prepro/*/energy": MatrixProfile( cols=["Expectation (MWh)", "Std Deviation (MWh)", "Min. (MWh)", "Max. (MWh)", "ROR Share"], - rows=[ - "January", - "February", - "March", - "April", - "May", - "June", - "July", - "August", - "September", - "October", - "November", - "December", - ], + rows=calendar.month_name[1:], stats=False, ), "input/thermal/prepro/*/*/modulation": MatrixProfile( From bff6b0cf8af7daf1acde24631303a471e836a726 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Fri, 9 Feb 2024 01:07:10 +0100 Subject: [PATCH 13/31] refactor(api-download): simplify unit tests --- .../raw_studies_blueprint/test_download_matrices.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integration/raw_studies_blueprint/test_download_matrices.py b/tests/integration/raw_studies_blueprint/test_download_matrices.py index 14b8aa9a58..3a565ecc84 100644 --- a/tests/integration/raw_studies_blueprint/test_download_matrices.py +++ b/tests/integration/raw_studies_blueprint/test_download_matrices.py @@ -183,8 +183,7 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st content = io.BytesIO(res.content) dataframe = pd.read_csv(content, index_col=0, sep="\t") assert list(dataframe.index) == list(dataframe.columns) == ["de", "es", "fr", "it"] - for i in range((len(dataframe))): - assert dataframe.iloc[i, i] == 1.0 + assert all(dataframe.iloc[i, i] == 1.0 for i in range(len(dataframe))) # test for empty matrix res = client.get( From 5a868ceae13e40802133e5c34dc9ae160c7f55d3 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Fri, 9 Feb 2024 01:09:04 +0100 Subject: [PATCH 14/31] refactor(api-download): use `user_access_token` instead of the admin token in unit tests --- .../test_download_matrices.py | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/tests/integration/raw_studies_blueprint/test_download_matrices.py b/tests/integration/raw_studies_blueprint/test_download_matrices.py index 3a565ecc84..53cde0a5b2 100644 --- a/tests/integration/raw_studies_blueprint/test_download_matrices.py +++ b/tests/integration/raw_studies_blueprint/test_download_matrices.py @@ -15,8 +15,8 @@ class TestDownloadMatrices: Checks the retrieval of matrices with the endpoint GET studies/uuid/raw/download """ - def test_download_matrices(self, client: TestClient, admin_access_token: str, study_id: str) -> None: - admin_headers = {"Authorization": f"Bearer {admin_access_token}"} + def test_download_matrices(self, client: TestClient, user_access_token: str, study_id: str) -> None: + user_headers = {"Authorization": f"Bearer {user_access_token}"} # ============================= # STUDIES PREPARATION @@ -25,20 +25,20 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st # Manage parent study and upgrades it to v8.2 # This is done to test matrix headers according to different versions copied = client.post( - f"/v1/studies/{study_id}/copy", params={"dest": "copied", "use_task": False}, headers=admin_headers + f"/v1/studies/{study_id}/copy", params={"dest": "copied", "use_task": False}, headers=user_headers ) parent_id = copied.json() - res = client.put(f"/v1/studies/{parent_id}/upgrade", params={"target_version": 820}, headers=admin_headers) + res = client.put(f"/v1/studies/{parent_id}/upgrade", params={"target_version": 820}, headers=user_headers) assert res.status_code == 200 task_id = res.json() assert task_id - task = wait_task_completion(client, admin_access_token, task_id, timeout=20) + task = wait_task_completion(client, user_access_token, task_id, timeout=20) assert task.status == TaskStatus.COMPLETED # Create Variant res = client.post( f"/v1/studies/{parent_id}/variants", - headers=admin_headers, + headers=user_headers, params={"name": "variant_1"}, ) assert res.status_code == 200 @@ -48,19 +48,19 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st area_name = "new_area" res = client.post( f"/v1/studies/{variant_id}/areas", - headers=admin_headers, + headers=user_headers, json={"name": area_name, "type": "AREA", "metadata": {"country": "FR"}}, ) assert res.status_code in {200, 201} # Change study start_date res = client.put( - f"/v1/studies/{variant_id}/config/general/form", json={"firstMonth": "july"}, headers=admin_headers + f"/v1/studies/{variant_id}/config/general/form", json={"firstMonth": "july"}, headers=user_headers ) assert res.status_code == 200 # Really generates the snapshot - client.get(f"/v1/studies/{variant_id}/areas", headers=admin_headers) + client.get(f"/v1/studies/{variant_id}/areas", headers=user_headers) assert res.status_code == 200 # ============================= @@ -76,7 +76,7 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st res = client.get( f"/v1/studies/{uuid}/raw/download", params={"path": path}, - headers=admin_headers, + headers=user_headers, ) assert res.status_code == 200 # noinspection SpellCheckingInspection @@ -104,7 +104,7 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st actual_matrix = dataframe.to_dict(orient="split") # asserts that the result is the same as the one we get with the classic get /raw endpoint - res = client.get(f"/v1/studies/{uuid}/raw", params={"path": path, "formatted": True}, headers=admin_headers) + res = client.get(f"/v1/studies/{uuid}/raw", params={"path": path, "formatted": True}, headers=user_headers) expected_matrix = res.json() assert actual_matrix == expected_matrix @@ -119,7 +119,7 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st res = client.get( f"/v1/studies/{parent_id}/raw/download", params={"path": raw_matrix_path, "format": "TSV", "header": header, "index": index}, - headers=admin_headers, + headers=user_headers, ) assert res.status_code == 200 assert res.headers["content-type"] == "text/tab-separated-values; charset=utf-8" @@ -140,7 +140,7 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st res = client.get( f"/v1/studies/{study_id}/raw/download", params={"path": "input/links/de/fr", "format": "tsv", "index": False}, - headers=admin_headers, + headers=user_headers, ) assert res.status_code == 200 content = io.BytesIO(res.content) @@ -160,7 +160,7 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st res = client.get( f"/v1/studies/{parent_id}/raw/download", params={"path": "input/links/de/fr_parameters", "format": "tsv"}, - headers=admin_headers, + headers=user_headers, ) assert res.status_code == 200 content = io.BytesIO(res.content) @@ -177,7 +177,7 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st # allocation and correlation matrices for path in ["input/hydro/allocation", "input/hydro/correlation"]: res = client.get( - f"/v1/studies/{parent_id}/raw/download", params={"path": path, "format": "tsv"}, headers=admin_headers + f"/v1/studies/{parent_id}/raw/download", params={"path": path, "format": "tsv"}, headers=user_headers ) assert res.status_code == 200 content = io.BytesIO(res.content) @@ -189,7 +189,7 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st res = client.get( f"/v1/studies/{study_id}/raw/download", params={"path": "input/hydro/common/capacity/waterValues_de", "format": "tsv"}, - headers=admin_headers, + headers=user_headers, ) assert res.status_code == 200 content = io.BytesIO(res.content) @@ -200,7 +200,7 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st res = client.get( f"/v1/studies/{parent_id}/raw/download", params={"path": "input/thermal/prepro/de/01_solar/modulation", "format": "tsv"}, - headers=admin_headers, + headers=user_headers, ) assert res.status_code == 200 content = io.BytesIO(res.content) @@ -222,7 +222,7 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st "path": "output/20201014-1422eco-hello/economy/mc-ind/00001/links/de/fr/values-hourly", "format": "tsv", }, - headers=admin_headers, + headers=user_headers, ) assert res.status_code == 200 content = io.BytesIO(res.content) @@ -245,7 +245,7 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st res = client.get( f"/v1/studies/{study_id}/raw/download", params={"path": "input/hydro/prepro/de/energy", "format": "tsv"}, - headers=admin_headers, + headers=user_headers, ) assert res.status_code == 200 content = io.BytesIO(res.content) @@ -262,7 +262,7 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st res = client.get( f"/v1/studies/{fake_str}/raw/download", params={"path": raw_matrix_path, "format": "tsv"}, - headers=admin_headers, + headers=user_headers, ) assert res.status_code == 404 assert res.json()["exception"] == "StudyNotFoundError" @@ -271,7 +271,7 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st res = client.get( f"/v1/studies/{parent_id}/raw/download", params={"path": f"input/links/de/{fake_str}", "format": "tsv"}, - headers=admin_headers, + headers=user_headers, ) assert res.status_code == 404 assert res.json()["exception"] == "ChildNotFoundError" @@ -280,7 +280,7 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st res = client.get( f"/v1/studies/{parent_id}/raw/download", params={"path": "settings/generaldata", "format": "tsv"}, - headers=admin_headers, + headers=user_headers, ) assert res.status_code == 404 assert res.json()["exception"] == "IncorrectPathError" @@ -290,7 +290,7 @@ def test_download_matrices(self, client: TestClient, admin_access_token: str, st res = client.get( f"/v1/studies/{parent_id}/raw/download", params={"path": raw_matrix_path, "format": fake_str}, - headers=admin_headers, + headers=user_headers, ) assert res.status_code == 422 assert res.json()["exception"] == "RequestValidationError" From 56802d47790a7912d625f65275e055c2a3684f8c Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Fri, 9 Feb 2024 01:10:22 +0100 Subject: [PATCH 15/31] refactor(api-download): correct spelling of the month name "July" --- .../raw_studies_blueprint/test_download_matrices.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/raw_studies_blueprint/test_download_matrices.py b/tests/integration/raw_studies_blueprint/test_download_matrices.py index 53cde0a5b2..00bfc4a767 100644 --- a/tests/integration/raw_studies_blueprint/test_download_matrices.py +++ b/tests/integration/raw_studies_blueprint/test_download_matrices.py @@ -55,7 +55,9 @@ def test_download_matrices(self, client: TestClient, user_access_token: str, stu # Change study start_date res = client.put( - f"/v1/studies/{variant_id}/config/general/form", json={"firstMonth": "july"}, headers=user_headers + f"/v1/studies/{variant_id}/config/general/form", + json={"firstMonth": "July"}, + headers=user_headers, ) assert res.status_code == 200 From af99e6ba063dca105b75403353aa102dc2d84f72 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Fri, 9 Feb 2024 01:11:35 +0100 Subject: [PATCH 16/31] refactor(api-download): add missing "res" variable in unit tests --- .../integration/raw_studies_blueprint/test_download_matrices.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/raw_studies_blueprint/test_download_matrices.py b/tests/integration/raw_studies_blueprint/test_download_matrices.py index 00bfc4a767..f8d89f282f 100644 --- a/tests/integration/raw_studies_blueprint/test_download_matrices.py +++ b/tests/integration/raw_studies_blueprint/test_download_matrices.py @@ -62,7 +62,7 @@ def test_download_matrices(self, client: TestClient, user_access_token: str, stu assert res.status_code == 200 # Really generates the snapshot - client.get(f"/v1/studies/{variant_id}/areas", headers=user_headers) + res = client.get(f"/v1/studies/{variant_id}/areas", headers=user_headers) assert res.status_code == 200 # ============================= From 5049503e3defb2f088be8a902debc25fa5f6eb85 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Fri, 9 Feb 2024 01:17:39 +0100 Subject: [PATCH 17/31] refactor(api-download): turn `_SPECIFIC_MATRICES` into a protected variable --- antarest/study/storage/matrix_profile.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/antarest/study/storage/matrix_profile.py b/antarest/study/storage/matrix_profile.py index 24efebfeb2..b2f8bb4ab9 100644 --- a/antarest/study/storage/matrix_profile.py +++ b/antarest/study/storage/matrix_profile.py @@ -49,7 +49,7 @@ def handle_links_columns(self, matrix_path: str) -> t.Sequence[str]: # noinspection SpellCheckingInspection -SPECIFIC_MATRICES = { +_SPECIFIC_MATRICES = { "input/hydro/common/capacity/creditmodulations_*": MatrixProfile( cols=[str(i) for i in range(101)], rows=["Generating Power", "Pumping Power"], @@ -120,8 +120,8 @@ def handle_links_columns(self, matrix_path: str) -> t.Sequence[str]: ), } -SPECIFIC_MATRICES_820 = copy.deepcopy(SPECIFIC_MATRICES) -SPECIFIC_MATRICES_820["input/links/*/*"] = MatrixProfile( +_SPECIFIC_MATRICES_820 = copy.deepcopy(_SPECIFIC_MATRICES) +_SPECIFIC_MATRICES_820["input/links/*/*"] = MatrixProfile( cols=[ "Hurdle costs direct", "Hurdle costs indirect", @@ -134,15 +134,15 @@ def handle_links_columns(self, matrix_path: str) -> t.Sequence[str]: stats=False, ) -SPECIFIC_MATRICES_870 = copy.deepcopy(SPECIFIC_MATRICES_820) +_SPECIFIC_MATRICES_870 = copy.deepcopy(_SPECIFIC_MATRICES_820) # noinspection SpellCheckingInspection -SPECIFIC_MATRICES_870["input/bindingconstraints/*"] = MatrixProfile(cols=[], rows=[], stats=False) +_SPECIFIC_MATRICES_870["input/bindingconstraints/*"] = MatrixProfile(cols=[], rows=[], stats=False) def get_matrix_profiles_by_version(study_version: int) -> t.Dict[str, MatrixProfile]: if study_version < 820: - return SPECIFIC_MATRICES + return _SPECIFIC_MATRICES elif study_version < 870: - return SPECIFIC_MATRICES_820 + return _SPECIFIC_MATRICES_820 else: - return SPECIFIC_MATRICES_870 + return _SPECIFIC_MATRICES_870 From 27af88ba0d22095e15c2d2b84896cf33cfb2d14d Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Fri, 9 Feb 2024 08:46:37 +0100 Subject: [PATCH 18/31] refactor(api-download): simplify and document the `matrix_profile` module --- antarest/study/service.py | 19 ++-- antarest/study/storage/matrix_profile.py | 124 +++++++++++++++-------- 2 files changed, 89 insertions(+), 54 deletions(-) diff --git a/antarest/study/service.py b/antarest/study/service.py index 05c7b24595..910cf62027 100644 --- a/antarest/study/service.py +++ b/antarest/study/service.py @@ -1,6 +1,5 @@ import base64 import contextlib -import fnmatch import io import json import logging @@ -97,7 +96,7 @@ StudySimResultDTO, ) from antarest.study.repository import StudyFilter, StudyMetadataRepository, StudyPagination, StudySortBy -from antarest.study.storage.matrix_profile import get_matrix_profiles_by_version +from antarest.study.storage.matrix_profile import adjust_matrix_columns_index from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfigDTO from antarest.study.storage.rawstudy.model.filesystem.folder_node import ChildNotFoundError from antarest.study.storage.rawstudy.model.filesystem.ini_file_node import IniFileNode @@ -2417,14 +2416,12 @@ def get_matrix_with_index_and_header( ) df_matrix.index = time_column - matrix_profiles = get_matrix_profiles_by_version(int(study.version)) - for pattern, matrix_profile in matrix_profiles.items(): - if fnmatch.fnmatch(path, pattern): - return matrix_profile.handle_specific_matrices( - df_matrix, - path, - with_index=with_index, - with_header=with_header, - ) + adjust_matrix_columns_index( + df_matrix, + path, + with_index=with_index, + with_header=with_header, + study_version=int(study.version), + ) return df_matrix diff --git a/antarest/study/storage/matrix_profile.py b/antarest/study/storage/matrix_profile.py index b2f8bb4ab9..080b5ffe13 100644 --- a/antarest/study/storage/matrix_profile.py +++ b/antarest/study/storage/matrix_profile.py @@ -1,31 +1,42 @@ import calendar import copy +import fnmatch import typing as t from pathlib import Path import pandas as pd -class MatrixProfile(t.NamedTuple): +class _MatrixProfile(t.NamedTuple): """ - Matrix profile for time series or classic tables. + Matrix profile for time series or specific matrices. """ cols: t.Sequence[str] rows: t.Sequence[str] - stats: bool - def handle_specific_matrices( + def process_dataframe( self, df: pd.DataFrame, matrix_path: str, *, with_index: bool, with_header: bool, - ) -> pd.DataFrame: + ) -> None: + """ + Adjust the column names and index of a dataframe according to the matrix profile. + + *NOTE:* The modification is done in place. + + Args: + df: The dataframe to process. + matrix_path: The path of the matrix file, relative to the study directory. + with_index: Whether to set the index of the dataframe. + with_header: Whether to set the column names of the dataframe. + """ if with_header: if Path(matrix_path).parts[1] == "links": - cols = self.handle_links_columns(matrix_path) + cols = self._process_links_columns(matrix_path) else: cols = self.cols if cols: @@ -33,29 +44,36 @@ def handle_specific_matrices( rows = self.rows if with_index and rows: df.index = rows # type: ignore - return df - def handle_links_columns(self, matrix_path: str) -> t.Sequence[str]: + def _process_links_columns(self, matrix_path: str) -> t.Sequence[str]: + """Process column names specific to the links matrices.""" path_parts = Path(matrix_path).parts - area_id_1 = path_parts[2] - area_id_2 = path_parts[3] + area1_id = path_parts[2] + area2_id = path_parts[3] result = list(self.cols) for k, col in enumerate(result): if col == "Hurdle costs direct": - result[k] = f"{col} ({area_id_1}->{area_id_2})" + result[k] = f"{col} ({area1_id}->{area2_id})" elif col == "Hurdle costs indirect": - result[k] = f"{col} ({area_id_2}->{area_id_1})" + result[k] = f"{col} ({area2_id}->{area1_id})" return result +_SPECIFIC_MATRICES: t.Dict[str, _MatrixProfile] +""" +The dictionary ``_SPECIFIC_MATRICES`` maps file patterns to ``_MatrixProfile`` objects, +representing non-time series matrices. +It's used in the `adjust_matrix_columns_index` method to fetch matrix profiles based on study versions. +""" + + # noinspection SpellCheckingInspection _SPECIFIC_MATRICES = { - "input/hydro/common/capacity/creditmodulations_*": MatrixProfile( + "input/hydro/common/capacity/creditmodulations_*": _MatrixProfile( cols=[str(i) for i in range(101)], rows=["Generating Power", "Pumping Power"], - stats=False, ), - "input/hydro/common/capacity/maxpower_*": MatrixProfile( + "input/hydro/common/capacity/maxpower_*": _MatrixProfile( cols=[ "Generating Max Power (MW)", "Generating Max Energy (Hours at Pmax)", @@ -63,48 +81,40 @@ def handle_links_columns(self, matrix_path: str) -> t.Sequence[str]: "Pumping Max Energy (Hours at Pmax)", ], rows=[], - stats=False, ), - "input/hydro/common/capacity/reservoir_*": MatrixProfile( + "input/hydro/common/capacity/reservoir_*": _MatrixProfile( cols=["Lev Low (p.u)", "Lev Avg (p.u)", "Lev High (p.u)"], rows=[], - stats=False, ), - "input/hydro/common/capacity/waterValues_*": MatrixProfile( + "input/hydro/common/capacity/waterValues_*": _MatrixProfile( cols=[f"{i}%" for i in range(101)], rows=[], - stats=False, ), - "input/hydro/series/*/mod": MatrixProfile(cols=[], rows=[], stats=True), - "input/hydro/series/*/ror": MatrixProfile(cols=[], rows=[], stats=True), - "input/hydro/common/capacity/inflowPattern_*": MatrixProfile(cols=["Inflow Pattern (X)"], rows=[], stats=False), - "input/hydro/prepro/*/energy": MatrixProfile( + "input/hydro/series/*/mod": _MatrixProfile(cols=[], rows=[]), + "input/hydro/series/*/ror": _MatrixProfile(cols=[], rows=[]), + "input/hydro/common/capacity/inflowPattern_*": _MatrixProfile(cols=["Inflow Pattern (X)"], rows=[]), + "input/hydro/prepro/*/energy": _MatrixProfile( cols=["Expectation (MWh)", "Std Deviation (MWh)", "Min. (MWh)", "Max. (MWh)", "ROR Share"], rows=calendar.month_name[1:], - stats=False, ), - "input/thermal/prepro/*/*/modulation": MatrixProfile( + "input/thermal/prepro/*/*/modulation": _MatrixProfile( cols=["Marginal cost modulation", "Market bid modulation", "Capacity modulation", "Min gen modulation"], rows=[], - stats=False, ), - "input/thermal/prepro/*/*/data": MatrixProfile( + "input/thermal/prepro/*/*/data": _MatrixProfile( cols=["FO Duration", "PO Duration", "FO Rate", "PO Rate", "NPO Min", "NPO Max"], rows=[], - stats=False, ), - "input/reserves/*": MatrixProfile( + "input/reserves/*": _MatrixProfile( cols=["Primary Res. (draft)", "Strategic Res. (draft)", "DSM", "Day Ahead"], rows=[], - stats=False, ), - "input/misc-gen/miscgen-*": MatrixProfile( + "input/misc-gen/miscgen-*": _MatrixProfile( cols=["CHP", "Bio Mass", "Bio Gaz", "Waste", "GeoThermal", "Other", "PSP", "ROW Balance"], rows=[], - stats=False, ), - "input/bindingconstraints/*": MatrixProfile(cols=["<", ">", "="], rows=[], stats=False), - "input/links/*/*": MatrixProfile( + "input/bindingconstraints/*": _MatrixProfile(cols=["<", ">", "="], rows=[]), + "input/links/*/*": _MatrixProfile( cols=[ "Capacités de transmission directes", "Capacités de transmission indirectes", @@ -116,12 +126,11 @@ def handle_links_columns(self, matrix_path: str) -> t.Sequence[str]: "P.Shift Max", ], rows=[], - stats=False, ), } _SPECIFIC_MATRICES_820 = copy.deepcopy(_SPECIFIC_MATRICES) -_SPECIFIC_MATRICES_820["input/links/*/*"] = MatrixProfile( +_SPECIFIC_MATRICES_820["input/links/*/*"] = _MatrixProfile( cols=[ "Hurdle costs direct", "Hurdle costs indirect", @@ -131,18 +140,47 @@ def handle_links_columns(self, matrix_path: str) -> t.Sequence[str]: "P.Shift Max", ], rows=[], - stats=False, ) _SPECIFIC_MATRICES_870 = copy.deepcopy(_SPECIFIC_MATRICES_820) # noinspection SpellCheckingInspection -_SPECIFIC_MATRICES_870["input/bindingconstraints/*"] = MatrixProfile(cols=[], rows=[], stats=False) +_SPECIFIC_MATRICES_870["input/bindingconstraints/*"] = _MatrixProfile(cols=[], rows=[]) + +def adjust_matrix_columns_index( + df: pd.DataFrame, matrix_path: str, with_index: bool, with_header: bool, study_version: int +) -> None: + """ + Adjust the column names and index of a dataframe according to the matrix profile. + + *NOTE:* The modification is done in place. -def get_matrix_profiles_by_version(study_version: int) -> t.Dict[str, MatrixProfile]: + Args: + df: The dataframe to process. + matrix_path: The path of the matrix file, relative to the study directory. + with_index: Whether to set the index of the dataframe. + with_header: Whether to set the column names of the dataframe. + study_version: The version of the study. + """ + # Get the matrix profiles for a given study version if study_version < 820: - return _SPECIFIC_MATRICES + matrix_profiles = _SPECIFIC_MATRICES elif study_version < 870: - return _SPECIFIC_MATRICES_820 + matrix_profiles = _SPECIFIC_MATRICES_820 else: - return _SPECIFIC_MATRICES_870 + matrix_profiles = _SPECIFIC_MATRICES_870 + + # Apply the matrix profile to the dataframe to adjust the column names and index + for pattern, matrix_profile in matrix_profiles.items(): + if fnmatch.fnmatch(matrix_path, pattern): + matrix_profile.process_dataframe( + df, + matrix_path, + with_index=with_index, + with_header=with_header, + ) + return + + # The matrix may be a time series, in which case we don't need to adjust anything + # (the "Time" columns is already the index) + return None From 511df6986b2003bee5a841490f7d6537d889b299 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Mon, 12 Feb 2024 11:49:08 +0100 Subject: [PATCH 19/31] refactor(api-download): add missing header to classic times series - Ensure time series have "TS-NNN" headers. - Output headers are not changed. --- antarest/study/storage/matrix_profile.py | 33 ++- .../test_download_matrices.py | 220 +++++++++++++----- 2 files changed, 190 insertions(+), 63 deletions(-) diff --git a/antarest/study/storage/matrix_profile.py b/antarest/study/storage/matrix_profile.py index 080b5ffe13..7dc137dc10 100644 --- a/antarest/study/storage/matrix_profile.py +++ b/antarest/study/storage/matrix_profile.py @@ -41,9 +41,11 @@ def process_dataframe( cols = self.cols if cols: df.columns = pd.Index(cols) - rows = self.rows - if with_index and rows: - df.index = rows # type: ignore + else: + df.columns = pd.Index([f"TS-{i}" for i in range(1, len(df.columns) + 1)]) + + if with_index and self.rows: + df.index = pd.Index(self.rows) def _process_links_columns(self, matrix_path: str) -> t.Sequence[str]: """Process column names specific to the links matrices.""" @@ -83,6 +85,7 @@ def _process_links_columns(self, matrix_path: str) -> t.Sequence[str]: rows=[], ), "input/hydro/common/capacity/reservoir_*": _MatrixProfile( + # Values are displayed in % in the UI, but the actual values are in p.u. (per unit) cols=["Lev Low (p.u)", "Lev Avg (p.u)", "Lev High (p.u)"], rows=[], ), @@ -130,6 +133,8 @@ def _process_links_columns(self, matrix_path: str) -> t.Sequence[str]: } _SPECIFIC_MATRICES_820 = copy.deepcopy(_SPECIFIC_MATRICES) +"""Specific matrices for study version 8.2.""" + _SPECIFIC_MATRICES_820["input/links/*/*"] = _MatrixProfile( cols=[ "Hurdle costs direct", @@ -142,8 +147,19 @@ def _process_links_columns(self, matrix_path: str) -> t.Sequence[str]: rows=[], ) +# Specific matrices for study version 8.6 +_SPECIFIC_MATRICES_860 = copy.deepcopy(_SPECIFIC_MATRICES_820) +"""Specific matrices for study version 8.6.""" + +# noinspection SpellCheckingInspection +# +_SPECIFIC_MATRICES_860["input/hydro/series/*/mingen"] = _MatrixProfile(cols=[], rows=[]) + _SPECIFIC_MATRICES_870 = copy.deepcopy(_SPECIFIC_MATRICES_820) +"""Specific matrices for study version 8.7.""" + # noinspection SpellCheckingInspection +# Scenarized RHS for binding constraints _SPECIFIC_MATRICES_870["input/bindingconstraints/*"] = _MatrixProfile(cols=[], rows=[]) @@ -165,8 +181,10 @@ def adjust_matrix_columns_index( # Get the matrix profiles for a given study version if study_version < 820: matrix_profiles = _SPECIFIC_MATRICES - elif study_version < 870: + elif study_version < 860: matrix_profiles = _SPECIFIC_MATRICES_820 + elif study_version < 870: + matrix_profiles = _SPECIFIC_MATRICES_860 else: matrix_profiles = _SPECIFIC_MATRICES_870 @@ -181,6 +199,13 @@ def adjust_matrix_columns_index( ) return + if fnmatch.fnmatch(matrix_path, "output/*"): + # Outputs already have their own column names + return + # The matrix may be a time series, in which case we don't need to adjust anything # (the "Time" columns is already the index) + # Column names should be Monte-Carlo years: "TS-1", "TS-2", ... + df.columns = pd.Index([f"TS-{i}" for i in range(1, len(df.columns) + 1)]) + return None diff --git a/tests/integration/raw_studies_blueprint/test_download_matrices.py b/tests/integration/raw_studies_blueprint/test_download_matrices.py index f8d89f282f..ca2c501374 100644 --- a/tests/integration/raw_studies_blueprint/test_download_matrices.py +++ b/tests/integration/raw_studies_blueprint/test_download_matrices.py @@ -1,4 +1,6 @@ +import datetime import io +import typing as t import numpy as np import pandas as pd @@ -9,6 +11,96 @@ from tests.integration.utils import wait_task_completion +class Proxy: + def __init__(self, client: TestClient, user_access_token: str): + self.client = client + self.user_access_token = user_access_token + self.headers = {"Authorization": f"Bearer {user_access_token}"} + + +class PreparerProxy(Proxy): + def copy_upgrade_study(self, ref_study_id, target_version=820): + """ + Copy a study in the managed workspace and upgrade it to a specific version + """ + # Prepare a managed study to test specific matrices for version 8.2 + res = self.client.post( + f"/v1/studies/{ref_study_id}/copy", + params={"dest": "copied-820", "use_task": False}, + headers=self.headers, + ) + res.raise_for_status() + study_820_id = res.json() + + res = self.client.put( + f"/v1/studies/{study_820_id}/upgrade", + params={"target_version": target_version}, + headers=self.headers, + ) + res.raise_for_status() + task_id = res.json() + assert task_id + + task = wait_task_completion(self.client, self.user_access_token, task_id, timeout=20) + assert task.status == TaskStatus.COMPLETED + return study_820_id + + def upload_matrix(self, study_id: str, matrix_path: str, df: pd.DataFrame) -> None: + tsv = io.BytesIO() + df.to_csv(tsv, sep="\t", index=False, header=False) + tsv.seek(0) + # noinspection SpellCheckingInspection + res = self.client.put( + f"/v1/studies/{study_id}/raw", + params={"path": matrix_path, "create_missing": True}, + headers=self.headers, + files={"file": tsv, "create_missing": "true"}, + ) + res.raise_for_status() + + def create_variant(self, parent_id: str, *, name: str) -> str: + res = self.client.post( + f"/v1/studies/{parent_id}/variants", + headers=self.headers, + params={"name": name}, + ) + res.raise_for_status() + variant_id = res.json() + return variant_id + + def generate_snapshot(self, variant_id: str, denormalize=False, from_scratch=True) -> None: + # Generate a snapshot for the variant + res = self.client.put( + f"/v1/studies/{variant_id}/generate", + headers=self.headers, + params={"denormalize": denormalize, "from_scratch": from_scratch}, + ) + res.raise_for_status() + task_id = res.json() + assert task_id + + task = wait_task_completion(self.client, self.user_access_token, task_id, timeout=20) + assert task.status == TaskStatus.COMPLETED + + def create_area(self, parent_id, *, name: str, country: str = "FR") -> str: + res = self.client.post( + f"/v1/studies/{parent_id}/areas", + headers=self.headers, + json={"name": name, "type": "AREA", "metadata": {"country": country}}, + ) + res.raise_for_status() + area_id = res.json()["id"] + return area_id + + def update_general_data(self, study_id: str, **data: t.Any): + res = self.client.put( + f"/v1/studies/{study_id}/config/general/form", + json=data, + headers=self.headers, + ) + res.raise_for_status() + + @pytest.mark.integration_test class TestDownloadMatrices: """ @@ -18,61 +110,47 @@ class TestDownloadMatrices: def test_download_matrices(self, client: TestClient, user_access_token: str, study_id: str) -> None: user_headers = {"Authorization": f"Bearer {user_access_token}"} - # ============================= + # ===================== # STUDIES PREPARATION - # ============================= + # ===================== - # Manage parent study and upgrades it to v8.2 - # This is done to test matrix headers according to different versions - copied = client.post( - f"/v1/studies/{study_id}/copy", params={"dest": "copied", "use_task": False}, headers=user_headers - ) - parent_id = copied.json() - res = client.put(f"/v1/studies/{parent_id}/upgrade", params={"target_version": 820}, headers=user_headers) - assert res.status_code == 200 - task_id = res.json() - assert task_id - task = wait_task_completion(client, user_access_token, task_id, timeout=20) - assert task.status == TaskStatus.COMPLETED + preparer = PreparerProxy(client, user_access_token) + + study_820_id = preparer.copy_upgrade_study(study_id, target_version=820) # Create Variant - res = client.post( - f"/v1/studies/{parent_id}/variants", - headers=user_headers, - params={"name": "variant_1"}, - ) - assert res.status_code == 200 - variant_id = res.json() + variant_id = preparer.create_variant(study_820_id, name="New Variant") # Create a new area to implicitly create normalized matrices - area_name = "new_area" - res = client.post( - f"/v1/studies/{variant_id}/areas", - headers=user_headers, - json={"name": area_name, "type": "AREA", "metadata": {"country": "FR"}}, - ) - assert res.status_code in {200, 201} + area_id = preparer.create_area(variant_id, name="Mayenne", country="France") # Change study start_date - res = client.put( - f"/v1/studies/{variant_id}/config/general/form", - json={"firstMonth": "July"}, - headers=user_headers, - ) - assert res.status_code == 200 + preparer.update_general_data(variant_id, firstMonth="July") # Really generates the snapshot - res = client.get(f"/v1/studies/{variant_id}/areas", headers=user_headers) - assert res.status_code == 200 + preparer.generate_snapshot(variant_id) - # ============================= + # Prepare a managed study to test specific matrices for version 8.6 + study_860_id = preparer.copy_upgrade_study(study_id, target_version=860) + + # Import a Min Gen. matrix: shape=(8760, 3), with random integers between 0 and 1000 + min_gen_df = pd.DataFrame(np.random.randint(0, 1000, size=(8760, 3))) + preparer.upload_matrix(study_860_id, "input/hydro/series/de/mingen", min_gen_df) + + # ============================================= # TESTS NOMINAL CASE ON RAW AND VARIANT STUDY - # ============================= + # ============================================= raw_matrix_path = r"input/load/series/load_de" - variant_matrix_path = f"input/load/series/load_{area_name}" + variant_matrix_path = f"input/load/series/load_{area_id}" + + raw_start_date = datetime.datetime(2018, 1, 1) + variant_start_date = datetime.datetime(2019, 7, 1) - for uuid, path in [(parent_id, raw_matrix_path), (variant_id, variant_matrix_path)]: + for uuid, path, start_date in [ + (study_820_id, raw_matrix_path, raw_start_date), + (variant_id, variant_matrix_path, variant_start_date), + ]: # Export the matrix in xlsx format (which is the default format) # and retrieve it as binary content (a ZIP-like file). res = client.get( @@ -89,26 +167,35 @@ def test_download_matrices(self, client: TestClient, user_access_token: str, stu dataframe = pd.read_excel(io.BytesIO(res.content), index_col=0) # check time coherence - generated_index = dataframe.index + actual_index = dataframe.index # noinspection PyUnresolvedReferences - first_date = generated_index[0].to_pydatetime() + first_date = actual_index[0].to_pydatetime() # noinspection PyUnresolvedReferences - second_date = generated_index[1].to_pydatetime() - assert first_date.month == second_date.month == 1 if uuid == parent_id else 7 + second_date = actual_index[1].to_pydatetime() + first_month = 1 if uuid == study_820_id else 7 # July + assert first_date.month == second_date.month == first_month assert first_date.day == second_date.day == 1 assert first_date.hour == 0 assert second_date.hour == 1 - # reformat into a json to help comparison - new_cols = [int(col) for col in dataframe.columns] - dataframe.columns = new_cols - dataframe.index = range(len(dataframe)) - actual_matrix = dataframe.to_dict(orient="split") - # asserts that the result is the same as the one we get with the classic get /raw endpoint - res = client.get(f"/v1/studies/{uuid}/raw", params={"path": path, "formatted": True}, headers=user_headers) + res = client.get( + f"/v1/studies/{uuid}/raw", + params={"path": path, "formatted": True}, + headers=user_headers, + ) expected_matrix = res.json() - assert actual_matrix == expected_matrix + expected_matrix["columns"] = [f"TS-{n + 1}" for n in expected_matrix["columns"]] + time_column = pd.date_range( + start=start_date, + periods=len(expected_matrix["data"]), + freq="H", + ) + expected_matrix["index"] = time_column + expected = pd.DataFrame(**expected_matrix) + assert dataframe.index.tolist() == expected.index.tolist() + assert dataframe.columns.tolist() == expected.columns.tolist() + assert (dataframe == expected).all().all() # ============================= # TESTS INDEX AND HEADER PARAMETERS @@ -119,7 +206,7 @@ def test_download_matrices(self, client: TestClient, user_access_token: str, stu for header in [True, False]: index = not header res = client.get( - f"/v1/studies/{parent_id}/raw/download", + f"/v1/studies/{study_820_id}/raw/download", params={"path": raw_matrix_path, "format": "TSV", "header": header, "index": index}, headers=user_headers, ) @@ -160,7 +247,7 @@ def test_download_matrices(self, client: TestClient, user_access_token: str, stu # tests links headers after v8.2 res = client.get( - f"/v1/studies/{parent_id}/raw/download", + f"/v1/studies/{study_820_id}/raw/download", params={"path": "input/links/de/fr_parameters", "format": "tsv"}, headers=user_headers, ) @@ -179,7 +266,7 @@ def test_download_matrices(self, client: TestClient, user_access_token: str, stu # allocation and correlation matrices for path in ["input/hydro/allocation", "input/hydro/correlation"]: res = client.get( - f"/v1/studies/{parent_id}/raw/download", params={"path": path, "format": "tsv"}, headers=user_headers + f"/v1/studies/{study_820_id}/raw/download", params={"path": path, "format": "tsv"}, headers=user_headers ) assert res.status_code == 200 content = io.BytesIO(res.content) @@ -200,7 +287,7 @@ def test_download_matrices(self, client: TestClient, user_access_token: str, stu # modulation matrix res = client.get( - f"/v1/studies/{parent_id}/raw/download", + f"/v1/studies/{study_820_id}/raw/download", params={"path": "input/thermal/prepro/de/01_solar/modulation", "format": "tsv"}, headers=user_headers, ) @@ -254,6 +341,21 @@ def test_download_matrices(self, client: TestClient, user_access_token: str, stu dataframe = pd.read_csv(content, index_col=0, sep="\t") assert dataframe.empty + # test the Min Gen of the 8.6 study + res = client.get( + f"/v1/studies/{study_860_id}/raw/download", + params={"path": "input/hydro/series/de/mingen", "format": "tsv"}, + headers=user_headers, + ) + assert res.status_code == 200 + content = io.BytesIO(res.content) + dataframe = pd.read_csv(content, index_col=0, sep="\t") + assert dataframe.shape == (8760, 3) + assert dataframe.columns.tolist() == ["TS-1", "TS-2", "TS-3"] + assert dataframe.index[0] == "2018-01-01 00:00:00" + # noinspection PyUnresolvedReferences + assert (dataframe.values == min_gen_df.values).all() + # ============================= # ERRORS # ============================= @@ -271,7 +373,7 @@ def test_download_matrices(self, client: TestClient, user_access_token: str, stu # fake path res = client.get( - f"/v1/studies/{parent_id}/raw/download", + f"/v1/studies/{study_820_id}/raw/download", params={"path": f"input/links/de/{fake_str}", "format": "tsv"}, headers=user_headers, ) @@ -280,7 +382,7 @@ def test_download_matrices(self, client: TestClient, user_access_token: str, stu # path that does not lead to a matrix res = client.get( - f"/v1/studies/{parent_id}/raw/download", + f"/v1/studies/{study_820_id}/raw/download", params={"path": "settings/generaldata", "format": "tsv"}, headers=user_headers, ) @@ -290,7 +392,7 @@ def test_download_matrices(self, client: TestClient, user_access_token: str, stu # wrong format res = client.get( - f"/v1/studies/{parent_id}/raw/download", + f"/v1/studies/{study_820_id}/raw/download", params={"path": raw_matrix_path, "format": fake_str}, headers=user_headers, ) From 4c60fa20ab6f1a54574bf8a2b899b31f94fad0b1 Mon Sep 17 00:00:00 2001 From: Samir Kamal <1954121+skamril@users.noreply.github.com> Date: Tue, 13 Feb 2024 17:56:41 +0100 Subject: [PATCH 20/31] ci: add commitlint GitHub action (#1933) lints Pull Request commits with commitlint --- .github/workflows/commitlint.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/workflows/commitlint.yml diff --git a/.github/workflows/commitlint.yml b/.github/workflows/commitlint.yml new file mode 100644 index 0000000000..8e08ce865c --- /dev/null +++ b/.github/workflows/commitlint.yml @@ -0,0 +1,13 @@ +name: Lint Commit Messages +on: [pull_request] + +permissions: + contents: read + pull-requests: read + +jobs: + commitlint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: wagoid/commitlint-github-action@v5 From bd76b9acd326d59c1836516ef011b8c61388da24 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Mon, 12 Feb 2024 11:51:36 +0100 Subject: [PATCH 21/31] feat(hydro): add the "Min Gen." tab for hydraulic generators --- .../explore/Modelization/Areas/Hydro/index.tsx | 4 +++- .../explore/Modelization/Areas/Hydro/utils.ts | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/Hydro/index.tsx b/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/Hydro/index.tsx index 289913bcbe..ffa2df3620 100644 --- a/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/Hydro/index.tsx +++ b/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/Hydro/index.tsx @@ -11,6 +11,7 @@ import { getCurrentAreaId } from "../../../../../../../redux/selectors"; function Hydro() { const { study } = useOutletContext<{ study: StudyMetadata }>(); const areaId = useAppSelector(getCurrentAreaId); + const studyVersion = parseInt(study.version, 10); const tabList = useMemo(() => { const basePath = `/studies/${study?.id}/explore/modelization/area/${encodeURI( @@ -30,7 +31,8 @@ function Hydro() { { label: "Water values", path: `${basePath}/watervalues` }, { label: "Hydro Storage", path: `${basePath}/hydrostorage` }, { label: "Run of river", path: `${basePath}/ror` }, - ]; + studyVersion >= 860 && { label: "Min Gen.", path: `${basePath}/mingen` }, + ].filter(Boolean); }, [areaId, study?.id]); //////////////////////////////////////////////////////////////// diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/Hydro/utils.ts b/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/Hydro/utils.ts index 8fc143d280..2a75d23f9d 100644 --- a/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/Hydro/utils.ts +++ b/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/Hydro/utils.ts @@ -15,6 +15,7 @@ export enum HydroMatrixType { WaterValues, HydroStorage, RunOfRiver, + MinGen, InflowPattern, OverallMonthlyHydro, Allocation, @@ -99,6 +100,10 @@ export const HYDRO_ROUTES: HydroRoute[] = [ path: "ror", type: HydroMatrixType.RunOfRiver, }, + { + path: "mingen", + type: HydroMatrixType.MinGen, + }, ]; export const MATRICES: Matrices = { @@ -144,6 +149,11 @@ export const MATRICES: Matrices = { url: "input/hydro/series/{areaId}/ror", stats: MatrixStats.STATS, }, + [HydroMatrixType.MinGen]: { + title: "Min Gen.", + url: "input/hydro/series/{areaId}/mingen", + stats: MatrixStats.STATS, + }, [HydroMatrixType.InflowPattern]: { title: "Inflow Pattern", url: "input/hydro/common/capacity/inflowPattern_{areaId}", From 2889240620f45b6bfc82f3b39646343127bf968a Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Tue, 13 Feb 2024 13:57:06 +0100 Subject: [PATCH 22/31] docs(hydro): add a screenshot of the "Min Gen." tab in the documentation --- .../areas/05-hydro.min-generation.series.png | Bin 0 -> 66421 bytes docs/user-guide/study/areas/05-hydro.md | 6 ++++++ 2 files changed, 6 insertions(+) create mode 100644 docs/assets/media/user-guide/study/areas/05-hydro.min-generation.series.png diff --git a/docs/assets/media/user-guide/study/areas/05-hydro.min-generation.series.png b/docs/assets/media/user-guide/study/areas/05-hydro.min-generation.series.png new file mode 100644 index 0000000000000000000000000000000000000000..23372726ab0fa8d10a328ee0897a82a7a99eea8c GIT binary patch literal 66421 zcmd?QQ*>v|*Y6vrlaB4Ala6iMwr$(C)v;~cwr%4V+s5hVy!(CL|GC+BXW#6LwZ@n= zR@JOkb5yOW@BFNAIT=w{Xe?+TARt(AF(CyYATR?UAmFec5PwUc7ezLKfRHdfl+>IQ z^j!(;9BfU@t&9nr-0h4BjNQylfPmcAi_+BN4cQ%pzw4l>ff;uv`pk-PAMTuKMn}sl zlQou>`=sLeyZiPAa1pLP06jJDv_1~yTZi)=O&Z+1YxP5Y8CSf#0B*>fo%{E)+t1=1 z-}lYOFK=vj>CWOA^e!J-mlrMvuMeMYo*Mld_t%Bs3$LEk9$)%y^oUQ-!0Y2L&ps3$ z)|+?FyxT6hh#gK}kmq~PV~eg&cc`8F#*Z6s@9(T^2aCF7ER7>}(H#xuZP)p^(G8Vc zdvDnfotv7TjvPTE`k@4YyX!XJe5tpb1-F{O87QrpPTC#Qcd8Y627%rFr>RTVvhgF^ zwpaI0UZ2e!_2m{^AIj$%*Bur+UR{)T<)oeU-0W+_Q`)799-W=zs~X*)yAYN|FL^v8z5-G>;^lJwmy6?djD^zD{sI=n1< z^@r7`30XW!wXnpO5F8hby>`3p3{*7wBR2y8^Dyedd!^#fNXoJk^;kPY$z+!^ront= zq?}>TE!O0lE6ZyiN~fz%(9q%DA&`=f9{6WhxHr{UPiDzqorrl4dy2cI^uJ-0WoB?h zg!?~9nmWP0xqNI#ZS|3fsc5INz~Oj#Q&NdP<}^=HIqbFe9wQ~JeWzD~8J=mtkdcUQ zm7ODR=#0GSN-ZReJ)mP`Bo zyd)W(p)8N}(~;@chjnSz9fx!47Tpej>QcAzXURy9eA7*phF8+O{iW*COJmbah399# zuo5E$nq%(1G-_^MfyI&MD0{g>d0NY;Qli)&;ik2|SUE!%@*IQ-*dB{#(ZjE*_O|qI zYrIzm^3(dwZogp5@(#*HPhb=YJsW$bfwSbOFYS*5pQ}dNE zLbe=#%*$yn**v{)Y$0D!V*#oxRy#Mk-}4odu;i8W&QEF0&B5R;5~ zxn(%1P25kCX`XBBCJ?n&p< zgRUowc+DF~_WCCB^WrX*RO(r#<{#WS?y{|hzN&OFfUqpITLm+KRKU8a|8O{ivQ1Ir ziXJ!3>PQ5n*3o97#mk)7Z^r>K7*Tk$!!}`oR=p?g!WhPL`KOh#t`aZJ*f?E&7PqV>sQ?{)$0s&Hc`ZG zOzO6#8k%}+3@3916w+eM#@Me6AVD2Ayxh~BaEAOl+dtu-lp9EAyKtV#OhqI_)7EuD zGWqSByWZ1*dCph|$JMU1#WirgLpu4V zhvK2aVU|?aiCnnnmoG<^ZHZjd*K@MNmhI*_^l|_X!+pAOhSOW{!BPhg8nfg(i&3JN zEnQO%GUt|}rRS}x5pJ0xZW{P!3e;O&z$`Cyi+@;+F*;CZYK+m58%-ke!}M3n-(#yr z+89H;IiSmc2io8n{ZS)T*W316;gPrQEv#B0NYM(;F@$Gs=kYh29$81LG8~=NK(hwV zrgL9gc=Nux;-~b1_OtuYD69`i%jJj2K#!I;A%&byB_=tTZZ$V-gn^8;l8}qMk>dre zP1TcCBj`0a>oqBu$xZNgBP!x6YD-bTCMXmMPbGK_T;{R28<{NYw!CBnwFpcTk4qBa z-M~vT_qU5NO66SOv1?Wqz5?E&TLoilbq6LFADct;UdeZO)UwW0C!l;Qf$r?XP?ZH@ zBYP_6o99>0lrnkECOOcl9cj;vFVaHlqK9eRL#NRKq6jj8Z&7z~J*yMHTiq3_h74Tt z&J0v}j*G;W?n5jjIjHkO8@EV}*&~(E%FWgt6CM^_6(CEg`MDMl;3*Rc9aN`7dY3-V zOj)Xc+bB2TG=Hf%v57hCnA@vY{tP@rSSS|k^jy{9%FB9V?wF&6zLnM`eNL{o4|JUS zGu+x-%FUWSV-mRS42EZ}8LHP8pod@@YdEF3QJLKfLbv#q#<^x}!b;H3TLJefq8%2_ z(#J^LSF%qY#E3dw&mySx=z+yVMFPLw-X2!VqZ6xm3rM1~)^LS+}Zln5dOTr&Dp7JqG}lAPY? z&&S<%`dT`4%Ia|W8P#1dY88@TAVRl#GFU7Rsih)kBhGZ7)}-^iB=a-v<8oWGQM~LS1x`eYCZG&hK|oXd&PNJMIP#BC5FE=h#=A7=@TElp zH6UCFM^#epX9D7xvUSlXeSg+~7jr|w7Hy>A(eHj&po;Moz*Q5ATr^G`$Zs=~#U(E* zD9hTvdq}}7^$}m`+u#$15{3CETl2dzSvjz0GYgRR!k~=inzYsy2cHmz(lcOIZPU#l zvmRmC^qM~@a0pl%g67@OUlogDhtJgLI*{VJwfP`RRQeTTBMHOS>dZ#?*saI^T7qS# zgGdwo5}={1kdXun`@tvThxr>8#}KCUNr=R+969gTQD6}+282m=J!4%A!2btZhI3$N zsd&60=v;GQ&@t|55(#j#7zQC8x!ebCiJ*y<8&$~sFTUkbWhU<*SwZcZrOONV#qU6u z*hM%#WI-;%qi^fA9X~145lQl4*Gpr<5jGDc>iLWmxqA$L>9q=WH4hE}2qq#cS=~i4 z{(!n!gU}$x{cOU?i>9Y{`XLAwGMl#x(g0jUqeT6~CI5+vjL4lW)lA*t2Dq;}1=xyl zK7$@M7x-KWq6tbk4}n-uWapHIQ6zwH!=GlN*J~JP&Jl%5z%l^y5f|HOeFCBkSSU@r z@z3!aE~J5v!@CXx>{GLw9R-yyZavUq|DS8ZQ37sC=GOp*hlz2!ETc!@9ISm-z8noc zXwxOqlA|Dw^YAM=CD#JX5@8`rofoW+Rq$PTCZayJzH9JLLX%@MM<*}Dgt;I(b0E5P z8(<66p~$eCy;LZA$U;c(#CKs25i04<+N)w~ziDqX(GM-bJy645W`fQb&=|Qn?9ch# zfm%z`K{}xtkn6RiYb9M5G{xtyL0u8>8hQg zT$oo#8FN4a6Gm5(ZX7d5sp>>BhnAzcpaPLFb5D8}p5evGu@cz_b*G<|SuaJ$rF_Bwa%j~<}7Rh~+7!f;Tp3D!MVyBG|bR!%JwP|7Gt<;cGi zTkq^DovHX~7_nsbR{_@{Xx#Q}B`Glo3ZY(n)1I_?TXQdHu@|JCaP9Wh>WCzK_IdY2m&AI?<29{1mf1>EUR4_ksJT00kUp@lE2j z#3Cf)gJC%i)L=ZO;y!|#i!l2Gph3FqA!@F<&>2Jw^Vg&EPw23Ea=k!QNS0b=&QXC} zc~p0IVUSqrF&Xq8f!e)mgmcqII&Rd?iLhJEQV$}nmZGih`qAvT2_SaAN`$$Dz%YIi z?fqD#G=N4JsJ*nu7{#L9v|M0>`mk_P`Lon~n&IUmL7$9M%)3vSgc8~G$ZjyHd z-a;J57Ka#-7{1C-I6tV+L)|p^HVW#;)%x*9Nmy6p=!a%aVYP@bD3jn%SqcKwk;Z>t zx+55a74{qR$E`n;6kY&cjM*-+eK zl21_LTIIMHNNsuKr`}92=U2P*6c)aYg<-Z)Wt{AL<4S_$rqiF>5Ll9KdC~TSRc@if z1b6#!06%Sl0g<-){`Hx=z-ppMo< z&CcrJwnd=m_5-;aQ5Cspo$q^WOhjY3_m9%UR^*x;g0sxKv_{_ireJcF;bXY@v94A~ zpkn$t%RJ3Tw32P$on%g11xt-aSRkN|n<^S#zK%<=veS1E2NVQku`d&r_nc#Fx*uMf zSm~6~gJefG5Ee#=X0xstUXWg5h^;NBTs4v2^>F>i^SHOhRs<|&qrpRF2>MiY=K!5&|aG_o-RR7%^4O}ZFg z8UQ?iffjCafYPt+;oE9!cx;;&2>p_c7vdKq${jsEw@v(8Z~dkYw@?9FKc8va7EYcF z$bYf3P>}@;z{OBiIxqCmNW-y7m&a5E0hFA#=idn)FL9p_-R>&5Y8fimsNSX@Loll- z*jxeY6pmhrc8+8`=Mdl*%Ln9|hjqcB&|f8z*5pkRG&ouie9X9c9TMg;NzU?qHp3+L zs$!Pq?(MhPyH-8~Y=TO^zKw5t292uQ{8kM~#-9N~umrM?6UcOv#d6a((vJ~A{cypE#1H|uRCT<50#=i9kOj}3^Z2cc>e1XrB{qm|n)Qeyt6!4>kmktFJro)0-@WF<_ za?a?$U%VGsh=sKYzVUw*qd;>(K{;_j!T%PC{z^lc-f`Sw1AI6`x{5=D)Nn2c_M>uH zR4>777N`+&zZRjZI8wFVz@@R`QP4&Ens#?b43_$a)Kwu>`4K*V>|N{>oZ^taQ&2Pp z-Rw4FU7zt>uYvO9_0yM_Ab$$^9&)OXY5856Qe zV~`?;;8ZwLlcb{EBG`&8I1YcR$0|>2W;RJgH(ibCVoaY?PH2^|2mMB?1A`(&S(Fn2 zXW4H&D^wF(@aZ4nJ$WN`AU~i8j57pQHv6N^`-1;*wM-!;<0ifbHv75HhCjeJLo-{+ zIsIG*V3PIQWwa}5`CNjayBGK9sv3&%>$cwAZ?7Wv>^+2|yD=IqzL4Ng7svzZ%$Z|7 z^!w7xb)7n|YKmQeBoNdtvhfQP)GHKQJLkEEbAEcdPE+ya6`^%V8V^&-_$%E(7)Xi= z0TBWb0WprG=`{W=fwB`*cLV~WA^PV6b}Qm_`CAC-BrYusc>)3p!vZXBw(I=22-8Ve z%}LPK+S=I02}sbvSl`Lmh``m{$&5f$Tv|@m9}XP|hyX}jh+oNV{bI}2OG$aX_jV^s zoHQB;DhNSHNsnf;jvN~z0sj|Hqe50JKosuFqpG?>T3PFN`y6}11 zb7pe$ITlPzV0@PFmmd>!M$T#x8w6SoxXd(uKU1}f^1P{ZrVR=J$-e+_65g0B+~PqtJJ52h#Qh77N& zica5jR^Q+VsdoHz(to_cmr}88&SYy{!SuPSz(D?L8{N_7BOk-T@zk7x)ADzgWxmkC|14mPh!R3}TN58Fe}bU9FC;(sPA z*}u8)7BoE@jc1lgVO)hIR6(_m6Dr6C)jzf%>`OczHL3y;qtjvr9x!^*b-RwES95*|7s*%Y9;8^{biZzf==SnLjcZqd z-2H;T>OC6qz5DutR*Ew7jIj514Zei%}OwF?G8HydF zK9j36`4F6F^>hRMh2JyQQi3%dje4m)ftugo(>CFRckQCCOM9p821<6s2B|f&Sw?h& zJBuuNMm8oxXs{yFEPNnW;es%NWl26IPU18n!h%aJu<~}mLB6qoZ%=GCj8c8;I?|>J zFSVp0;m8;s$1P;?bt(4WO)zSSx5gaH96-&dk{VHUI!SGT~jP?ew|)+#;-HL z%)!SWrLN%(Tk&^wH9f$q)Dctl;4h?*fF#}Jajbi1oGA$ ze4IE*D$+-cehuc(KskCWN&5Hz1Ift599Jf@e{!N{M7@4Khs_P$4P^8nRW0bvZDgFX zP96G(?(Wps{;D!JBl5kb;MvmjvoH|1y$+{RcY=Rj?4Ek>BbN5x9!xWjddgsLA2UvLTnHIn9Ava#AFH~%XK|n?y6$nS~_He4~Mgb7_*T|@Aj1}?qxV) zJ+(LxYmbEm}1(UNRt8g;HDPz!_#D(n94@g7Xs>BgGrx;b;}pMoi(L_d_vD@&B7z*;L6m-SjZ5j!{wt_7SnjK%w+Y+dC2tu zbb;ehJ;5qi$AR|K3it`Ov`Q7>{zj6OxFw>gQc}7B_X%UTtc_88EM&9hJ-MY! zwVMa$-=!$BBbIV~bg;O)(7A79BI<)kJh5W<8DH=WF4-4XJh)tMeE{m4JyPv%RzO1M zBc@b?h*Q&a#Sd+T{rPs3DT(NOrIw~HNW@X|jSaqkrVm09{H%H&xSXX5pZ20$?{X^b zLL7gvIilF*nuk2AM9bsmXv2GyuG9^vz0qg$wF-2oR85_gZnY$c^N6_Ktda5AYdr-( zeZ`^pjpR$6yqPMwPu=?USlEAu`)}7#-Q%PJQ z(YM|g=ihC!+B9zCikFdC{pKwKCR$Iz$UGaVi)5fd42iOl2L`)@^bC-XP&#=IO{>5p zqb(bvv;%_+g_qm(QqVXC0(n=KSF!f)18kpKa&dNN8ob+DzR*`>LMUp!y+`TYQ8SW9 zyob;gFwh^h&Zv(YI(o(-*N5h~6vyk930fZdMV8Z0Uy2VIx<_d@F`7e{vkb z#qOU2m`5pr1wPtdUfh~9f?n??Y|AP*u~Wy#Da2%F4!Ym3*hn!G;^Lg;Ysf#p!ZDh% z=Q}~}ocMI`cGBDEFMN1Ip0-3ZbndxY%7Z>SK_2*QT!>=|MCwOZU>kOk&gAVnU4H%K zl3crcf@Ues>3*LkWh77Q?D!s@@0mO@`+yf($-?!rkhmM0SO9%B@-+>-&iHwQZ#9+u z=jghUjDYffvm~I(;T$&;y-(gKnb}v&QaFjfD6iA6axwr787u)Vq97`>@}fB0ZNq+M zJWC%>L#FC24tFG-mV`B>F-Xz3GCNO=(PRunKE~^jF4me{p1W$ z#C!9>G;GAnT>7Yn2>q#KA{6+Z6oQ_eFkYPItiMvoWtu3;yQ3~DFVK)3iV{#UGF>;$%coc$hZsr49^#lnqA>?@r?AE5 zLDPAneqP8Ln6bI6z?x^c447Z2X(2t=ImA+D(?%I4Zb1o*SI3yg(SO#TzkyVpJ8!Zy zz?+|MA181JPt}peW=0vsPh0_pG;Ix*Z(Y@L%58aL4j~x1PH0~|HtN_b*iSc<*C>|3 zHak=4-{^SWo@WLdJ3VKlh)BG6Bk~*5b6s$-n_6KW`?`03P^&OY?ph9Ng0>ei94Rl6I@^Qe=D<8-~@!ynABstkgQcjqrYb$Y_hwRgtP^^^~s66qi z&BLzAm3EVg-tEmiXp9ksM`>-ZjDcLmC`sW_V- z<0vK>Mq*SEkdK9JoAEBGP31Z8MsNJ}+eO%xNHG>+U)JdDVeI(B>Udi3?CcDa%?{}P zXbMPgq4=_}#I)YgluiwpW)(0Gbp-ccd`s;pNAX_JLDbjMSX{t;_L-g3!AZ&V+$NG?v~?>v^s+AxU_ zNZ6tEFhT`Oq_s9)P{kbm7j;UcBpUGy)v`!fA_ahKuB>9u0^AS1OhKH`SmmtuMe%b1 zJv~{20fRCsuHiT-)&LNkRt*cl{DUNHH89bMD)LlyV6e5#Zx|F@IFkB$)&fw7SsOE^ zx<2cEu}x_hCZ>}p8qFaKI>XTP1Q&wTRk&n=vhq6CQwRzPO8HC3eb(NyacnIPi*ekf zthqoH8c!CXT-eO110QscB?@J6aia{lep5X~SW8Hcefbm#DL2AXxGT3c3crBp6v*l5 zvcarrCvCoO`AJWDs$S%caVLccnz^td6?v8F0f$^x9hY@poyi6DyF=H{QH;zXO8-MH zJ53r;Lx>xRAoUbOAM0YGihWeC&=kPB4zfJV!y}bRNP~o6LGh~_g`h~%zP#bgO6Zhk zk4>0>Nhy(L#aSA2{ZaYLeTqg!H(1p3V2B9TG$0zjDL)gs2z}cpMvW|`VW%QdBnYtp zQ)*T$qv2B&FxZ;wuOu3bXnnviRbgITtde;kCnY-^?P`OiIoSmTEf>a-xp+z`IILA* za5#~ghgNtEPg4E1S+7`8QN~g(#@q@KRj4~9?h=C1`0b~pOsgj>2^zlPVpo35yveqW zmj(=dd%SneTeubNzOcXuoXTno@#NCHy0MlmKAP#HTt@84sp;j#Mf<{4oqMa<*H}Wz zXaX}-zRw2bUQ)uM@iya`E_)0DO-3e{GnVWP#ROu8bFu*`0e{%^P7%mM8>-a>E)&B- zhLVH5P3^(xb@kheDfmK$Msx^mcAw!IQ)SZfbxb72`w=N8Xc^m82c1LxR(+B)ht`If zzXs+tz_dGDa0g@N=o)U{Yb2|ay*Rz|0Y+g#AuK%VH$zbXuLOhc!UUqIH7@xY@uifV z^0>oY>Ac(CqNKkB>d<&B)EBIfcYFvOD7(HWc`8dX&^uW`ht__+LcAtl)o>Eelk~{D zAU$qhU`~wLb#=h%y}9S{#u4@c8gsV7z-%bbI){RTMkUCc$88F0ibIw60A20veK2cq0u`Z+#h=nS3>p~+XP&G}R2MTEL9 zbO_q6IMQwCLMqaP#ziJ18C<~VElzpOZa0^k3k+1lMjCHoqYrZd{Ysp1#>DIEvQcp| z=gkQ^H#uFgdWA;3(m8}IwL_mQ5d}&kA5q1>5=X{`g*HTi4txwD$D@t5i zwv2@Wp{QUilI~``UQ}v}^IveJfZgo_0`CWo_;uhrnG*jwO9|wrA>%DG;%)T(~L;-+16kypzrpt2>Uj$p=>X$8G50+4`49Ps#VwjXVCt#s(lj zb?D zg;TN3TjsN`ByX(r%oiETg-Eoq*bx1;xNZ7226PNkGJABdz4vNQRc{!S-+W`+K7O6s z`UaqXFf!OT!*O~%VtQlywb`8;GcPWYhY{kN2f|AfJ4`E1=#%|t7BvRaL9D|C=l)qn zcf-CJn0=%P5}O(8d3;UC!+z0aiUp~4<0E9nkMAeP=ECyS=O^Rke#C?7?a495Yfeo0 z=1Xo6rosl}*NzKDdgIafkyd$mjU`LdI{B&zy_E=G#Cc1gY`kh1Cr8uO&5ktYd$w2r8-$gltMUPeyB@2gy53 zzl|kq_BNbRQ1P5z=`ZrK`13AEJ*1+Mje*3>&oQOa;}u_+vJUqi_t@0pPx_-;x6{kS zo_kbCXc&={zo(hpiJt)98e9N6Ix80>Z12=S==oLFcx8~T+jBzTaq)y)C>^Xrb;XiO?7 zaS2(G*;@78^LM+EEslm~@0F5PK!Q-O!gSrfx|NAhj%Z9~?;H!VnxY>13&`+AhkBX$ zf;(!G>f+$eZ_PXAgLw2=Su4SC)FKxP{?!qiLC6OrnFK~7GEX}%zh`DLBfhi05mh`N zUEu`ne+7H|%+&VC3%AJ-ldpBXifqbD{V+gH)Byu?*Pq)-LFdid_E@$=?)A&k6kBoS z;l4V&2n)_j%%A7gqbTm;rjhcH2R*jRwphfYKb|{+b(hB(OB;7~U7zh>hTe$v$l2;jw%O`RCc9 zqeytQVS$9UJ;dA)rch{bt?|1Vp4XwukrXtEh2RtF9aHOl{dwcJ(<1BqQ~m9#+Sm2) zEzYV`$JkiRGZm28Z(<(YfvIe_^`%{af>{3YeTS7K1X+VE`xXFM z>l{hVUr;H&g(cp+2hXLtP-IWmnOtJBHOY5u1y@Nv$QK)l_uL=(lWbOVcXJ0=+G$U? zGGb0%O%6im^TZT7c*>h!S6je_*(~YIMI|2_u|j6W*czwDS{iWp%7}^U@_21V9CVWY zHwjR{(_((_vJXhGb8t9F7xzbZu6+9Vs<~x_Uu0*0MQKb3Vm}5Pj;wm9>Cg8=JuCrP zK#dU2_qh@(YHfV5qYb6+Ee!7}*~izugg8P1GQIAgXQcbJW{n%UYp$^Q$!RDAmo)VB zpGMpUl&ivtK9smYNp%8ajWaCk?M&C3Y?uax5Vec)FL~3UK=0RX=Q=u{);UeT-=9tP zaY26EFMSVGJq_|^}JIPAwax;G`>2$r$AKa*aI{a$->Owi*NjWxED#?bw` z&r*4Oesd+Ajlr3atEgy_&av%b`h(~mv{ues7bLm@U@>P1q*YX}P$o`X3@9oZLeq~evW56;W&bA z>GWHIrdayB4NrqJ7&X%6vQQAGvTAppDdpC9)KeUCEa3a1I;nW|nS2dQ{3?mTbJq+}$lL zC5K7Z?3cEDWEEX6;F>fz_`vNc=A1Lush^DAk}j&tFE36MQ8XlciM?RcvtnxhB1Qve zID$xLLiJ@LjuS!F21aR8bI_LVe?f$Z(&|3TNgptZ^cb&Ipn)tZ*|m4 zfmq}mc|LpQRXB|4N%!p&1r{XVrAyoVp>|nc2&EP+j`D(ff}F4%7P_<347ImJs0@}+ zme1ES9z3wAv>~BLMX!59l*4kUY=2<3!=O81DY!OjN|7=`s#^~tf5EfFIXn{zNhc{Wnp)-0$3t1##?E^f<$6~8YSGxrt!(mb+~#2x2%#0kj-+Q(B* z^{#;wQS|Smo^>1jXFt6`wUoAvt*ynT>vId63+)ZZ}FL#y1oXsIf3CjEhY1Gr(M5zu{ z?4CNWA+;fRQJt?e#mmhWMonUpj0MvuM+XpkB@#{ zMM8+heKHqO?V0&Duz{+797oP{7;FTG@w?@sjBNa*}*uU8!;rj(oPPRV^&e`IK z)G{ky__R73Br-W_0}J0!CBv)t?S36_(Hl(JUgjF-0F}_&+XYMZwLep1irZ2ctdYI$ zPimS1-tNE3NLg<7h7s;yRn>+6QSxqo16R>Bhy7Q6QQzxUgh>@Yq&BFZF{o#i5%9RBF(RvjwY_Ki+_KlD9>JYh5=|(uzd->wXu>u zh6@Rwt6{OlZ$myS!s_i$#iM82e10||dfl}w;e2yn-_j`eC|QcXJTiV|dE+`3PwSmN zWUDF2in|aZ?s!c$*Jjd?j8ZI(JUPCEh^9JESiy)SUGhLj7QCEI8IVk7jt8Z+Fm$D$ z7Kd<1I5387^hB@*$|_hYSw-cteW9!=iqc&si$63s^e@T@%9|j0s@NR1q*A^TH#8Wt zI)-rO?UXRZ{yA=DhY7ZNf7blST(>bK(BJr*+v*5LL=ze0eE9wSfcCvT_o0!>lkzf% zWg^o^K>q9U!qO~6AwHgx&hc8@!V6lN!VD5?!xxU_@tS|-ezIZ(lmbE<_pq%j=);y8 zTZ6mEtj6cDu-Z4HrQq+52U;PfWS@uUkMPPven}Iig*7|7IAZ0S9;E)EZ;Vt+xF(II z?zpfr>Y0Vq*!U&?sEO>C>uB-?&DA~(F`Sn-Q%bA}YAr)M{w)NxK6Y0dIcaQY&D0^F z3|6gxn06#&XZfv`aPy90L$}*Ms0(r|oh_~HEA*A2Ja3|ol!G<~AdHN`ZVA=kv8GpK zURY0;pqSBpu#$yIz3y{--^@|)=FI@>r{R$qSm)R}(i8JXuk4@6p!_?6r~sCuiv&V!G>ulk#S%c6=?r?_T0q48F6TP$aA zk{r-#eyVjrm77-|G+Bt@45hQZijs7wbmq10-0eyK z4W;uh@A0vR%j*+Se-{zSBctv<1Wu~YheZD`UGkE^_GwzidH04r;rD32Xk2HV9%L*u=edgiZ`;Usx3~W& ztbesEm$$b1J7||HnC_D#x)SmKbfNJV+i>&roLw2aoy=*Oo}v9s+vE2S>XGcnF8v=} zMc5KzIY<4s=O-lbVgEzP{GYc-6u|!JOY1zbFA{n1EG!ou9(C|PgpZa<{eQe(e#So} zP-$r?1UH%tw=y}>zkvyx5L6lbcccZJkj4Ku7HkAcz`qd}a{Yt)KOsceDkMiNVEQKg zH!iMfB#=>Nqvc-Ga7}_*405_z=TNJF@aa)zPnvl_nqvo+^^puKrby6 zv1GpUw&Erz|HGLKT>I{=`&0Qm+MBRZ_(hbj$ZN_A*mxkxLq0cL3jse`WbDx6PiEEZ zjDRI5emoa_BWe^iTc8bG`y_?pG~e{!1tykRf5VbJMd`kw_}2YA+PS{^rMZ*82|=s3 zHozm~`@zFIxRTdw?(k0)QIIM#4oz<7L-60C7keeTM5EQb-WZOVpw)&~i-<39C*j1- ziAJP|b(W+Xgbt<3z*UefonHhxklTmR)UQlS?GVT8vUjoA9*B>x6<&z$Z+<3Gt;fcA zcB|8o*SNVfH~$b#*PC#;k_R2eIdwHrM{l-oQ4tZ{%>kVU$QN2YX#?u8rCCjQ6aG?N z&cG}0&m$^|lJ+V{9m|y@Y>K{k&TH`!r`xjb`qoJ00F-P`XP!Wdn7X?p+aEPnJjg>K z;;}u0AN{oMHqIb#EXG^$JdTt#J#~Rp18tA;y7EHn;ej7_jz7qksB7j?M)X(c9LLE% zARha+1@iuDAruIFc4X_oZ=veIjxC^kS&=1QYtaO^MUu zwFD{Xe4pD`NAXss$Gjz7?cq+Xoy1B!I(J!NNea!YOPIz1V8shlYUq6H02DsG3T-IMyBHw8VI94f{_gr*Bo5qQbZNNRUphhJI zFC4BCiUc66x$RMQtklVpb)-U7j9jbR65)!SY!=j zSy=u%WA%@9#;<3#WcE2 zSW-=7rzGLR>>tMGw!a3G&N#yy$HAZ7qyHJ^Lk$L=H_{5g18x*CZX#f(r|658)~;jC zMCWZ6p~$*h2;jg?OY@WPGJ*A#q&mm`HgCWYxbf*lI%i-Ls=M{6IjEgt9J*eNtv&(& zxmfYpvqk#&-2Haw_P5s!#11J_PLIJvA*s%ROAXU9)qB)#{gJ?YyW{;iJpIckw-qKzR3sTb^u}a#uJLWJr_{iyBSgewv`{TVuXsP)%sEo9w`-vb> zkoRComvIhMqlLZmW7GLTVy&0*`ihhD`J}u`9pdJugSC&9_Ey}?dqTo=G^^%DtY_;3 zaH$8aY&Z-7E}23C)l7%_j=0%!f#w0}JK3=z-P$+mo@jr2@AUa( z@bZ8ug?~KIyRBk|O))0TGxkOO&(?Emj4S{Cj`!3n64AQdJ;q{@J=63DUQR5)1Q3a|xa@HUWPYPIZ)tF4< z>_{K~L~)$y7*p{~?;{3PbAEK$xqYQ2{r zbkm#Eegbyd0l@!q`Qr#IjyZcfMk#S|{|XSn!cb>Ux6r-|Fc$fW`#r8q2RTbzdzBSE z*oOm;?Gj!6BFV&$>EZ0J54APv8*30}Yv5ma?ZI3Ys{RNl!2PXX`DwFj>kHe@rO&(3{TaO&{}17H zi-9=W84Uc}r!j{Cs42d-Mzxk)ICi-|h9CLJVd2v?&(o1Tqh3R6)_ti8LXi^tt@RDT zZHq7)_RjS3?z^70<&jCOD%xZ!=4g&Q>EmG=ZU0f)xv++!-{}UH%f5%V z$dl#v^$cblCi%h1VR&Q)B7>6LNGqaV$%E1TQ~_;cn1n;WMJL5SJnH@z%WcLMVv)P)k$au-@jgH0+;c=vUPbev( zSQY4*+wrGQ5~7txLwsw1$0RN{R61iIX|}8B5@!Y5=9=^Y_@g_KlTr+75{7_`{Q{^V z#=`LrPH31agR$IRpiIepHOHfg=x})45t9E6LgjPJKm8|1NZ{3}v+FHaeAKdC#F5oY ztAf1hA~v>!y${iOiWLmcZB01=BWJ~HVg75i_+Q#h+VcaAw;yCdyR(NSJ^kpeg4`k~ zF2ztwjvNM_$ma_dEV}XNB=2uxGCz1n4aBNp&AE_0E{I%*pJ)zea{}rcX!w!0_o#A$ z@#j^gECtKg=Iw#BTk*Bg7OCY}xk9arol$NLwu_4D!iAjd_BOsc?*$av4w_iMFPyv{ z5DMq;cD*ZeA`)znLF-v`eBL)=>i#j$n$qYwfdlHde) zO>lP!fk5zt;2IbtxXZu*Aq2PJGDvWD7~I|6-EA0L2ECJW&hx&{hx_4wtM0AaRWnsH zy?V>q%X;;1?RH8byvkTRh%#ca&v;47t4=?0v z!YDLDobu`B^U^T|iJSn2U(N~}OFygUt%(Z-l3@YQzRUx+d_CgNuJA1sXXj5>mr;Ns zlVS4H-?P5(`98u$DaZU|u@h4ZHm6WvqPWH@nIo@BnTK^5ZVz0-S(7#UYJy%Mh0Tq>ZLhrO}zn7P-@;ffD&x>(`r3|qUed0k426{D49 zaO2=eKCKoQPMIq^f!RGg+Si{YSGEG#%{`Sk3{tsCdmkEgmlMenLWM@ItpZObngw`R zb^!djU~cPEyq;OqDiC1sHSW>9trRLM5H+^oSHd03q29f_yYzAW5O(wv*iG^MqRckN~J;Ec7$PM#aP_C0Xiu z{Q}GSEfU8oUG%6sg%OK=)6aA4U1~8YzWa`SO;9aZMm~X;i7{XJJD=GSO4u@gJiO^P zwB1D}fe#?=TM{YjzVDm8ch4+*=t`+45?JEG1;<#qKZNEJ?j}iDLLIQQbd3eic~JNm z;9J}KzMx7cKN89dB2e?uT#CBK)y=MM?N(G$bl-USb6c-jT7%TDhmgJJIp> z2R{~!28rXr`oWv|QLJY!oveZ(Q=L&;N+b%uulZGtN0l9`A5e%Teuo5Io=!2(gy~j( zk80=y)q%t+@-(Toc78?wQtCX%b{HJa3WD?gaJ*E#0s@|yG1+i4hV>1g|Mdzw2*YH% zZB9a$L>B88!x`sopC%=t5(b6MylkaWo{|A|C$H7*FxA6Na_Ai|Lh9{CXvGVB+Dj=qLhpCu-ag;F``b^pOo=KG59yLhLp$OHYB?4$C{NO%Lkf-xn-B89= z!}SoIAi+ZK)1Znyrw64QdPLj$3GxV{K)(0{n>++~W=Ae0hNOGTX=H+eOLa^u#}lIa zOn<(CJ$#Vsa<5n=`PCU9$P^7c1Ep@thocB-)62-}d&Nk5(cHBz51p7g)q6SOQNiAJ zvDa)2}z3H+G#FDA8!BhtHZba8b#(7$wf{e(spvpLo4Kzuf4uD30ut_dG zPK}CWR(^@?W6_JM{-$WAxt@pw8LE*RA-%yp?-iSYM0c|bLA{BQCIj=OxmO8nEsx>^ zXOTjmxJ!QmmA$m0W~)QKb=Bc{$cW%TKew%w4hYo?tX{v_h)RmHepm^x%xcl-g?=Cz z;=9;bxz@ud5V~vKsPeFABZNO9;IkMO3PaqJAfb6b%jO!?fcdHTn9DEJOASjPy2^d9%8N?8Cx>l;VV1W zXh#EWE0hKqif>Asl73cgf89#e0=F6c-m~Y7ifbmJ;6!5kUnJHf{Qm}tr8)Ui zjQyws-J?X^@L7+iu@oI8n;s^mBv8MON{u4 zmgiVb?<}l;XTZ%7Sh9Uhr|47~)FX5!bsMZD7kcX(-{o*Xomc-~G9#@?1&kTRk3ta# zShS4=vu|V;6YNcPPhD;QQadQJT=3G#PMllgH=>6zqLGra51*OVFh-@`@o-*}1fmx*b$_BeoVgMAsnEV4zzQIo zbb}J3)%?}1A=S$@AnfOtQLh)hpQrOdyB=%Mcg20)qHJtt?xQ0}xDd=lxubNNmtHH9 z+mAsWtR#E59$W<}fGvCJ(TGK5Hz|of+?Kny{*MAnUDZaCOOU=) zsG*OEoq?v)kE`9!`aLQ+M32a}I7A&QVLeW_>%sc#TZ>qLu-f@;mZwnBK&(Z`jDK5O zT30}~seEt)d!6HDJ+D*b%fwE76EA}hhq`4- z+gcn5TW7(Vylh0$J&O<~x2f-Kxd*fq1OIj&0R}R&BAJhRD;RbDNl74e3 zXTT(akp@82q-rL!`5zx7ju(q>9AQZ`i0pTQx+^ zE^w0_sowy*p^sUR|3YBRiW;rTn7Lq`*1tSGt^R6Phm1|7`EgNEe$csP|1ZjWXX!(v z_{LD58c^%(R64r2_KI`ie}GS}m7tIN>)lKKo%u?AT~EF_-&%C~M=OSlB!|!(kHTBj zp!4nHnGu;egbBdWa$|!-a16k(26(*ssqfKHD(U`)HTNh40MI~}J^u%Cm-_DI--tVr z7`5?R){Rvb!qj@@e458Vy6gjRhcuO}=bJu;vQum9iu8LNyOo3adks0{#Z_Y zX6Xr;tYn+>_sfzg$8|9Ry=pNDX_A&YFSw$jq;u7( z=b#2nl`)byW$qT`$Bg~(UIku=ey&4?ID1%mWh=Kg$OrP2988HB0q9**ry3^e1z)>- zf|7`}`!$_}Z;`b9B9>FzY<`Zptf;t^dP@{CEAj38vRU{OK!an}WlL8jK`OWEJ(`^y5IPltk}@jXA@X6?7; zS=ePv0z6zp=@Vv5(R>W~{T_9YDBd1E4lg!_sF>3y-A^?t+PXLv+@DPJLYY;|N>9(4 zU|ZqfPJKNuNU(W>*5mb%JeInf=o8eL+*ZKEO^n|Vc5{38N?qrQqGHn@97@tW9a2Zr ztytu;wnE@f7A7ifYAqqI5V0OypL2g7hqA5D=bLs}XIVo&C~|jzyC81>aRMP3>=Mb2Z`Nq8?kNW$;GT5LpYf`Qo5_ioc7;< zH6{EjG$~9uxj4OJF{GF`Vk-#ofU4w96EYk%7cFSX6``9nxO;n8(Q%=Dfr>I!x>x#! zThYFS*>CZ6n?JHCj-#I6nzv8gt#++ay8l)6-x8-0~BN8T#jYK+`$kzxb9)}+i> zz#O-}dOj-cq?p23ej#obOZa;7S3@90Uaoxt+#AAR&~YR&&D_!Ykiu5LywmKJRAc|V zzDU5SvQWt%q3rq>NiS3N3S3B_I;R}(JuurQJL7z9VXwfR0##;|yRFB?G))&TlC%t> zD|tUTLWx#FR^%ky+(ei3?5`<$k4BZv4{}F=Pqi033Q3K&fgUfeyaspM{lD%@x82Xz z$v(B&^<$cg8+!AG?R}`_HmasDx*Mn<6%UAtFSsXNn9^kLUkE&ho%c}Ip7UGug>HQs zaNa?OpM4p^u%wW|*|J*OdQye}!v}6|#$Xm({->?85vQS{WNChAYdh zcm0xdcYWKYZ{!&(oEg2vmA~EGp71ua=dj5pY9MVPh z!8)Y92_XWVzSuS{_xpIc9~o4eDEj$9H{oA~cXJ&`hkyFclRq|Cv{GG={FG5UyZC7| z_rc%wR%aD8&dJ`j)0}m28Z2o5tE;cy5c8l^8A%&jQF-c&$V3l$g|gr1L z>apF#(*_^>pun=jPgBa&$L)Dk*k<7xrV|~4X{u8iR91G8i6POhW(w0+pz5OU;j_Ek z3?Sd8-V83y?(OM5vF%mYWa$p{ zCk4tphQ&8R;KoLA=OX0C%}wSE%8$>-g)^U{V-z4*l~vwbWOsS)TTeaF@bs}Obkq_$ zadG~wz}`>41-=I9^rAERtQXu-DIoDS{09rLGoo%-(fY&*-Eegia(9EEOCO#WQCoL< zmwI{&S9wMETOZTO9N~UZHgIJ(Wi~PO4xIiH>2!R-x-uW{M@iw2YPvQZ>7Dy&PL%Sn zNTII;tweQsBnD6Fi|*KAV;x%DzzyzP(jnwr+XbKgWlt;I~9Bs{MvDAnVh z9PY&!n%zFWe~IIfVDLN}DVrq45Gip!2Mzv-s zW$7Y^tpJG#pV;7eM3bRnM=S4o0)yd`>6wE-f|pn-E9=DkR1rq!c7xh=`<7{7{t(F2ylg!Nl;u-!Q-{%;mW9iFTCy`~?9V-08tY@G3MDp{#jWl1$C_E~ua$58k znF<;6!Z{85oTR`Cuev2)1>NjlMdfS=iAB?UUDPa+L>0i?f(2c;3XvFm~TMCl0iMuR^ra&ve~_$R6FUXshNv%EAqv(A9wCaz7x z=cs6B3q>lMo10!Vlj|mBx_twg%6eQ43_cOcx`gl1^yO6|^;~TgdTbpLwNIsqwcdk8 z9K%~VSw2r|eybt-oWDg{E_n%?3fI$Rcg#@cjMfPE-BwW-dbs|e<&+agD z63hv8`Ke09^N9(1yQ8+ILX@QzCmZN}=Y|S*64+?cq@VcpCi2*n1~%q>R+xZf=m{K5 zG{&zqogGBrOB@lZAt7x{ZPu4)j|cdP8L_GV+3l)#0@!iICl zY=Z+CIGRgKqyr4T;WvAJf;!UN`MS@QPaUS07b)nwDZ7>+bbgD00QxRmL*)C6T?kJZ zoy?BLjEu;n*E=^4OsKEn1mg^a5QoqQ&fk$#T&t~}gT0Rd2?&sxJP+E1jmPaveoy{= zZN?5A|Jf#)3f>G;9;Of;?$P5V00%Wo9$pOi@B^3KkdVG?>i+3upV8fp-CN`VQ+GJ}Zm%pk@B`*&=KT`OKS>ZNPO^Qq$SIGOO6_*!;4z=@36; z`)=|nd-o!=Gk90Q4?4vP6(9bF*x||nb91GMq}?PYZC|j}s_y0IyperT5O+;F_;B;) zwwDu^;`KY@(D%u`=qC!W`secl8~!xXwI4Sh*{>ExX&A&==d3aY+njF=gYst8-A`9I zR(={Jcc^>IM^6PxJba0Xpa(o&-JmpGY~YK+r&wZRWAzd_ibT7jfsG{gi`2qG?WJgh zFPT()-Y3s-vMRgZ@a5PzgsBR)+F5a+CrgI|3nW-J(=U+Po4i5IVVk9-vhmo1NOSWKR&2 z6zw)3I~{+dJA9B5NGwwM8r&lEZac#ibfrL#ttS>P`#!z`q)WSx%3o~YTfIN)i!}Kb zc9jx*rsaJLqOgJ0NAwS=S0ui~Z zXY<7)Vxg6M^=q^AiH!XC8k%-LbRk(!C+*G-;X8wN{0_Dh#zWP_o+9`tc=OdfQL)fx z%qb|}1l(Cs9$w5<^8}-L#;ofZTkMYvV>{=$Phw?aoBq;&wKCkmy%X}$8hr=XOR=bI z#>4%L4jEmugwbg3O^@q*-_ue40g17~QddBAdsvgh^AS`q+~}CAmpkj%@am)V=>Dwz zHJyfcT5APjxg<+SIAM-x^|1BHvf8xUV7cAOQL4$W=dWI5X`fasAKDO@nW$Z^h{OnM zlMXMieGuh+v`iMst8#y6kC{oBq@q%;U`vW~z8h3q$hHIMD!w2A=4g58PPpCY@N&$L zPTT}=BPUDB>563yyAk-VCdOt1>L#6lugrs%~W6~xKpeJVg}Ps*M+8xEU>)|BX-8XZ5`TtSK`8! zyv{!yN?erOk7bx_8PdKO|vV9TQ zUK(jeUSW72d&G=lU$UuQJVpg=*EmtkQfXo6sg$IXQhrM!a_(le5=n!~H5UgR@th!O zuA|Mne5b~@NRx!3BRqK$H?{!>Y$LRP`QdiwlQ<J5y)7ZrAz=Oasxg%)kc1eE z%+w#4P2W?#)m)}k&*SIVDQ$l&UR@Msb|q(9v4HPZDQ3s{a2Hr{(;FiJ99VLEjx8aQ zx#R10tton?tVR)7ADc+^G)UH;z-4>$4Mw>iNVdI2-&=`ra#{p&s!o~Qp3LJF3}GZ& z@_IwZL{LG;;82b>p^nI(dnin1J0($i4_gx{XW`$foydm>ji0>@bDudf>O-H%&AHH{|(`q6AZ z^=^sM>M!a<*G4!JACHv~Rd1y8pYQ7&IU^%6#IqcELbBY=Zxf)x(-^~P0|&1w!kWwo zP~uxh3!#o1I}d@H*jPqG45-Q3NzFAs;V6?&;xDa!2Ewfr|U z9Af@;N2@$e^NbtkqYt$vvd1()cQ+Y5fuHZF+)~IHOei7%aHyeHd2JIimE0&zd|>o^ z4O6tQ7f0UH>lXJ0NuW0)hKP^xOcNg;>C2j#&M!8fuQcLqD7u$7(JS1qjiaIJE>4>b zQen-xs3b-F4)-V|(4^0x6A$*zUj3fmWH95(1bj%QBAy$&>5OM<1;@4Z;Fa^;ebh@u z754glBldz*lu9k`P0XEih%)7TuDtQA4Go~r^mz}58K81=DFQohDc+~jTlN|5!z~Zc z9?3gxgg6JD-DR^AQ%i&&w`zauMK+Hf)L%Iv{3a=}ZWUk~OOF#{onsYBZfsoTV9&n` z>C994Y4Gd3g9{^9|FbpQeh>BTa`AGCp_p75(=wV%hmb2}qw*UR^rS`CDCh03Xw`;Y zzC#NG*Dfsvr(AX-JzPe6)1so|_)MM{+QD~PCooVv{ldu2Qn`qYgh(aY#O7I1y=dD@ zHzx-Fq`>*}^T7MMEgT~Avdc+C7%_axlNK^S5PyHj#w{CewiQ(HD(xVmKJ$BZ-7@HI z$5?|(@UomSZz1O{<%d!l?^s{dgJMwNK#L7OFW zZGIE}fN|y=aQ1!zySu`4q*a?R&QJ0Z_+WBLB67OgGC~$wMY@;etfIP-GY)#V>LpMj zZ@+0oUpq#74JYZcehjJIwJGv)cW0ZmchODTb!06CQYx=c&4$;cf;sL_FC&dJv$sIt zNi%*n+kwKB;4nd@UJK6%iO3OWqk%r>Qc9O!iw`z}BCOdtz;aiY=P};rC0v4q(`zXA zyf<~rE}9Cr%|!3oFl&O zxOBAt9I7g@LI0BD7BI#Lnftr!*@n6IIQl~xjkY?r#bBMIG$eB}p*0lw3nm#D9!{!Fru%E~Re*RlF3qHILABojS+@C7`nMfeet+HKm7LWGl%AJ1(Zt)c{%BsUpn^tNt;VsFc zUylFM=p$(UU0uwueooTulNX)8m3OYc6`y_qhEA!ZiWvSM2t`*R&flEp#qwGt<@M z)Snh&C9}X1rpIMAm*_D#xIKFn}rh2Bj)J5*~G;(rm&T6nt9L#%T77%iOhuMB=< zYg#@6J$`+){9@@fYRfuj#RR;!bA9rE?hw38o>w&;9g|luS9(V@z^2 zJ^wyR6()3S62|Z*wSF|aYIm7klnU@nflEacAF-*uS-j6oa3jB<5xMr9-UtQu@p|xI zbMP;+r3TQDhrY9Rbp;`)nIpHD7KAyBq)4n!uVZen-BNe>o2-X zC3IY8_ye`YOJcDNL$)TDVy32JPHdvOtGSZGto3E^iSU8iu;O`w4v|5SpnJ6ZILMv1 zRgrb)WaBFAX`W({>p1lj5esX!c|(7B0CVW;&V_Y=7is@0L>PcB*jZU+ z?v5LmE}zbCsjOnxQI^y9Slr?woD3>YRUANS7eV>i78D7mx;fihoubNVVLbF=yAcza=_0y-jB%p&u8M&*kmY?uw(P5h>*030T8W|oWouA8QKLQe za9ka|%Fd`=57S#$G`F`MufK`ka8BxqoTooQwzv7jp&uPg4O6I&een7(j<9m|5;|}a z05Q)ACnoiSRh&%iQY5Mf@1hTN`@U221ja1SgVipHV$6H&wIYGT;d7$r#~2g^>XORp zVSs2+t!9Rw2`d((01*m3pbK|``tIdO9TW<)4ywf&hXl199IQ_j|Dt4Q0oBb1!;a+b z3m*-hX8fxQE=)SxR}Vj%f~XC*2^^mi-;0iH_tp`doYZK$A6xb-Jy~`95Omv66}z#~ z7p!e2lQ%9RFd)^7*iMWTtQpvmcQb5x;QM80U9V_iC*ja6dzeXfhu5rCEga_gIG)Tg z_|o0jkza}ZhcXwDIc?8G)PAciy=&*>E7yq;_CNOf*^~u#L?|#Vb`zg1=FW7IruZk8 z5rZNu!E3_w;73+p!ts?RXNH@F`sQ@~FFCQb+3=N=E8SN-rg2{@h!YjJ!=T#Fx!Q+> zKzcEw<(1YzBn$&!Sfq1`v%1E$YL9_xkOZSV3&-Pwu}Q5cC2_(4%BHGHVm{GtJCpJ8 zYgyO9qD2mYlp7Zy61vz6;^_rsb9GlUNOv88mm@om=ewk|32rHd!F}~AidE73g+yfX zi*LGN`@e{E2O_o85&KSgU-eE1B#|Yjcrg#>D6waOQs5T#93;J7O`;4zDp$#S3oHpc zvOWQRj}ML&&=aP@snqmZVXk5yac}cqT3qr?@NYHfBQ|HmTc^8aPMCI;24xjZt?LXj zb>Ba?#A(W`cJ|6K3BA*z1)k&yiH@y2lR=hP5)0TLd`E8yxh_cTg3pZRA#s|zxo!8oAbEj-PZMULEIdIKhF|(m<*nS`=OieIMP0&aq(k; zGg%lePA&&kEn48J1t%TYa?R+OHP^&xf1fw`+9ADg6+fgPpntL5%rz5=-kUpyc&L_r zd~Kuq5#e&63XYqbuiIT7A0W<}Sc1@>XmgzDrt{mBC$v36F)hQG-oaJljY>kE{6JOv zzIOIjR*$>3mLGVxHLf_dGcm=RGum}!Z=b?f2cAmS{Oe_XOuLhO;E z>Lt10!vbO2#ly=>g4BsbByR_L&I&J1-P_l{n(8U;ppY0w;|0T(NG||VjPE?GE5e1% z-LmE-*1+gv=+R2kX{5b|2>DNGld~Hf{Hm5K31Q9Q?&9BF9hX%Lr`>+z&VO;3+C_`p zMHhL1juafTq}<=$MK9HPYlys%XC?Y=gt42fQEm>zjRSYAK}@x<62=9V;_}Pb1m#78 z#;O%NK>+M0XdF{OUSozaKdi!z}iQZZ-W?-SX z#pnsZz>ze@BjI#qvqXRaUs~z$sIhK1y94Z1qG(8Y zh)wYvX}>8-1V=hv^fi%MoWAM+Rv0=Oxf; z-1FN+#78>le%{nJ~V-)JIB1@I+gfGyNb{LYH8N;4j&iDgeqyE!= z`7iLH;#Cy#Ge#o3N~8DRNW>Yj9V4vjb>7Qc04mBwCikxiLyTw`ro)NJy678Yb{iw- z1xzZQn%_l}YIElT+%ZNK#ltYzL3n88XbHGRkaD)gkWPnqQ*$=}pvfP+NQmqDI~K-} z21>Db)|oVXs$m<>$~cwvYi2ubNZ9#oN)FzRV|FyObIDS9W44}T=jNLG(tn3`fsc!N zz0L(pUX5Awoz(}6?0Z3q!^4XcOP#jJs)-2g5u{OTVb+&`YYT2JC2;Jzk=u@aDdS=D zcl6)V-nX_QalpkHfGc{5A08^( zHA1W94!govwz_@y*pw!ERb-+F?C!vJ@oy)&W$p48huHR5T!SgC zF!T2=LVw;!?6m})0P9$y2Ar4!8NFLBIbx*FBnDkEYHmMpGR3Ss*Lnt&@rF$^}M zJE(e#Ie#TQSGiJ`7-kk8K~Xpam0D^3pc_A|8^P?IAz}$73cy*W4Zmwl0Rj`c;Z=3N ze_(W;{Ur{QV$VogS{nblODKUQ)P|SbnQdkZx=F_hd%ENDd3puNGQ^6ZvplZ;>d3Kn zCCL!m3$M)G8a13P#&nw`&t^^RO-48PIuwgMq}5pL5tfqlYmmvlz?}cQRbOglPImOR zjgS*+2NJ4g$~l}V|7uIH5`{u629;SkAP%zWaJoTzy5EI$9Z}v(Mqe}#*|J9~rk+GJ-eyl{Ux13Td+0oX`HqYa`W=x+l%|4yWghzJ z&aI+2NlK`ZxHrEZO6zY`;hB%$6Z=O_c1r4zFPhn+glgk?4h5);Lokye1M8)yB-M(4 zZRSq1&vETdbAW@U-cc?bYgDl(6%U0f06dRo{76!Aw6>)x7fOh#*u?8+ z#J@}ExITWNLlKAmod3a6x=5Ehrr z!!djDrcKM0xkN+0mZ7bFfl*LL3YcT_3gftH%SWyTo7VB@V1#bK)nchX%Wc-P_`YmQBQWbPR^fYpL?y}kvGQ>xLS(Sg zY0Dii&UB2YFE#)9;P`czs!FCR`71kEN3P)Rk=kI7lrasdJ$*=YYCEP#lVBt)c0Q}zVPZHUK$y!(vtx{%r| zhXbQmKm(IW!}@xwK6qYlXMIo}$qIQ?yav4!NjE3%;*|gd`mM)>1!Ydd-r-`@axR!S zMEl`LRkf@s_>U|yQAi`THuYykUV;geHHgL_2<{{<%N>KTs&MdFZHOb;%6zxPIw zQtGFdfZaWD;6n=kstk1GTLjhee%}E7q-|zgEfI|IjmWNZS<>$`K%K4So7Uq7EJs!r z;CUX(Tm{~I79Ay{`K{OkA2?0?(iIKu&WQ0TacJBBAM=3=-^d`oP1~}NW2!=gX@{#3 zYGZ0EwC2U_OvdFO&O5O^>kr2VlSKEByaC`(Esu2G;vg7`{lFlLGH84~|KNogB*hgC zAgfHM9f!u4N1gmUF{9^%h74@j9Nni`^tj7!i>7OaDYoOqv(l2582(}Bs_gy zgBgOAn?B|R1=Z^RofopK<>){9!OFsHhkL_AZ)|UGzcOlle~7Q>afz#UAENbm^n&}T z&w$F*bG~7;Co%1acy2a9NMb9c+g%R5`$LYP;-er@PldNlz>f3ZNhphV29J=WAHUBC z>%Tq{P`n}EAG|SB$6ZnS;n1(wQ_J%5aV|KMynyxI7XL$?)4sldQ}FJ*BC!dR$DO!5 zCd)EAm3(4D`JcG?qnLcDA6z;5(0zmeH>ziAGzyLLO#AG%bIJ2KYsqQVdM5sHoNSBw zD6kgjDMb^{KYN}&?_;WH6&dqFr+&`H3jwFrNuch`Y+rJ77A zmOU-^oOz+#H>PfT#)@RNA3x@x;ZLVX*PyZ6CcL-(^CL^=l2)jyh_sIO6DyfcTD4z} z9OgGaX6%#x2kZ^rb3RHESDq&WLH~=rWoVcJQ|(mSJkzgu3gO9|dZL{lg7J2O_jfqi z8%jfKfpFwV@2&}lLoEnR+K?j?2rM(!?86~^3tOwtD4J7uE+zYxaardHyd_m} zz4u~kg}#07bQ6LYQXe8`6d*eL!50S??x*$ohm-l%vGLXNY^BMGdnp)k2O_q>y?3Ty zpOKQeq=+R`w!JB&0AJ7JgN};1F6jGZQlvXB{yb8wcVWg4Z~+WWA=e3o^otU2+Y`CP1|YStF18G)&CV) zmdvd2?^XYkh@EI7X2j2xxv55tDLANQ@n*J*tL5Dic5C@JIcxD1Z$f`>qN#5D0meQY zD)-Ut^WV)`jJd!-Z~w`3PP8F1k6hJG$ojej0wE#~jQGwrpNGaqP!WH>{inq}VYijV z|49?1;iSvoR;JVxu$9z8VtIKk^S_B$-1@K4F<+j>j%%5T|EDi0shEFR_WxTl`u}OJdnTZoi>6)j zzos?8!~OTPs2wX_VqmE@C9AE!O5(Oyb<% zgI;qMBzT~YHM(JoKS9MHHu~-H!-H@SnCvUcspINHN^TRRMM&w_k-VqR)`xE(CAzgH zSLk*XRuPBq)mBQ`26qk7dw%>o!wd3nB(BVZszUf4OT%S8Or3t>$N zMQ_@c!^i~AzDX%nlz`_(Qch`8g=^@TEm_!fcmWrC>vatcj*CawZy>uQi-^tu{9x@0 zW6Whw=k@sfvt03O3&tf=>+-OT^JG<$Zs&tn|o^ZI+)uD5iV!Ls{9xVH6D-S!^0llTJ% zE-u^NtPd}uzTCg*?lPRoDg|!Ypd&F z($;(;`?RjHvC62^%)rBfai1jVMa#kR+r#Cc!GJXwY@}4*v)*!J7zqKmRsH%oOWELZm_0XRWKEU&$clu45%?DtmdQiI2WRMjS&fW>NyO;&Lu%+}T&2z-WvN8XhkyqS1+8Ch*Pof1VCNU6{M=FP96A?q}cGn!oG z)Gj}4vF>|#C~$i_kC?ZP3K`j3Ylx8Fc4>%b^f7g7Yf8qe$!3E7X0P?QZj({`X+sz4 zite@yJU`|VGuu0Dd`fF?^YyZKbE9M6OVze(?ZuX0H(BtiJ?*2rKAm2oa=+czB$4nY zs9Nz3r}Eb8yVxJy?_!jeuKtQpq+*?#2rLTPBvJT_70|f^0U@Eou~;p)*DbI3B`oVHs$?Xp}BHLMq^ z<9_|3HXCvG{rwvWiPGo3Y{}<|Vry%w(q>V)CyZo7*RHkjoqS1V)@FWjakfH2#E{1w z9~u@^N?TjVwQ(~xUAxI0HA_Zi9Zt0J3F%M}f>_#*9|fPcz24zJ0r1HQH(sd;0RP3v_D!E2Qpjh9!@~OQa=_!OKKNMu^R#c1=p_ zs+`1XtsO_Qe;{6s0}DD9J-108lEKqmRd|a5+H6mm9C6;^RLbUK)e> zOCr2&w+CvVi$qc%KA@WBm_p~P(&6Vm-6q@)yQuQ z{{F~!WG{PH*B)aoo;$zV`m+Hp>*WSND3l!5bZ!P~yw-S~smAn2iRIdHq5_!M`20MN zJ;SuCR?ydW;jM@_%wv~Z!!3(7SW-pY+Lo77!FCM+9UZi`t(%gRzzc1D8Jirt^fUxB zSb`ko17lR{u3LH$tI>6Hnssv#^Hjc2(rHzot*^_49IV6;=gWs ziL*#26HeO0!Xfd&)fKC(Ra~LHIBV|e&<#UiiPIn9sns+VPOi=@V^7b^l+m(tb5EDA z&n*`De~f1Z@;FQgYLv~lK@Mm>dOy~TYF>54aBx|ivBk#4O(GU?Xs5@^XR_79yY^b* zh5$U=WBBMufoK-BXnH{8c&IdFpUbID>~u>(vGK`)(tC@f1#U)37@Bl-v}jx_;_iG~ zg=px!%;$l^(2$DK6AFCsGPD{|mc~A!m?9X_)C7v>FdW8~U`6_R;DV1F6skha$e4yO z-3Sd`t6W#_+#r0cya+2JO5iZ;6uVa5P4mn{;oPk%t3SFzzgfiRINea-LEc-c7h72| z(BlXF1q8ws>u>oD+>Oxc>*2DqJoaBD8eEV17HU3=3O@+_;kdlKd>vv1nLmz`bJ3nL z$@OKMBCYN3e|~s!Mrvwq{_DkyHpJ$E2%YP}47CEW^HyW*?~}avoiY*f+DZ%I9-2K| zHW7LV*cyxr;m4bXSau=6OE&NGDIR&RDZvO4ZamqR=3|%lq9r?y0pA>gZQ~O zkpq}^Tv^G}*F;FHkWl^40RHyQ&gRcdFF}R)-UxGoc-<41rV_D2AXXG~sXl?vN?^lY z@Z_{Wr)POMJi!nc{W&XO9k%0IEs#)D^b!dji`?Y$;4+MqFKliOMcQT|H`U;Kp>%U( zq=1~b6Np_uY|Hx!2T;0BJyw;a@*H=j|hyTBrrNTks_2M zJT9IRKPo8n+PZ!E{J9+kTU^e;p?`d|yQhc6UA5@f8Z{#Fny)lP9~FBNZY{Y@3JQwX zVl~TMeyV$>7Flt$P(x?ebSer20`)wXDd-s~RdEo^HLD^C%sLRP_kLhR)KHwBp7vPQ z7+1}cszaNai*^FvGBY!Bo#GT}mYMhcLPlILB(q>y)1kVS5poYRX29N7^ z4looG?SbJvSNWei`-X-XxhwhHmb~ezBvMe`?T%2KTwa4co9k5mw%U~s*gO(Q)Xl>ep@taP0hzgK}58xQ%2O;p{&^B48`fEi(HabR1}Sr zczlMK{(3MI<#TE=F+UV^EIi!tACXVTtJyg@R21CUzfx5B?4}+cDw;0NB)lce1Yf<* z{sSKI@!sBk4Z409)pNRhZs4_T;d-)h3bLM z`%B0I0-74dT9bRxoJeVD>KO48mE1_;x2wC=*7GUzRl8q_(M+eysFKF6t^dTX*DFpe zhzYO7>%#0z97+x-2$NLrek$)zKEs!1B2SGDc37MOfm=#QTn^PD;wacf2+gEC%i}R z7SP?1XePgnx-Y+Da(Vi+@=)ZENV`mR(e7G|c>*@Y^ z7M-ict^XVv0gr@)|#|j&l%xi2hdk>csKRHa2lT&AX|BeBcpFWnkjyk8n7cel| zlY_j^A))ak_eECpVJ}QB{2miK4stmTyCJc_Qcgj#+u$kybXutV``O4AeLx{CMsw}7l(WQ9>F&4(-gP=k!;?mf!o%*gMW`3V{ zHD#gaGQXWrSm5W61eXkS$n=-5G@PwrpQ+Y_H+rC!X~uVv&cMAxNrdWROJ(D7DbLJ} zi91e2u{RGO&RvZirmr5%G-0`h$}Dqj(1C%8DL1MCwr81As9GHr8beXtF(WRA0R7UB z1mn8<-jPsxqhZJHo*t)%JL$_!j}%I+GsH*ob~0@;F!E1n`4%4DenMN<50C(?1g@o>UG@cW(hJz*jLgg+vzd#CqH(#&Hp;B%b(6XfUa4(>(jezUM^J~FZqn;hqBq@8TCkf1ZX+M8{cv29e z9=g8=0|y5d9T^#!-jF0ZUA?t2r9$O0L=|DLm7Se7pWs?BQ`onzXJ^$67aC9{6Q0Yc zsPqR!B@8%n-QUlbn@oiNg3v@l$SyWfOdMDB-GMY5s)KR6w$;Bnmek!?HGxJX*2Zds zwA;zO-I?Rq(i790$%b#f30c~@NbIax7F{=>Kpi#-MLU_JVXE+7b?TqS{J5 zwnd%Kkd6<7-FDLR=a(l<&=0x?4H)A^3z@&ve@(Kpu^r93SRCJ7oKyn_Lw3ld(V;q`V$C0Nl@#zBO@)ib&}1)!;_rI1m8 z8((q`B_gV|HS+xQeU4bHl`)$UUOihP+qX6emtlXS24Q&XtXGZurU$VQAGx`i&4?8x z)25GoaiN+g#<`8b!j0Yj*K8)?iVNH1VUIgKyv6j(O%W3h#o0t^*6sk2z(9R*Ta5xP zmc#@`;$9J*xdu~MT$%ElH`KyR8QurH_G?(IE;Vv!jS*4f<*LBx@y4qgbPw#*+oG2^_ykS;E-Xx=rVd{zmCk|$P2MCEl}(Vu9D+S!Kcyh*Ev zIM`wpLwKLMykgONpZ7jdp5gFdD3rUQD_) zi0zU()WHO%c@Ir(WL{xSSc@om-NQYpM7=rEBR~T`UZITxp|G&u%Ll5=b_*&%ynC{d zC(KXb6L@x!KQCKg2c3>jZ3$`a-zfxoD-`gVkZ8Yc_7C9}(Qtpgif*+jgH@pF3$_{L zHynB2tItq-(W-+1#bMw6&K$g^r=E+9j7&jm$*>5M<;%B3M4#Kl znYGHMOw^^pu$nAmNqz<$F6~%Uc?NLuV3~nSH1&c6FK^hmYR3*_u?p4c{+$1JNf!Wb z5-94)UXM6hL%Q2;PCgIhiol(bE7sAusKlUnKoYB!daWe$cD=c69@E0Zpwz)s*TCnC zxR&^}+OYqtshz2?HC8?Aj*%H;`6U9_ALw*qfn7zGZ5=bP)lsXsegp`X!K{I1bYiUK zc5wb#sEb39$fRw#=d9hY^)m#Q=)$mwcF>QvF4?P!Ti%qVi>?60A2v?Y6g z`Cv9`X$IxrttM)pNNOZIVhXmRd^ETX);>OdIeqDr=4J zW7AXB7bh#tM*_H9&Q~ly?ETEm-CA8=AIi3)B-Z|{P@tlP@#=gp^p4Bj<$R<@Qv74h zi4@AuXF?z8Q&r402sGyRVWn_chc8X%*z4BW7Id3?dXi>mzb>pKt2)}U(OMu_+rWjK zKP4k42OEAPSGAQ|S@Xn1 zAag;&5Lgxh0>mckIg%qsdq)a$lpSCAJn@9GZ)t1upPl8aG@p^?wye?mE-_a@li(J5ujfvu@NSP2!7HI^kfI-Oh)Y9@q0GN-%5yE2+ zBtAEa7a(t{BvNTr9o-m>3uV2`SLkt)s58l6ZS)$3#m=hkBGSchbjG#$Vw{(n2|k*x zroB>Td3m#FV*M>D(VLL<+Z#BoV-m(z31@4(ob64sQl;X_y~$8fpGnhKy$Wz7f#BVc z8N=?oF7}qCT6AokfeS6uxR{X`IsUu_JF>whyLEM(B}!D0k+<`14d1`N@bJ)t+<4kC zzAz9I9v+UB0!URm&oAkwXFzcV__AP#ss4L0hKEPnhHLdt))Y(9n$l*Uz(71JM*=oB zWx6;CCT&e+W&glHWQanV{>iTSg1wAv)--g+{pj{lXme(4r!AWo4&) zrRunpjiT)0Vz2wV8smDqat#Yw1RzHh8x)xf`61B-zeA7_lP!5>*)mf9Bg)v$p#u^+ zOR#McHSI*#vYi1BJ1SBULn*%^r*SvRzE~-A%U-{&OC`#bb)p}SgvEpXbf;yK(udxv z`3;Scag%L8rnjuDsCU~}$YQsqjR6RkU2R90_}=;tI1sq zdk*FZRMMW31Lf4?hpF0=!^&=khX)JlaedL~H71P};u74MBQ#$ksi)7$5XlA^ZM)xB z#ff^1NTo5D<=E1W=w)Y7m)%a&m-FXHq`|#36D8LZf1#@>;ukY1Why+sDp_LMHQwO* zV41$?GbyFZ_v>P(CVZzm%ynlqgODfGoQvJSc}0I5o+dp+$#Hfcrg`ZdlqYv(pCKbX zJ3GlzUE5R$d&fJB?4tMYu#hpi_971Nv zjMb>wkbkWJ3&k&xF=RAFN<_4HA)Qpzw}FrGfISSaep*pZL;V7%~~6oG98fGx3f_`fcqBF53x(Gjg}9r-{2)XU9aBePC$4 z-_TsUkUm&Dd~Y~@Km0+sRn$S#t9O=>-g-oK8X6;cNjPI4W)_wZZ6)qKioa4g%FlLE zKOYi|6p3G^AnP^kG zC3`2L#42kvl;CB17ZEAuj!I=J61Dx@BHU{A2qMx1?LJ{oUSFXI#9H-Ry=3_~p#h0z z5-tr13#IyexxD#P93s3>{CW+J0)lKgav0&}vhldYQ*HX>+C9@?OV+n`8)@cJ_UvOH z#j9;j&StBxoJT28v6)9B;^HWv>%Ix~)@u6csCr0eC<>0%^Q=yUa&>#mMFeY~esn*Z z{g#Nop6(4Ji1beN^gHT1C+DEgOo78x$WZlYo>E*LMF`}g6ch%MAK9-PFTeas#jVwT zy_TgA1V!kCT1`|7wTTy)(UU48!K4@|_JLx(ApjT+GY}1e;@^8-j_BARJw=j3FEjo~ zu;o!w!@7;x&YYf=`Zgv~fQwvE5M?uI^7>97!yJh5#Wd=C8sV>WuC=4r$ckr#Ufc@e zG3dY3!6jH!Y`n+CUpOWZ6OSp@?-Mhx-6KhM)XsZ;f@W|te69!s#eX}(bH}Bu9Ort~ zlLAbr3~TpT2AmMtN(&F~n}f#C#6+q})2nw1CiEVT*UB)YQrwylwQgWLVr?Ykv&cv) zp}eudIfgf24zbr>lPSjw%Y}!X+EO|>#o5}QKap2_9w3k&8H29jJ&{gmx8Kmv>L3Qj%-Rt)wp?AR(u~Llu=6l~;=tw>Wt7&JH!%$H({s z1#wGD1~`zSJ8;mLDUzxW8l|MCYT!doj>ES|6gcpGh80-d-LM!~Sj31g#O9p7*Pi9> z-P-Snm++)n#^Q3kQa?mmbtZFFsAETrWs^A)EyzlX}-r5#_ zjoD#uotW3%LypHSFw;r?sxnGeRaH7nh?jNKpm`{m2bRG>7iF^#Z8)5C2x5c=3e)oN z`d)r?T;VWTj<{ju84mkk@J@O%P1`7`%uud=x1|gAoLyUftrHAP_dDLklp$mr)lbN{ zRU>P?Ibu_iF*e&g+03z`k{{{w72e@EwuA_}O@=T=D5*A2U!ZszMD{-`04`oi=h%EN z!5n?GW*L-vt7mW%{3%;6WdCr080?(@5X1%ig;4@LV>9t_Fz45X@-3 z>m=nqVr#hDqb*xI;BFn7NVuHk@lfY;@ho!RTezI}L^xcAQ!SZ+zG@}qA&n_aCKV2; zlx@_;WSRSk1!~y5TgU1SL`=r(k*UBgL#>vIGq09U@d+H-q{P7jZ+GWhf@*5$#iG}3 zoNW_5eN~9c85rZAN26?LGh@T}wgiRLIND9@l#HPysk?2Ss$h*%krMC*_c6a$bzyl| zQA5Vm_Em-J47F~~B}7(LzPmjE`_Qp&cixTB{&wI4cqhl#XV3{s3JRS8HF`m;i8!32 zI4<{@=B{?$j+A895#l_^#A1v{?ji~CrTss01CTsWN$@gITgJb}&D( zxpuc;x+BO}?Z}8!yA(=M!wYmNy&QcTqxA%X5EY3a$4jHs=1Ee{+9G_wu|^Fk@+{9AXvqT5dOz57t8e3%Z2eoe{Gm> zq!5-zqWa z;pgzALg27JA|<8AgXh*QAOKcLrBv=|_17$=i0^rnExPoU&ZBK<{L~Us&N@|;C^Cvw z@Z#ZNF_rkPBUtrr{3Gt`)J_EYw606|$~EPFh4OQ*TjC4pxuT^>2)Vfmc(?n{ zfVcUy*6mvn&mr#lE*n{sT6T%BT73BjxSkkVV)8COm*$qGmJTa+b$YxG(X`L;+j6C4 zBeHoPf2u{`m~eh+p<~{2udc>KL0t!e-lRurToZ3c!uZHeMPxKuF&P77(ZoOx2f{#g z$e3|DBIhyI5eSAE7=04{*WamJS?pimj{Vr?k){vFclNUy%2$^6?$bB8Nap?#O{=Hp=Cq6M?pzbq9iSZ~bQcsHy}$pZO7FqM zL`;?ph4bN7!}^3?xvVC@;J!Ri>z&t-VQPBax1b;tlmH;=eJ6|AO|-S&=M0j{$x2HL zC^>@h@zooyN5lvgqM+!$+!)~bH8}XKE6A>t}3>>0jy&W(34- z-O}#Ih#jiZ#TE+$FcxT_*z??Nc^uXs+xnMvR&A)m8cB<)OLipRMW|6uYZadpO&p zr^zU_udoVaXyIF0mS*^h5Q!OWX)BD!ob0AtwC>9*ss@b33-lb(BucSxryM-BaUzMr_fvTX16#~z)B{$7c-^O?esJD0f{Ec1Fj+=GSkFBkJQ z$;->L$|m~MVuHCJhG&QUJ62YdR$%1hsR8aXnD^S=+T6{muC{4sH^KfvuBm`=b76X;3o*m-xTOb3=w3)zgt3DgI zW56-NDlHv0GUaT_>=0H>c-hGF-hDoOa&M(j)P9fKd46F0Lrn3~^fjBF%!gQh7Z(T2r!31r8cALd>2uN#363G%h;s zzyP~rpweu>>S8W|_$S-<9~97w=%GAald*xC59=4KNjj#J#Zslg^2R@+b_#3b4_r5R zHuuU-ItfGR*tb_hW>~~ntp;rL@xTKuQ|W`SwLM3A;kC&=7P^4{hLTdUeZ}b=>*lY$ zb3}L_848r2bICR*P5F%U^m2|)8=rR1U)O2iqdVg_mNq0tuOuyn=<2F)@lrS?3pkB<*)GA7!cKAh<9Km^98_|<4+tz5S+k|!qzd>v@ z6>D9K&&~r)-s=vkyRQsCvg<+mQC5sY+g*!1aHT9pHpPeP?`^vVx zTb?$u^;2)d4B@uDl;mzIZx~_Po6$M!*DOT{{wEeM~JGB`KDs8moGULo7Ax%;b zlfPKjTNYAJ_nQ}3!SSG&rt^mKrF}bc#LL&H09w)MNyEOJ%r|eoG_94smt-igP*6BH zZuNI{A2rYdVFKF(tYm4%J`zhvI2xMow{pLuH%XdJgin`MnvwBVNccRmwca_ztX3%c zNHU&uQwCt**mhFT(OKvU_HszfV3W8nA0Ip4Io5MKJ40b&+eE}ibFSRksk~gWbeygP zl}RDT--M^VefBp*ho@sX$SW64XRk_z)3kH8WG!jEEC5Qi{LSP751+oinCY2>f&zv8 zjW;%(N&(5Zg#Vt)8!rWpn}!)vP*pkf``!Skz?T+#&f27fSfEg-TY)&|e$!XyF;ehY z`k(&NrB9d3J#_4MGqcdvewsJ-_ER6Pv^4(3!UIIgRWB*)`L$HC5mwZZWAg6I zLT5_yT;Jg#nPO3qU{<_yf}SY-r?RQ56B`$&rM73W68;|)P*N^V!2Ku$=4*tVeCn&}NZ#{Hiw|VcpRl!+&?s6wy(jPqTLF+p4SGEMf7zFDx z5*zGS;@p~AT77U3tG^HdA)z+_!p0TxX_)NS?j8^o*R<#A8SMv+KiyKuJ+9i0l}L zYI$8#3w?OwY4aoF1I5wRGsk*&zO;8{d(40_oH`ebCi&L%bi*0(3{h|VKmssGy8-em zuPu6s95Gffz3M{yUOW-o6iPxqHat7PqH6nH;`0N?dV-s$SjjH#_|g_G-d~<{z~8XC zdAK-3clGoT(b7iD&!=HZGcwaHJwZ{T%ATy$4TP!cv5~#ukcS#FM5&tLw=QL~_W+z! zk$XAGVC)B~>ltOrS_XbCi@hNJXD|~L6J^nLSO~=|8KdPPY-HGs}S8;j99Kf?uS;WtLn||Z9)#p;JR7(8Y#)76q2@h0e*fv z)2m(Oe@F{0Wu1DXc?}ftiN#Mu#OTA5&56OS?(gSjV@3Z~ra|INs-nUHf5@Pb*u+0_=V+h)rA)HhQq$eF+!pl z@eRwMQnwpQkOgyFUDYko>+uCs`>JJR04!bs2D`7z#6@IuD&nh+^&kPZ!{MIA;t0zP z>Nr}9PicHn#`d~wN<`gkE3W99wEFbV9GA~uznX?e^bwJ4lhbr(0ZdxwGzn(EyT|8% z0=5EFujdRy>yV7Jh z`W+$s=ylJl5p4(nZGclSF+oQ~6@-5oY7GcecJ`}iMH=3m5wL=4Kif174-XpQ7G$SH z@-KEcpxfT=28?Qk@3xHbuBjK4w`uUJoA-3eSAkED1_5R}1b~(K%?)u~7-XYUkANNi8cwj~M#p zW3mHS8d#b;MF-IkBf3P4wvZZ5vpkl?HLr6f&j%JBXu;MM_)fP+q{@jYJrRvgD2)pP z2=0f3)pf^$ms1%JUFm{xIGm1Nk_p_+N8`vPGek3&u4g6>?H72ipV2iZ1Ln8^1HEFa zwmsvl9xV~>HJ_U{hKJ%JB$TSO;9=d%?4-xjWDh3!Lflpz+b0C1xr_`Njkz;tQzk($ za@gymeVBSg!6PhtIXFJ=HqY66fKco*Kxb?sHsxsZ3@hOSoKT@z4{%|%)eF3`Q-?Byd9&$T&YV1Pw zrlE|z)AN^I|CB9$*VBVUL0ucPa2TARmBjdikU6r*WTYmOmEtM;QA+5@LkCOZBK=AG z>H;N_Ud!@w!H$dL{0F(Sz^yh8|7+*HR=u}i4fyjDemw$VCe(Lhad{+M`I4f}YkRYC zL8pvPLVq6c;wbw6MJD-)IpK2GfTycHp-4DqWO#hM$pQbC_7%RndUG}3e~(l`0>Bx3 ztmh5ZTL3>o@DI{!4Gz;8WXMu$et6y93JSb^Jw0uj$T~#`-TyRAj@)T+=fbAt<%AX^ ziS@BA`ojH$uXf`IV4Rv)#|Bg7TM9rW3FSP-2)*BhfiVvO;Oy}e^78UcgWo(K+*db1 z*io`H3HqNf%epq5P`}VF-PG)w$Ve0h4U5l}_Snn`>VVvkqsYjsi`m<=8&|-_n1t&D zlUlO2XV1~Huz037<4~a?-ykqDux>hzI6K!z4-Y>@)5~5wyLNKgSYn$fQRxGk8VL|Z z3TC9_<$Xt!I%*&Q;mKE1ZVJKmW#wY0p@kp_j`d!QNV&wgm~|tSlt>`dr8<>^@mAQ`kMpIQj zfG=(FYU3h2A*L&S>?7w%j{0*?0zgj;P0HxDZNp+>MBUsVUv*$a;4KIOJ`yQzFU8K` zfF}f=@SsfFnbw@e5BoI?hZGV8hDL_t`5c`%@sSgLSvJ``SAB?j(8AZQEEJED?rm^WGy{?HeccMg^Hff6Fs( z+v<$}HP7^B)J6+`xEv4$0+;*we$w^mUKYBCFfXwErUJo28U1u_v^dNtdl9 zwHY6Yr(l7ctjssNrbq4x*M|S4FUg3?p}UZ6Ec~or>=t z#Bf6BUg2_w02Vv0IzBW!+?C6a;;Ltm1BQ4r!@#cdNXO_Fqpn`o$*}lM-03BF*s@CB zB;DuVva`Fexn4T%Htg-qeTYQ=L{Ch=btEgV&Sza3hf9ke&-6Hqr)O$0u~R-D!2lva zPSG9azwq>UJJ02bh*V{&90i-*_AF_!gPSEX&=F`^=bfK^Hk(Oa*{@|0FVlK;>Wtea z2zk50;~ocCENx?5!j~@uuyDslW?TeQ?F39?-F6e|eYg&@{1z!m!gx+WwcJ1bIm{P2 zY4JSdo(8m-f~lv|T%_=cnb-vr|7)T`=j#fNrGLG%+U?Lc?&5J53K3tu2V{QXS#n?oAs$u0F&4kxz(nbx0aZ~_U zU|->}OY!qZ;hgee(ZTr-Y=km>3J8P+$tYdjRYoqZJ23jVk17wRO$vg7g2JNpmwrT{ zvI-^2BS*u%i$ey_sHCvEV}%g$g*LM!6G8v0r-FiP=oPd@xJGR|XF7~nfN}M0jjT+X z+6VbNgGFzku{g!+Icfz{)+nI<1_yN4do8-UnXOoKm|F_FcZBG!@EJo80y%dNi8|&I zpeG|YBJ2|}27Ftz0ei=6$rICT_*ruVl87fcBX;|AiTi>5{BO&N=Y@kkO3F+aN)tj zmnC%f)VBNO)yv4kW)p)Zw`~c4-3SpCO|#Usz$s>I4Eu0p7GALRQLvaE5Hz_75?0mjmbhpLbYjzbR~Li$`~cjPil= zLPCiG8Q;Rfz`6hl>ii%)u3IC|XO%njn?hM>4)ieMu1+S!iJ{q9QO$=#K?4Ak{e&{W zeI_7y(#S&wz48<}SZM12f00u2PLZBr1i*Y;^NZ(8mKdJA__ZlRhPt-45>FFG4q#s* z0FArhy22v^D3$l?76&Wy(s#HMnpGO5!4V^DK252iOqEZ99^Kv~;6bj_pZkd?9;UDv z%!Fl_J+6wo({7Of{OMAQ0cGG{5?YI69P)pU&nkb1hkC~2a>_%O(D%3|I$dw^h?AWV zA;PEE`bANIIzM&IY(7EM)s;P|JEt|6Hbk^&Ev<3v9!*QhmosrVdvfFGJVOZ&&#FJ! zv2`;u!XW}U3UJ6y&tLU9rkD7`Af*1%rfaJtR+advF2CS3zQy<`*4l`Zs^?tOl6x95 z2F&acuuZuZR_oBUMTI&V1PY`BZT0QYi!c$d!a{Qj%9tdM`yTPFO-^FoK3+XN%s5P= zN1pJ+aV@GIgk`xEiV9;)S8TUGu)unU12R`)h6;GG{Uj0437El+HK>Rm+F>loBYOEN zS>Mty^3*mjsN;kRD=FSVFfQIe(zz~HVxvJL=l_I*%V#-b#sUIyAuUUb*E{1U_0agS zHO-wBCd7&2U^o>kWye1c38Tv&C$hF-7~HYjJhQRrMnTgpbG-^bshokv5qK-x-2(@l z{0|?g+mHc z$1S0!N9XVFEGCAQF=o8EA6N5LUfJl0R1RE1Ux-+iK$cuPvPVq9pILx-YKvZ-O89@^ zrFJz{zmM%BnZIaIC8JQ0u(BdpxL!xc#I>_NHAIGJygoa)yo^SWRjKh}GEuMwSA5ip ztO``$&8;m$B!BCHIR|Ze%-q5ssqTU=sXJzfXIR;AGE5ilVJ{Xu@Ig++bn5t1{pHBemxz~)oXJeOf=V+J8YLQ*FeDVI2!x6L_M+lo9rz|CpA zT~ly>xZx44JXy@sykTj$SbPRz`U<<}yxd1!16^5p8(k6rClz&7IA{{~PzL7ak^>EP zWNb{>;o@^pht;qll9F6v;FKv5&MOA??d37`BCtWXt^HZrh>4L>St2Oj%rg*XdvhCo zEw}d&wHG)UmD!QOEG&yhj#KqFhP#(3x&4=y&!Jy2m*nSvL$&L{uwrF%{LaRV9#L~N zhPqsk+F22OPazg*NOW7wOuQ9UWn(};Jc5CR7iJ*XY1KArEtVwJ<71*|d1}Z3%*10| zSSMUq;Lb+<=83`*%r6z`f!*d?QH_#MX3qqDXhmbDX9>T_6cLKQ1~=_!wbHd5(C3nq>|6C ztZ1z&!Ieq$Pj3}*nLbP?=%4doBwD}fmBbDjA`$)d17d<1wOha1-PtLAqQ@${ zyi=QzkyKXx!ujcs4<6B^M!%S68!ysqS$Mb;&Q;L>V1DUW6QS#lDvy?>rP6<&Q>XqU ziy;=b29*sAWdP~7*vCAEFv(D4)IaipfCiW|W@gM%iMwIX44%+qz;j+DJbY@kiVt<$ z)qwaXRNCZ~@ej(Wnlq1qnfnak?h;oY4=I2<{h8K!))P(1RxMFJHNw<3wuB zFP8|nvfxh@r`^}>6__!DT!AiUh$r6@rm-wtgPaV7Jj-elwsFU&zB;DUcc46zQGj=@ z(3L#AXYlk4VJXO&Fd?U=){@nS)hJ2<5__@@Zr+%LXqUE}Jk?~)9wU>T$edF?+_#jN zuI?z*-AOCIk&Kn&r-;LT$}dVw>B%Y7dXwv+J5?gP`dHL`hhjU0d9KM}HefS9P$-vW z!zT3g?{m3*pw{f}zCNC$y#>LRDAYEL;A>25>bbdR7CrY9l<{(W<;y}n4F)}jzu5If2b?H-_ zEiL22{UB&XE1_Rz#htMq;KE;QPhhs3P*)WZEjf-z;cNxk+U|^l$kPNt9eFsWii!4n zb%n~vhpUu_#VG^LNYjlbmc}E&QDY{S`JZvYRkObUjplv{+T!e`MOZDCfPFOg-6oNT zv!3DHBCxG>x?&8#fo?~foPU=k{}*oT2mZ3s(pW=A3g`xqpyE58S-AfKn(bf1dcTVc zs&ls`%_0^jigo(mrN=farm_^kGFXq6N*^NYz>>D?X|DjTEWqp-!OnoM%I1AQLD5w& z?gs&fL@9LYj#`MH<}G4CP4(gLIRcyzi;Tu(Ix1^ zu`4gPweHr3egh7Ok4H*&(w|7+&X&p{prlNS(U>0i&7%!&{%U%Bf;KcXg?;n z4B6q;Q;^Jq0FEKZrIoF$@t5iUhh6)Qn^$_^4X9hd2-MoIF+ukChjPE4EK;g4ZmZtI zpKX=)xP}8F2!z&5<;KsO{k~zUSWipVuRUuacQ+>~jWNOx8&=cPQh3qk>wj+E-jNgU3qlSXyFXxp zNuFf!eVN%~W5MqgBLprHy2dPi0;PH<&_h&09EQj2?kQH{{+JLmCi?hL zux&od9n-Bf`T4O3vrZg6dEHMgyQJ|Vv}k%-XPQllD(LCyg;e6b^Ar>oF|4E{m1utH zO0{m;w$-fTg@(R=<2S`P=EchK0!XK7qYZWTnW%uV4;gBi@iXN}hw79~(UEKeC7Sc0 z*&PY#!{~UVoV984h+$ba`@e{I<&xxnY#q@N%jm||`j%{m-(I6bU~TKBYLYtMj8 zrnmPMBXebI<)3W0Jh0pyP?U|0qcGtIUZDR$LS6}EyjB@ttbmADohGBupjv7*q5&(b zroKJ}iO$LToCpdrg1Lkk6jI3Tc>kpQoS& z@tij8bWn4*e;|Q~BT(J7@iF2mq|BwDhRA>;&6g($PS|GE5SwNq^AQX|tO2Ls6?-Nd z`Fb$Ot>Nl}gctt(p{A_*M_yK(CPR8LvKaq~osONep%I48$=HO^@>R@(q7ma4zBvw~ zo=u18yw<6#NL<^X=2m$*tBIhX+uQ3?;xQ z6d4|B$aqam0j}%rVxi;V{x)WE&vT?QK|<&Q1uEV^bvZjdURTu zRGzY_f3__K4>gx>2V_^EVA39N_Tr~ZdC%;9I{UFE=a9Vpy9?W@kOK!)RmjbO0W$-m zE-~i$V_x9LU7ot(8*B}~h0(`L+>-6}A3%>i8=m0yfQS-fGE?RIiq%MK*>&@M=)1Dr zN8+k_2QJY3`Z5qL3$Oy3!%KI@_*_8?U~{C)c6N$@l+9O_0)0cDkcI{P_XYQXZWG^+ zZgLJ>-XN~FcHx1lx0(<1JROluIJ$i@&{3#R-BF07ySip=hzB!`*G(>@R9N zqtA$iiHjStNW-V10;{P05i~#CQ%&_F{}x=vH!*#NA^E2pTBZ;&S{tq zPozrGZm0a8VX(<89#n6{PSfduFaY+K1?FAJ4Ujl~9=|@II;dX40;rR9zBHykq{Vd! zE=B*YW(-i(0slWU%Db~ini^ul8{{ziZ)YuAm;n42G{~W@lq?M%$d#WXLjjGE14$VL z>2;C)uM)hFa{_T)THAX1r8?H)fw`T?EDQv%dw~r3xa1`C|FgvP@BqOraDnR#gXeb1 z;c8Ak)M~6-#m5)7yu4hLHRjEKZRws!$D86?K5E~zw5?i$vL7o@9#PclMml-+y&QQI zSyTXvnY)?@@ZYDDjaJw6TWOe*{xx|8xDZUH-#DI)sHAB6Lob`bd~Am)Cg2XTrHr9K zO5fU2I6kJ|+TD%twE1#*)WVbT-BECNz@gzJrLM*Vyp;Jt_{rrzaeAB7y{6;W7+I>4 ztJF*DZC`be%~UlugQ_AdC4~mkf*5&BaXg%N`QSNBwd1{RL@^mIw`ocwUOCDFK5xX| zS-O*{e_6WnC)w$^=@G~q z13mp9Hb?g5z07t$|6GBz*+!MBX22bccME`#n_pwzplj0rs&ViJEMKMCf=sBWdXDuP z8q@a_3lH}QItw@G=s6V?fgn`k<0H(*#SZKoXdsrRju^dw;xBn(%PcC|Zj|yi3DsX< zvi1VQ&Q1@6nm0~2S)mas)Lah)={~}x7K{nGxv{a#)j>tcf$Fok+Lg>UhKZj7R*5vA zJVdbbYvml>w6`TF3J z_}}FP4i`Gim!>}}lPy^#Y;1fJpG&rK{v-0a@fH;w{V>NsGu(7zO0n_W!u6mD+VRM{ z0F-yv<%pCBam9g%IPe^J%Gko@842L{|9`>FRWsQtN*C}KYDO@T57%(wa3CS% z3Qk*k?WEv^qbNb0GrCo7`qLg#IJ9zPhJ{Ev``bj?pJXn)0C5@jUL_ipj!!KuItmq7 zTKtW3yqb-SpZ*|&%X9Ame^X1tN?tKhvn7XDN4!}eOZCrn-oP zT!}#1|F56dsw&+7U!;xyj**sVfo2Kdh5SX2TlENoF-o-5J?1{f2Ki9B^Gq;*kXKN6 zBOUw4chtds2jR(Vk4xXiHHL>%nD_+M>bnpG(Km|UQB07)D2Nzoh0Rz148<2>%v;k+&*1u()H*W_wb z_+U9&b)LUznNwj@(|2G4NRE;5#%oOn+`oW{Q4BKe*UuI{4J3Oa_3AyQQe|@^sg%Qo z&CN+b>Tl-$5_ZW_064aQ@cz#`wyTvh=F5+F6lNJV=f6oW$?Q|I#E(0BMXFcf0|KnO z`b7;k`dc<_Rj{yrU|^?cIVzFN!FdBZ`{h~DQAuMpOJ&~y0)*v1XO&+Qo_RL?DssPi z!Bu~E+tbJiRs|5i)dT=|;kF%JHg9a!aYdhb-ufBB!{rTQ{+!UcxSZp)ti-uD#%+UfBBxyKHC`#t$ebK>*AP>4B9ETUt(Kop zMS>OF{e>1d5rb43i5*pqXT+wJct9JBc)#BDfxq=v!(A|hu70-lShscZ{5dSd&iL3j zUFP&L62uo?hGras!omkZOl7~JG$ZpZBCKHR{G;bprfPuz%z%L9(D$5-GV}^DL0;Z* z$Q^*n+qZ0otn1)D?=a^Rk#~uZp`{?ziMMl+0!ej9oH>tj)$+Rn+ z6XfH%y)_+R##vAh`;ywpMp2k@7F-{1$k0E1mWm-J&^Gc?6F_0gdQ<6PC z@xZ`PfPheLF3qQbd7aZ4->yy(eBx+*fxb)_>S%-xVr@J$K@G>-E zMl28(ms5(B*3l8)ZRqPWd%XEDX0L!amcTp*+MUGYSMpesR&Li>8aFx+9%x~``AUQN zd3m{q48h=z*f=&Z0^q3R4>U7?g9HY0^9I~u14khN;)@|5*P0}}^yVX2IRe{I7_+gl zZEd*CWu?A>1Gf?M%E*wR0GkI?*c+!j@ZJbyv_SvmbhRIOVzIG@gR`^#Zd$tI?A2_T z0vX^Y!4@zBceoF@SwJ{>jHx}1R79a;-Tbzm8Ovf_I`8m}gJq#T0Cmju;D#r&(c}Ho z>fSmH9?-CuoXiVCy_wm>1CAa^3jC~`^%;2ZK7-qgR(l=B8(s0#xAm4Zlb-yHkapK@{LBKrkS`SpnRb4o|P>1Q^ zQ@4D3%yfZHNN)3{Uvg??Bo1MaizdOp z+f+yD=VblI#;WFG_=W7hZL9)4K!zeJG743K<7g^N_wP+r>s=){;4#R%o~ZIHo^x=w z2D5<3{+{(;&9nBJ8AF|$o3UQZ&@cw>JV~rmAS98KyT!_(G2b?Amj}|;Y)Lv8@YhN@ z*uMNW-|dl=$;!$C zQSnEat?uFO0P>fy&gXOW~3~+uB3#bP`7J_7gFf??9cIQ+C|Pg*kZSPi`91TKO?DW_apT1PBO>03mSYOe(2( zR97s>Og`@e2vcK*ZYVRdGaUrbGY!O_I?~1UyVXl&Db#PViE!faIM>^&@JnRLA%%Du z6fm%Dwdry?%uHbcWc0{;5_fHXnXps5wBYN2~Xvw;bu0&SiE=!I=mIAee*PW)O=DM?AjF6Vavrt}sBS*A^ zWN1*vP2*YH2AozEg1oY_gm!vzcz7F#57)Rr0fj4Z!Q74fd?RZQ8qoX4=1;GdFr)t> zgRE-dh5)iiC}`^^R~`-M!9sl|!r4;scX^r*A)yvmh*??UN@e=QRde>p(wrq9`4UDF zXz&Mk9_TOTZya>>^h%xWxF*!fq79Or9#aG@blvBBMV1eD7hw?*ZPC>D?O>cCNb^ws zG4x4MU`HlcdQ0^n!9@>`6abhqJi|fuPdwEa=KcTVsp?cWTK!v=>ixrX?a5{3zV*Td zexE0?a&(7dkZ-HKX$SwZgj7y>mXV4fnK!=$T*Avp7cKckbJkkh7y{qsFO}sR^Ddv0 zpF-Afyce%YGa|J*K*c52kv%2U>G zFFzg&_u%2VHYy;uS@75Twl|YE-~wk&pwCY;PPtyt-p(8lXtRZnu{=8X!p@GGidxxA zqlg$W)(<|F!9SQu(g}SlNQSI6nM+u?GXL9cn{px+*3x^s7VirUln5ZkV$I~{Q)%B7 zn8=VvUAnG-@D5uZJ0W4l@22hQS688oWUmqYkFZcVG}|C#FBfCkJI#$5bl_NC&GZ2(_DCq&YK`Ck&UrmzcGS>ijJ80Vt8`27rpUR^7;lE z!g0SkMkzL`Nt~8D7#Ori9UkNkXVnl0ZU>|w#OPUf4z~6Vg`OgS`vB=H0Zz>iC$=s&)q+T^YYGcJ6nw#q|UQ5aP?tH3H$A==zudra!X) zN33EXST5*<309D=O>g-5Y365~h)UzrD+;jdd4I&GCj0pKo{)>YD5qDnKSN=xssG*pH)rG%~}`AF+0j*|LoMG zSBuK0w)Uq>EBcAKx_aw$64&-ZD!TMun0UiH}D_0T#|drYoo{QEt$>(qH;0G)*WN%{HIm_dWd$+#p- zVXG;<#!*Lg<&U7|W>cr;KQ#fu_tdY|)a~m|U@d`_w2$$5pESr^3sVewQ znXZn03*6p@YHzO#E22axHl0GsFCnQk=Q#Yr1ny{#>%dz2gh{TN26)vL4j7&Bgp#Kh z2$J`UGI>|4?K?TUjSd#nb6n5RBfs!H(aXBh)uVoc0y{K1Ds|#7#b&jRS+CK7-eAd6 zHFr)|imqQ;cP!+}%|j6te!z8V2Hw?b50cK(Txamo5Bp9qrqSaCwLS~C*B(++m37A? ziT$6tn8jdVnDN|nk@FO&JIoJOd=uQ;_vfIAV#PVn{Wfzyh@nw;6A=d}GHmoEa}clL2$H-SmT;{$<}x;>GpY0LfHK?R%KqPSYO9Qq1%jyQv% z`lEA4`z?EPBm0N8Xh&k9{qz%9_-xmqZBemooV?%PldVyGtlaIvhsx(bBiVf}P(>a% zTnq|v!5#s>yp%wJD0k@(~D1xFrEMk$Xk;tyFAaZ{Q%&D&3J z?1&VqTIBkh7i51mayttmjh5-SstJ3C;#(?AH`MPL_DY$VE8J^U&KBfz&|mLr2@e^$A84S z8VsMR@?|5(AT2emMkr-aXJqGTX!fAFl$@Nw;($kxi#s<@X+%VlgzDUJ4UOuLru}%w zpSIzx9JF_=`v;a39waSLaJ6+wVYb(ZUrI{o8%2WU$PU^2c#1F7eNVSu+Rcq@Vt=4M zWuvrO)Qz!zr>lh0+=4)_Bbu)QlU$vl&LLBD>Ga+1$@?4P#YtD-?Xap@6M#=K`BsA1 zSINn1Q=WIc=&4DHo7C}u4_Kj%ysZ*(db$pje?g4-EowKmY%&LleuKrKeuu3U$qgt=TGc z@5@IxTgXom_|)tSh;U!NcySiq@Yxc#9wpZ$-0O@e$;+H*Dc&VvAB+c41p4Ijto^;j zCacb9wD|%Io`5=pu=@%v!yz|W+*~PQpK}{WUrhNSQumxzV0bg5k zhlYe0l&Hf2=W2ptOZHfyH@ImG##+h!DKBBWT{1Q_?`2zCRUzut$jtfl6;6vX1R}zM&DP6UULqCqo!j*XQHK{;T*_Zo=WrVm`C#r z$huW0)z=sKmXD8z4pZKhvK{B2ur7^X1C9EKE{a9d(Wa)-r$0xAMc#hHdv1LUMmW>y4nXaa2s*zTC>tHvhl_%dgG;yzLz^$~ro< zlbug{g#*;rc3MqpewOx#ddf%0rYlWv1?&FcG%Gbz&10P;rKyh&cGBv3hEE8D*@jS^ z=x0g_N_!IGyLNU$Yip_?xH;&x|8a0=+qd9_aLf}}KYs$O=Obw9g)15^i_6&TCOhP3 zb|xP&y=OKEp;=q*%V#{j#N=CD?jU!0`rOklrt~Oik8zgOmqe5f=6+bF1{&6Y`!elb zf-D1^gmBdbT;D$>({YaT_1Z@`S?p7mkJ{m0yeLCI!o;7{L>~O|K0s`v5Pj3O8?BYen)H^C0jGOG>hPdg-bD;(U(~P`&;AugalDiXIsNvQ>5>USPY?KIx3~ z)yk2E&=XmCJcZ;X90qpB8);nBCsXUaqnN>1ABUAHliS+PwHb|>_JIV}rV_=riwKOK zN#iqhneM8sDW27Kb))K~obH=mUcyq(FF=X_$S^a4njUyaK-M6oVB%$Zaaio(mVI&ez0n;3w5H^$%lD1V@2Llv~!+-GM_ z&HR~x=XBoH)s=$6HX<>SX;@-@22Fh76g@tEAngcPJ4`K~#Mq?SUxOorLuYjcU#4p= zwv89&z}SArIpjrC1Tvi?Vk4I`l&oZDNlQCZ)#ED<7C;mDSV{i^&_wtqxm>(+EfxZM zrCdDE8ejD#`Bcq6k&29Bo9lHNYtCA&A}p0|sd9+Q$hh7{2qSDRn*GJ2jfXm(J7@Q& z$U?tkVmwhO5l38*jF_33(I*C;I|-@@d0s+FpC+jKMn>16QgD(RPeh{8@`m%D#W5D; zTLLGcMYuSt$Gre1r=TXzppJ_z;M(1x0nSi4+mC3yui@ck4a0f`}uQ1Jj=Nxc^OH}3{|4C@)W#V&E!9g9>0MMSux#-Y~(r? zEtOfVlbMiQF{izOmM}Ixm6tbWXV;yeOXLpw0|dc>8v`|3A4C`V^`18iYBw2hy!Ly& z)}!`BByTs;4&V^Stq8@oO)mwN@(j9S>)6>paxIf`oJ#ozTc$3E1QcP&OX z+0F`mKl8a)rE;~d$FQVjZBR`I?hkgr!8YD}@72o6v1nFwD?G}PMs)P_Pa#u{A~ZeY z(_3`AhIb2zPm6(px_IM!=qrWsFBK)}F-uF_Q&X(X&E#zzU4Ao%hb}{K?sy&{TEr`% ze8DeuETC<9BrB(PxL?w}CJG)wdx4uTUJOGk4yE%|6;jb@qhn-yV=gaDl(g~cNV!_d z)+NZ3da0GW#XPm2Z*R>~D+ob5BOl3|FYn&|Ss9R6g4Q95hi1Qgm*>UWi^9gXh?w29I?ul;S z2oXk^ZwN2UAbLtBnBkY@};#hyx#M)|)wm_SnI{DU$Qo2a9w4&QF~x&EQBF9kq8| zBCc}FAf)>f3ew;MRGd2XP;>B(rFJ0RUv6HSYj#CCe|>QWtiAtP8vHj5zQz{8vXcK# z@ES16Gk@yzJ@+4Nc?2(j!Zu?0E5Pd9RHyk)69dgZNaYIINT#+3@_&3gVbD+i{iSW3 ztPY(-qn-bM6DCi64!_80FkY|&&1jp%_#dPdKWp~ zN7C5OC@2^KVN6!<26BxWpBY6*>i{&2)XNuJD|-}ulLfGaomrc>g9EOi;dtBN#Z!;U zEC>f4k!qGxZ8uzo%d1j{u>;EJ0UJ9lCnqt~%&sto-96toOQ83I>AQexPMpWy;`|~~ zOA+#;BBSOaqDs9LneM5m2zD_F%r?g{ws%Mm)$D9xERJRiJW-?y`=_F$(OcBJyiVgT zf}Zs$S;?+1v%I!y@SL6km}9Tq)+C_H+QxX*(TEiXE8lq*EJ-um{O(~#(a7HR>sh2TP$jEN2*jcQZ^DcH{TJ7pHJ_{&s;CV@#9$$tn5yw97S#(S9&S8YFC}L1 zufpN4J6d%p0c&jZ;n8+*2&T--1E-V>-Q1t*e*l`0@lnFDD>zej{yL|(uE z#Cl`--roL)Clg9nNyi(^yIp)+ufpO!#P?^`Y?deC9Vf+M&0%UV4$PceNU9 zi4U)$lbb`RPJCK)cHdpCH##>>T2_EWl!fSNQ`_Tn_c~3s8&{m5cpa-$X!z+kn}S8RO4A z9dV}XpQu*K)sU=syuU2}G1yj_4bYpm*vtVn(bJq2z`Cm%dEywu_#G+y!nTp7(Hf7%`6S<>$?{o zF%4~`hpeouqLTjaM z;%Iqd1v5V(BB|0#h_iCIlmrxHk6msNh4)R(@vq5lE8)hC_`rt<`ulUP`!rthdIdV?8TLnaXoZynkpzr zmiO#qp^)ylhol;L1b>K)1!)qEyMKhLKDhMBCO4|AEE$Q3>g~gNs2*G>wA{O`m9La7 zi26klM@5NKz7>&p_%J&W>$crMn~0R0&r_9=4GOD2oK=UOZD*QBch0Ha>-LAr8B%Ll z-@$VWfq>b6R1`~qEs+2tBP&L~tof^b6)iUR3AOEv6X;S*Xls0~txK*x)sWx5;yfTP zw2_hJLy9c<;E&^0s0)T5pIe{P-1}(y0A{9&I?+8V!jkhvuuFuJ=1KE)#^J&(qU0@3 zSq8du$?K~~;%ZJ!hJx&fq$IG*>ow&)gm5KjTl*;a<*#x)MGjH@7Vk#H&x z{rFLj6Ooe*&Cc6AOTQ%*-HB%5txv>&QchPP2YmmdO&k=;a`j!uG2qX~z2|O9k0LNIi;0cFg0Rc!jWU;1>3lK_6gHJ!TO~Gmc_w>i2nk;_UyDhf(syD5WIkR={vT4Fr6sC zBdeR@GxSJh#+($JcY7A##N3=b2?;}J5d@e;Mml4K#@X3~L(3TJ=5O|2kmF}g1}UWW z%28MU1^`myEH67QENB{cilr+7?0q97fAySLR)nB>m!xjxDiZSXQa}ld3;X3>|4M`M zw6M0fhqX=2`dy!K&$|0e98KacZPwkuUfAXq^)=@jmFPF2Rj9(@sq2YMUEG^~m0|UW zh=GCpHuhMM#s=5!neB`$R+E#HPgR+-fCP!FJMOUP{PfS=q4N3=wX&tJ&d3uvYBA7q zo8-c{z!ve+#RPR*$HS5lSiSz7Xz4Njcjla^S7FJN>Pc+RVR%DPYHW9`*V=P~HB@N# zXk(g2lvuVdf;G*lN|KXBWMVt0yl!&`rIh_ZrR)heWInA0ib&+CcD;+Dcb}RbEM5y) zR4>GyOt#6Wt@M8RWU#-;f6%`#AHpay9-xu;#w#9l#iAh^2WL_F52v>jsH-6W!oE)i zZVe0^^70ZQBezetUp*Ilm6BM&g)&&eXM*Ncgl)2vY>>%p`(prMt6?B6W`dxt^iSu8 zjc>(4kYjX%@ilgK--kId8=YKU0vPjfOM4g?2T9z`C_1Be3&p`b6ZuC=o(4u7_*Us9 zIjbIWWI#9uK<>atAo?YQ{j(;B5|$ICfV(~Q!nU#%4k##IQ(GsDAO6ZW4u#7a7kl!v zeGOG0J%P_{Mh8slN_ZM&2xQWfzQbO-obC`SoapFF$6!#w=r#Ov7w%f{iR0hc^)jUq zl*VuBf=75Hv=aGmsqQLlcN%@=w=){!&PaAR?PJDuP~0fQPiMjEUC<(2 z1CR--l_%`nyaEdl_>?!-R))x-C^ui{>M^8dH{KY|oMJ9rz!g;x)Fyz56U0AhCDIqy z_nU<<)7Uyx?aVFm+olB%3~aWQK}Vvgd4ir^raSpa9RL0MZNGWN)qvC^5>$xn4ZGO_3k9Jb#yH!*{#4(wap+Z)crqTLA)(_dl2 zffOfEO@7TN9%^twh#ypgoJf4Y8mHeKk)$9f>0hW6KUNV>&O;xQCz zFec!IH0oQFJ%SNFWPp(ql~}jW7-bdN&!-{FM%<`ijsdzG7`sG9v$al=C*Rts)C)i# zy^WLxkEGG7l;E;5tIx`$=+66ZmS<*uJ^}gz+6?jNp5)1mV8imWipt@VvO!OwGXlIN zAi%i#wQu;7jH4q=jxHG(5P|;q-b&5)Lm}h#)Y80!{gybGe%+TY4vUMsV{Tran7AP; zzN5Ubc%^6f5G8uRh<)>6FmBKlf6;x4bkXH-r;B8Wb$lVhHlsArAacp- z1vQYI*U`D@>0{XY!dG7tENVS|d57yfQL%^HPZw*jvs(Nb6rw_?F!i241>F}Dj`#+3 z>!ZiVMLhQ-7!i(370;39|8GM_sn}0%87F^8Mm=lHmZ?8G$gNcAPPk~Tj?Ibc8BY(R zkISE1>xJ>U_$R9(1f2T}+*Vl{T$kc?JGm2y{;$ZQCtwWa~DQY_O zNO@&}OY-{2H}|6xqO(Pkd~2JU3D}N%VX%&_JU^MPH&~d~6?!b~F2Fz2 z18jA}=mTW0cBIKntMNWlt~v=`)Y2SdRb)X}n9?TATSF;*;*OY``X47}#2h?~*ccqr zb6A=)Q`_z{5&1SvM#B~jMKY6}C0`c~w3PpkvqbGBv@OUAzJbu#?}9v3@nZRuPS9lTv=YjjBEdZf;ZsbH`hD{6TTVG-#Jx&9TJxT4OY#! zM#ajtvK%4?Ril(1#oqi|8|;n;+Ta|N%F&)JEWoQmmev2js5yUc^rcT_QG}t=#QRHf+7C7*tIs)&@1~pBuDIqib z4-c>5h&W&bBHOb)w0sR?fhDX-CAx5miKUB?#r)W7MZY(19OD)2+x@vhu@P_czGoG6 zbY#8ieU1+LXohNBPD9|tZ|P3UOIRhI=Qa-Cgu5_<3K66H`?ZLpHSE%@Fbj;>e*pnW zLWOAsj7!4Si;quD-*m2>=BBdFsvTpSwvB`zik;=^&6|X!8ZSC{T0ea;rVN?9vG>^M zdPG75!~z>n9J<2fp5{@{KXBm@KQd>2VaVdr`UY9}IqS^YCV6e`z`mrWDjfB=8@Z2^ zIyaa5LEaEX<_&g9_d@HVYptz9Ve_u`YB)H={ZjXCKdARwGUU_a_{Q&2eD}lu0UA_O z$76Nr?D^7wD}#Bd=_RYrfG_L1dQ;dRg3{9l7a=P4v9nR+(_?Y=J!A?ZJFsBNBXqBR zO`DlHdVQvVI{o!)|JGLHmD(+?(PB?3rwI?5wKZPPgGENfz)d2*VxPW<(H|9Me%si>}0RV3Nghs)J2hW`rV>?Cr)o05R5fC!8cV)m4?e zyDrH9Ci^2j7}va|HNIjq{*-*#79_?g2_rP>f}2!Mlgr`uao~J*U#dd54~RG`Et@o0qsgJ^mSw=bnWqf| zUvY^_kERJg68Wsamxh)&qZL;8MMba-j^SI;S-Wi;Qh=a<6Cq0hVAHQeC;fM4K_INVO^Ceh& zEMAk4(5l(!W^;AhL!-OXii+a?E6#Maw)}*51%ib+GdzN^r>^?u|16l~oK(vxj5QpZ z(C*oG_69Car+}sejTrU|TSG9LsLDZ47YSibTf=-{q9K)%55eS>Z*g*-Pb}*WXs5j# zQVL0P$dwwqcmW7%2naY{W+@6vZe|Be1)H}xmr!9O%RA zp%*1-gVi3I5dRw1*r-oJnwP7k6lD03pI@KY11uf<#hL5 zty-+7ag%DMan&<$Rb-nET=XFl>62qyo2d_&(P(s9S=qH>FAopWtx;qQp3^bUoj^(T zdQU2#5kxk0>+r4)OWgP!U21`%ww+{Pcm~Q{b&X;_AK#F$SaRPxuTu+-E`U(Eb(+7> zwO{#jZ$$S_1$1i6K$Jq!1>^I#Wd}v%%r<{c4PZ@}(?K3+yYvPH)lWs;iGW%sn^pme zdGzz9!m@W4lH`3LRTu`-Y=f~_J_JX+rSiv5eypW62s=7|Z#R4E)M+y0_pj$Sm%@57 zbnA34AApcnKyVLIt0G>J^DQ5`CPnMnm`MtsixtZnlyIPIYO(@xg?!TGHw8VnEpq3V z?y-=6zy=HlV`GBue3;?f{6Q2RP7->D(o1m@#Fj(YBwt@ zfhPH_$e!Ef6gAyy@eS=N`b%)o&!z}=j7TGx4jbAjjOXitd zGvG}*a88oW+ifs^<@~>gPwOC8@4Cpigt>Y3Xwo%lxH+neMiC91h#;uv5)!uve5if& zH|63Im$;+ZwC%ec-3ol5_$w(9gUIvq=N1qFRA?#5T3m=YQu<|VZM!OZ1#}-7nGE4^ zVX`A=^kQ%Gh(Y!{=0MgI;U>%#lod@X)caU+T+kzYnul@vW}US%S;tUp%-ULm%P(Q* zZ>dhwq@*!G!dQ7I_zc>m13|YZ-&8b>v%jWrgjQSGqXa$~B*XYA+s&@YUY;d*e5YYE zIP6#8wL_AH$HX*+(s%;B#mK|s01q=|$h9&{x0n;u+XVCT+2Aa}6b(;pFR~Q;bsOr? z7m4oz83JkV&YG-*YpS5R`J3mo**(Kd+QEEE-s`#h`ao+ERV|2G&Xbv3V`vrAu_DmU zQX{SRqUPn`cs?Nt`Xq4WMT3sko7Qe`bAAEry60)E>AAS@f%Mha=RG*M3)__r3H#rD?f~&`5^ze&6xh6)-fU1|8q;(5rPwXBZ4+ZP(=Th26~6HNqubTtq-8 z0mq7ffGbSI!&P(im2?Qel>=*dKE4Xva-{apePe35C}I6ksnqxe+||uZo%j%^M6RN{E*#k7AVLj8%cW6Pl_(a4i{ z_Lpncb{OO-otm)Eh@=GV+(Lu-@(x;#lG(r4;Dq9o-m`Jo*+yAULLffvJVmy>qy0vp z?tU~Vk>s9L-f0;z01^7RlHs=OYh?ZXXM%!V=ycVX^oEP+9~sKQ4w$dF=JpO-3OmUi z;st}sl{J3N%?L1I^GL>jTcwHLZ+xw)q#CcFp+VcI=$D`G38GA>jOD)=c2intP%Om+{aYf<0Nr~OQ^J&rDPlkO1(q+hN*BV>XLoHzM`|QxN$G%N( z&rt22Hy+X-MSVW08IMIKhK+%D;JI60Q=<SecwSMnAlc{+@hQnSEKzUk(Y=4CCTf#d5M=!z u6IwhDyj`JzI45W(kY=X$ba1B94ro#`Y)yU0F5JU`4=Hi^d--?tp8W@T_ZnpY literal 0 HcmV?d00001 diff --git a/docs/user-guide/study/areas/05-hydro.md b/docs/user-guide/study/areas/05-hydro.md index 40a02d4eca..c2a535bb2c 100644 --- a/docs/user-guide/study/areas/05-hydro.md +++ b/docs/user-guide/study/areas/05-hydro.md @@ -71,3 +71,9 @@ This tab allows you to configure the hydro storage time series of the hydraulic ## Run of River This tab allows you to configure the run of river time series of the hydraulic generators. + +## Minimum Generation + +The "Min Gen." tab is dedicated to configuring the minimum generation levels of the hydraulic generators. This tab presents a time series that represents the minimum hourly production for one or more Monte-Carlo years. + +![05-hydro.min-generation.series.png](../../../assets/media/user-guide/study/areas/05-hydro.min-generation.series.png) \ No newline at end of file From e57610836d43712c59b4d807003e6228ead64bd4 Mon Sep 17 00:00:00 2001 From: hatim dinia Date: Tue, 13 Feb 2024 18:35:59 +0100 Subject: [PATCH 23/31] fix(ui-hydro): remove dots from labels and add `studyVersion` missing dep --- .../Singlestudy/explore/Modelization/Areas/Hydro/index.tsx | 4 ++-- .../App/Singlestudy/explore/Modelization/Areas/Hydro/utils.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/Hydro/index.tsx b/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/Hydro/index.tsx index ffa2df3620..0baab19ab8 100644 --- a/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/Hydro/index.tsx +++ b/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/Hydro/index.tsx @@ -31,9 +31,9 @@ function Hydro() { { label: "Water values", path: `${basePath}/watervalues` }, { label: "Hydro Storage", path: `${basePath}/hydrostorage` }, { label: "Run of river", path: `${basePath}/ror` }, - studyVersion >= 860 && { label: "Min Gen.", path: `${basePath}/mingen` }, + studyVersion >= 860 && { label: "Min Gen", path: `${basePath}/mingen` }, ].filter(Boolean); - }, [areaId, study?.id]); + }, [areaId, study?.id, studyVersion]); //////////////////////////////////////////////////////////////// // JSX diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/Hydro/utils.ts b/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/Hydro/utils.ts index 2a75d23f9d..ed8457afe4 100644 --- a/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/Hydro/utils.ts +++ b/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/Hydro/utils.ts @@ -150,7 +150,7 @@ export const MATRICES: Matrices = { stats: MatrixStats.STATS, }, [HydroMatrixType.MinGen]: { - title: "Min Gen.", + title: "Min Gen", url: "input/hydro/series/{areaId}/mingen", stats: MatrixStats.STATS, }, From 23a813c8eee271d0c33f8d7f82d2994e5640a6a6 Mon Sep 17 00:00:00 2001 From: mabw-rte <41002227+mabw-rte@users.noreply.github.com> Date: Wed, 14 Feb 2024 17:57:16 +0100 Subject: [PATCH 24/31] feat(tags-db): populate `tag` and `study_tag` tables using pre-existing patch data (#1929) Context: Currently, tags do not have a specific table but are directly retrieved from Patches using Python code. Issue: This coding paradigm results in filtering that cannot occur at the database level but rather post-query (posing a problem for pagination). It can also potentially slightly slow down API queries. Solution in following steps: - Create two tables, `tag` and `study_tag`, to manage the many-to-many relationships between studies and tags. This step requires data migration. - Update endpoints and services - Create an update script to populate the newly created tables with pre-existing data. Note: This PR deals with the last step --- ...populate_tag_and_study_tag_tables_with_.py | 101 ++++++++++++++++++ scripts/rollback.sh | 2 +- 2 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 alembic/versions/dae93f1d9110_populate_tag_and_study_tag_tables_with_.py diff --git a/alembic/versions/dae93f1d9110_populate_tag_and_study_tag_tables_with_.py b/alembic/versions/dae93f1d9110_populate_tag_and_study_tag_tables_with_.py new file mode 100644 index 0000000000..0cfc66e8b0 --- /dev/null +++ b/alembic/versions/dae93f1d9110_populate_tag_and_study_tag_tables_with_.py @@ -0,0 +1,101 @@ +""" +Populate `tag` and `study_tag` tables from `patch` field in `study_additional_data` table + +Revision ID: dae93f1d9110 +Revises: 3c70366b10ea +Create Date: 2024-02-08 10:30:20.590919 +""" +import collections +import itertools +import json +import secrets + +import sqlalchemy as sa # type: ignore +from alembic import op +from sqlalchemy.engine import Connection # type: ignore + +from antarest.study.css4_colors import COLOR_NAMES + +# revision identifiers, used by Alembic. +revision = "dae93f1d9110" +down_revision = "3c70366b10ea" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """ + Populate `tag` and `study_tag` tables from `patch` field in `study_additional_data` table + + Four steps to proceed: + - Retrieve study-tags pairs from patches in `study_additional_data`. + - Delete all rows in `tag` and `study_tag`, as tag updates between revised 3c70366b10ea and this version, + do modify the data in patches alongside the two previous tables. + - Populate `tag` table using unique tag-labels and by randomly generating their associated colors. + - Populate `study_tag` using study-tags pairs. + """ + + # create connexion to the db + connexion: Connection = op.get_bind() + + # retrieve the tags and the study-tag pairs from the db + study_tags = connexion.execute("SELECT study_id,patch FROM study_additional_data") + tags_by_ids = {} + for study_id, patch in study_tags: + obj = json.loads(patch or "{}") + study = obj.get("study") or {} + tags = frozenset(study.get("tags") or ()) + tags_by_ids[study_id] = tags + + # delete rows in tables `tag` and `study_tag` + connexion.execute("DELETE FROM study_tag") + connexion.execute("DELETE FROM tag") + + # insert the tags in the `tag` table + labels = set(itertools.chain.from_iterable(tags_by_ids.values())) + bulk_tags = [{"label": label, "color": secrets.choice(COLOR_NAMES)} for label in labels] + sql = sa.text("INSERT INTO tag (label, color) VALUES (:label, :color)") + connexion.execute(sql, *bulk_tags) + + # Create relationships between studies and tags in the `study_tag` table + bulk_study_tags = ({"study_id": id_, "tag_label": lbl} for id_, tags in tags_by_ids.items() for lbl in tags) + sql = sa.text("INSERT INTO study_tag (study_id, tag_label) VALUES (:study_id, :tag_label)") + connexion.execute(sql, *bulk_study_tags) + + +def downgrade() -> None: + """ + Restore `patch` field in `study_additional_data` from `tag` and `study_tag` tables + + Three steps to proceed: + - Retrieve study-tags pairs from `study_tag` table. + - Update patches study-tags in `study_additional_data` using these pairs. + - Delete all rows from `tag` and `study_tag`. + """ + # create a connection to the db + connexion: Connection = op.get_bind() + + # Creating the `tags_by_ids` mapping from data in the `study_tags` table + tags_by_ids = collections.defaultdict(set) + study_tags = connexion.execute("SELECT study_id, tag_label FROM study_tag") + for study_id, tag_label in study_tags: + tags_by_ids[study_id].add(tag_label) + + # Then, we read objects from the `patch` field of the `study_additional_data` table + objects_by_ids = {} + study_tags = connexion.execute("SELECT study_id, patch FROM study_additional_data") + for study_id, patch in study_tags: + obj = json.loads(patch or "{}") + obj["study"] = obj.get("study") or {} + obj["study"]["tags"] = obj["study"].get("tags") or [] + obj["study"]["tags"] = sorted(tags_by_ids[study_id] | set(obj["study"]["tags"])) + objects_by_ids[study_id] = obj + + # Updating objects in the `study_additional_data` table + sql = sa.text("UPDATE study_additional_data SET patch = :patch WHERE study_id = :study_id") + bulk_patches = [{"study_id": id_, "patch": json.dumps(obj)} for id_, obj in objects_by_ids.items()] + connexion.execute(sql, *bulk_patches) + + # Deleting study_tags and tags + connexion.execute("DELETE FROM study_tag") + connexion.execute("DELETE FROM tag") diff --git a/scripts/rollback.sh b/scripts/rollback.sh index bf92685dc4..46d04a7966 100755 --- a/scripts/rollback.sh +++ b/scripts/rollback.sh @@ -12,5 +12,5 @@ CUR_DIR=$(cd "$(dirname "$0")" && pwd) BASE_DIR=$(dirname "$CUR_DIR") cd "$BASE_DIR" -alembic downgrade 1f5db5dfad80 +alembic downgrade 3c70366b10ea cd - From 2cfa5529b28596584210cac682aa465c008e6d6f Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Wed, 14 Feb 2024 23:25:15 +0100 Subject: [PATCH 25/31] fix(tags-db): correct `tag` and `study_tag` migration script Avoid bulk insertion if the list of values to insert is empty. --- ...0_populate_tag_and_study_tag_tables_with_.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/alembic/versions/dae93f1d9110_populate_tag_and_study_tag_tables_with_.py b/alembic/versions/dae93f1d9110_populate_tag_and_study_tag_tables_with_.py index 0cfc66e8b0..c6c29d3716 100644 --- a/alembic/versions/dae93f1d9110_populate_tag_and_study_tag_tables_with_.py +++ b/alembic/versions/dae93f1d9110_populate_tag_and_study_tag_tables_with_.py @@ -54,13 +54,15 @@ def upgrade() -> None: # insert the tags in the `tag` table labels = set(itertools.chain.from_iterable(tags_by_ids.values())) bulk_tags = [{"label": label, "color": secrets.choice(COLOR_NAMES)} for label in labels] - sql = sa.text("INSERT INTO tag (label, color) VALUES (:label, :color)") - connexion.execute(sql, *bulk_tags) + if bulk_tags: + sql = sa.text("INSERT INTO tag (label, color) VALUES (:label, :color)") + connexion.execute(sql, *bulk_tags) # Create relationships between studies and tags in the `study_tag` table - bulk_study_tags = ({"study_id": id_, "tag_label": lbl} for id_, tags in tags_by_ids.items() for lbl in tags) - sql = sa.text("INSERT INTO study_tag (study_id, tag_label) VALUES (:study_id, :tag_label)") - connexion.execute(sql, *bulk_study_tags) + bulk_study_tags = [{"study_id": id_, "tag_label": lbl} for id_, tags in tags_by_ids.items() for lbl in tags] + if bulk_study_tags: + sql = sa.text("INSERT INTO study_tag (study_id, tag_label) VALUES (:study_id, :tag_label)") + connexion.execute(sql, *bulk_study_tags) def downgrade() -> None: @@ -92,9 +94,10 @@ def downgrade() -> None: objects_by_ids[study_id] = obj # Updating objects in the `study_additional_data` table - sql = sa.text("UPDATE study_additional_data SET patch = :patch WHERE study_id = :study_id") bulk_patches = [{"study_id": id_, "patch": json.dumps(obj)} for id_, obj in objects_by_ids.items()] - connexion.execute(sql, *bulk_patches) + if bulk_patches: + sql = sa.text("UPDATE study_additional_data SET patch = :patch WHERE study_id = :study_id") + connexion.execute(sql, *bulk_patches) # Deleting study_tags and tags connexion.execute("DELETE FROM study_tag") From 596c486cd285b58591826bade63990478e66b71d Mon Sep 17 00:00:00 2001 From: "olfa.mizen_externe@rte-france.com" Date: Mon, 8 Jan 2024 15:30:54 +0100 Subject: [PATCH 26/31] perf(watcher): change db queries to improve Watcher scanning perfs --- ...fd73601a9075_add_delete_cascade_studies.py | 69 +++++++++++++++++++ antarest/study/model.py | 4 +- antarest/study/repository.py | 15 +++- antarest/study/service.py | 25 +++---- .../storage/variantstudy/model/dbmodel.py | 2 +- tests/storage/test_service.py | 32 ++++++--- 6 files changed, 117 insertions(+), 30 deletions(-) create mode 100644 alembic/versions/fd73601a9075_add_delete_cascade_studies.py diff --git a/alembic/versions/fd73601a9075_add_delete_cascade_studies.py b/alembic/versions/fd73601a9075_add_delete_cascade_studies.py new file mode 100644 index 0000000000..ac063fe516 --- /dev/null +++ b/alembic/versions/fd73601a9075_add_delete_cascade_studies.py @@ -0,0 +1,69 @@ +""" +Add delete cascade constraint to study foreign keys + +Revision ID: fd73601a9075 +Revises: 3c70366b10ea +Create Date: 2024-02-12 17:27:37.314443 +""" +from alembic import op + +# revision identifiers, used by Alembic. +revision = "fd73601a9075" +down_revision = "3c70366b10ea" +branch_labels = None +depends_on = None + +# noinspection SpellCheckingInspection +RAWSTUDY_FK = "rawstudy_id_fkey" + +# noinspection SpellCheckingInspection +VARIANTSTUDY_FK = "variantstudy_id_fkey" + +# noinspection SpellCheckingInspection +STUDY_ADDITIONAL_DATA_FK = "study_additional_data_study_id_fkey" + + +def upgrade() -> None: + dialect_name: str = op.get_context().dialect.name + if dialect_name == "postgresql": + with op.batch_alter_table("rawstudy", schema=None) as batch_op: + batch_op.drop_constraint(RAWSTUDY_FK, type_="foreignkey") + batch_op.create_foreign_key(RAWSTUDY_FK, "study", ["id"], ["id"], ondelete="CASCADE") + + with op.batch_alter_table("study_additional_data", schema=None) as batch_op: + batch_op.drop_constraint(STUDY_ADDITIONAL_DATA_FK, type_="foreignkey") + batch_op.create_foreign_key(STUDY_ADDITIONAL_DATA_FK, "study", ["study_id"], ["id"], ondelete="CASCADE") + + with op.batch_alter_table("variantstudy", schema=None) as batch_op: + batch_op.drop_constraint(VARIANTSTUDY_FK, type_="foreignkey") + batch_op.create_foreign_key(VARIANTSTUDY_FK, "study", ["id"], ["id"], ondelete="CASCADE") + + elif dialect_name == "sqlite": + # Adding ondelete="CASCADE" to a foreign key in sqlite is not supported + pass + + else: + raise NotImplementedError(f"{dialect_name=} not implemented") + + +def downgrade() -> None: + dialect_name: str = op.get_context().dialect.name + if dialect_name == "postgresql": + with op.batch_alter_table("rawstudy", schema=None) as batch_op: + batch_op.drop_constraint(RAWSTUDY_FK, type_="foreignkey") + batch_op.create_foreign_key(RAWSTUDY_FK, "study", ["id"], ["id"]) + + with op.batch_alter_table("study_additional_data", schema=None) as batch_op: + batch_op.drop_constraint(STUDY_ADDITIONAL_DATA_FK, type_="foreignkey") + batch_op.create_foreign_key(STUDY_ADDITIONAL_DATA_FK, "study", ["study_id"], ["id"]) + + with op.batch_alter_table("variantstudy", schema=None) as batch_op: + batch_op.drop_constraint(VARIANTSTUDY_FK, type_="foreignkey") + batch_op.create_foreign_key(VARIANTSTUDY_FK, "study", ["id"], ["id"]) + + elif dialect_name == "sqlite": + # Removing ondelete="CASCADE" to a foreign key in sqlite is not supported + pass + + else: + raise NotImplementedError(f"{dialect_name=} not implemented") diff --git a/antarest/study/model.py b/antarest/study/model.py index fe10b4f211..df36efa856 100644 --- a/antarest/study/model.py +++ b/antarest/study/model.py @@ -130,7 +130,7 @@ class StudyAdditionalData(Base): # type:ignore study_id = Column( String(36), - ForeignKey("study.id"), + ForeignKey("study.id", ondelete="CASCADE"), primary_key=True, ) author = Column(String(255), default="Unknown") @@ -230,7 +230,7 @@ class RawStudy(Study): id = Column( String(36), - ForeignKey("study.id"), + ForeignKey("study.id", ondelete="CASCADE"), primary_key=True, ) content_status = Column(Enum(StudyContentStatus)) diff --git a/antarest/study/repository.py b/antarest/study/repository.py index 3aa6e60681..7728e7068b 100644 --- a/antarest/study/repository.py +++ b/antarest/study/repository.py @@ -272,10 +272,10 @@ def get_all_raw(self, exists: t.Optional[bool] = None) -> t.Sequence[RawStudy]: studies: t.Sequence[RawStudy] = query.all() return studies - def delete(self, id: str) -> None: + def delete(self, id_: str, *ids: str) -> None: + ids = (id_,) + ids session = self.session - u: Study = session.query(Study).get(id) - session.delete(u) + session.query(Study).filter(Study.id.in_(ids)).delete(synchronize_session=False) session.commit() def update_tags(self, study: Study, new_tags: t.Sequence[str]) -> None: @@ -292,3 +292,12 @@ def update_tags(self, study: Study, new_tags: t.Sequence[str]) -> None: study.tags = [Tag(label=tag) for tag in new_labels] + existing_tags self.session.merge(study) self.session.commit() + + def list_duplicates(self) -> t.List[t.Tuple[str, str]]: + """ + Get list of duplicates as tuples (id, path). + """ + session = self.session + subquery = session.query(Study.path).group_by(Study.path).having(func.count() > 1).subquery() + query = session.query(Study.id, Study.path).filter(Study.path.in_(subquery)) + return t.cast(t.List[t.Tuple[str, str]], query.all()) diff --git a/antarest/study/service.py b/antarest/study/service.py index 910cf62027..9b22ae7638 100644 --- a/antarest/study/service.py +++ b/antarest/study/service.py @@ -1,4 +1,5 @@ import base64 +import collections import contextlib import io import json @@ -701,20 +702,16 @@ def get_input_matrix_startdate( return get_start_date(file_study, output_id, level) def remove_duplicates(self) -> None: - study_paths: t.Dict[str, t.List[str]] = {} - for study in self.repository.get_all(): - if isinstance(study, RawStudy) and not study.archived: - path = str(study.path) - if path not in study_paths: - study_paths[path] = [] - study_paths[path].append(study.id) - - for studies_with_same_path in study_paths.values(): - if len(studies_with_same_path) > 1: - logger.info(f"Found studies {studies_with_same_path} with same path, de duplicating") - for study_name in studies_with_same_path[1:]: - logger.info(f"Removing study {study_name}") - self.repository.delete(study_name) + duplicates = self.repository.list_duplicates() + ids: t.List[str] = [] + # ids with same path + duplicates_by_path = collections.defaultdict(list) + for study_id, path in duplicates: + duplicates_by_path[path].append(study_id) + for path, study_ids in duplicates_by_path.items(): + ids.extend(study_ids[1:]) + if ids: # Check if ids is not empty + self.repository.delete(*ids) def sync_studies_on_disk(self, folders: t.List[StudyFolder], directory: t.Optional[Path] = None) -> None: """ diff --git a/antarest/study/storage/variantstudy/model/dbmodel.py b/antarest/study/storage/variantstudy/model/dbmodel.py index bbe264f89f..9272eb797f 100644 --- a/antarest/study/storage/variantstudy/model/dbmodel.py +++ b/antarest/study/storage/variantstudy/model/dbmodel.py @@ -77,7 +77,7 @@ class VariantStudy(Study): id: str = Column( String(36), - ForeignKey("study.id"), + ForeignKey("study.id", ondelete="CASCADE"), primary_key=True, ) generation_task: t.Optional[str] = Column(String(), nullable=True) diff --git a/tests/storage/test_service.py b/tests/storage/test_service.py index 12f61e6489..fa7ed5c62d 100644 --- a/tests/storage/test_service.py +++ b/tests/storage/test_service.py @@ -350,18 +350,30 @@ def test_partial_sync_studies_from_disk() -> None: ) -@pytest.mark.unit_test -def test_remove_duplicate() -> None: - ma = RawStudy(id="a", path="a") - mb = RawStudy(id="b", path="a") +@with_db_context +def test_remove_duplicate(db_session: Session) -> None: + with db_session: + db_session.add(RawStudy(id="a", path="/path/to/a")) + db_session.add(RawStudy(id="b", path="/path/to/a")) + db_session.add(RawStudy(id="c", path="/path/to/c")) + db_session.commit() + study_count = db_session.query(RawStudy).filter(RawStudy.path == "/path/to/a").count() + assert study_count == 2 # there are 2 studies with same path before removing duplicates - repository = Mock() - repository.get_all.return_value = [ma, mb] - config = Config(storage=StorageConfig(workspaces={DEFAULT_WORKSPACE_NAME: WorkspaceConfig()})) - service = build_study_service(Mock(), repository, config) + with db_session: + repository = StudyMetadataRepository(Mock(), db_session) + config = Config(storage=StorageConfig(workspaces={DEFAULT_WORKSPACE_NAME: WorkspaceConfig()})) + service = build_study_service(Mock(), repository, config) + service.remove_duplicates() - service.remove_duplicates() - repository.delete.assert_called_once_with(mb.id) + # example with 1 duplicate with same path + with db_session: + study_count = db_session.query(RawStudy).filter(RawStudy.path == "/path/to/a").count() + assert study_count == 1 + # example with no duplicates with same path + with db_session: + study_count = db_session.query(RawStudy).filter(RawStudy.path == "/path/to/c").count() + assert study_count == 1 # noinspection PyArgumentList From 1ce2bebd681cda5ad25d1a49e3d37933e427269e Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Wed, 14 Feb 2024 23:01:45 +0100 Subject: [PATCH 27/31] refactor(studies-db): change signature of `get` method in `StudyMetadataRepository` class Replace `id` parameter with `study_id`. --- antarest/study/repository.py | 6 ++-- tests/storage/repository/test_study.py | 42 ++++++++++++++------------ 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/antarest/study/repository.py b/antarest/study/repository.py index 7728e7068b..9d6c9317fb 100644 --- a/antarest/study/repository.py +++ b/antarest/study/repository.py @@ -138,7 +138,7 @@ def save( def refresh(self, metadata: Study) -> None: self.session.refresh(metadata) - def get(self, id: str) -> t.Optional[Study]: + def get(self, study_id: str) -> t.Optional[Study]: """Get the study by ID or return `None` if not found in database.""" # todo: I think we should use a `entity = with_polymorphic(Study, "*")` # to make sure RawStudy and VariantStudy fields are also fetched. @@ -146,13 +146,11 @@ def get(self, id: str) -> t.Optional[Study]: # When we fetch a study, we also need to fetch the associated owner and groups # to check the permissions of the current user efficiently. study: Study = ( - # fmt: off self.session.query(Study) .options(joinedload(Study.owner)) .options(joinedload(Study.groups)) .options(joinedload(Study.tags)) - .get(id) - # fmt: on + .get(study_id) ) return study diff --git a/tests/storage/repository/test_study.py b/tests/storage/repository/test_study.py index f865ab613a..bb4ea795e3 100644 --- a/tests/storage/repository/test_study.py +++ b/tests/storage/repository/test_study.py @@ -1,22 +1,24 @@ from datetime import datetime +from sqlalchemy.orm import Session # type: ignore + from antarest.core.cache.business.local_chache import LocalCache from antarest.core.model import PublicMode from antarest.login.model import Group, User from antarest.study.model import DEFAULT_WORKSPACE_NAME, RawStudy, Study, StudyContentStatus from antarest.study.repository import StudyMetadataRepository from antarest.study.storage.variantstudy.model.dbmodel import VariantStudy -from tests.helpers import with_db_context -@with_db_context -def test_lifecycle() -> None: - user = User(id=0, name="admin") +def test_lifecycle(db_session: Session) -> None: + repo = StudyMetadataRepository(LocalCache(), session=db_session) + + user = User(id=1, name="admin") group = Group(id="my-group", name="group") - repo = StudyMetadataRepository(LocalCache()) + a = Study( name="a", - version="42", + version="820", author="John Smith", created_at=datetime.utcnow(), updated_at=datetime.utcnow(), @@ -26,7 +28,7 @@ def test_lifecycle() -> None: ) b = RawStudy( name="b", - version="43", + version="830", author="Morpheus", created_at=datetime.utcnow(), updated_at=datetime.utcnow(), @@ -36,7 +38,7 @@ def test_lifecycle() -> None: ) c = RawStudy( name="c", - version="43", + version="830", author="Trinity", created_at=datetime.utcnow(), updated_at=datetime.utcnow(), @@ -47,7 +49,7 @@ def test_lifecycle() -> None: ) d = VariantStudy( name="d", - version="43", + version="830", author="Mr. Anderson", created_at=datetime.utcnow(), updated_at=datetime.utcnow(), @@ -57,30 +59,32 @@ def test_lifecycle() -> None: ) a = repo.save(a) - b = repo.save(b) + a_id = a.id + + repo.save(b) repo.save(c) repo.save(d) - assert b.id - c = repo.one(a.id) - assert a == c + + c = repo.one(a_id) + assert a_id == c.id assert len(repo.get_all()) == 4 assert len(repo.get_all_raw(exists=True)) == 1 assert len(repo.get_all_raw(exists=False)) == 1 assert len(repo.get_all_raw()) == 2 - repo.delete(a.id) - assert repo.get(a.id) is None + repo.delete(a_id) + assert repo.get(a_id) is None + +def test_study_inheritance(db_session: Session) -> None: + repo = StudyMetadataRepository(LocalCache(), session=db_session) -@with_db_context -def test_study_inheritance() -> None: user = User(id=0, name="admin") group = Group(id="my-group", name="group") - repo = StudyMetadataRepository(LocalCache()) a = RawStudy( name="a", - version="42", + version="820", author="John Smith", created_at=datetime.utcnow(), updated_at=datetime.utcnow(), From 403087e728b8ba7e3efafccab98a6e90b2d16f7e Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Wed, 14 Feb 2024 23:01:55 +0100 Subject: [PATCH 28/31] fix(studies-db): correct the many-to-many relationship between `Study` and `Group` --- antarest/study/model.py | 45 +++++++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/antarest/study/model.py b/antarest/study/model.py index df36efa856..5079198296 100644 --- a/antarest/study/model.py +++ b/antarest/study/model.py @@ -16,7 +16,6 @@ Integer, PrimaryKeyConstraint, String, - Table, ) from sqlalchemy.orm import relationship # type: ignore @@ -50,12 +49,31 @@ NEW_DEFAULT_STUDY_VERSION: str = "860" -groups_metadata = Table( - "group_metadata", - Base.metadata, - Column("group_id", String(36), ForeignKey("groups.id")), - Column("study_id", String(36), ForeignKey("study.id")), -) + +class StudyGroup(Base): # type:ignore + """ + A table to manage the many-to-many relationship between `Study` and `Group` + + Attributes: + study_id: The ID of the study associated with the group. + group_id: The IS of the group associated with the study. + """ + + __tablename__ = "group_metadata" + __table_args__ = (PrimaryKeyConstraint("study_id", "group_id"),) + + group_id: str = Column(String(36), ForeignKey("groups.id", ondelete="CASCADE"), index=True, nullable=False) + study_id: str = Column(String(36), ForeignKey("study.id", ondelete="CASCADE"), index=True, nullable=False) + + def __str__(self) -> str: # pragma: no cover + cls_name = self.__class__.__name__ + return f"[{cls_name}] study_id={self.study_id}, group={self.group_id}" + + def __repr__(self) -> str: # pragma: no cover + cls_name = self.__class__.__name__ + study_id = self.study_id + group_id = self.group_id + return f"{cls_name}({study_id=}, {group_id=})" class StudyTag(Base): # type:ignore @@ -63,8 +81,8 @@ class StudyTag(Base): # type:ignore A table to manage the many-to-many relationship between `Study` and `Tag` Attributes: - study_id (str): The ID of the study associated with the tag. - tag_label (str): The label of the tag associated with the study. + study_id: The ID of the study associated with the tag. + tag_label: The label of the tag associated with the study. """ __tablename__ = "study_tag" @@ -74,7 +92,8 @@ class StudyTag(Base): # type:ignore tag_label: str = Column(String(40), ForeignKey("tag.label", ondelete="CASCADE"), index=True, nullable=False) def __str__(self) -> str: # pragma: no cover - return f"[StudyTag] study_id={self.study_id}, tag={self.tag}" + cls_name = self.__class__.__name__ + return f"[{cls_name}] study_id={self.study_id}, tag={self.tag}" def __repr__(self) -> str: # pragma: no cover cls_name = self.__class__.__name__ @@ -90,8 +109,8 @@ class Tag(Base): # type:ignore This class is used to store tags associated with studies. Attributes: - label (str): The label of the tag. - color (str): The color code associated with the tag. + label: The label of the tag. + color: The color code associated with the tag. """ __tablename__ = "tag" @@ -174,7 +193,7 @@ class Study(Base): # type: ignore tags: t.List[Tag] = relationship(Tag, secondary=StudyTag.__table__, back_populates="studies") owner = relationship(Identity, uselist=False) - groups = relationship(Group, secondary=lambda: groups_metadata, cascade="") + groups = relationship(Group, secondary=StudyGroup.__table__, cascade="") additional_data = relationship( StudyAdditionalData, uselist=False, From 7c3c5cde824875b48c9c2cf019f826a3974ec09b Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Wed, 14 Feb 2024 23:21:03 +0100 Subject: [PATCH 29/31] fix(db): add a migration script to correct the many-to-many relationship between `Study` and `Group` --- ...fd73601a9075_add_delete_cascade_studies.py | 43 +++++++++++++------ 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/alembic/versions/fd73601a9075_add_delete_cascade_studies.py b/alembic/versions/fd73601a9075_add_delete_cascade_studies.py index ac063fe516..3f9f42c684 100644 --- a/alembic/versions/fd73601a9075_add_delete_cascade_studies.py +++ b/alembic/versions/fd73601a9075_add_delete_cascade_studies.py @@ -5,11 +5,12 @@ Revises: 3c70366b10ea Create Date: 2024-02-12 17:27:37.314443 """ +import sqlalchemy as sa # type: ignore from alembic import op # revision identifiers, used by Alembic. revision = "fd73601a9075" -down_revision = "3c70366b10ea" +down_revision = "dae93f1d9110" branch_labels = None depends_on = None @@ -25,6 +26,8 @@ def upgrade() -> None: dialect_name: str = op.get_context().dialect.name + + # SQLite doesn't support dropping foreign keys, so we need to ignore it here if dialect_name == "postgresql": with op.batch_alter_table("rawstudy", schema=None) as batch_op: batch_op.drop_constraint(RAWSTUDY_FK, type_="foreignkey") @@ -38,16 +41,25 @@ def upgrade() -> None: batch_op.drop_constraint(VARIANTSTUDY_FK, type_="foreignkey") batch_op.create_foreign_key(VARIANTSTUDY_FK, "study", ["id"], ["id"], ondelete="CASCADE") - elif dialect_name == "sqlite": - # Adding ondelete="CASCADE" to a foreign key in sqlite is not supported - pass - - else: - raise NotImplementedError(f"{dialect_name=} not implemented") + with op.batch_alter_table("group_metadata", schema=None) as batch_op: + batch_op.alter_column("group_id", existing_type=sa.VARCHAR(length=36), nullable=False) + batch_op.alter_column("study_id", existing_type=sa.VARCHAR(length=36), nullable=False) + batch_op.create_index(batch_op.f("ix_group_metadata_group_id"), ["group_id"], unique=False) + batch_op.create_index(batch_op.f("ix_group_metadata_study_id"), ["study_id"], unique=False) + if dialect_name == "postgresql": + batch_op.drop_constraint("group_metadata_group_id_fkey", type_="foreignkey") + batch_op.drop_constraint("group_metadata_study_id_fkey", type_="foreignkey") + batch_op.create_foreign_key( + "group_metadata_group_id_fkey", "groups", ["group_id"], ["id"], ondelete="CASCADE" + ) + batch_op.create_foreign_key( + "group_metadata_study_id_fkey", "study", ["study_id"], ["id"], ondelete="CASCADE" + ) def downgrade() -> None: dialect_name: str = op.get_context().dialect.name + # SQLite doesn't support dropping foreign keys, so we need to ignore it here if dialect_name == "postgresql": with op.batch_alter_table("rawstudy", schema=None) as batch_op: batch_op.drop_constraint(RAWSTUDY_FK, type_="foreignkey") @@ -61,9 +73,14 @@ def downgrade() -> None: batch_op.drop_constraint(VARIANTSTUDY_FK, type_="foreignkey") batch_op.create_foreign_key(VARIANTSTUDY_FK, "study", ["id"], ["id"]) - elif dialect_name == "sqlite": - # Removing ondelete="CASCADE" to a foreign key in sqlite is not supported - pass - - else: - raise NotImplementedError(f"{dialect_name=} not implemented") + with op.batch_alter_table("group_metadata", schema=None) as batch_op: + # SQLite doesn't support dropping foreign keys, so we need to ignore it here + if dialect_name == "postgresql": + batch_op.drop_constraint("group_metadata_study_id_fkey", type_="foreignkey") + batch_op.drop_constraint("group_metadata_group_id_fkey", type_="foreignkey") + batch_op.create_foreign_key("group_metadata_study_id_fkey", "study", ["study_id"], ["id"]) + batch_op.create_foreign_key("group_metadata_group_id_fkey", "groups", ["group_id"], ["id"]) + batch_op.drop_index(batch_op.f("ix_group_metadata_study_id")) + batch_op.drop_index(batch_op.f("ix_group_metadata_group_id")) + batch_op.alter_column("study_id", existing_type=sa.VARCHAR(length=36), nullable=True) + batch_op.alter_column("group_id", existing_type=sa.VARCHAR(length=36), nullable=True) From d0370cd8a84b148f65bf8a1602cc5324ca67f1ac Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Fri, 16 Feb 2024 09:35:40 +0100 Subject: [PATCH 30/31] test: correct issue with test_synthesis.py --- tests/integration/studies_blueprint/test_synthesis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/studies_blueprint/test_synthesis.py b/tests/integration/studies_blueprint/test_synthesis.py index 70f5f0c907..9afd66be9b 100644 --- a/tests/integration/studies_blueprint/test_synthesis.py +++ b/tests/integration/studies_blueprint/test_synthesis.py @@ -108,4 +108,4 @@ def test_variant_study( ) assert res.status_code == 200, res.json() duration = time.time() - start - assert 0 <= duration <= 0.1, f"Duration is {duration} seconds" + assert 0 <= duration <= 0.2, f"Duration is {duration} seconds" From ca92f1a88d18102dd8096d39ecfad4638ee21b8f Mon Sep 17 00:00:00 2001 From: hatim dinia Date: Tue, 20 Feb 2024 17:45:39 +0100 Subject: [PATCH 31/31] fix(ui-hydro): add missing matrix path encoding --- webapp/src/services/api/matrix.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/src/services/api/matrix.ts b/webapp/src/services/api/matrix.ts index 9b2d4f5213..1eff05f7c9 100644 --- a/webapp/src/services/api/matrix.ts +++ b/webapp/src/services/api/matrix.ts @@ -97,7 +97,7 @@ export const editMatrix = async ( matrixEdit: MatrixEditDTO[], ): Promise => { const res = await client.put( - `/v1/studies/${sid}/matrix?path=${path}`, + `/v1/studies/${sid}/matrix?path=${encodeURIComponent(path)}`, matrixEdit, ); return res.data;