From 36410b54aa05e880c957abb16c5fdb59dd507335 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE <43534797+laurent-laporte-pro@users.noreply.github.com> Date: Thu, 28 Sep 2023 13:18:40 +0200 Subject: [PATCH 01/16] feat(antares-launcher): use antares launcher v1.3.1 (#1743) --- requirements.txt | 2 +- tests/launcher/test_slurm_launcher.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e44a33d502..5c00c7cedd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Antares-Launcher~=1.3.0 +Antares-Launcher~=1.3.1 aiofiles~=0.8.0 alembic~=1.7.5 diff --git a/tests/launcher/test_slurm_launcher.py b/tests/launcher/test_slurm_launcher.py index 615700f727..7820abcdea 100644 --- a/tests/launcher/test_slurm_launcher.py +++ b/tests/launcher/test_slurm_launcher.py @@ -420,6 +420,7 @@ def test_kill_job( slurm_launcher.kill_job(job_id=launch_id) launcher_arguments = Namespace( + antares_version=0, check_queue=False, job_id_to_kill=42, json_ssh_config=None, From 9155566ec6b2d52c97c4d9fc5bbefbada20a91bf Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Thu, 14 Sep 2023 18:44:20 +0200 Subject: [PATCH 02/16] fix(matrix): create empty file if the matrix is empty --- antarest/matrixstore/repository.py | 29 +++++---- antarest/matrixstore/service.py | 19 ++++-- antarest/study/service.py | 1 + antarest/tools/lib.py | 1 + tests/matrixstore/test_repository.py | 93 +++++++++++++++++----------- 5 files changed, 90 insertions(+), 53 deletions(-) diff --git a/antarest/matrixstore/repository.py b/antarest/matrixstore/repository.py index f96e80de67..6301e39c7f 100644 --- a/antarest/matrixstore/repository.py +++ b/antarest/matrixstore/repository.py @@ -1,7 +1,7 @@ import hashlib import logging +import typing as t from pathlib import Path -from typing import List, Optional, Union import numpy as np from filelock import FileLock @@ -31,19 +31,19 @@ def save(self, matrix_user_metadata: MatrixDataSet) -> MatrixDataSet: logger.debug(f"Matrix dataset {matrix_user_metadata.id} for user {matrix_user_metadata.owner_id} saved") return matrix_user_metadata - def get(self, id: str) -> Optional[MatrixDataSet]: + def get(self, id: str) -> t.Optional[MatrixDataSet]: matrix: MatrixDataSet = db.session.query(MatrixDataSet).get(id) return matrix - def get_all_datasets(self) -> List[MatrixDataSet]: - matrix_datasets: List[MatrixDataSet] = db.session.query(MatrixDataSet).all() + def get_all_datasets(self) -> t.List[MatrixDataSet]: + matrix_datasets: t.List[MatrixDataSet] = db.session.query(MatrixDataSet).all() return matrix_datasets def query( self, - name: Optional[str], - owner: Optional[int] = None, - ) -> List[MatrixDataSet]: + name: t.Optional[str], + owner: t.Optional[int] = None, + ) -> t.List[MatrixDataSet]: """ Query a list of MatrixUserMetadata by searching for each one separately if a set of filter match @@ -59,7 +59,7 @@ def query( query = query.filter(MatrixDataSet.name.ilike(f"%{name}%")) # type: ignore if owner is not None: query = query.filter(MatrixDataSet.owner_id == owner) - datasets: List[MatrixDataSet] = query.distinct().all() + datasets: t.List[MatrixDataSet] = query.distinct().all() return datasets def delete(self, dataset_id: str) -> None: @@ -83,7 +83,7 @@ def save(self, matrix: Matrix) -> Matrix: logger.debug(f"Matrix {matrix.id} saved") return matrix - def get(self, matrix_hash: str) -> Optional[Matrix]: + def get(self, matrix_hash: str) -> t.Optional[Matrix]: matrix: Matrix = db.session.query(Matrix).get(matrix_hash) return matrix @@ -130,6 +130,7 @@ def get(self, matrix_hash: str) -> MatrixContent: matrix_file = self.bucket_dir.joinpath(f"{matrix_hash}.tsv") matrix = np.loadtxt(matrix_file, delimiter="\t", dtype=np.float64, ndmin=2) + matrix = matrix.reshape((1, 0)) if matrix.size == 0 else matrix data = matrix.tolist() index = list(range(matrix.shape[0])) columns = list(range(matrix.shape[1])) @@ -148,7 +149,7 @@ def exists(self, matrix_hash: str) -> bool: matrix_file = self.bucket_dir.joinpath(f"{matrix_hash}.tsv") return matrix_file.exists() - def save(self, content: Union[List[List[MatrixData]], npt.NDArray[np.float64]]) -> str: + def save(self, content: t.Union[t.List[t.List[MatrixData]], npt.NDArray[np.float64]]) -> str: """ Saves the content of a matrix as a TSV file in the bucket directory and returns its SHA256 hash. @@ -188,8 +189,12 @@ def save(self, content: Union[List[List[MatrixData]], npt.NDArray[np.float64]]) # Ensure exclusive access to the matrix file between multiple processes (or threads). lock_file = matrix_file.with_suffix(".tsv.lock") with FileLock(lock_file, timeout=15): - # noinspection PyTypeChecker - np.savetxt(matrix_file, matrix, delimiter="\t", fmt="%.18f") + if matrix.size == 0: + # If the array or dataframe is empty, create an empty file instead of + # traditional saving to avoid unwanted line breaks. + open(matrix_file, mode="wb").close() + else: + np.savetxt(matrix_file, matrix, delimiter="\t", fmt="%.18f") # IMPORTANT: Deleting the lock file under Linux can make locking unreliable. # See https://github.com/tox-dev/py-filelock/issues/31 diff --git a/antarest/matrixstore/service.py b/antarest/matrixstore/service.py index c20b0197dc..d10b92ff50 100644 --- a/antarest/matrixstore/service.py +++ b/antarest/matrixstore/service.py @@ -187,6 +187,7 @@ def _file_importation(self, file: bytes, is_json: bool = False) -> str: return self.create(MatrixContent.parse_raw(file).data) # noinspection PyTypeChecker matrix = np.loadtxt(BytesIO(file), delimiter="\t", dtype=np.float64, ndmin=2) + matrix = matrix.reshape((1, 0)) if matrix.size == 0 else matrix return self.create(matrix) def get_dataset( @@ -380,8 +381,13 @@ def create_matrix_files(self, matrix_ids: Sequence[str], export_path: Path) -> s name = f"matrix-{mtx.id}.txt" filepath = f"{tmpdir}/{name}" array = np.array(mtx.data, dtype=np.float64) - # noinspection PyTypeChecker - np.savetxt(filepath, array, delimiter="\t", fmt="%.18f") + if array.size == 0: + # If the array or dataframe is empty, create an empty file instead of + # traditional saving to avoid unwanted line breaks. + open(filepath, mode="wb").close() + else: + # noinspection PyTypeChecker + np.savetxt(filepath, array, delimiter="\t", fmt="%.18f") zip_dir(Path(tmpdir), export_path) stopwatch.log_elapsed(lambda x: logger.info(f"Matrix dataset exported (zipped mode) in {x}s")) return str(export_path) @@ -467,5 +473,10 @@ def download_matrix( raise UserHasNotPermissionError() if matrix := self.get(matrix_id): array = np.array(matrix.data, dtype=np.float64) - # noinspection PyTypeChecker - np.savetxt(filepath, array, delimiter="\t", fmt="%.18f") + if array.size == 0: + # If the array or dataframe is empty, create an empty file instead of + # traditional saving to avoid unwanted line breaks. + open(filepath, mode="wb").close() + else: + # noinspection PyTypeChecker + np.savetxt(filepath, array, delimiter="\t", fmt="%.18f") diff --git a/antarest/study/service.py b/antarest/study/service.py index 332fc384da..b9ec491c18 100644 --- a/antarest/study/service.py +++ b/antarest/study/service.py @@ -1378,6 +1378,7 @@ def _create_edit_study_command( if isinstance(data, bytes): # noinspection PyTypeChecker matrix = np.loadtxt(io.BytesIO(data), delimiter="\t", dtype=np.float64, ndmin=2) + matrix = matrix.reshape((1, 0)) if matrix.size == 0 else matrix return ReplaceMatrix( target=url, matrix=matrix.tolist(), diff --git a/antarest/tools/lib.py b/antarest/tools/lib.py index b27e4dcee4..2d2953e3f5 100644 --- a/antarest/tools/lib.py +++ b/antarest/tools/lib.py @@ -77,6 +77,7 @@ def apply_commands( matrix_dataset: List[str] = [] for matrix_file in matrices_dir.iterdir(): matrix = np.loadtxt(matrix_file, delimiter="\t", dtype=np.float64, ndmin=2) + matrix = matrix.reshape((1, 0)) if matrix.size == 0 else matrix matrix_data = matrix.tolist() res = self.session.post(self.build_url("/v1/matrix"), json=matrix_data) res.raise_for_status() diff --git a/tests/matrixstore/test_repository.py b/tests/matrixstore/test_repository.py index 9d1254953a..3973a18d39 100644 --- a/tests/matrixstore/test_repository.py +++ b/tests/matrixstore/test_repository.py @@ -1,9 +1,10 @@ +import typing as t from datetime import datetime from pathlib import Path -from typing import Optional import numpy as np import pytest +from numpy import typing as npt from antarest.core.config import Config, SecurityConfig from antarest.core.utils.fastapi_sqlalchemy import db @@ -12,16 +13,15 @@ from antarest.matrixstore.model import Matrix, MatrixContent, MatrixDataSet, MatrixDataSetRelation from antarest.matrixstore.repository import MatrixContentRepository, MatrixDataSetRepository, MatrixRepository +ArrayData = t.Union[t.List[t.List[float]], npt.NDArray[np.float64]] + class TestMatrixRepository: - def test_db_lifecycle(self): + def test_db_lifecycle(self) -> None: with db(): # sourcery skip: extract-method repo = MatrixRepository() - m = Matrix( - id="hello", - created_at=datetime.now(), - ) + m = Matrix(id="hello", created_at=datetime.now()) repo.save(m) assert m.id assert m == repo.get(m.id) @@ -29,11 +29,11 @@ def test_db_lifecycle(self): repo.delete(m.id) assert repo.get(m.id) is None - def test_bucket_lifecycle(self, tmp_path: Path): + def test_bucket_lifecycle(self, tmp_path: Path) -> None: repo = MatrixContentRepository(tmp_path) - a = [[1, 2], [3, 4]] - b = [[5, 6], [7, 8]] + a: ArrayData = [[1, 2], [3, 4]] + b: ArrayData = [[5, 6], [7, 8]] matrix_content_a = MatrixContent(data=a, index=[0, 1], columns=[0, 1]) matrix_content_b = MatrixContent(data=b, index=[0, 1], columns=[0, 1]) @@ -51,7 +51,7 @@ def test_bucket_lifecycle(self, tmp_path: Path): with pytest.raises(FileNotFoundError): repo.get(aid) - def test_dataset(self): + def test_dataset(self) -> None: with db(): # sourcery skip: extract-duplicate-method, extract-method repo = MatrixRepository() @@ -66,15 +66,9 @@ def test_dataset(self): dataset_repo = MatrixDataSetRepository() - m1 = Matrix( - id="hello", - created_at=datetime.now(), - ) + m1 = Matrix(id="hello", created_at=datetime.now()) repo.save(m1) - m2 = Matrix( - id="world", - created_at=datetime.now(), - ) + m2 = Matrix(id="world", created_at=datetime.now()) repo.save(m2) dataset = MatrixDataSet( @@ -94,7 +88,7 @@ def test_dataset(self): dataset.matrices.append(matrix_relation) dataset = dataset_repo.save(dataset) - dataset_query_result: Optional[MatrixDataSet] = dataset_repo.get(dataset.id) + dataset_query_result = dataset_repo.get(dataset.id) assert dataset_query_result is not None assert dataset_query_result.name == "some name" assert len(dataset_query_result.matrices) == 2 @@ -106,12 +100,12 @@ def test_dataset(self): updated_at=datetime.now(), ) dataset_repo.save(dataset_update) - dataset_query_result: Optional[MatrixDataSet] = dataset_repo.get(dataset.id) + dataset_query_result = dataset_repo.get(dataset.id) assert dataset_query_result is not None assert dataset_query_result.name == "some name change" assert dataset_query_result.owner_id == user.id - def test_datastore_query(self): + def test_datastore_query(self) -> None: # sourcery skip: extract-duplicate-method with db(): user_repo = UserRepository(Config(security=SecurityConfig())) @@ -121,15 +115,9 @@ def test_datastore_query(self): user2 = user_repo.save(User(name="hello", password=Password("world"))) repo = MatrixRepository() - m1 = Matrix( - id="hello", - created_at=datetime.now(), - ) + m1 = Matrix(id="hello", created_at=datetime.now()) repo.save(m1) - m2 = Matrix( - id="world", - created_at=datetime.now(), - ) + m2 = Matrix(id="world", created_at=datetime.now()) repo.save(m2) dataset_repo = MatrixDataSetRepository() @@ -176,14 +164,19 @@ def test_datastore_query(self): assert repo.get(m1.id) is not None assert ( len( - db.session.query(MatrixDataSetRelation).filter(MatrixDataSetRelation.dataset_id == dataset.id).all() + # fmt: off + db.session + .query(MatrixDataSetRelation) + .filter(MatrixDataSetRelation.dataset_id == dataset.id) + .all() + # fmt: on ) == 0 ) class TestMatrixContentRepository: - def test_save(self, matrix_content_repo: MatrixContentRepository): + def test_save(self, matrix_content_repo: MatrixContentRepository) -> None: """ Saves the content of a matrix as a TSV file in the directory and returns its SHA256 hash. @@ -192,6 +185,7 @@ def test_save(self, matrix_content_repo: MatrixContentRepository): bucket_dir = matrix_content_repo.bucket_dir # when the data is saved in the repo + data: ArrayData data = [[1, 2, 3], [4, 5, 6]] matrix_hash = matrix_content_repo.save(data) # then a TSV file is created in the repo directory @@ -224,12 +218,37 @@ def test_save(self, matrix_content_repo: MatrixContentRepository): other_matrix_file = bucket_dir.joinpath(f"{other_matrix_hash}.tsv") assert set(matrix_files) == {matrix_file, other_matrix_file} - def test_get(self, matrix_content_repo): + def test_save_and_retrieve_empty_matrix(self, matrix_content_repo: MatrixContentRepository) -> None: + """ + Test saving and retrieving empty matrices as TSV files. + Il all cases the file must be empty. + """ + bucket_dir = matrix_content_repo.bucket_dir + + # Test with an empty matrix + empty_array: ArrayData = [] + matrix_hash = matrix_content_repo.save(empty_array) + matrix_file = bucket_dir.joinpath(f"{matrix_hash}.tsv") + retrieved_matrix = matrix_content_repo.get(matrix_hash) + + assert not matrix_file.read_bytes() + assert retrieved_matrix.data == [[]] + + # Test with an empty 2D array + empty_2d_array: ArrayData = [[]] + matrix_hash = matrix_content_repo.save(empty_2d_array) + matrix_file = bucket_dir.joinpath(f"{matrix_hash}.tsv") + retrieved_matrix = matrix_content_repo.get(matrix_hash) + + assert not matrix_file.read_bytes() + assert retrieved_matrix.data == [[]] + + def test_get(self, matrix_content_repo: MatrixContentRepository) -> None: """ Retrieves the content of a matrix with a given SHA256 hash. """ # when the data is saved in the repo - data = [[1, 2, 3], [4, 5, 6]] + data: ArrayData = [[1, 2, 3], [4, 5, 6]] matrix_hash = matrix_content_repo.save(data) # then the saved matrix object can be retrieved content = matrix_content_repo.get(matrix_hash) @@ -243,12 +262,12 @@ def test_get(self, matrix_content_repo): missing_hash = "8b1a9953c4611296a827abf8c47804d7e6c49c6b" matrix_content_repo.get(missing_hash) - def test_exists(self, matrix_content_repo): + def test_exists(self, matrix_content_repo: MatrixContentRepository) -> None: """ Checks if a matrix with a given SHA256 hash exists in the directory. """ # when the data is saved in the repo - data = [[1, 2, 3], [4, 5, 6]] + data: ArrayData = [[1, 2, 3], [4, 5, 6]] matrix_hash = matrix_content_repo.save(data) # then the saved matrix object exists assert matrix_content_repo.exists(matrix_hash) @@ -258,12 +277,12 @@ def test_exists(self, matrix_content_repo): missing_hash = "8b1a9953c4611296a827abf8c47804d7e6c49c6b" assert not matrix_content_repo.exists(missing_hash) - def test_delete(self, matrix_content_repo): + def test_delete(self, matrix_content_repo: MatrixContentRepository) -> None: """ Deletes the tsv file containing the content of a matrix with a given SHA256 hash. """ # when the data is saved in the repo - data = [[1, 2, 3], [4, 5, 6]] + data: ArrayData = [[1, 2, 3], [4, 5, 6]] matrix_hash = matrix_content_repo.save(data) # then the saved matrix object can be deleted matrix_content_repo.delete(matrix_hash) From c156c3bc8b312a0932b2b25dc5c835b8e12d2cfc Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Thu, 28 Sep 2023 10:26:41 +0200 Subject: [PATCH 03/16] refactor(matrix): improve implementation of dataframe saving --- antarest/matrixstore/service.py | 5 +- antarest/matrixstore/uri_resolver_service.py | 24 +- .../filesystem/matrix/output_series_matrix.py | 19 +- tests/matrixstore/test_service.py | 215 ++++++++++++------ .../matrix/output_series_matrix_test.py | 6 +- 5 files changed, 177 insertions(+), 92 deletions(-) diff --git a/antarest/matrixstore/service.py b/antarest/matrixstore/service.py index d10b92ff50..4f44a0a471 100644 --- a/antarest/matrixstore/service.py +++ b/antarest/matrixstore/service.py @@ -1,4 +1,5 @@ import contextlib +import json import logging import tempfile from abc import ABC, abstractmethod @@ -184,7 +185,9 @@ def _file_importation(self, file: bytes, is_json: bool = False) -> str: A SHA256 hash that identifies the imported matrix. """ if is_json: - return self.create(MatrixContent.parse_raw(file).data) + obj = json.loads(file) + content = MatrixContent(**obj) + return self.create(content.data) # noinspection PyTypeChecker matrix = np.loadtxt(BytesIO(file), delimiter="\t", dtype=np.float64, ndmin=2) matrix = matrix.reshape((1, 0)) if matrix.size == 0 else matrix diff --git a/antarest/matrixstore/uri_resolver_service.py b/antarest/matrixstore/uri_resolver_service.py index a13d6c6ad0..01717c57bd 100644 --- a/antarest/matrixstore/uri_resolver_service.py +++ b/antarest/matrixstore/uri_resolver_service.py @@ -11,7 +11,7 @@ class UriResolverService: def __init__(self, matrix_service: ISimpleMatrixService): self.matrix_service = matrix_service - def resolve(self, uri: str, formatted: bool = True) -> Optional[SUB_JSON]: + def resolve(self, uri: str, formatted: bool = True) -> SUB_JSON: res = UriResolverService._extract_uri_components(uri) if res: protocol, uuid = res @@ -52,19 +52,17 @@ def _resolve_matrix(self, id: str, formatted: bool = True) -> SUB_JSON: index=data.index, columns=data.columns, ) - if not df.empty: - return ( - df.to_csv( - None, - sep="\t", - header=False, - index=False, - float_format="%.6f", - ) - or "" - ) - else: + if df.empty: return "" + else: + csv = df.to_csv( + None, + sep="\t", + header=False, + index=False, + float_format="%.6f", + ) + return csv or "" raise ValueError(f"id matrix {id} not found") def build_matrix_uri(self, id: str) -> str: diff --git a/antarest/study/storage/rawstudy/model/filesystem/matrix/output_series_matrix.py b/antarest/study/storage/rawstudy/model/filesystem/matrix/output_series_matrix.py index 13ea8e0400..6f82eaab34 100644 --- a/antarest/study/storage/rawstudy/model/filesystem/matrix/output_series_matrix.py +++ b/antarest/study/storage/rawstudy/model/filesystem/matrix/output_series_matrix.py @@ -98,15 +98,16 @@ def _dump_json(self, data: JSON) -> None: matrix = pd.concat([time, matrix], axis=1) head = self.head_writer.build(var=df.columns.size, end=df.index.size) - self.config.path.write_text(head) - - matrix.to_csv( - open(self.config.path, "a", newline="\n"), - sep="\t", - index=False, - header=False, - line_terminator="\n", - ) + with self.config.path.open(mode="w", newline="\n") as fd: + fd.write(head) + if not matrix.empty: + matrix.to_csv( + fd, + sep="\t", + header=False, + index=False, + float_format="%.6f", + ) def check_errors( self, diff --git a/tests/matrixstore/test_service.py b/tests/matrixstore/test_service.py index a251cda4c5..bcef2c96b6 100644 --- a/tests/matrixstore/test_service.py +++ b/tests/matrixstore/test_service.py @@ -1,9 +1,10 @@ import datetime import io +import json import time import typing as t from unittest.mock import ANY, Mock -from zipfile import ZIP_DEFLATED, ZipFile +import zipfile import numpy as np import pytest @@ -26,12 +27,14 @@ ) from antarest.matrixstore.service import MatrixService +MatrixType = t.List[t.List[float]] + class TestMatrixService: - def test_create__nominal_case(self, matrix_service: MatrixService): + def test_create__nominal_case(self, matrix_service: MatrixService) -> None: """Creates a new matrix object with the specified data.""" # when a matrix is created (inserted) in the service - data = [[1, 2, 3], [4, 5, 6]] + data: MatrixType = [[1, 2, 3], [4, 5, 6]] matrix_id = matrix_service.create(data) # A "real" hash value is calculated @@ -52,7 +55,7 @@ def test_create__nominal_case(self, matrix_service: MatrixService): now = datetime.datetime.utcnow() assert now - datetime.timedelta(seconds=1) <= obj.created_at <= now - def test_create__from_numpy_array(self, matrix_service: MatrixService): + def test_create__from_numpy_array(self, matrix_service: MatrixService) -> None: """Creates a new matrix object with the specified data.""" # when a matrix is created (inserted) in the service data = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float64) @@ -76,13 +79,13 @@ def test_create__from_numpy_array(self, matrix_service: MatrixService): now = datetime.datetime.utcnow() assert now - datetime.timedelta(seconds=1) <= obj.created_at <= now - def test_create__side_effect(self, matrix_service: MatrixService): + def test_create__side_effect(self, matrix_service: MatrixService) -> None: """Creates a new matrix object with the specified data, but fail during saving.""" # if the matrix can't be created in the service matrix_repo = matrix_service.repo matrix_repo.save = Mock(side_effect=Exception("database error")) with pytest.raises(Exception, match="database error"): - data = [[1, 2, 3], [4, 5, 6]] + data: MatrixType = [[1, 2, 3], [4, 5, 6]] matrix_service.create(data) # the associated matrix file must not be deleted @@ -94,10 +97,10 @@ def test_create__side_effect(self, matrix_service: MatrixService): with db(): assert not db.session.query(Matrix).count() - def test_get(self, matrix_service): + def test_get(self, matrix_service: MatrixService) -> None: """Get a matrix object from the database and the matrix content repository.""" # when a matrix is created (inserted) in the service - data = [[1, 2, 3], [4, 5, 6]] + data: MatrixType = [[1, 2, 3], [4, 5, 6]] matrix_id = matrix_service.create(data) # nominal_case: we can retrieve the matrix and its content @@ -120,10 +123,10 @@ def test_get(self, matrix_service): obj = matrix_service.get(missing_hash) assert obj is None - def test_exists(self, matrix_service): + def test_exists(self, matrix_service: MatrixService) -> None: """Test the exists method.""" # when a matrix is created (inserted) in the service - data = [[1, 2, 3], [4, 5, 6]] + data: MatrixType = [[1, 2, 3], [4, 5, 6]] matrix_id = matrix_service.create(data) # nominal_case: we can retrieve the matrix and its content @@ -132,10 +135,10 @@ def test_exists(self, matrix_service): missing_hash = "8b1a9953c4611296a827abf8c47804d7e6c49c6b" assert not matrix_service.exists(missing_hash) - def test_delete__nominal_case(self, matrix_service: MatrixService): + def test_delete__nominal_case(self, matrix_service: MatrixService) -> None: """Delete a matrix object from the matrix content repository and the database.""" # when a matrix is created (inserted) in the service - data = [[1, 2, 3], [4, 5, 6]] + data: MatrixType = [[1, 2, 3], [4, 5, 6]] matrix_id = matrix_service.create(data) # When the matrix id deleted @@ -151,7 +154,7 @@ def test_delete__nominal_case(self, matrix_service: MatrixService): with db(): assert not db.session.query(Matrix).count() - def test_delete__missing(self, matrix_service: MatrixService): + def test_delete__missing(self, matrix_service: MatrixService) -> None: """Delete a matrix object from the matrix content repository and the database.""" # When the matrix id deleted with db(): @@ -167,8 +170,139 @@ def test_delete__missing(self, matrix_service: MatrixService): with db(): assert not db.session.query(Matrix).count() + @pytest.mark.parametrize( + "data", + [ + pytest.param([[1, 2, 3], [4, 5, 6]], id="classic-array"), + pytest.param([[]], id="2D-empty-array"), + ], + ) + @pytest.mark.parametrize("content_type", ["application/json", "text/plain"]) + def test_create_by_importation__nominal_case( + self, + matrix_service: MatrixService, + data: MatrixType, + content_type: str, + ) -> None: + """ + Create a new matrix by importing a file. + The file is either a JSON file or a CSV file. + """ + # Prepare the matrix data to import + matrix = np.array(data, dtype=np.float64) + if content_type == "application/json": + # JSON format of the array using the dataframe format + index = list(range(matrix.shape[0])) + columns = list(range(matrix.shape[1])) + content = json.dumps({"index": index, "columns": columns, "data": matrix.tolist()}) + buffer = io.BytesIO(content.encode("utf-8")) + filename = "matrix.json" + json_format = True + else: + # CSV format of the array (without header) + buffer = io.BytesIO() + np.savetxt(buffer, matrix, delimiter="\t") + buffer.seek(0) + filename = "matrix.txt" + json_format = False + + # Prepare a UploadFile object using the buffer + upload_file = _create_upload_file(filename=filename, file=buffer, content_type=content_type) + + # when a matrix is created (inserted) in the service + info_list: t.Sequence[MatrixInfoDTO] = matrix_service.create_by_importation(upload_file, json=json_format) + + # Then, check the list of created matrices + assert len(info_list) == 1 + info = info_list[0] -def test_dataset_lifecycle(): + # A "real" hash value is calculated + assert info.id, "ID can't be empty" + + # The matrix is saved in the content repository as a TSV file + bucket_dir = matrix_service.matrix_content_repository.bucket_dir + content_path = bucket_dir.joinpath(f"{info.id}.tsv") + actual = np.loadtxt(content_path) + assert actual.all() == matrix.all() + + # A matrix object is stored in the database + with db(): + obj = matrix_service.repo.get(info.id) + assert obj is not None, f"Missing Matrix object {info.id}" + assert obj.width == matrix.shape[1] + assert obj.height == matrix.shape[0] + now = datetime.datetime.utcnow() + assert now - datetime.timedelta(seconds=1) <= obj.created_at <= now + + @pytest.mark.parametrize("content_type", ["application/json", "text/plain"]) + def test_create_by_importation__zip_file(self, matrix_service: MatrixService, content_type: str) -> None: + """ + Create a ZIP file with several matrices, using either a JSON format or a CSV format. + All matrices of the ZIP file use the same format. + Check that the matrices are correctly imported. + """ + # Prepare the matrix data to import + data_list: t.List[MatrixType] = [ + [[1, 2, 3], [4, 5, 6]], + [[7, 8, 9, 10, 11], [17, 18, 19, 20, 21], [27, 28, 29, 30, 31]], + [[]], + ] + matrix_list: t.List[np.ndarray] = [np.array(data, dtype=np.float64) for data in data_list] + if content_type == "application/json": + # JSON format of the array using the dataframe format + index_list = [list(range(matrix.shape[0])) for matrix in matrix_list] + columns_list = [list(range(matrix.shape[1])) for matrix in matrix_list] + data_list = [matrix.tolist() for matrix in matrix_list] + content_list = [ + json.dumps({"index": index, "columns": columns, "data": data}).encode("utf-8") + for index, columns, data in zip(index_list, columns_list, data_list) + ] + json_format = True + else: + # CSV format of the array (without header) + content_list = [] + for matrix in matrix_list: + buffer = io.BytesIO() + np.savetxt(buffer, matrix, delimiter="\t") + content_list.append(buffer.getvalue()) + json_format = False + + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, mode="w", compression=zipfile.ZIP_DEFLATED) as zf: + for i, content in enumerate(content_list): + suffix = {True: "json", False: "txt"}[json_format] + zf.writestr(f"matrix-{i:1d}.{suffix}", content) + buffer.seek(0) + + # Prepare a UploadFile object using the buffer + upload_file = _create_upload_file(filename="matrices.zip", file=buffer, content_type="application/zip") + + # When matrices are created (inserted) in the service + info_list: t.Sequence[MatrixInfoDTO] = matrix_service.create_by_importation(upload_file, json=json_format) + + # Then, check the list of created matrices + assert len(info_list) == len(data_list) + for info, matrix in zip(info_list, matrix_list): + # A "real" hash value is calculated + assert info.id, "ID can't be empty" + + # The matrix is saved in the content repository as a TSV file + bucket_dir = matrix_service.matrix_content_repository.bucket_dir + content_path = bucket_dir.joinpath(f"{info.id}.tsv") + actual = np.loadtxt(content_path) + assert actual.all() == matrix.all() + + # A matrix object is stored in the database + with db(): + obj = matrix_service.repo.get(info.id) + assert obj is not None, f"Missing Matrix object {info.id}" + assert obj.width == (matrix.shape[1] if matrix.size else 0) + assert obj.height == matrix.shape[0] + now = datetime.datetime.utcnow() + assert now - datetime.timedelta(seconds=1) <= obj.created_at <= now + + +def test_dataset_lifecycle() -> None: content = Mock() repo = Mock() dataset_repo = Mock() @@ -347,7 +481,7 @@ def test_dataset_lifecycle(): dataset_repo.delete.assert_called_once() -def _create_upload_file(filename: str, file: t.IO = None, content_type: str = "") -> UploadFile: +def _create_upload_file(filename: str, file: io.BytesIO, content_type: str = "") -> UploadFile: if hasattr(UploadFile, "content_type"): # `content_type` attribute was replace by a read-ony property in starlette-v0.24. headers = Headers(headers={"content-type": content_type}) @@ -356,54 +490,3 @@ def _create_upload_file(filename: str, file: t.IO = None, content_type: str = "" else: # noinspection PyTypeChecker,PyArgumentList return UploadFile(filename=filename, file=file, content_type=content_type) - - -def test_import(): - # Init Mock - repo_content = Mock() - repo = Mock() - - file_str = "1\t2\t3\t4\t5\n6\t7\t8\t9\t10" - matrix_content = str.encode(file_str) - - # Expected - matrix_id = "123" - exp_matrix_info = [MatrixInfoDTO(id=matrix_id, name="matrix.txt")] - exp_matrix = Matrix(id=matrix_id, width=5, height=2) - # Test - service = MatrixService( - repo=repo, - repo_dataset=Mock(), - matrix_content_repository=repo_content, - file_transfer_manager=Mock(), - task_service=Mock(), - config=Mock(), - user_service=Mock(), - ) - service.repo.get.return_value = None - service.matrix_content_repository.save.return_value = matrix_id - service.repo.save.return_value = exp_matrix - - # CSV importation - matrix_file = _create_upload_file( - filename="matrix.txt", - file=io.BytesIO(matrix_content), - content_type="test/plain", - ) - matrix = service.create_by_importation(matrix_file) - assert matrix[0].name == exp_matrix_info[0].name - assert matrix[0].id is not None - - # Zip importation - zip_content = io.BytesIO() - with ZipFile(zip_content, "w", ZIP_DEFLATED) as output_data: - output_data.writestr("matrix.txt", file_str) - - zip_content.seek(0) - zip_file = _create_upload_file( - filename="Matrix.zip", - file=zip_content, - content_type="application/zip", - ) - matrix = service.create_by_importation(zip_file) - assert matrix == exp_matrix_info diff --git a/tests/storage/repository/filesystem/matrix/output_series_matrix_test.py b/tests/storage/repository/filesystem/matrix/output_series_matrix_test.py index e93b0006f8..d739e73b0d 100644 --- a/tests/storage/repository/filesystem/matrix/output_series_matrix_test.py +++ b/tests/storage/repository/filesystem/matrix/output_series_matrix_test.py @@ -9,7 +9,7 @@ from antarest.study.storage.rawstudy.model.filesystem.matrix.output_series_matrix import OutputSeriesMatrix MATRIX_DAILY_DATA = """\ -DE area va hourly +DE\tarea\tva\thourly \tVARIABLES\tBEGIN\tEND \t2\t1\t2 @@ -21,7 +21,7 @@ """ -def test_get(tmp_path: Path): +def test_get(tmp_path: Path) -> None: file = tmp_path / "matrix-daily.txt" file.write_text("\n\n\n\nmock\tfile\ndummy\tdummy\ndummy\tdummy\ndummy\tdummy") config = FileStudyTreeConfig(study_path=file, path=file, study_id="id", version=-1) @@ -55,7 +55,7 @@ def test_get(tmp_path: Path): assert node.load() == matrix.to_dict(orient="split") -def test_save(tmp_path: Path): +def test_save(tmp_path: Path) -> None: file = tmp_path / "matrix-daily.txt" config = FileStudyTreeConfig(study_path=file, path=file, study_id="id", version=-1) From b054b8b72d49966ed46f2110f2a24aedb4549d2b Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Thu, 28 Sep 2023 13:17:15 +0200 Subject: [PATCH 04/16] refactor(matrix-service): improve implementation of `create_by_importation` --- antarest/matrixstore/service.py | 63 +++++++++++++++++++++---------- antarest/matrixstore/web.py | 2 +- tests/matrixstore/test_service.py | 14 +++---- 3 files changed, 52 insertions(+), 27 deletions(-) diff --git a/antarest/matrixstore/service.py b/antarest/matrixstore/service.py index 4f44a0a471..639084b587 100644 --- a/antarest/matrixstore/service.py +++ b/antarest/matrixstore/service.py @@ -1,13 +1,13 @@ import contextlib +import io import json import logging import tempfile +import zipfile from abc import ABC, abstractmethod from datetime import datetime, timezone -from io import BytesIO from pathlib import Path from typing import List, Optional, Sequence, Tuple, Union -from zipfile import ZipFile import numpy as np from fastapi import UploadFile @@ -37,6 +37,18 @@ ) from antarest.matrixstore.repository import MatrixContentRepository, MatrixDataSetRepository, MatrixRepository +# List of files to exclude from ZIP archives +EXCLUDED_FILES = { + "__MACOSX", + ".DS_Store", + "._.DS_Store", + "Thumbs.db", + "desktop.ini", + "$RECYCLE.BIN", + "System Volume Information", + "RECYCLER", +} + logger = logging.getLogger(__name__) @@ -151,29 +163,42 @@ def create(self, data: Union[List[List[MatrixData]], npt.NDArray[np.float64]]) - self.repo.save(matrix) return matrix_id - def create_by_importation(self, file: UploadFile, json: bool = False) -> List[MatrixInfoDTO]: + def create_by_importation(self, file: UploadFile, is_json: bool = False) -> List[MatrixInfoDTO]: + """ + Imports a matrix from a TSV or JSON file or a collection of matrices from a ZIP file. + + TSV-formatted files are expected to contain only matrix data without any header. + + JSON-formatted files are expected to contain the following attributes: + + - `index`: The list of row labels. + - `columns`: The list of column labels. + - `data`: The matrix data as a nested list of floats. + + Args: + file: The file to import (TSV, JSON or ZIP). + is_json: Flag indicating if the file is JSON-encoded. + + Returns: + A list of `MatrixInfoDTO` objects containing the SHA256 hash of the imported matrices. + """ with file.file as f: if file.content_type == "application/zip": - input_zip = ZipFile(BytesIO(f.read())) - files = { - info.filename: input_zip.read(info.filename) for info in input_zip.infolist() if not info.is_dir() - } + with contextlib.closing(f): + buffer = io.BytesIO(f.read()) matrix_info: List[MatrixInfoDTO] = [] - for name in files: - if all( - [ - not name.startswith("__MACOSX/"), - not name.startswith(".DS_Store"), - ] - ): - matrix_id = self._file_importation(files[name], json) - matrix_info.append(MatrixInfoDTO(id=matrix_id, name=name)) + with zipfile.ZipFile(buffer) as zf: + for info in zf.infolist(): + if info.is_dir() or info.filename in EXCLUDED_FILES: + continue + matrix_id = self._file_importation(zf.read(info.filename), is_json=is_json) + matrix_info.append(MatrixInfoDTO(id=matrix_id, name=info.filename)) return matrix_info else: - matrix_id = self._file_importation(f.read(), json) + matrix_id = self._file_importation(f.read(), is_json=is_json) return [MatrixInfoDTO(id=matrix_id, name=file.filename)] - def _file_importation(self, file: bytes, is_json: bool = False) -> str: + def _file_importation(self, file: bytes, *, is_json: bool = False) -> str: """ Imports a matrix from a TSV or JSON file in bytes format. @@ -189,7 +214,7 @@ def _file_importation(self, file: bytes, is_json: bool = False) -> str: content = MatrixContent(**obj) return self.create(content.data) # noinspection PyTypeChecker - matrix = np.loadtxt(BytesIO(file), delimiter="\t", dtype=np.float64, ndmin=2) + matrix = np.loadtxt(io.BytesIO(file), delimiter="\t", dtype=np.float64, ndmin=2) matrix = matrix.reshape((1, 0)) if matrix.size == 0 else matrix return self.create(matrix) diff --git a/antarest/matrixstore/web.py b/antarest/matrixstore/web.py index a97ae7d45b..4b47135b52 100644 --- a/antarest/matrixstore/web.py +++ b/antarest/matrixstore/web.py @@ -55,7 +55,7 @@ def create_by_importation( ) -> Any: logger.info("Importing new matrix dataset", extra={"user": current_user.id}) if current_user.id is not None: - return service.create_by_importation(file, json) + return service.create_by_importation(file, is_json=json) raise UserHasNotPermissionError() @bp.get("/matrix/{id}", tags=[APITag.matrix], response_model=MatrixDTO) diff --git a/tests/matrixstore/test_service.py b/tests/matrixstore/test_service.py index bcef2c96b6..db26e6403a 100644 --- a/tests/matrixstore/test_service.py +++ b/tests/matrixstore/test_service.py @@ -3,8 +3,8 @@ import json import time import typing as t -from unittest.mock import ANY, Mock import zipfile +from unittest.mock import ANY, Mock import numpy as np import pytest @@ -186,7 +186,7 @@ def test_create_by_importation__nominal_case( ) -> None: """ Create a new matrix by importing a file. - The file is either a JSON file or a CSV file. + The file is either a JSON file or a TSV file. """ # Prepare the matrix data to import matrix = np.array(data, dtype=np.float64) @@ -199,7 +199,7 @@ def test_create_by_importation__nominal_case( filename = "matrix.json" json_format = True else: - # CSV format of the array (without header) + # TSV format of the array (without header) buffer = io.BytesIO() np.savetxt(buffer, matrix, delimiter="\t") buffer.seek(0) @@ -210,7 +210,7 @@ def test_create_by_importation__nominal_case( upload_file = _create_upload_file(filename=filename, file=buffer, content_type=content_type) # when a matrix is created (inserted) in the service - info_list: t.Sequence[MatrixInfoDTO] = matrix_service.create_by_importation(upload_file, json=json_format) + info_list: t.Sequence[MatrixInfoDTO] = matrix_service.create_by_importation(upload_file, is_json=json_format) # Then, check the list of created matrices assert len(info_list) == 1 @@ -237,7 +237,7 @@ def test_create_by_importation__nominal_case( @pytest.mark.parametrize("content_type", ["application/json", "text/plain"]) def test_create_by_importation__zip_file(self, matrix_service: MatrixService, content_type: str) -> None: """ - Create a ZIP file with several matrices, using either a JSON format or a CSV format. + Create a ZIP file with several matrices, using either a JSON format or a TSV format. All matrices of the ZIP file use the same format. Check that the matrices are correctly imported. """ @@ -259,7 +259,7 @@ def test_create_by_importation__zip_file(self, matrix_service: MatrixService, co ] json_format = True else: - # CSV format of the array (without header) + # TSV format of the array (without header) content_list = [] for matrix in matrix_list: buffer = io.BytesIO() @@ -278,7 +278,7 @@ def test_create_by_importation__zip_file(self, matrix_service: MatrixService, co upload_file = _create_upload_file(filename="matrices.zip", file=buffer, content_type="application/zip") # When matrices are created (inserted) in the service - info_list: t.Sequence[MatrixInfoDTO] = matrix_service.create_by_importation(upload_file, json=json_format) + info_list: t.Sequence[MatrixInfoDTO] = matrix_service.create_by_importation(upload_file, is_json=json_format) # Then, check the list of created matrices assert len(info_list) == len(data_list) From 58656e06ae5d44d50193b3ce8911e7d1e6f55df1 Mon Sep 17 00:00:00 2001 From: abdoulbari zakir <32519851+a-zakir@users.noreply.github.com> Date: Thu, 28 Sep 2023 19:19:44 +0200 Subject: [PATCH 05/16] fix(ui-xpansion): add the missing `batch_size` field in the Xpansion parameters view (#1739) --- webapp/public/locales/en/main.json | 1 + webapp/public/locales/fr/main.json | 1 + .../explore/Xpansion/Settings/SettingsForm.tsx | 10 ++++++++++ .../App/Singlestudy/explore/Xpansion/types.ts | 1 + 4 files changed, 13 insertions(+) diff --git a/webapp/public/locales/en/main.json b/webapp/public/locales/en/main.json index 5e5d65edbd..f7651f99b1 100644 --- a/webapp/public/locales/en/main.json +++ b/webapp/public/locales/en/main.json @@ -644,6 +644,7 @@ "xpansion.amplPresolve": "Ampl presolve", "xpansion.amplSolverBoundsFrequency": "Ampl solve bounds frequency", "xpansion.relativeGap": "Relative gap", + "xpansion.batchSize": "Batch size", "xpansion.solver": "Solver", "xpansion.timeLimit": "Time limit (in hours)", "xpansion.logLevel": "Log level", diff --git a/webapp/public/locales/fr/main.json b/webapp/public/locales/fr/main.json index 314d3c9b03..d60556d964 100644 --- a/webapp/public/locales/fr/main.json +++ b/webapp/public/locales/fr/main.json @@ -644,6 +644,7 @@ "xpansion.amplPresolve": "Ampl presolve", "xpansion.amplSolverBoundsFrequency": "Ampl solve bounds frequency", "xpansion.relativeGap": "Gap d'optimalité relatif", + "xpansion.batchSize": "Taille du batch", "xpansion.solver": "Solveur", "xpansion.timeLimit": "Limite de temps (en heures)", "xpansion.logLevel": "Niveau de log", diff --git a/webapp/src/components/App/Singlestudy/explore/Xpansion/Settings/SettingsForm.tsx b/webapp/src/components/App/Singlestudy/explore/Xpansion/Settings/SettingsForm.tsx index 4afd50488d..b03440c7b4 100644 --- a/webapp/src/components/App/Singlestudy/explore/Xpansion/Settings/SettingsForm.tsx +++ b/webapp/src/components/App/Singlestudy/explore/Xpansion/Settings/SettingsForm.tsx @@ -185,6 +185,16 @@ function SettingsForm(props: PropType) { }} optional /> + + handleChange("batch_size", parseInt(e.target.value, 10)) + } + sx={{ mb: 1 }} + /> Date: Fri, 1 Sep 2023 17:49:52 +0200 Subject: [PATCH 06/16] build: prepare next minor release v2.15.0 (unreleased) --- antarest/__init__.py | 4 ++-- docs/CHANGELOG.md | 4 ++++ setup.py | 2 +- sonar-project.properties | 2 +- webapp/package-lock.json | 2 +- webapp/package.json | 2 +- 6 files changed, 10 insertions(+), 6 deletions(-) diff --git a/antarest/__init__.py b/antarest/__init__.py index 4ebdf105eb..28bcfd952d 100644 --- a/antarest/__init__.py +++ b/antarest/__init__.py @@ -7,9 +7,9 @@ # Standard project metadata -__version__ = "2.14.6" +__version__ = "2.15.0" __author__ = "RTE, Antares Web Team" -__date__ = "2023-09-01" +__date__ = "(unreleased)" # noinspection SpellCheckingInspection __credits__ = "(c) Réseau de Transport de l’Électricité (RTE)" diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index c80a89d441..4537dbaa82 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,6 +1,10 @@ Antares Web Changelog ===================== +v2.15.0 (unreleased) +-------------------- + + v2.14.6 (2023-09-01) -------------------- diff --git a/setup.py b/setup.py index 2218925929..9dd8abaca5 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="AntaREST", - version="2.14.6", + version="2.15.0", description="Antares Server", long_description=long_description, long_description_content_type="text/markdown", diff --git a/sonar-project.properties b/sonar-project.properties index 54924fc307..3fb85c7559 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -6,5 +6,5 @@ sonar.exclusions=antarest/gui.py,antarest/main.py sonar.python.coverage.reportPaths=coverage.xml sonar.python.version=3.8 sonar.javascript.lcov.reportPaths=webapp/coverage/lcov.info -sonar.projectVersion=2.14.6 +sonar.projectVersion=2.15.0 sonar.coverage.exclusions=antarest/gui.py,antarest/main.py,antarest/singleton_services.py,antarest/worker/archive_worker_service.py,webapp/**/* \ No newline at end of file diff --git a/webapp/package-lock.json b/webapp/package-lock.json index 7e05d48f8b..af03a6bde3 100644 --- a/webapp/package-lock.json +++ b/webapp/package-lock.json @@ -1,6 +1,6 @@ { "name": "antares-web", - "version": "2.14.6", + "version": "2.15.0", "lockfileVersion": 3, "requires": true, "packages": { diff --git a/webapp/package.json b/webapp/package.json index 620245c9b7..3a5251d7f8 100644 --- a/webapp/package.json +++ b/webapp/package.json @@ -1,6 +1,6 @@ { "name": "antares-web", - "version": "2.14.6", + "version": "2.15.0", "private": true, "engines": { "node": "18.16.1" From c4495a07db92d90cc38b26ebad5973742713fe3f Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Sun, 3 Sep 2023 08:44:31 +0200 Subject: [PATCH 07/16] build: remove unused libraries from project requirements --- requirements.txt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5c00c7cedd..a82424f3b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ Antares-Launcher~=1.3.1 -aiofiles~=0.8.0 alembic~=1.7.5 asgi-ratelimit[redis]==0.7.0 bcrypt~=3.2.0 @@ -26,8 +25,6 @@ python-multipart~=0.0.5 PyYAML~=5.4.1 redis~=4.1.2 requests~=2.27.1 -scandir~=1.10.0 SQLAlchemy~=1.4.46 starlette~=0.17.1 -uvicorn[standard]~=0.15.0 -xarray \ No newline at end of file +uvicorn[standard]~=0.15.0 \ No newline at end of file From f8d036e021c534eaeed4885397073e1f6e3b9d53 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Sun, 3 Sep 2023 08:47:38 +0200 Subject: [PATCH 08/16] build: move `checksumdir` and `locust` libraries from project requirements to tests requirements (only used in unit tests) --- requirements-dev.txt | 1 - requirements-test.txt | 4 +++- requirements.txt | 2 -- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index c8bf292f4b..56b4d22438 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,7 +4,6 @@ black~=23.7.0 isort~=5.12.0 mypy~=1.4.1 pyinstaller~=4.8 -checksumdir~=1.2.0 # Extra requirements installed by `mypy --install-types`. # IMPORTANT: Make sure the versions of these typing libraries match the versions diff --git a/requirements-test.txt b/requirements-test.txt index 088f207160..c752146feb 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,3 +1,5 @@ -r requirements.txt +checksumdir~=1.2.0 +locust~=2.7.0 pytest~=6.2.5 -pytest-cov~=4.0.0 +pytest-cov~=4.0.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index a82424f3b9..8931c773c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ Antares-Launcher~=1.3.1 alembic~=1.7.5 asgi-ratelimit[redis]==0.7.0 bcrypt~=3.2.0 -checksumdir~=1.2.0 click~=8.0.3 contextvars~=2.4 fastapi-jwt-auth~=0.5.0 @@ -12,7 +11,6 @@ filelock~=3.4.2 gunicorn~=20.1.0 Jinja2~=3.0.3 jsonref~=0.2 -locust~=2.7.0 MarkupSafe~=2.0.1 numpy~=1.22.1 pandas~=1.4.0 From 00ae9923abcb83e5764eaa09a56fdceb3cc6cfde Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Sun, 3 Sep 2023 08:58:06 +0200 Subject: [PATCH 09/16] build: include `typing_extensions` in the project requirements (new direct usage) --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 8931c773c5..bd0455b56f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,4 +25,5 @@ redis~=4.1.2 requests~=2.27.1 SQLAlchemy~=1.4.46 starlette~=0.17.1 +typing_extensions~=4.7.1 uvicorn[standard]~=0.15.0 \ No newline at end of file From 6c3f4bff861ab4f5af36eb3c22dcbf33032a91ee Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Sun, 3 Sep 2023 09:14:17 +0200 Subject: [PATCH 10/16] build: remove unused Sphinx-related libraries from docs requirements --- requirements-doc.txt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/requirements-doc.txt b/requirements-doc.txt index 66863fde4a..21f876b437 100644 --- a/requirements-doc.txt +++ b/requirements-doc.txt @@ -1,5 +1,2 @@ mkdocs -mkdocs-material -sphinx -myst-parser - +mkdocs-material \ No newline at end of file From 1de508e4974bbc13ec6dbab86a2799bd5ff01005 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Sun, 3 Sep 2023 09:02:47 +0200 Subject: [PATCH 11/16] build: update project's metadata in `setup.py` (author, author email, license, platforms, classifiers) --- setup.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 9dd8abaca5..28d31b53d5 100644 --- a/setup.py +++ b/setup.py @@ -1,18 +1,33 @@ -import setuptools +from pathlib import Path -with open("README.md", "r") as fh: - long_description = fh.read() +from setuptools import setup, find_packages -setuptools.setup( +setup( name="AntaREST", version="2.15.0", description="Antares Server", - long_description=long_description, + long_description=Path("README.md").read_text(encoding="utf-8"), long_description_content_type="text/markdown", + author="RTE, Antares Web Team", + author_email="andrea.sgattoni@rte-france.com", url="https://github.com/AntaresSimulatorTeam/api-iso-antares", - packages=setuptools.find_packages(), + packages=find_packages(), + license="Apache Software License", + platforms=[ + "linux-x86_64", + "macosx-10.14-x86_64", + "macosx-10.15-x86_64", + "macosx-11-x86_64", + "macosx-12-x86_64", + "macosx-13-x86_64", + "win-amd64", + ], classifiers=[ - "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "License :: Apache License :: 2.0", "Operating System :: OS Independent", ], From 236a2d712858ee23abb7710ef7f92e19042d48cb Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Sun, 3 Sep 2023 09:07:40 +0200 Subject: [PATCH 12/16] build: update project configuration, specify exclusion of non-package directories when using `find_packages` --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 28d31b53d5..323d8deb7b 100644 --- a/setup.py +++ b/setup.py @@ -2,6 +2,8 @@ from setuptools import setup, find_packages +excluded_dirs = {"alembic", "conf", "docs", "examples", "resources", "scripts", "tests", "venv", "webapp"} + setup( name="AntaREST", version="2.15.0", @@ -11,7 +13,7 @@ author="RTE, Antares Web Team", author_email="andrea.sgattoni@rte-france.com", url="https://github.com/AntaresSimulatorTeam/api-iso-antares", - packages=find_packages(), + packages=find_packages(exclude=excluded_dirs), license="Apache Software License", platforms=[ "linux-x86_64", From 09be12a3989dab5a4cdaa6d73f8e77d2e89c521f Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Mon, 11 Sep 2023 10:00:17 +0200 Subject: [PATCH 13/16] refactor(tools): correct migration from `Requests` to `httpx` --- antarest/gui.py | 4 +++- antarest/tools/lib.py | 17 ++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/antarest/gui.py b/antarest/gui.py index 2a408610ad..399b0a2518 100644 --- a/antarest/gui.py +++ b/antarest/gui.py @@ -9,9 +9,11 @@ try: # `httpx` is a modern alternative to the `requests` library import httpx as requests + from httpx import ConnectError as ConnectionError except ImportError: # noinspection PyUnresolvedReferences, PyPackageRequirements import requests + from requests import ConnectionError import uvicorn # type: ignore from PyQt5.QtGui import QIcon @@ -88,7 +90,7 @@ def main() -> None: ) server.start() for _ in range(30, 0, -1): - with contextlib.suppress(requests.ConnectionError): + with contextlib.suppress(ConnectionError): res = requests.get("http://localhost:8080") if res.status_code == 200: break diff --git a/antarest/tools/lib.py b/antarest/tools/lib.py index 2d2953e3f5..5ade3d214b 100644 --- a/antarest/tools/lib.py +++ b/antarest/tools/lib.py @@ -57,9 +57,20 @@ def __init__( session: Optional[Session] = None, ): self.study_id = study_id - self.session = session or Session() - # TODO fix this - self.session.verify = False + + # todo: find the correct way to handle certificates. + # By default, Requests/Httpx verifies SSL certificates for HTTPS requests. + # When verify is set to `False`, requests will accept any TLS certificate presented + # by the server,and will ignore hostname mismatches and/or expired certificates, + # which will make your application vulnerable to man-in-the-middle (MitM) attacks. + # Setting verify to False may be useful during local development or testing. + if Session.__name__ == "Client": + # noinspection PyArgumentList + self.session = session or Session(verify=False) + else: + self.session = session or Session() + self.session.verify = False + self.host = host if session is None and host is None: raise ValueError("Missing either session or host") From ccd6496266dea01ff952c41565d8940391f0f317 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Wed, 13 Sep 2023 17:09:49 +0200 Subject: [PATCH 14/16] docs: update changelog for the v2.15 release --- docs/CHANGELOG.md | 85 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 4537dbaa82..35402c8f26 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -4,6 +4,91 @@ Antares Web Changelog v2.15.0 (unreleased) -------------------- +### Features + +* **business:** add xpansion batch size back office [`#1485`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1485) +* **commands:** add ST-Storage commands [`#1630`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1630) +* **matrix:** improve unexpected exception error message [`#1635`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1635) +* **matrix:** improve matrix read/write using NumPy [`#1452`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1452) — was introduced in v2.14.2 +* **model:** handle 8.6 study model and study upgrader [`#1489`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1489) +* **service:** assign to a copied study all groups of the user [`#1721`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1721) +* **service:** set the user's name as author in `study.antares` when creating a study [`#1632`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1632) +* **st-storage:** add ST Storage API endpoints [`#1697`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1697) +* **ui-common:** add `GroupedDataTable` used in the new design for listing clusters (thermal / renewables) and ST Storage [`#1633`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1633) +* **ui-config:** enable undo/redo in adequacy patch form [`#1543`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1543) +* **ui-i18n:** add missing adequacy patch translations [`#1680`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1680) — was introduced in v2.14.5 +* **ui-map:** add map zoom buttons [`#1518`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1518) +* **ui-study:** enhance study upgrade dialog [`#1687`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1687) + + +### Bug Fixes + +* **api:** correct API endpoints and CLI tools [`#1726`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1726) +* **api:** add missing `use_leeway` field and validation rules [`#1650`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1650) +* **api:** standardize `areas_ui` to dict format for single area case [`#1557`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1557) +* **build:** fix pyinstaller build [`#1551`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1551) — was resolved in v2.14.2 +* **docs:** update README, removing obsolete instructions [`#1552`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1552) +* **export:** ZIP outputs are now uncompressed before export [`#1656`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1656) +* **filesystem:** correct the handling of default matrices in `InputSeriesMatrix` node [`#1608`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1608) +* **launcher:** fixing launcher versions display and API endpoint for solver versions [`#1671`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1671) +* **log-parser:** simplify parsing and make it accurate [`#1682`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1682) +* **model:** wrong frequency for `hydro energy credits` matrices [`#1708`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1708) +* **packaging:** update the packaging script to use Antares Solver v8.6.1 [`#1652`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1652) +* **service:** set public mode to `NONE` and assign user groups to the study during import [`#1619`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1619) +* **service:** set public mode to `NONE` and assign user groups to the study during copying [`#1620`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1620) +* **st-storage:** allow DELETE endpoint to accept multiple IDs as parameters [`#1716`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1716) +* **st-storage:** update "group" parameter values for Short-Term Storage [`#1665`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1665) +* **st-storage:** fixing archived study configuration reading for short-term storages [`#1673`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1673) +* **st-storage:** replace inflow.txt by inflows.txt according to the changing doc [`#1618`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1618) +* **storage:** fix INI file writing anomaly in Antares study update [`#1542`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1542) +* **table-mode:** issue to read area information in the case where the study has only one area [`#1690`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1690) — was resolved in v2.14.5 +* **table-mode:** fixes the reading of UI information in the case where the study has only one area [`#1674`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1674) +* **ui-api:** resolve Area deletion & Layers/Districts UI modification issues [`#1677`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1677) +* **ui-common:** prevent null values for string cells in HandsonTable component [`#1689`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1689) +* **variant:** fixed TS deletion of renewable clusters and short-term storage [`#1693`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1693) +* **variant:** enhance `VariantCommandsExtractor` for Renewable Clusters and Short-Term Storage [`#1688`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1688) +* **web:** change API response_model to avoid ValidationError [`#1526`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1526) +* **worker:** archive worker must be kept alive for processing [`#1558`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1558) — was resolved in v2.14.2 +* **xpansion:** correct field names and types in Xpansion settings [`#1684`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1684) + + +### Documentation + +* correct and improve the "Variant Manager" documentation [`#1637`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1637) + + +### Tests + +* improve fixtures in unit tests for better reuse [`#1638`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1638) +* **api:** simplify initialization of the database in unit tests [`#1540`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1540) + + +### Refactoring + +* refactoring of unit tests and Model improvement [`#1647`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1647) +* **api-model:** improve `transform_name_to_id` implementation for more efficiency [`#1546`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1546) +* **command:** rename "cluster" as "thermal cluster" [`#1639`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1639) +* **command:** improve implementation of area management [`#1636`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1636) +* **variant:** enhance implementation of variant commands [`#1539`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1539) + + +### Styles + +* **ui:** correct spelling mistake in webapp code [`#1570`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1570) + + +### Build System + +* specify the node versions supported [`#1553`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1553) +* **launcher:** use Antares-Launcher v1.3.0 which preserve the "output" directory required by the `--step=sensitivity` Xpansion option [`#1606`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1606) + + +### Continuous Integration + +* upgrade mypy to v1.4.1 and Black to v23.7.0 for improved typing and formatting [`#1685`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1685) + + + v2.14.6 (2023-09-01) -------------------- From acf3c07eb9959bada064fa2a29b5f5824980af1c Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Fri, 29 Sep 2023 12:09:06 +0200 Subject: [PATCH 15/16] build(deploy): update the GitHub deploy workflow to generate bin artefacts --- .github/workflows/deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c05f960674..b6c40edec7 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -5,6 +5,7 @@ on: branches: - "master" - "hotfix/**" + - "release/2.15" jobs: binary: From d1832c507384b00277d5d1c12e107bb0d7733098 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Fri, 29 Sep 2023 12:26:59 +0200 Subject: [PATCH 16/16] docs: update changelog and release date --- antarest/__init__.py | 2 +- docs/CHANGELOG.md | 159 ++++++++++++++++++++++++++++++++++---- scripts/update_version.py | 36 ++++++--- 3 files changed, 168 insertions(+), 29 deletions(-) diff --git a/antarest/__init__.py b/antarest/__init__.py index 28bcfd952d..17b75e0279 100644 --- a/antarest/__init__.py +++ b/antarest/__init__.py @@ -9,7 +9,7 @@ __version__ = "2.15.0" __author__ = "RTE, Antares Web Team" -__date__ = "(unreleased)" +__date__ = "2023-09-29" # noinspection SpellCheckingInspection __credits__ = "(c) Réseau de Transport de l’Électricité (RTE)" diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 35402c8f26..9ce5a1be89 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,93 +1,220 @@ Antares Web Changelog ===================== -v2.15.0 (unreleased) +v2.15.0 (2023-09-29) -------------------- ### Features +* **antares-launcher:** use antares launcher v1.3.1 [`#1743`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1743) +* **api:** add Properties form endpoint [`31898a1`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/31898a1306d7a2115de20ca0226f8fc436377588) +* **api:** add api doc and integration test [`51db54f`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/51db54fec714cf9aac30ce1f7543e58b5ab1ff6e) +* **api:** add new pollutants in thermal manager [`bcd5a01`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/bcd5a01e6878048512550d26834d4dad335b975e) +* **api:** change the response model of "launcher/_versions" endpoint to return a list of versions [`e7089a1`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/e7089a1a5407640fa49f0230c812257f5ac7cecf) +* **api:** correct default values for "/launcher/versions" endpoint [`f87ccca`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/f87ccca0c1b6c41fb9e0cb6e0e7435b4df59500f) +* **api:** improve error handling in "launcher/_versions" endpoint [`f84ff84`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/f84ff849b0ab69a0a0e4344cfd408880e843b0e9) +* **api:** improve exception handling in "launcher/_versions" endpoint [`1263181`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/12631818a1d515485af2827c76acf2b91ad66245) +* **api:** improves "launcher/_versions" endppoint [`ae6c7d6`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/ae6c7d6cb1b4d3de2a96b3ed9f7b46db712eae6d) +* **api:** replace "launcher/_versions" endpoint by "/launcher/versions" [`fb294dd`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/fb294dd81e520834998883deb1d3561c07484b60) +* **api-layers:** minor improvements [`ef7e21f`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/ef7e21fc612acf53e642b7bbc334baedf52e952c) * **business:** add xpansion batch size back office [`#1485`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1485) * **commands:** add ST-Storage commands [`#1630`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1630) +* **common:** add undo/redo to Form component [`fc02c00`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/fc02c00aea2c9b7d6cd9464346e8962d5304ca1c) +* **common:** multiple updates on Form component [`2193e43`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/2193e431d67d0f1807a2d1cc0853329871c63479) +* **matrix:** improve matrix read/write using NumPy [`#1452`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1452) +* **matrix:** improve matrix service read/write using NumPy [`8c2f14f`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/8c2f14fa027ac5e0ceba9a5c027fd409feff49ce) * **matrix:** improve unexpected exception error message [`#1635`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1635) -* **matrix:** improve matrix read/write using NumPy [`#1452`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1452) — was introduced in v2.14.2 +* **matrix-service:** improve the importation function to handle NumPy arrays directly without the necessity of converting them [`ecf971a`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/ecf971a142533f6c8d4103cd554ebd8278b86227) +* **matrix-service:** avoid having to save the matrix again [`a40ad09`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/a40ad09a61fe5292ec2cfca08cea383868006e54) +* **matrix-service:** enhance the `ISimpleMatrixService` class and sub-classes to support creating from NumPy array [`93737ae`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/93737aecc130f991e8cebfb5d042eab031c18046) +* **matrix-service:** enhance the `MatrixContentRepository` class to support saving NumPy array directly [`1bf113b`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/1bf113bec8fbd6aa868752342d3727b2c2a8792f) +* **matrix-service:** ensure exclusive access to the matrix file between multiple processes (or threads) [`66ce1bc`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/66ce1bc42f7512469352e899b4dce25289790a52) * **model:** handle 8.6 study model and study upgrader [`#1489`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1489) * **service:** assign to a copied study all groups of the user [`#1721`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1721) +* **service:** set `PublicMode.READ` when importing is done by user with no groups [`#1649`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1649) +* **service:** set `PublicMode` to `READ` when importing is done by user with no groups [`04d70ed`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/04d70ed6be492fc860701273cce36b631798c568) * **service:** set the user's name as author in `study.antares` when creating a study [`#1632`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1632) * **st-storage:** add ST Storage API endpoints [`#1697`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1697) -* **ui-common:** add `GroupedDataTable` used in the new design for listing clusters (thermal / renewables) and ST Storage [`#1633`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1633) +* **storage:** ini reader can now take ioTextWrapper as input [`015bc12`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/015bc12a5e1530144133cb072633a8ec4f2e3e23) +* **style:** code style and refactoring enhancements [`8b4ec2e`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/8b4ec2e4f5264a87da78a41443b28daef6303d00) +* **ui:** add and use correct launcher verisons instead of study's [`1878617`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/1878617c2911f653f72bd8f13d9180cac225f42e) +* **ui:** enable undo/redo on some forms [`d0d07e9`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/d0d07e92b17151ef1c650b5cb6b12b67f3b6e6e3) +* **ui-common:** add GroupedDataTable [`#1633`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1633) +* **ui-common:** add submitButtonIcon prop in Form [`6b16e01`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/6b16e010f143b859d6214d2f7cd850e7b72f44ec) +* **ui-common:** display submit error in Form [`92d5a1e`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/92d5a1ed3c2c98802e3a65f49985960183bb6ac5) +* **ui-common:** increase the width of the first column on tables [`d3d70ff`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/d3d70ff6d1e271378cc360cd28953bdb42e90c98) +* **ui-common:** update ColorPickerFE [`8347556`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/8347556d947dc5c005492421cc4722c360ce0373) * **ui-config:** enable undo/redo in adequacy patch form [`#1543`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1543) -* **ui-i18n:** add missing adequacy patch translations [`#1680`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1680) — was introduced in v2.14.5 +* **ui-i18n:** add missing adequacy patch translations [`#1680`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1680) +* **ui-lib:** add material-react-table package [`e88fb69`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/e88fb69b30c319f27a37b3c298a8aaff6bd98f34) +* **ui-login:** update form [`9631045`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/9631045d9f40446264dcad896e5672f3aff4a77d) * **ui-map:** add map zoom buttons [`#1518`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1518) * **ui-study:** enhance study upgrade dialog [`#1687`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1687) +* **ui-thermal:** add new pollutants [`5164a4c`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/5164a4cb5a894f5b96f6c0fd21b358e4df072aec) ### Bug Fixes -* **api:** correct API endpoints and CLI tools [`#1726`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1726) +* **api:** Correct endpoint for `HTTPStatus.NO_CONTENT` (204) [`b70bc97`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/b70bc977f2a3245944f54d19ebb6e792345b03df) * **api:** add missing `use_leeway` field and validation rules [`#1650`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1650) -* **api:** standardize `areas_ui` to dict format for single area case [`#1557`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1557) -* **build:** fix pyinstaller build [`#1551`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1551) — was resolved in v2.14.2 +* **api:** correct API endpoints and CLI tools [`#1726`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1726) +* **api:** correct `update_area_ui` endpoint response model [`963f5ee`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/963f5ee84fa7409a8fcf4effa88007362305d6ab) +* **api:** correct exception handling in `extract_file_to_tmp_dir` [`af38216`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/af38216e685b9f9d37e14cb66aa1300fe1cc592c) +* **api:** correct response model of "/studies/_versions" endpoint [`91da3e7`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/91da3e775086d4f19207bce2d2f684729b694c0d) +* **api:** correct the regex in `IniReader` to match a section in square brackets [`123d716`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/123d716c68584c8c2ce5d907991c7379e21f0a4a) +* **api:** enhance Python detection in version info endpoint [`86fb0e5`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/86fb0e5baa59e1508e239d195894ab186b47189d) +* **api:** sanitize and avoid duplicates in group IDs lists [`8c9ac99`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/8c9ac998fbd4610ca8c7de338adeff53ca08b2cd) +* **api:** standardize 'areas_ui' to dict format for single area case [`#1557`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1557) +* **api:** study data retrieval now considers document types, enabling retrieval of Microsoft Office documents. Binary data support has also been enhanced [`8204d45`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/8204d45867f54337306d463e217ae259b3cc761f) +* **api,tablemode:** wrong adequacy patch mode path and clean code [`0d00432`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/0d00432910cbd6c761a17e69571ef46ec1839dc2) +* **api-layers:** fixed issue preventing layers deletion when no areas [`b3f98f2`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/b3f98f2c12b0042d35f9421388b5f33858f18189) +* **api-layers:** remove unnecessary parameter that causes tests to fail [`052a9f3`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/052a9f38ea5c96d0aac0a0f050cabbe21153a761) +* **build:** fix pyinstaller build [`#1551`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1551) +* **command:** change code according to comments [`f3c68d2`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/f3c68d2d8cf323a36582a5e5a165c1ddf23c0d7c) +* **command:** fix update_config to support API PUT v1/studies/{uuid}/raw [`ffed124`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/ffed124cb17965e41409812a83da2bf36da8b014) * **docs:** update README, removing obsolete instructions [`#1552`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1552) * **export:** ZIP outputs are now uncompressed before export [`#1656`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1656) * **filesystem:** correct the handling of default matrices in `InputSeriesMatrix` node [`#1608`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1608) * **launcher:** fixing launcher versions display and API endpoint for solver versions [`#1671`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1671) * **log-parser:** simplify parsing and make it accurate [`#1682`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1682) +* **matrix:** correct the loading and saving of empty matrices as TSV files [`#1746`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1746) +* **matrix:** create empty file if the matrix is empty [`9155566`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/9155566ec6b2d52c97c4d9fc5bbefbada20a91bf) +* **matrix-service:** correct the `MatrixService.create` so that the matrix file is not deleted in case of exception [`1860379`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/1860379a96cd7ab87c150788c5d2efedd91d5859) +* **matrix-service:** prevent the use of scientific notation when saving matrices [`7387248`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/73872489fb57560e74172b4d5dfc9532ce6660bb) * **model:** wrong frequency for `hydro energy credits` matrices [`#1708`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1708) * **packaging:** update the packaging script to use Antares Solver v8.6.1 [`#1652`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1652) -* **service:** set public mode to `NONE` and assign user groups to the study during import [`#1619`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1619) * **service:** set public mode to `NONE` and assign user groups to the study during copying [`#1620`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1620) +* **service:** set public mode to `NONE` and assign user groups to the study during import [`#1619`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1619) +* **st-storage:** add default value for st storage group [`f9a3af7`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/f9a3af75bc564e75296de35814ee4d625d822c84) * **st-storage:** allow DELETE endpoint to accept multiple IDs as parameters [`#1716`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1716) -* **st-storage:** update "group" parameter values for Short-Term Storage [`#1665`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1665) +* **st-storage:** correct docstring [`60bdbbf`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/60bdbbfe805bff8df7a6619b0b3227b14e0d4ed5) * **st-storage:** fixing archived study configuration reading for short-term storages [`#1673`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1673) +* **st-storage:** parse_st_storage works with zipped study with version previous 860 [`0cae9f2`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/0cae9f2e535b29e11b1ef62234dd3ef4f1afcbfe) * **st-storage:** replace inflow.txt by inflows.txt according to the changing doc [`#1618`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1618) +* **st-storage:** update "group" parameter values for Short-Term Storage [`#1665`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1665) * **storage:** fix INI file writing anomaly in Antares study update [`#1542`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1542) -* **table-mode:** issue to read area information in the case where the study has only one area [`#1690`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1690) — was resolved in v2.14.5 * **table-mode:** fixes the reading of UI information in the case where the study has only one area [`#1674`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1674) +* **table-mode:** issue to read area information in the case where the study has only one area [`#1690`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1690) +* **tests:** use deepcopy for dict to only modify the copy [`bca0d60`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/bca0d6062f9fef7d672a4059163e14d21d4624fe) +* **thematic-trimming:** correct the name of the "Profit by plant" variable [`7190ee1`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/7190ee12fc16ea9617888574fa20e86306fa421d) * **ui-api:** resolve Area deletion & Layers/Districts UI modification issues [`#1677`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1677) +* **ui-common:** error with formState in Form [`121d4e5`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/121d4e5343a87f15f34a1a63108d6888232ffef4) * **ui-common:** prevent null values for string cells in HandsonTable component [`#1689`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1689) -* **variant:** fixed TS deletion of renewable clusters and short-term storage [`#1693`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1693) +* **ui-i18n:** add missing key [`685c487`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/685c487f0f73d8eb37b1594c5e46b283dd55d8d4) +* **ui-i18n:** missing key [`7c8de2a`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/7c8de2a883e646d927b52e28baa0184a13146625) +* **ui-map:** area persistence issue after deletion [`47d6bb6`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/47d6bb666aa3d98ff98724fccd6b8504f9d27d29) +* **ui-model:** update Properties form with the new endpoint [`15d5964`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/15d596494827e3f14eaf77d65e0a8c6db411802b) +* **ui-redux:** missing state update for layers/districts after node deletion [`42d698c`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/42d698cf89a8eec41475c7f3640e0d29b69bba14) +* **ui-redux:** remove eslint no-param-reassign warning [`3f88166`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/3f88166c18337f2a4bf096d1a28664d793198d46) +* **ui-xpansion:** add the missing `batch_size` field in the Xpansion parameters view [`#1739`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1739) * **variant:** enhance `VariantCommandsExtractor` for Renewable Clusters and Short-Term Storage [`#1688`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1688) +* **variant:** fixed TS deletion of renewable clusters and short-term storage [`#1693`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1693) +* **variant-manager:** correct the variant manager [`5cc467f`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/5cc467f48a2f6b73d9abf652391c14bef76d40fc) * **web:** change API response_model to avoid ValidationError [`#1526`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1526) -* **worker:** archive worker must be kept alive for processing [`#1558`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1558) — was resolved in v2.14.2 +* **worker:** archive worker must be kept alive for processing [`#1558`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1558) * **xpansion:** correct field names and types in Xpansion settings [`#1684`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1684) +* correct the import of `declarative_base` to avoid a deprecation warning. [`c8c8388`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/c8c8388e5db898da897f28f5a4cd76c3485571da) ### Documentation +* **api:** improve the documentation of the study upgrade and import endpoints [`a8691bb`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/a8691bb7963fc5d9cfa6e06175f4827e455eee44) * correct and improve the "Variant Manager" documentation [`#1637`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1637) +* update changelog for the v2.15 release [`ccd6496`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/ccd6496266dea01ff952c41565d8940391f0f317) ### Tests -* improve fixtures in unit tests for better reuse [`#1638`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1638) +* **api:** correct JSON payloads for "table mode" [`41b2cc8`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/41b2cc8a58373c41e54c452d2667206680d39fb8) +* **api:** correct unit tests for Xpansion-related endpoints to use bytes for file upload [`7936312`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/7936312c1caa178e59f79b6cf8516a4c6bea855e) +* **api:** improve unit test of `get_dependencies` endpoint [`0524e91`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/0524e91d343b9a73414110c25094064342d25a77) * **api:** simplify initialization of the database in unit tests [`#1540`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1540) +* **api:** update `UploadFile` object construction for `Starlette` >= 0.24 [`d9c5152`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/d9c5152110420a10a4e73de7db6d6316c6a707f6) +* **api:** update integration tests for Starlette 0.21.0 [`ec3c7b7`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/ec3c7b76f82c612fb40937dddce14207735be38f) +* **api:** update stream file download tests for compatibility with Requests and Httpx [`ce6bdc5`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/ce6bdc539a1ba8ae1d4d81342e90e5369706f426) +* **api-layers:** add layers management tests [`4523022`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/4523022d7f7df8a5b2a35f8aba1687de3c854427) +* **export:** correct unit tests for `VariantStudyService.generate_task` method [`f533fdc`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/f533fdc6776bc2b4201023df391e8054b1f406b2) +* **ini:** correct the `IniFileNode` unit tests [`b96a92e`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/b96a92eadb6ca8b69b1bfc2e335a5535d67f52b1) +* **matrix:** use NumPy arrays in `parse_commands` UT and correct path on Windows [`2528cce`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/2528cce0226096f717df432f2a6b56248da57330) +* **matrix-service:** improve unit tests for `MatrixContentRepository` class [`e35639f`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/e35639f5d26d93e3619c3ff6335db33d6f8e9b56) +* **matrix-service:** improve unit tests for `MatrixService` class [`8cf9f94`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/8cf9f94e8e5b79a966552e9728d50455b885b117) +* **st-storage:** improve unit tests to check "group" default value and add typing [`a3051ac`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/a3051ac581df99f366ec6ff9595015cecdeec1d8) +* **thematic-trimming:** correct unit test (`study.version` was not set) [`28720f5`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/28720f5d86fad43e3359beb7bdfdbc6c386c7619) +* add Properties form test and fix others one [`5267276`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/5267276ce6ea59a36680d242fd939bd295b6a7b6) +* improve and correct unit tests for `AreaManager` [`e6f7ed2`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/e6f7ed2e22a51356c039ff3dda42ea33439b9c48) +* improve fixtures in unit tests for better reuse [`#1638`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1638) ### Refactoring -* refactoring of unit tests and Model improvement [`#1647`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1647) +* **allocation:** remove deprecated class `FixedAllocationKeyIniReader` [`b1299fe`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/b1299fea4149a57a56bf4e1e002597d7f35a7625) +* **api:** refactor data extraction logic in `_extract_data_from_file` function [`219990b`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/219990b14d3a4c9f64189fb7c9d245e9c0afba9c) +* **api-model:** improve `transform_name_to_id` implementation and add UT [`24b1059`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/24b10595cdf7c054815834181a82d65f653f0270) * **api-model:** improve `transform_name_to_id` implementation for more efficiency [`#1546`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1546) -* **command:** rename "cluster" as "thermal cluster" [`#1639`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1639) * **command:** improve implementation of area management [`#1636`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1636) +* **command:** rename "cluster" as "thermal cluster" [`#1639`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1639) +* **matrix:** improve implementation of dataframe saving [`c156c3b`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/c156c3bc8b312a0932b2b25dc5c835b8e12d2cfc) +* **matrix-service:** improve implementation of `create_by_importation` [`b054b8b`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/b054b8b72d49966ed46f2110f2a24aedb4549d2b) +* **matrix-service:** simplify timestamp calculation used during `MatrixDTO` creation [`be5aafe`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/be5aafe2fb64bdae2f58c67fa7f69df26277f18d) +* **matrix-service:** use `matrix_id` in favor of `matrix_hash` as matrix hashing is an implementation detail [`fc9e48c`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/fc9e48ca7e05c40a25d95c4dd9cd093dc8beb7a3) +* **rawstudy-filesystem:** improve error message raised by `ChildNotFoundError` [`9a6c4f9`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/9a6c4f92b704219836e42b15c196f4c317ee3fad) +* **tools:** correct migration from `Requests` to `httpx` [`09be12a`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/09be12a3989dab5a4cdaa6d73f8e77d2e89c521f) +* **tools:** prepare for migration from `Requests` to `httpx` [`f494188`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/f494188b97dfa34c0981875d647ac700c44a02f3) * **variant:** enhance implementation of variant commands [`#1539`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1539) +* avoid always returning the same value (reported by SonarCloud) [`6e4d8cb`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/6e4d8cb2f2cc532fe5a48f97e8b94d2df7f1f37a) +* correct SonarCloud issues [`39f584d`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/39f584d1733ebfd3dc9b438a4a6819990c4007ec) +* enhance code quality in variant commands implementation [`608f0b3`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/608f0b38ce92b8186678ab39d8e0a6364d97a7de) +* fix lambda capturing `name` variable reported by SonarCloud [`cb3022c`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/cb3022cd5bdd7a3e4b550e028669feba49dd4618) +* refactoring of unit tests and Model improvement [`#1647`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1647) ### Styles +* **api:** change field version check [`d99be3f`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/d99be3f272fe879edf650e987850d9a4864579b1) +* **matrix-service:** correct the Google docstring format [`26bfc2d`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/26bfc2d02026590c1b30e4adaf119aeaf7ec8c27) * **ui:** correct spelling mistake in webapp code [`#1570`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1570) +* **ui-redux-map:** minor improvements [`27cb4be`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/27cb4befffab872a403c5ca5b9f15c95a250ac61) +* add configuration for iSort [`cd6ae15`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/cd6ae1591d4026b9b5bf9159c31737ed0b1d0c18) +* correct mypy issues [`8f5f97b`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/8f5f97b18298c27818b903c8f3475452dfdf5ad3) +* ignore unused imports in `dbmodel.py` [`853af50`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/853af500b7c246cf1f6a32222d71c0c8a45a62b7) +* reindent `ini_reader.py` [`d5fa83d`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/d5fa83d7371b533a0b6aac38aac924c3e3d8a0de) +* reindent project base code, `scripts/` and unit tests using `line-length = 120` [`51a0f27`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/51a0f27cebd1216668f3da91b43cd1903b07cbca) +* reindent source code in unit tests [`4515451`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/4515451313b5ab3935dfbb360eedce794bfe5d3e) +* remove unused imports in production base code [`9dc34ce`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/9dc34cec9f8e719477049a751f2a0d3fa49c9889) +* remove unused imports in unit tests [`496f164`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/496f1640fccf09c63242985f8d41aceba7f78f05) +* sort imports in project base code and unit tests using `line_length = 120` [`13a5929`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/13a59293dba60e709c995cffa15cd27cd830bc35) +* sort imports in project base code and unit tests using iSort [`44ac3d8`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/44ac3d81e8beefe570d536478393c315c7e8d5fa) ### Build System -* specify the node versions supported [`#1553`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1553) * **launcher:** use Antares-Launcher v1.3.0 which preserve the "output" directory required by the `--step=sensitivity` Xpansion option [`#1606`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1606) +* add iSort in the `requirements-dev.txt` [`e86b58f`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/e86b58fe84360ce37786aace48aa0a32f01f8859) +* include `typing_extensions` in the project requirements (new direct usage) [`00ae992`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/00ae9923abcb83e5764eaa09a56fdceb3cc6cfde) +* move `checksumdir` and `locust` libraries from project requirements to tests requirements (only used in unit tests) [`f8d036e`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/f8d036e021c534eaeed4885397073e1f6e3b9d53) +* remove unused Sphinx-related libraries from docs requirements [`6c3f4bf`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/6c3f4bff861ab4f5af36eb3c22dcbf33032a91ee) +* remove unused libraries from project requirements [`c4495a0`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/c4495a07db92d90cc38b26ebad5973742713fe3f) +* specify the node versions supported [`#1553`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1553) +* update project configuration, specify exclusion of non-package directories when using `find_packages` [`236a2d7`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/236a2d712858ee23abb7710ef7f92e19042d48cb) +* update project's metadata in `setup.py` (author, author email, license, platforms, classifiers) [`1de508e`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/1de508e4974bbc13ec6dbab86a2799bd5ff01005) ### Continuous Integration +* change the configuration of mypy to ignore `httpx` package [`6eab8ac`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/6eab8acd491de95113b1cadebb62a04aa73a3d89) +* change the main GitHub action to run iSort for code style checking [`837a229`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/837a2298b781afcfc87c9bef009b899b451a9351) +* correct the main GitHub action to use `Black~=23.7.0` [`609d334`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/609d334b7a3d32d04faad0260fe8ab4a6a1add09) * upgrade mypy to v1.4.1 and Black to v23.7.0 for improved typing and formatting [`#1685`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1685) +### Chore + +* **github-actions:** update Node.js version to 18.16.1 [`b9988f6`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/b9988f6dedc5a653de00bf5becc917487ce589e6) +* correct handling of base class with no `__annotations__` in `AllOptionalMetaclass` [`d9ed61f`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/d9ed61fdaaa32974431b41e9cce44be09bb92e79) +* correct indentation [`4af01b4`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/4af01b4023cb828093f8fd9b139f76429806c7b8) +* increase line length limit to 120 characters for Black and iSort [`586fb43`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/586fb438607824899c66db66f183451fbe4a88e4) +* remove `# fmt: off`/`# fmt: on` Black directives [`c970257`](https://github.com/AntaresSimulatorTeam/AntaREST/commit/c970257087031236d33c089aeb459d1eb3f419a2) + v2.14.6 (2023-09-01) diff --git a/scripts/update_version.py b/scripts/update_version.py index 07070fc582..f2d5e32512 100755 --- a/scripts/update_version.py +++ b/scripts/update_version.py @@ -7,17 +7,15 @@ import datetime import pathlib import re -import typing +import typing as t try: from antarest import __version__ except ImportError: __version__ = "(unknown): use the virtualenv's Python to get the actual version number." -# fmt: off HERE = pathlib.Path(__file__).parent.resolve() PROJECT_DIR = next(iter(p for p in HERE.parents if p.joinpath("antarest").exists())) -# fmt: on TOKENS = [ ("H1", r"^([^\n]+)\n={3,}$"), @@ -64,7 +62,7 @@ def __str__(self) -> str: return "#" * self.level + " " + title -def parse_changelog(change_log: str) -> typing.Generator[Token, None, None]: +def parse_changelog(change_log: str) -> t.Generator[Token, None, None]: for mo in re.finditer(ANY_TOKEN_RE, change_log, flags=re.MULTILINE): kind = mo.lastgroup if kind in {"H1", "H2", "H3", "H4"} and mo.lastindex is not None: @@ -78,18 +76,31 @@ def parse_changelog(change_log: str) -> typing.Generator[Token, None, None]: raise NotImplementedError(kind, mo.group()) -def update_changelog(change_log: str, new_version: str, new_date: str) -> typing.Generator[Token, None, None]: - title_found = False +def update_changelog(change_log: str, new_version: str, new_date: str) -> t.Generator[Token, None, None]: new_title = f"v{new_version} ({new_date})" + + first_release_found = False for token in parse_changelog(change_log): - if not title_found and isinstance(token, TitleToken) and token.level == 2: - title_found = True - if token.text != new_title: + if first_release_found: + yield token + continue + + is_release_title = isinstance(token, TitleToken) and token.level == 2 + if is_release_title: + first_release_found = True + if token.text.split()[0] == new_title.split()[0]: + # Update the release date + yield TitleToken(kind=token.kind, text=new_title) + else: + # Insert a new release title before the current one yield TitleToken(kind=token.kind, text=new_title) yield NewlineToken() yield NewlineToken() yield NewlineToken() - yield token + yield token + + else: + yield token def upgrade_version(new_version: str, new_date: str) -> None: @@ -181,7 +192,7 @@ def __call__(self, string: str) -> str: Use this script to update the version number and release date of the AntaREST application. It is designed to be executed before releasing the application on GitHub, specifically -when a new version is completed in the `master` branch or `hotfix` branch. +when a new version is completed in the `release` or `hotfix` branch. """ @@ -204,7 +215,7 @@ def main() -> None: help=f"new release date, using the format '{date_type.regex.pattern}'", metavar="ISO_DATE", ) - version_type = RegexType(regex=r"\d+(?:\.\d+)+") + version_type = RegexType(regex=r"v?\d+(?:\.\d+)+") parser.add_argument( "new_version", type=version_type, @@ -213,6 +224,7 @@ def main() -> None: ) args = parser.parse_args() + args.new_version = args.new_version.lstrip("v") upgrade_version(args.new_version, args.new_date)