diff --git a/antarest/study/storage/variantstudy/business/command_reverter.py b/antarest/study/storage/variantstudy/business/command_reverter.py index 1ac83c2704..28d2102273 100644 --- a/antarest/study/storage/variantstudy/business/command_reverter.py +++ b/antarest/study/storage/variantstudy/business/command_reverter.py @@ -6,6 +6,7 @@ from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy from antarest.study.storage.rawstudy.model.filesystem.folder_node import ChildNotFoundError from antarest.study.storage.variantstudy.business.utils import strip_matrix_protocol +from antarest.study.storage.variantstudy.business.utils_binding_constraint import reformat_values from antarest.study.storage.variantstudy.model.command.common import CommandName from antarest.study.storage.variantstudy.model.command.create_area import CreateArea from antarest.study.storage.variantstudy.model.command.create_binding_constraint import CreateBindingConstraint @@ -107,7 +108,7 @@ def _revert_update_binding_constraint( time_step=command.time_step, operator=command.operator, coeffs=command.coeffs, - values=strip_matrix_protocol(command.values), + values=reformat_values(command.values), filter_year_by_year=command.filter_year_by_year, filter_synthesis=command.filter_synthesis, comments=command.comments, diff --git a/antarest/study/storage/variantstudy/business/matrix_constants_generator.py b/antarest/study/storage/variantstudy/business/matrix_constants_generator.py index 46b702e4f9..26689a8fe0 100644 --- a/antarest/study/storage/variantstudy/business/matrix_constants_generator.py +++ b/antarest/study/storage/variantstudy/business/matrix_constants_generator.py @@ -35,9 +35,11 @@ ONES_SCENARIO_MATRIX = "ones_scenario_matrix" # Binding constraint aliases -BINDING_CONSTRAINT_HOURLY = "empty_2nd_member_hourly" -BINDING_CONSTRAINT_DAILY = "empty_2nd_member_daily" -BINDING_CONSTRAINT_WEEKLY = "empty_2nd_member_daily" +BINDING_CONSTRAINT_HOURLY_v86 = "empty_2nd_member_hourly_v86" +BINDING_CONSTRAINT_DAILY_WEEKLY_v86 = "empty_2nd_member_daily_or_weekly_v86" + +BINDING_CONSTRAINT_HOURLY_v87 = "empty_2nd_member_hourly_v87" +BINDING_CONSTRAINT_DAILY_WEEKLY_v87 = "empty_2nd_member_daily_or_weekly_v87" # Short-term storage aliases ST_STORAGE_PMAX_INJECTION = ONES_SCENARIO_MATRIX @@ -94,10 +96,17 @@ def init_constant_matrices( self.hashes[MISCGEN_TS] = self.matrix_service.create(FIXED_8_COLUMNS) # Binding constraint matrices - series = matrix_constants.binding_constraint.series_before_v87 - self.hashes[BINDING_CONSTRAINT_HOURLY] = self.matrix_service.create(series.default_bc_hourly) - self.hashes[BINDING_CONSTRAINT_DAILY] = self.matrix_service.create(series.default_bc_weekly_daily) - self.hashes[BINDING_CONSTRAINT_WEEKLY] = self.matrix_service.create(series.default_bc_weekly_daily) + series_before_87 = matrix_constants.binding_constraint.series_before_v87 + self.hashes[BINDING_CONSTRAINT_HOURLY_v86] = self.matrix_service.create(series_before_87.default_bc_hourly) + self.hashes[BINDING_CONSTRAINT_DAILY_WEEKLY_v86] = self.matrix_service.create( + series_before_87.default_bc_weekly_daily + ) + + series_after_87 = matrix_constants.binding_constraint.series_after_v87 + self.hashes[BINDING_CONSTRAINT_HOURLY_v87] = self.matrix_service.create(series_after_87.default_bc_hourly) + self.hashes[BINDING_CONSTRAINT_DAILY_WEEKLY_v87] = self.matrix_service.create( + series_after_87.default_bc_weekly_daily + ) # Some short-term storage matrices use np.ones((8760, 1)) self.hashes[ONES_SCENARIO_MATRIX] = self.matrix_service.create( @@ -156,17 +165,27 @@ def get_default_reserves(self) -> str: def get_default_miscgen(self) -> str: return MATRIX_PROTOCOL_PREFIX + self.hashes[MISCGEN_TS] - def get_binding_constraint_hourly(self) -> str: - """2D-matrix of shape (8760, 3), filled-in with zeros.""" - return MATRIX_PROTOCOL_PREFIX + self.hashes[BINDING_CONSTRAINT_HOURLY] - - def get_binding_constraint_daily(self) -> str: - """2D-matrix of shape (365, 3), filled-in with zeros.""" - return MATRIX_PROTOCOL_PREFIX + self.hashes[BINDING_CONSTRAINT_DAILY] - - def get_binding_constraint_weekly(self) -> str: - """2D-matrix of shape (52, 3), filled-in with zeros.""" - return MATRIX_PROTOCOL_PREFIX + self.hashes[BINDING_CONSTRAINT_WEEKLY] + def get_binding_constraint_hourly(self, version: int) -> str: + """ + Version 1 of the command corresponds to study versions prior to v8.7. + Version 2 corresponds to study versions v8.7 or later. + For version 1 : 2D-matrix of shape (8784, 3), filled-in with zeros. + For version 2 : 2D-matrix of shape (8784, 1), filled-in with zeros. + """ + if version == 1: + return MATRIX_PROTOCOL_PREFIX + self.hashes[BINDING_CONSTRAINT_HOURLY_v86] + return MATRIX_PROTOCOL_PREFIX + self.hashes[BINDING_CONSTRAINT_HOURLY_v87] + + def get_binding_constraint_daily_weekly(self, version: int) -> str: + """ + Version 1 of the command corresponds to study versions prior to v8.7. + Version 2 corresponds to study versions v8.7 or later. + For version 1 : 2D-matrix of shape (8784, 3), filled-in with zeros. + For version 2 : 2D-matrix of shape (8784, 1), filled-in with zeros. + """ + if version == 1: + return MATRIX_PROTOCOL_PREFIX + self.hashes[BINDING_CONSTRAINT_DAILY_WEEKLY_v86] + return MATRIX_PROTOCOL_PREFIX + self.hashes[BINDING_CONSTRAINT_DAILY_WEEKLY_v87] def get_st_storage_pmax_injection(self) -> str: """2D-matrix of shape (8760, 1), filled-in with ones.""" diff --git a/antarest/study/storage/variantstudy/business/utils_binding_constraint.py b/antarest/study/storage/variantstudy/business/utils_binding_constraint.py index db6ea4d1db..1e519e05d9 100644 --- a/antarest/study/storage/variantstudy/business/utils_binding_constraint.py +++ b/antarest/study/storage/variantstudy/business/utils_binding_constraint.py @@ -8,6 +8,7 @@ ) from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy +from antarest.study.storage.variantstudy.business.utils import strip_matrix_protocol from antarest.study.storage.variantstudy.model.command.common import BindingConstraintOperator, CommandOutput @@ -22,7 +23,7 @@ def apply_binding_constraint( freq: BindingConstraintFrequency, operator: BindingConstraintOperator, coeffs: Dict[str, List[float]], - values: Optional[Union[List[List[MatrixData]], str]], + values: Optional[Union[List[List[MatrixData]], str, Dict[str, List[List[MatrixData]]], Dict[str, str]]], filter_year_by_year: Optional[str] = None, filter_synthesis: Optional[str] = None, ) -> CommandOutput: @@ -74,9 +75,15 @@ def apply_binding_constraint( ["input", "bindingconstraints", "bindingconstraints"], ) if values: - if not isinstance(values, str): # pragma: no cover - raise TypeError(repr(values)) - study_data.tree.save(values, ["input", "bindingconstraints", bd_id]) + if study_data.config.version < 870: + if not isinstance(values, str): # pragma: no cover + raise TypeError(repr(values)) + study_data.tree.save(values, ["input", "bindingconstraints", bd_id]) + else: + if not isinstance(values, dict): # pragma: no cover + raise TypeError(repr(values)) + for term in ["lt", "gt", "eq"]: + study_data.tree.save(values[term], ["input", "bindingconstraints", f"{bd_id}_{term}"]) return CommandOutput(status=True) @@ -116,3 +123,13 @@ def remove_area_cluster_from_binding_constraints( selection = [b for b in study_data_config.bindings if area_id in b.areas] for binding in selection: study_data_config.bindings.remove(binding) + + +def reformat_values( + values: Optional[Union[List[List[MatrixData]], str, Dict[str, List[List[MatrixData]]], Dict[str, str]]] +) -> Optional[Union[str, Dict[str, str]]]: + if isinstance(values, str): + return strip_matrix_protocol(values) + elif isinstance(values, dict): + return {key: strip_matrix_protocol(values[key]) for key in values} + raise TypeError(repr(values)) diff --git a/antarest/study/storage/variantstudy/command_factory.py b/antarest/study/storage/variantstudy/command_factory.py index 5fe5c7d9cb..5cf298b15e 100644 --- a/antarest/study/storage/variantstudy/command_factory.py +++ b/antarest/study/storage/variantstudy/command_factory.py @@ -74,14 +74,11 @@ def __init__( patch_service=patch_service, ) - def _to_single_command(self, action: str, args: JSON) -> ICommand: + def _to_single_command(self, action: str, args: JSON, version: int) -> ICommand: """Convert a single CommandDTO to ICommand.""" if action in COMMAND_MAPPING: command_class = COMMAND_MAPPING[action] - return command_class( - **args, - command_context=self.command_context, - ) # type: ignore + return command_class(**args, command_context=self.command_context, version=version) # type: ignore raise NotImplementedError(action) def to_command(self, command_dto: CommandDTO) -> List[ICommand]: @@ -99,9 +96,9 @@ def to_command(self, command_dto: CommandDTO) -> List[ICommand]: """ args = command_dto.args if isinstance(args, dict): - return [self._to_single_command(command_dto.action, args)] + return [self._to_single_command(command_dto.action, args, command_dto.version)] elif isinstance(args, list): - return [self._to_single_command(command_dto.action, argument) for argument in args] + return [self._to_single_command(command_dto.action, argument, command_dto.version) for argument in args] raise NotImplementedError() def to_commands(self, cmd_dto_list: List[CommandDTO]) -> List[ICommand]: diff --git a/antarest/study/storage/variantstudy/model/command/create_binding_constraint.py b/antarest/study/storage/variantstudy/model/command/create_binding_constraint.py index 901294a73d..7136c61dab 100644 --- a/antarest/study/storage/variantstudy/model/command/create_binding_constraint.py +++ b/antarest/study/storage/variantstudy/model/command/create_binding_constraint.py @@ -13,6 +13,7 @@ from antarest.study.storage.variantstudy.business.utils_binding_constraint import ( apply_binding_constraint, parse_bindings_coeffs_and_save_into_config, + reformat_values, ) from antarest.study.storage.variantstudy.model.command.common import ( BindingConstraintOperator, @@ -27,7 +28,7 @@ MatrixType = List[List[MatrixData]] -def check_matrix_values(time_step: BindingConstraintFrequency, values: MatrixType) -> None: +def check_matrix_values(time_step: BindingConstraintFrequency, values: MatrixType, version: int) -> None: """ Check the binding constraint's matrix values for the specified time step. @@ -52,8 +53,13 @@ def check_matrix_values(time_step: BindingConstraintFrequency, values: MatrixTyp } # Check the matrix values and create the corresponding matrix link array = np.array(values, dtype=np.float64) - if array.shape != shapes[time_step]: - raise ValueError(f"Invalid matrix shape {array.shape}, expected {shapes[time_step]}") + expected_shape = shapes[time_step] + actual_shape = array.shape + if version == 1: + if actual_shape != expected_shape: + raise ValueError(f"Invalid matrix shape {actual_shape}, expected {expected_shape}") + elif actual_shape[0] != expected_shape[0]: + raise ValueError(f"Invalid matrix length {actual_shape[0]}, expected {expected_shape[0]}") if np.isnan(array).any(): raise ValueError("Matrix values cannot contain NaN") @@ -65,7 +71,9 @@ class BindingConstraintProperties(BaseModel): time_step: BindingConstraintFrequency operator: BindingConstraintOperator coeffs: Dict[str, List[float]] - values: Optional[Union[MatrixType, str]] = Field(None, description="2nd member matrix") + values: Optional[Union[MatrixType, str, Dict[str, MatrixType], Dict[str, str]]] = Field( + None, description="2nd member matrix" + ) filter_year_by_year: Optional[str] = None filter_synthesis: Optional[str] = None comments: Optional[str] = None @@ -87,11 +95,11 @@ def to_dto(self) -> CommandDTO: "filter_synthesis": self.filter_synthesis, } if self.values is not None: - args["values"] = strip_matrix_protocol(self.values) - return CommandDTO( - action=self.command_name.value, - args=args, - ) + if isinstance(self.values, str): + args["values"] = strip_matrix_protocol(self.values) + elif isinstance(self.values, dict): + args["values"] = {key: strip_matrix_protocol(self.values[key]) for key in self.values} # type: ignore + return CommandDTO(action=self.command_name.value, args=args, version=self.version or 1) def get_inner_matrices(self) -> List[str]: if self.values is not None: @@ -115,27 +123,35 @@ class CreateBindingConstraint(AbstractBindingConstraintCommand): @validator("values", always=True) def validate_series( cls, - v: Optional[Union[MatrixType, str]], + v: Optional[Union[MatrixType, str, Dict[str, MatrixType], Dict[str, str]]], values: Dict[str, Any], - ) -> Optional[Union[MatrixType, str]]: + ) -> Optional[Union[MatrixType, str, Dict[str, str]]]: constants: GeneratorMatrixConstants constants = values["command_context"].generator_matrix_constants time_step = values["time_step"] + version = values["version"] if v is None: # Use an already-registered default matrix methods = { - BindingConstraintFrequency.HOURLY: constants.get_binding_constraint_hourly, - BindingConstraintFrequency.DAILY: constants.get_binding_constraint_daily, - BindingConstraintFrequency.WEEKLY: constants.get_binding_constraint_weekly, + BindingConstraintFrequency.HOURLY: constants.get_binding_constraint_hourly(version), + BindingConstraintFrequency.DAILY: constants.get_binding_constraint_daily_weekly(version), + BindingConstraintFrequency.WEEKLY: constants.get_binding_constraint_daily_weekly(version), } - method = methods[time_step] - return method() + matrix = methods[time_step] + return matrix if version == 1 else {"lt": matrix, "gt": matrix, "eq": matrix} if isinstance(v, str): # Check the matrix link return validate_matrix(v, values) if isinstance(v, list): - check_matrix_values(time_step, v) + check_matrix_values(time_step, v, version) return validate_matrix(v, values) + if isinstance(v, dict): + new_values = {} + for key, value in v.items(): + if isinstance(value, list): + check_matrix_values(time_step, value, version) + new_values[key] = validate_matrix(value, values) + return new_values # Invalid datatype # pragma: no cover raise TypeError(repr(v)) @@ -201,7 +217,7 @@ def _create_diff(self, other: "ICommand") -> List["ICommand"]: time_step=other.time_step, operator=other.operator, coeffs=other.coeffs, - values=strip_matrix_protocol(other.values) if self.values != other.values else None, + values=reformat_values(other.values) if self.values != other.values else None, filter_year_by_year=other.filter_year_by_year, filter_synthesis=other.filter_synthesis, comments=other.comments, diff --git a/antarest/study/storage/variantstudy/model/command/update_binding_constraint.py b/antarest/study/storage/variantstudy/model/command/update_binding_constraint.py index ea52dca4c4..e3bff6742f 100644 --- a/antarest/study/storage/variantstudy/model/command/update_binding_constraint.py +++ b/antarest/study/storage/variantstudy/model/command/update_binding_constraint.py @@ -43,7 +43,8 @@ def validate_series( cls, v: Optional[Union[MatrixType, str]], values: Dict[str, Any], - ) -> Optional[Union[MatrixType, str]]: + ) -> Optional[Union[MatrixType, str, Dict[str, str]]]: + version = values["version"] time_step = values["time_step"] if v is None: # The matrix is not updated @@ -52,8 +53,15 @@ def validate_series( # Check the matrix link return validate_matrix(v, values) if isinstance(v, list): - check_matrix_values(time_step, v) + check_matrix_values(time_step, v, version) return validate_matrix(v, values) + if isinstance(v, dict): + new_values = {} + for key, value in v.items(): + if isinstance(value, list): + check_matrix_values(time_step, value, version) + new_values[key] = validate_matrix(value, values) + return new_values # Invalid datatype # pragma: no cover raise TypeError(repr(v)) diff --git a/antarest/study/storage/variantstudy/model/dbmodel.py b/antarest/study/storage/variantstudy/model/dbmodel.py index 1a88a76853..e9f9d66c7c 100644 --- a/antarest/study/storage/variantstudy/model/dbmodel.py +++ b/antarest/study/storage/variantstudy/model/dbmodel.py @@ -58,7 +58,7 @@ class CommandBlock(Base): # type: ignore args: str = Column(String()) def to_dto(self) -> CommandDTO: - return CommandDTO(id=self.id, action=self.command, args=json.loads(self.args)) + return CommandDTO(id=self.id, action=self.command, args=json.loads(self.args), version=self.version) @dataclass diff --git a/antarest/study/storage/variantstudy/variant_study_service.py b/antarest/study/storage/variantstudy/variant_study_service.py index f9d3eea0aa..1aa36acb23 100644 --- a/antarest/study/storage/variantstudy/variant_study_service.py +++ b/antarest/study/storage/variantstudy/variant_study_service.py @@ -182,9 +182,7 @@ def append_commands( # noinspection PyArgumentList new_commands = [ CommandBlock( - command=command.action, - args=json.dumps(command.args), - index=(first_index + i), + command=command.action, args=json.dumps(command.args), index=(first_index + i), version=command.version ) for i, command in enumerate(validated_commands) ] @@ -219,11 +217,7 @@ def replace_commands( validated_commands = transform_command_to_dto(command_objs, commands) # noinspection PyArgumentList study.commands = [ - CommandBlock( - command=command.action, - args=json.dumps(command.args), - index=i, - ) + CommandBlock(command=command.action, args=json.dumps(command.args), index=i, version=command.version) for i, command in enumerate(validated_commands) ] self.invalidate_cache(study, invalidate_self_snapshot=True) diff --git a/tests/study/storage/variantstudy/business/test_matrix_constants_generator.py b/tests/study/storage/variantstudy/business/test_matrix_constants_generator.py index d7ff90287b..3c3a205430 100644 --- a/tests/study/storage/variantstudy/business/test_matrix_constants_generator.py +++ b/tests/study/storage/variantstudy/business/test_matrix_constants_generator.py @@ -59,17 +59,12 @@ def test_get_binding_constraint_before_v87(self, tmp_path): generator.init_constant_matrices() series = matrix_constants.binding_constraint.series_before_v87 - hourly = generator.get_binding_constraint_hourly() + hourly = generator.get_binding_constraint_hourly(version) hourly_matrix_id = hourly.split(MATRIX_PROTOCOL_PREFIX)[1] hourly_matrix_dto = generator.matrix_service.get(hourly_matrix_id) assert np.array(hourly_matrix_dto.data).all() == series.default_bc_hourly.all() - daily = generator.get_binding_constraint_daily() - daily_matrix_id = daily.split(MATRIX_PROTOCOL_PREFIX)[1] - daily_matrix_dto = generator.matrix_service.get(daily_matrix_id) - assert np.array(daily_matrix_dto.data).all() == series.default_bc_weekly_daily.all() - - weekly = generator.get_binding_constraint_weekly() - weekly_matrix_id = weekly.split(MATRIX_PROTOCOL_PREFIX)[1] - weekly_matrix_dto = generator.matrix_service.get(weekly_matrix_id) - assert np.array(weekly_matrix_dto.data).all() == series.default_bc_weekly_daily.all() + daily_weekly = generator.get_binding_constraint_daily_weekly(version) + matrix_id = daily_weekly.split(MATRIX_PROTOCOL_PREFIX)[1] + matrix_dto = generator.matrix_service.get(matrix_id) + assert np.array(matrix_dto.data).all() == series.default_bc_weekly_daily.all() diff --git a/tests/study/storage/variantstudy/model/test_dbmodel.py b/tests/study/storage/variantstudy/model/test_dbmodel.py index 0715dec535..0109072303 100644 --- a/tests/study/storage/variantstudy/model/test_dbmodel.py +++ b/tests/study/storage/variantstudy/model/test_dbmodel.py @@ -156,7 +156,7 @@ def test_init(self, db_session: Session, variant_study_id: str) -> None: "id": command_id, "action": command, "args": json.loads(args), - "version": 1, + "version": 42, } diff --git a/tests/variantstudy/test_command_factory.py b/tests/variantstudy/test_command_factory.py index aac2be6c59..1588fbcbe4 100644 --- a/tests/variantstudy/test_command_factory.py +++ b/tests/variantstudy/test_command_factory.py @@ -410,19 +410,21 @@ def test_command_factory(self, command_dto: CommandDTO): commands = command_factory.to_command(command_dto=command_dto) if isinstance(command_dto.args, dict): - exp_action_args_list = [(command_dto.action, command_dto.args)] + exp_action_args_list = [(command_dto.action, command_dto.args, command_dto.version)] else: - exp_action_args_list = [(command_dto.action, args) for args in command_dto.args] + exp_action_args_list = [(command_dto.action, args, command_dto.version) for args in command_dto.args] actual_cmd: ICommand - for actual_cmd, exp_action_args in itertools.zip_longest(commands, exp_action_args_list): - assert actual_cmd is not None, f"Missing action/args for {exp_action_args=}" - assert exp_action_args is not None, f"Missing command for {actual_cmd=}" - expected_action, expected_args = exp_action_args + for actual_cmd, exp_action_args_version in itertools.zip_longest(commands, exp_action_args_list): + assert actual_cmd is not None, f"Missing action/args for {exp_action_args_version=}" + assert exp_action_args_version is not None, f"Missing command for {actual_cmd=}" + expected_action, expected_args, expected_version = exp_action_args_version actual_dto = actual_cmd.to_dto() actual_args = {k: v for k, v in actual_dto.args.items() if v is not None} + actual_version = actual_dto.version assert actual_dto.action == expected_action assert actual_args == expected_args + assert actual_version == expected_version self.command_class_set.discard(type(commands[0]).__name__)