From f3b9b12f5fe4048472aaa7b2314f3448ea83b73f Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 8 Apr 2024 16:48:23 -0600 Subject: [PATCH 01/66] Adding initial implementation of linear walker that only walks with respect to specified variables, treating others as data --- pyomo/repn/linear_wrt.py | 77 +++++++++++++++++++++++++++++ pyomo/repn/tests/test_linear_wrt.py | 40 +++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 pyomo/repn/linear_wrt.py create mode 100644 pyomo/repn/tests/test_linear_wrt.py diff --git a/pyomo/repn/linear_wrt.py b/pyomo/repn/linear_wrt.py new file mode 100644 index 00000000000..0d86528056c --- /dev/null +++ b/pyomo/repn/linear_wrt.py @@ -0,0 +1,77 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2024 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +from pyomo.common.collections import ComponentSet +from pyomo.core import Var +from pyomo.core.expr.logical_expr import _flattened +from pyomo.core.expr.numeric_expr import ( + LinearExpression, + MonomialTermExpression, + SumExpression, +) +from pyomo.repn.linear import LinearBeforeChildDispatcher, LinearRepnVisitor +from pyomo.repn.util import ExprType + + +class MultiLevelLinearBeforeChildDispatcher(LinearBeforeChildDispatcher): + def __init__(self): + super().__init__() + self[Var] = self._before_var + self[MonomialTermExpression] = self._before_monomial + self[LinearExpression] = self._before_linear + self[SumExpression] = self._before_general_expression + + @staticmethod + def _before_linear(visitor, child): + return True, None + + @staticmethod + def _before_monomial(visitor, child): + return True, None + + @staticmethod + def _before_general_expression(visitor, child): + return True, None + + @staticmethod + def _before_var(visitor, child): + if child in visitor.wrt: + # This is a normal situation + print("NORMAL: %s" % child) + _id = id(child) + if _id not in visitor.var_map: + if child.fixed: + return False, ( + ExprType.CONSTANT, + visitor.check_constant(child.value, child), + ) + MultiLevelLinearBeforeChildDispatcher._record_var(visitor, child) + ans = visitor.Result() + ans.linear[_id] = 1 + return False, (ExprType.LINEAR, ans) + else: + print("DATA: %s" % child) + # We aren't treating this Var as a Var for the purposes of this walker + return False, (ExprType.CONSTANT, child) + + +_before_child_dispatcher = MultiLevelLinearBeforeChildDispatcher() + + +class MultilevelLinearRepnVisitor(LinearRepnVisitor): + def __init__(self, subexpression_cache, var_map, var_order, sorter, wrt): + super().__init__(subexpression_cache, var_map, var_order, sorter) + self.wrt = ComponentSet(_flattened(wrt)) + + def beforeChild(self, node, child, child_idx): + print("before child %s" % child) + print(child.__class__) + return _before_child_dispatcher[child.__class__](self, child) diff --git a/pyomo/repn/tests/test_linear_wrt.py b/pyomo/repn/tests/test_linear_wrt.py new file mode 100644 index 00000000000..29c8f69ad03 --- /dev/null +++ b/pyomo/repn/tests/test_linear_wrt.py @@ -0,0 +1,40 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2024 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import pyomo.common.unittest as unittest +from pyomo.environ import Binary, ConcreteModel, Var +from pyomo.repn.linear_wrt import MultilevelLinearRepnVisitor +from pyomo.repn.tests.test_linear import VisitorConfig + + +class TestMultilevelLinearRepnVisitor(unittest.TestCase): + def make_model(self): + m = ConcreteModel() + m.x = Var(bounds=(0, 45)) + m.y = Var(domain=Binary) + + return m + + def test_walk_sum(self): + m = self.make_model() + e = m.x + m.y + cfg = VisitorConfig() + print("constructing") + visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + + repn = visitor.walk_expression(e) + + self.assertIsNone(repn.nonlinear) + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.x), repn.linear) + self.assertEqual(repn.linear[id(m.x)], 1) + self.assertIs(repn.constant, m.y) + self.assertEqual(repn.multiplier, 1) From 7614ff0367558558d27bb48d9ea1393278cd9291 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Tue, 9 Apr 2024 14:44:15 -0600 Subject: [PATCH 02/66] Overriding finalizeResult to account for the fact that mult might not be constant --- pyomo/repn/linear.py | 31 ++++++++++++++++------------- pyomo/repn/linear_wrt.py | 14 +++++++++++++ pyomo/repn/tests/test_linear_wrt.py | 16 ++++++++++++++- 3 files changed, 46 insertions(+), 15 deletions(-) diff --git a/pyomo/repn/linear.py b/pyomo/repn/linear.py index ba08c7ef245..913b36f6f16 100644 --- a/pyomo/repn/linear.py +++ b/pyomo/repn/linear.py @@ -803,6 +803,22 @@ def exitNode(self, node, data): self, node, *data ) + def _factor_multiplier_into_linear_terms(self, ans, mult): + linear = ans.linear + zeros = [] + for vid, coef in linear.items(): + if coef: + linear[vid] = coef * mult + else: + zeros.append(vid) + for vid in zeros: + del linear[vid] + if ans.nonlinear is not None: + ans.nonlinear *= mult + if ans.constant: + ans.constant *= mult + ans.multiplier = 1 + def finalizeResult(self, result): ans = result[1] if ans.__class__ is self.Result: @@ -831,20 +847,7 @@ def finalizeResult(self, result): else: # mult not in {0, 1}: factor it into the constant, # linear coefficients, and nonlinear term - linear = ans.linear - zeros = [] - for vid, coef in linear.items(): - if coef: - linear[vid] = coef * mult - else: - zeros.append(vid) - for vid in zeros: - del linear[vid] - if ans.nonlinear is not None: - ans.nonlinear *= mult - if ans.constant: - ans.constant *= mult - ans.multiplier = 1 + self._factor_mult_into_linear_terms(ans, mult) return ans ans = self.Result() assert result[0] is _CONSTANT diff --git a/pyomo/repn/linear_wrt.py b/pyomo/repn/linear_wrt.py index 0d86528056c..46451d3d64c 100644 --- a/pyomo/repn/linear_wrt.py +++ b/pyomo/repn/linear_wrt.py @@ -10,6 +10,7 @@ # ___________________________________________________________________________ from pyomo.common.collections import ComponentSet +from pyomo.common.numeric_types import native_numeric_types from pyomo.core import Var from pyomo.core.expr.logical_expr import _flattened from pyomo.core.expr.numeric_expr import ( @@ -66,6 +67,7 @@ def _before_var(visitor, child): _before_child_dispatcher = MultiLevelLinearBeforeChildDispatcher() +# LinearSubsystemRepnVisitor class MultilevelLinearRepnVisitor(LinearRepnVisitor): def __init__(self, subexpression_cache, var_map, var_order, sorter, wrt): super().__init__(subexpression_cache, var_map, var_order, sorter) @@ -75,3 +77,15 @@ def beforeChild(self, node, child, child_idx): print("before child %s" % child) print(child.__class__) return _before_child_dispatcher[child.__class__](self, child) + + def finalizeResult(self, result): + ans = result[1] + if ans.__class__ is self.Result: + mult = ans.multiplier + if not mult.__class__ in native_numeric_types: + # mult is an expression--we should push it back into the other terms + self._factor_multiplier_into_linear_terms(ans, mult) + return ans + + # In all other cases, the base class implementation is correct + return super().finalizeResult(result) diff --git a/pyomo/repn/tests/test_linear_wrt.py b/pyomo/repn/tests/test_linear_wrt.py index 29c8f69ad03..384412b25ce 100644 --- a/pyomo/repn/tests/test_linear_wrt.py +++ b/pyomo/repn/tests/test_linear_wrt.py @@ -27,7 +27,6 @@ def test_walk_sum(self): m = self.make_model() e = m.x + m.y cfg = VisitorConfig() - print("constructing") visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) repn = visitor.walk_expression(e) @@ -38,3 +37,18 @@ def test_walk_sum(self): self.assertEqual(repn.linear[id(m.x)], 1) self.assertIs(repn.constant, m.y) self.assertEqual(repn.multiplier, 1) + + def test_bilinear_term(self): + m = self.make_model() + e = m.x * m.y + cfg = VisitorConfig() + visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + + repn = visitor.walk_expression(e) + + self.assertIsNone(repn.nonlinear) + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.x), repn.linear) + self.assertIs(repn.linear[id(m.x)], m.y) + self.assertEqual(repn.constant, 0) + self.assertEqual(repn.multiplier, 1) From 0451815e320b77689bb56ab263b4c1c4134174e5 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Thu, 18 Apr 2024 14:30:14 -0600 Subject: [PATCH 03/66] Rewriting append and to_expression in LinearRepn to not assume that constant and multiplier are numeric types --- pyomo/repn/linear.py | 61 ++++++++++++------- pyomo/repn/linear_wrt.py | 4 +- pyomo/repn/tests/test_linear.py | 2 +- pyomo/repn/tests/test_linear_wrt.py | 91 ++++++++++++++++++++++++++++- 4 files changed, 131 insertions(+), 27 deletions(-) diff --git a/pyomo/repn/linear.py b/pyomo/repn/linear.py index 913b36f6f16..fd5eae13fc2 100644 --- a/pyomo/repn/linear.py +++ b/pyomo/repn/linear.py @@ -128,9 +128,10 @@ def to_expression(self, visitor): ans += e elif e.nargs() == 1: ans += e.arg(0) - if self.constant: + if self.constant.__class__ not in native_numeric_types or self.constant: ans += self.constant - if self.multiplier != 1: + if (self.multiplier.__class__ not in native_numeric_types or + self.multiplier != 1): ans *= self.multiplier return ans @@ -147,33 +148,49 @@ def append(self, other): callback). """ - # Note that self.multiplier will always be 1 (we only call append() - # within a sum, so there is no opportunity for self.multiplier to - # change). Omitting the assertion for efficiency. - # assert self.multiplier == 1 _type, other = other if _type is _CONSTANT: self.constant += other return mult = other.multiplier - if not mult: - # 0 * other, so there is nothing to add/change about - # self. We can just exit now. - return - if other.constant: - self.constant += mult * other.constant - if other.linear: - _merge_dict(self.linear, mult, other.linear) - if other.nonlinear is not None: - if mult != 1: + try: + _mult = bool(mult) + if not _mult: + return + if mult == 1: + _mult = False + except: + _mult = True + + const = other.constant + try: + _const = bool(const) + except: + _const = True + + if _mult: + if _const: + self.constant += mult * const + if other.linear: + _merge_dict(self.linear, mult, other.linear) + if other.nonlinear is not None: nl = mult * other.nonlinear - else: + if self.nonlinear is None: + self.nonlinear = nl + else: + self.nonlinear += nl + else: + if _const: + self.constant += const + if other.linear: + _merge_dict(self.linear, 1, other.linear) + if other.nonlinear is not None: nl = other.nonlinear - if self.nonlinear is None: - self.nonlinear = nl - else: - self.nonlinear += nl + if self.nonlinear is None: + self.nonlinear = nl + else: + self.nonlinear += nl def to_expression(visitor, arg): @@ -847,7 +864,7 @@ def finalizeResult(self, result): else: # mult not in {0, 1}: factor it into the constant, # linear coefficients, and nonlinear term - self._factor_mult_into_linear_terms(ans, mult) + self._factor_multiplier_into_linear_terms(ans, mult) return ans ans = self.Result() assert result[0] is _CONSTANT diff --git a/pyomo/repn/linear_wrt.py b/pyomo/repn/linear_wrt.py index 46451d3d64c..b58ff32ba71 100644 --- a/pyomo/repn/linear_wrt.py +++ b/pyomo/repn/linear_wrt.py @@ -74,15 +74,13 @@ def __init__(self, subexpression_cache, var_map, var_order, sorter, wrt): self.wrt = ComponentSet(_flattened(wrt)) def beforeChild(self, node, child, child_idx): - print("before child %s" % child) - print(child.__class__) return _before_child_dispatcher[child.__class__](self, child) def finalizeResult(self, result): ans = result[1] if ans.__class__ is self.Result: mult = ans.multiplier - if not mult.__class__ in native_numeric_types: + if mult.__class__ not in native_numeric_types: # mult is an expression--we should push it back into the other terms self._factor_multiplier_into_linear_terms(ans, mult) return ans diff --git a/pyomo/repn/tests/test_linear.py b/pyomo/repn/tests/test_linear.py index 0fd428fd8ee..badb7f407f5 100644 --- a/pyomo/repn/tests/test_linear.py +++ b/pyomo/repn/tests/test_linear.py @@ -1643,7 +1643,7 @@ def test_zero_elimination(self): self.assertEqual(repn.multiplier, 1) self.assertEqual(repn.constant, 0) self.assertEqual(repn.linear, {}) - self.assertEqual(repn.nonlinear, None) + self.assertIsNone(repn.nonlinear) m.p = Param(mutable=True, within=Any, initialize=None) e = m.p * m.x[0] + m.p * m.x[1] * m.x[2] + m.p * log(m.x[3]) diff --git a/pyomo/repn/tests/test_linear_wrt.py b/pyomo/repn/tests/test_linear_wrt.py index 384412b25ce..6bfbd51c01e 100644 --- a/pyomo/repn/tests/test_linear_wrt.py +++ b/pyomo/repn/tests/test_linear_wrt.py @@ -10,7 +10,8 @@ # ___________________________________________________________________________ import pyomo.common.unittest as unittest -from pyomo.environ import Binary, ConcreteModel, Var +from pyomo.core.expr.compare import assertExpressionsEqual +from pyomo.environ import Binary, ConcreteModel, Var, log from pyomo.repn.linear_wrt import MultilevelLinearRepnVisitor from pyomo.repn.tests.test_linear import VisitorConfig @@ -52,3 +53,91 @@ def test_bilinear_term(self): self.assertIs(repn.linear[id(m.x)], m.y) self.assertEqual(repn.constant, 0) self.assertEqual(repn.multiplier, 1) + + def test_distributed_bilinear_term(self): + m = self.make_model() + e = m.y * (m.x + 7) + cfg = VisitorConfig() + visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + + repn = visitor.walk_expression(e) + + self.assertIsNone(repn.nonlinear) + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.x), repn.linear) + self.assertIs(repn.linear[id(m.x)], m.y) + assertExpressionsEqual( + self, + repn.constant, + m.y * 7 + ) + self.assertEqual(repn.multiplier, 1) + + def test_monomial(self): + m = self.make_model() + e = 45 * m.y + cfg = VisitorConfig() + visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.y]) + + repn = visitor.walk_expression(e) + + self.assertIsNone(repn.nonlinear) + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.y), repn.linear) + self.assertEqual(repn.linear[id(m.y)], 45) + self.assertEqual(repn.constant, 0) + self.assertEqual(repn.multiplier, 1) + + def test_constant(self): + m = self.make_model() + e = 45 * m.y + cfg = VisitorConfig() + visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + + repn = visitor.walk_expression(e) + + self.assertIsNone(repn.nonlinear) + self.assertEqual(len(repn.linear), 0) + assertExpressionsEqual( + self, + repn.constant, + 45 * m.y + ) + self.assertEqual(repn.multiplier, 1) + + def test_fixed_var(self): + m = self.make_model() + m.x.fix(42) + e = (m.y ** 2) * (m.x + m.x ** 2) + + cfg = VisitorConfig() + visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + + repn = visitor.walk_expression(e) + + self.assertIsNone(repn.nonlinear) + self.assertEqual(len(repn.linear), 0) + assertExpressionsEqual( + self, + repn.constant, + (m.y ** 2) * 1806 + ) + self.assertEqual(repn.multiplier, 1) + + def test_nonlinear(self): + m = self.make_model() + e = (m.y * log(m.x)) * (m.y + 2) / m.x + + cfg = VisitorConfig() + visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + + repn = visitor.walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + print(repn.nonlinear) + assertExpressionsEqual( + self, + repn.nonlinear, + log(m.x) * (m.y *(m.y + 2))/m.x + ) From a1636f085c0dec9d8c7d320c1dfe021e25bde9d0 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Thu, 18 Apr 2024 14:31:43 -0600 Subject: [PATCH 04/66] black --- pyomo/repn/linear.py | 6 ++++-- pyomo/repn/tests/test_linear_wrt.py | 30 +++++++---------------------- 2 files changed, 11 insertions(+), 25 deletions(-) diff --git a/pyomo/repn/linear.py b/pyomo/repn/linear.py index fd5eae13fc2..c3b84940a71 100644 --- a/pyomo/repn/linear.py +++ b/pyomo/repn/linear.py @@ -130,8 +130,10 @@ def to_expression(self, visitor): ans += e.arg(0) if self.constant.__class__ not in native_numeric_types or self.constant: ans += self.constant - if (self.multiplier.__class__ not in native_numeric_types or - self.multiplier != 1): + if ( + self.multiplier.__class__ not in native_numeric_types + or self.multiplier != 1 + ): ans *= self.multiplier return ans diff --git a/pyomo/repn/tests/test_linear_wrt.py b/pyomo/repn/tests/test_linear_wrt.py index 6bfbd51c01e..fe159874186 100644 --- a/pyomo/repn/tests/test_linear_wrt.py +++ b/pyomo/repn/tests/test_linear_wrt.py @@ -66,11 +66,7 @@ def test_distributed_bilinear_term(self): self.assertEqual(len(repn.linear), 1) self.assertIn(id(m.x), repn.linear) self.assertIs(repn.linear[id(m.x)], m.y) - assertExpressionsEqual( - self, - repn.constant, - m.y * 7 - ) + assertExpressionsEqual(self, repn.constant, m.y * 7) self.assertEqual(repn.multiplier, 1) def test_monomial(self): @@ -95,20 +91,16 @@ def test_constant(self): visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) repn = visitor.walk_expression(e) - + self.assertIsNone(repn.nonlinear) self.assertEqual(len(repn.linear), 0) - assertExpressionsEqual( - self, - repn.constant, - 45 * m.y - ) + assertExpressionsEqual(self, repn.constant, 45 * m.y) self.assertEqual(repn.multiplier, 1) def test_fixed_var(self): m = self.make_model() m.x.fix(42) - e = (m.y ** 2) * (m.x + m.x ** 2) + e = (m.y**2) * (m.x + m.x**2) cfg = VisitorConfig() visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) @@ -117,17 +109,13 @@ def test_fixed_var(self): self.assertIsNone(repn.nonlinear) self.assertEqual(len(repn.linear), 0) - assertExpressionsEqual( - self, - repn.constant, - (m.y ** 2) * 1806 - ) + assertExpressionsEqual(self, repn.constant, (m.y**2) * 1806) self.assertEqual(repn.multiplier, 1) def test_nonlinear(self): m = self.make_model() e = (m.y * log(m.x)) * (m.y + 2) / m.x - + cfg = VisitorConfig() visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) @@ -136,8 +124,4 @@ def test_nonlinear(self): self.assertEqual(len(repn.linear), 0) self.assertEqual(repn.multiplier, 1) print(repn.nonlinear) - assertExpressionsEqual( - self, - repn.nonlinear, - log(m.x) * (m.y *(m.y + 2))/m.x - ) + assertExpressionsEqual(self, repn.nonlinear, log(m.x) * (m.y * (m.y + 2)) / m.x) From 549d8b00d25b5698f57100331a2fa7ab245a6741 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Thu, 18 Apr 2024 14:58:03 -0600 Subject: [PATCH 05/66] fixing another to_expression spot where we assume the coefficient is numeric --- pyomo/repn/linear.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/repn/linear.py b/pyomo/repn/linear.py index c3b84940a71..2a578827afd 100644 --- a/pyomo/repn/linear.py +++ b/pyomo/repn/linear.py @@ -122,7 +122,7 @@ def to_expression(self, visitor): var_map = visitor.var_map with mutable_expression() as e: for vid, coef in self.linear.items(): - if coef: + if coef.__class__ not in native_numeric_types or coef: e += coef * var_map[vid] if e.nargs() > 1: ans += e From 02be1cc6636a08276787a4707f211775783bd100 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Thu, 18 Apr 2024 15:24:40 -0600 Subject: [PATCH 06/66] Generalizing merge_dict for non-constant multipliers --- pyomo/repn/linear.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pyomo/repn/linear.py b/pyomo/repn/linear.py index 2a578827afd..78534789fbe 100644 --- a/pyomo/repn/linear.py +++ b/pyomo/repn/linear.py @@ -62,7 +62,12 @@ def _merge_dict(dest_dict, mult, src_dict): - if mult == 1: + try: + _mult = mult != 1 + except: + _mult = True + + if not _mult: for vid, coef in src_dict.items(): if vid in dest_dict: dest_dict[vid] += coef From a91eb4d5a3dee95c11148f4d32a03c9fcc1e2888 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Thu, 18 Apr 2024 15:25:06 -0600 Subject: [PATCH 07/66] Completely overriding finalizeResult because of non-constant coefficients --- pyomo/repn/linear_wrt.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/pyomo/repn/linear_wrt.py b/pyomo/repn/linear_wrt.py index b58ff32ba71..42543340f48 100644 --- a/pyomo/repn/linear_wrt.py +++ b/pyomo/repn/linear_wrt.py @@ -84,6 +84,33 @@ def finalizeResult(self, result): # mult is an expression--we should push it back into the other terms self._factor_multiplier_into_linear_terms(ans, mult) return ans + if mult == 1: + for vid, coef in ans.linear.items(): + if coef.__class__ in native_numeric_types and not coef: + del ans.linear[vid] + elif not mult: + # the mulltiplier has cleared out the entire expression. + # Warn if this is suppressing a NaN (unusual, and + # non-standard, but we will wait to remove this behavior + # for the time being) + if ans.constant != ans.constant or any( + c != c for c in ans.linear.values() + ): + deprecation_warning( + f"Encountered {str(mult)}*nan in expression tree. " + "Mapping the NaN result to 0 for compatibility " + "with the lp_v1 writer. In the future, this NaN " + "will be preserved/emitted to comply with IEEE-754.", + version='6.6.0', + ) + return self.Result() + else: + # mult not in {0, 1}: factor it into the constant, + # linear coefficients, and nonlinear term + self._factor_multiplier_into_linear_terms(ans, mult) + return ans - # In all other cases, the base class implementation is correct - return super().finalizeResult(result) + ans = self.Result() + assert result[0] is ExprType.CONSTANT + ans.constant = result[1] + return ans From c9aa8b8565521963e49ee304ab3fbc6637bc774a Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Thu, 18 Apr 2024 15:25:28 -0600 Subject: [PATCH 08/66] Testing to_expression --- pyomo/repn/tests/test_linear_wrt.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/pyomo/repn/tests/test_linear_wrt.py b/pyomo/repn/tests/test_linear_wrt.py index fe159874186..89bc9ee6abf 100644 --- a/pyomo/repn/tests/test_linear_wrt.py +++ b/pyomo/repn/tests/test_linear_wrt.py @@ -38,6 +38,27 @@ def test_walk_sum(self): self.assertEqual(repn.linear[id(m.x)], 1) self.assertIs(repn.constant, m.y) self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), m.x + m.y) + + def test_walk_triple_sum(self): + m = self.make_model() + m.z = Var() + e = m.x + m.z*m.y + m.z + + cfg = VisitorConfig() + visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x, m.y]) + + repn = visitor.walk_expression(e) + + self.assertIsNone(repn.nonlinear) + self.assertEqual(len(repn.linear), 2) + self.assertIn(id(m.x), repn.linear) + self.assertIn(id(m.y), repn.linear) + self.assertEqual(repn.linear[id(m.x)], 1) + self.assertIs(repn.linear[id(m.y)], m.z) + self.assertIs(repn.constant, m.z) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), m.x + m.z * m.y + m.z) def test_bilinear_term(self): m = self.make_model() @@ -53,6 +74,7 @@ def test_bilinear_term(self): self.assertIs(repn.linear[id(m.x)], m.y) self.assertEqual(repn.constant, 0) self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), m.y * m.x) def test_distributed_bilinear_term(self): m = self.make_model() @@ -68,6 +90,7 @@ def test_distributed_bilinear_term(self): self.assertIs(repn.linear[id(m.x)], m.y) assertExpressionsEqual(self, repn.constant, m.y * 7) self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), m.y * m.x + m.y * 7) def test_monomial(self): m = self.make_model() @@ -83,6 +106,7 @@ def test_monomial(self): self.assertEqual(repn.linear[id(m.y)], 45) self.assertEqual(repn.constant, 0) self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), 45 * m.y) def test_constant(self): m = self.make_model() @@ -96,6 +120,7 @@ def test_constant(self): self.assertEqual(len(repn.linear), 0) assertExpressionsEqual(self, repn.constant, 45 * m.y) self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), 45 * m.y) def test_fixed_var(self): m = self.make_model() @@ -111,6 +136,7 @@ def test_fixed_var(self): self.assertEqual(len(repn.linear), 0) assertExpressionsEqual(self, repn.constant, (m.y**2) * 1806) self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), (m.y**2) * 1806) def test_nonlinear(self): m = self.make_model() @@ -123,5 +149,6 @@ def test_nonlinear(self): self.assertEqual(len(repn.linear), 0) self.assertEqual(repn.multiplier, 1) - print(repn.nonlinear) assertExpressionsEqual(self, repn.nonlinear, log(m.x) * (m.y * (m.y + 2)) / m.x) + assertExpressionsEqual(self, repn.to_expression(visitor), + log(m.x) * (m.y * (m.y + 2)) / m.x) From 9fb61265d4a9975683c045dec3b5f5713d4f89b2 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Thu, 18 Apr 2024 15:26:04 -0600 Subject: [PATCH 09/66] Black --- pyomo/repn/tests/test_linear_wrt.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyomo/repn/tests/test_linear_wrt.py b/pyomo/repn/tests/test_linear_wrt.py index 89bc9ee6abf..35e7160a351 100644 --- a/pyomo/repn/tests/test_linear_wrt.py +++ b/pyomo/repn/tests/test_linear_wrt.py @@ -43,7 +43,7 @@ def test_walk_sum(self): def test_walk_triple_sum(self): m = self.make_model() m.z = Var() - e = m.x + m.z*m.y + m.z + e = m.x + m.z * m.y + m.z cfg = VisitorConfig() visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x, m.y]) @@ -150,5 +150,6 @@ def test_nonlinear(self): self.assertEqual(len(repn.linear), 0) self.assertEqual(repn.multiplier, 1) assertExpressionsEqual(self, repn.nonlinear, log(m.x) * (m.y * (m.y + 2)) / m.x) - assertExpressionsEqual(self, repn.to_expression(visitor), - log(m.x) * (m.y * (m.y + 2)) / m.x) + assertExpressionsEqual( + self, repn.to_expression(visitor), log(m.x) * (m.y * (m.y + 2)) / m.x + ) From 4ec32ffd5ecb6c683c072124f8a8aaa3d57f3911 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Fri, 19 Apr 2024 13:56:32 -0600 Subject: [PATCH 10/66] Fixing even more places where we assume constant and coefficients are not expressions --- pyomo/repn/linear.py | 6 +- pyomo/repn/linear_wrt.py | 11 +-- pyomo/repn/tests/test_linear_wrt.py | 111 +++++++++++++++++++++++++++- 3 files changed, 119 insertions(+), 9 deletions(-) diff --git a/pyomo/repn/linear.py b/pyomo/repn/linear.py index 78534789fbe..079fbb86489 100644 --- a/pyomo/repn/linear.py +++ b/pyomo/repn/linear.py @@ -831,15 +831,15 @@ def _factor_multiplier_into_linear_terms(self, ans, mult): linear = ans.linear zeros = [] for vid, coef in linear.items(): - if coef: - linear[vid] = coef * mult + if coef.__class__ not in native_numeric_types or coef: + linear[vid] = mult * coef else: zeros.append(vid) for vid in zeros: del linear[vid] if ans.nonlinear is not None: ans.nonlinear *= mult - if ans.constant: + if ans.constant.__class__ not in native_numeric_types or ans.constant: ans.constant *= mult ans.multiplier = 1 diff --git a/pyomo/repn/linear_wrt.py b/pyomo/repn/linear_wrt.py index 42543340f48..8fa1f9b4546 100644 --- a/pyomo/repn/linear_wrt.py +++ b/pyomo/repn/linear_wrt.py @@ -46,7 +46,6 @@ def _before_general_expression(visitor, child): def _before_var(visitor, child): if child in visitor.wrt: # This is a normal situation - print("NORMAL: %s" % child) _id = id(child) if _id not in visitor.var_map: if child.fixed: @@ -59,7 +58,6 @@ def _before_var(visitor, child): ans.linear[_id] = 1 return False, (ExprType.LINEAR, ans) else: - print("DATA: %s" % child) # We aren't treating this Var as a Var for the purposes of this walker return False, (ExprType.CONSTANT, child) @@ -85,14 +83,17 @@ def finalizeResult(self, result): self._factor_multiplier_into_linear_terms(ans, mult) return ans if mult == 1: - for vid, coef in ans.linear.items(): - if coef.__class__ in native_numeric_types and not coef: - del ans.linear[vid] + zeros = [(vid, coef) for vid, coef in ans.linear.items() if + coef.__class__ in native_numeric_types and not coef] + for vid, coef in zeros: + del ans.linear[vid] elif not mult: # the mulltiplier has cleared out the entire expression. # Warn if this is suppressing a NaN (unusual, and # non-standard, but we will wait to remove this behavior # for the time being) + # ESJ TODO: This won't work either actually... + # I'm not sure how to do it. if ans.constant != ans.constant or any( c != c for c in ans.linear.values() ): diff --git a/pyomo/repn/tests/test_linear_wrt.py b/pyomo/repn/tests/test_linear_wrt.py index 35e7160a351..d00668d5c00 100644 --- a/pyomo/repn/tests/test_linear_wrt.py +++ b/pyomo/repn/tests/test_linear_wrt.py @@ -9,9 +9,10 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ +from pyomo.common.log import LoggingIntercept import pyomo.common.unittest as unittest from pyomo.core.expr.compare import assertExpressionsEqual -from pyomo.environ import Binary, ConcreteModel, Var, log +from pyomo.environ import Any, Binary, ConcreteModel, log, Param, Var from pyomo.repn.linear_wrt import MultilevelLinearRepnVisitor from pyomo.repn.tests.test_linear import VisitorConfig @@ -153,3 +154,111 @@ def test_nonlinear(self): assertExpressionsEqual( self, repn.to_expression(visitor), log(m.x) * (m.y * (m.y + 2)) / m.x ) + + def test_finalize(self): + m = self.make_model() + m.z = Var() + m.w = Var() + + e = m.x + 2 * m.w**2 * m.y - m.x - m.w * m.z + + cfg = VisitorConfig() + repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x, m.y, m.z]).walk_expression(e) + self.assertEqual(cfg.subexpr, {}) + self.assertEqual(cfg.var_map, {id(m.x): m.x, id(m.y): m.y, id(m.z): m.z}) + self.assertEqual(cfg.var_order, {id(m.x): 0, id(m.y): 1, id(m.z): 2}) + self.assertEqual(repn.multiplier, 1) + self.assertEqual(repn.constant, 0) + self.assertEqual(len(repn.linear), 2) + self.assertIn(id(m.y), repn.linear) + assertExpressionsEqual( + self, + repn.linear[id(m.y)], + 2 * m.w ** 2 + ) + self.assertIn(id(m.z), repn.linear) + assertExpressionsEqual( + self, + repn.linear[id(m.z)], + -m.w + ) + self.assertEqual(repn.nonlinear, None) + + e *= 5 + + cfg = VisitorConfig() + repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x, m.y, m.z]).walk_expression(e) + self.assertEqual(cfg.subexpr, {}) + self.assertEqual(cfg.var_map, {id(m.x): m.x, id(m.y): m.y, id(m.z): m.z}) + self.assertEqual(cfg.var_order, {id(m.x): 0, id(m.y): 1, id(m.z): 2}) + self.assertEqual(repn.multiplier, 1) + self.assertEqual(repn.constant, 0) + self.assertEqual(len(repn.linear), 2) + self.assertIn(id(m.y), repn.linear) + print(repn.linear[id(m.y)]) + assertExpressionsEqual( + self, + repn.linear[id(m.y)], + 5 * (2 * m.w ** 2) + ) + self.assertIn(id(m.z), repn.linear) + assertExpressionsEqual( + self, + repn.linear[id(m.z)], + -5 * m.w + ) + self.assertEqual(repn.nonlinear, None) + + e = 5 * (m.w * m.y + m.z**2 + 3 * m.w * m.y**3) + + cfg = VisitorConfig() + repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x, m.y, m.z]).walk_expression(e) + self.assertEqual(cfg.subexpr, {}) + self.assertEqual(cfg.var_map, {id(m.y): m.y, id(m.z): m.z}) + self.assertEqual(cfg.var_order, {id(m.y): 0, id(m.z): 1}) + self.assertEqual(repn.multiplier, 1) + self.assertEqual(repn.constant, 0) + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.y), repn.linear) + assertExpressionsEqual( + self, + repn.linear[id(m.y)], + 5 * m.w + ) + assertExpressionsEqual(self, repn.nonlinear, (m.z**2 + 3 * m.w * m.y**3) * 5) + + def test_errors_propogate_nan(self): + m = ConcreteModel() + m.p = Param(mutable=True, initialize=0, domain=Any) + m.x = Var() + m.y = Var() + m.z = Var() + m.y.fix(1) + + expr = m.y + m.x + m.z + ((3 * m.z * m.x) / m.p) / m.y + cfg = VisitorConfig() + with LoggingIntercept() as LOG: + repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(expr) + self.assertEqual( + LOG.getvalue(), + "Exception encountered evaluating expression 'div(3, 0)'\n" + "\tmessage: division by zero\n" + "\texpression: 3/p\n", + ) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual( + self, + repn.constant, + 1 + m.z + ) + self.assertEqual(len(repn.linear), 1) + self.assertEqual(str(repn.linear[id(m.x)]), 'InvalidNumber(nan)') + self.assertEqual(repn.nonlinear, None) + + m.y.fix(None) + expr = m.z * log(m.y) + 3 + repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(expr) + self.assertEqual(repn.multiplier, 1) + self.assertEqual(str(repn.constant), 'InvalidNumber(nan)') + self.assertEqual(repn.linear, {}) + self.assertEqual(repn.nonlinear, None) From 259ee576ed2673fa30dacab41283e1c152b9ffe6 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Tue, 23 Apr 2024 15:09:30 -0600 Subject: [PATCH 11/66] Reverting my changes to linear--I'll override stuff that needs to handle expressions so that I don't kill performance --- pyomo/repn/linear.py | 101 ++++++++++++++++--------------------------- 1 file changed, 37 insertions(+), 64 deletions(-) diff --git a/pyomo/repn/linear.py b/pyomo/repn/linear.py index 079fbb86489..ba08c7ef245 100644 --- a/pyomo/repn/linear.py +++ b/pyomo/repn/linear.py @@ -62,12 +62,7 @@ def _merge_dict(dest_dict, mult, src_dict): - try: - _mult = mult != 1 - except: - _mult = True - - if not _mult: + if mult == 1: for vid, coef in src_dict.items(): if vid in dest_dict: dest_dict[vid] += coef @@ -127,18 +122,15 @@ def to_expression(self, visitor): var_map = visitor.var_map with mutable_expression() as e: for vid, coef in self.linear.items(): - if coef.__class__ not in native_numeric_types or coef: + if coef: e += coef * var_map[vid] if e.nargs() > 1: ans += e elif e.nargs() == 1: ans += e.arg(0) - if self.constant.__class__ not in native_numeric_types or self.constant: + if self.constant: ans += self.constant - if ( - self.multiplier.__class__ not in native_numeric_types - or self.multiplier != 1 - ): + if self.multiplier != 1: ans *= self.multiplier return ans @@ -155,49 +147,33 @@ def append(self, other): callback). """ + # Note that self.multiplier will always be 1 (we only call append() + # within a sum, so there is no opportunity for self.multiplier to + # change). Omitting the assertion for efficiency. + # assert self.multiplier == 1 _type, other = other if _type is _CONSTANT: self.constant += other return mult = other.multiplier - try: - _mult = bool(mult) - if not _mult: - return - if mult == 1: - _mult = False - except: - _mult = True - - const = other.constant - try: - _const = bool(const) - except: - _const = True - - if _mult: - if _const: - self.constant += mult * const - if other.linear: - _merge_dict(self.linear, mult, other.linear) - if other.nonlinear is not None: + if not mult: + # 0 * other, so there is nothing to add/change about + # self. We can just exit now. + return + if other.constant: + self.constant += mult * other.constant + if other.linear: + _merge_dict(self.linear, mult, other.linear) + if other.nonlinear is not None: + if mult != 1: nl = mult * other.nonlinear - if self.nonlinear is None: - self.nonlinear = nl - else: - self.nonlinear += nl - else: - if _const: - self.constant += const - if other.linear: - _merge_dict(self.linear, 1, other.linear) - if other.nonlinear is not None: + else: nl = other.nonlinear - if self.nonlinear is None: - self.nonlinear = nl - else: - self.nonlinear += nl + if self.nonlinear is None: + self.nonlinear = nl + else: + self.nonlinear += nl def to_expression(visitor, arg): @@ -827,22 +803,6 @@ def exitNode(self, node, data): self, node, *data ) - def _factor_multiplier_into_linear_terms(self, ans, mult): - linear = ans.linear - zeros = [] - for vid, coef in linear.items(): - if coef.__class__ not in native_numeric_types or coef: - linear[vid] = mult * coef - else: - zeros.append(vid) - for vid in zeros: - del linear[vid] - if ans.nonlinear is not None: - ans.nonlinear *= mult - if ans.constant.__class__ not in native_numeric_types or ans.constant: - ans.constant *= mult - ans.multiplier = 1 - def finalizeResult(self, result): ans = result[1] if ans.__class__ is self.Result: @@ -871,7 +831,20 @@ def finalizeResult(self, result): else: # mult not in {0, 1}: factor it into the constant, # linear coefficients, and nonlinear term - self._factor_multiplier_into_linear_terms(ans, mult) + linear = ans.linear + zeros = [] + for vid, coef in linear.items(): + if coef: + linear[vid] = coef * mult + else: + zeros.append(vid) + for vid in zeros: + del linear[vid] + if ans.nonlinear is not None: + ans.nonlinear *= mult + if ans.constant: + ans.constant *= mult + ans.multiplier = 1 return ans ans = self.Result() assert result[0] is _CONSTANT From 7886519da92c7da509aad346304df7daa5f9a2b1 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Tue, 23 Apr 2024 15:20:54 -0600 Subject: [PATCH 12/66] Adding a new LinearSubsystemRepn class to handle LinearRepns with 'constants' that are actually Pyomo expressions --- pyomo/repn/linear_wrt.py | 156 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 152 insertions(+), 4 deletions(-) diff --git a/pyomo/repn/linear_wrt.py b/pyomo/repn/linear_wrt.py index 8fa1f9b4546..bbd4c6a7d25 100644 --- a/pyomo/repn/linear_wrt.py +++ b/pyomo/repn/linear_wrt.py @@ -9,6 +9,8 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ +import copy + from pyomo.common.collections import ComponentSet from pyomo.common.numeric_types import native_numeric_types from pyomo.core import Var @@ -16,10 +18,122 @@ from pyomo.core.expr.numeric_expr import ( LinearExpression, MonomialTermExpression, + mutable_expression, + ProductExpression, SumExpression, ) -from pyomo.repn.linear import LinearBeforeChildDispatcher, LinearRepnVisitor +from pyomo.repn.linear import ( + ExitNodeDispatcher, + _initialize_exit_node_dispatcher, + LinearBeforeChildDispatcher, + LinearRepn, + LinearRepnVisitor, +) from pyomo.repn.util import ExprType +from . import linear + +_CONSTANT = ExprType.CONSTANT + + +def _merge_dict(dest_dict, mult, src_dict): + if mult.__class__ not in native_numeric_types or mult != 1: + for vid, coef in src_dict.items(): + if vid in dest_dict: + dest_dict[vid] += mult * coef + else: + dest_dict[vid] = mult * coef + else: + for vid, coef in src_dict.items(): + if vid in dest_dict: + dest_dict[vid] += coef + else: + dest_dict[vid] = coef + + +class LinearSubsystemRepn(LinearRepn): + def to_expression(self, visitor): + if self.nonlinear is not None: + # We want to start with the nonlinear term (and use + # assignment) in case the term is a non-numeric node (like a + # relational expression) + ans = self.nonlinear + else: + ans = 0 + if self.linear: + var_map = visitor.var_map + with mutable_expression() as e: + for vid, coef in self.linear.items(): + if coef.__class__ not in native_numeric_types or coef: + e += coef * var_map[vid] + if e.nargs() > 1: + ans += e + elif e.nargs() == 1: + ans += e.arg(0) + if self.constant.__class__ not in native_numeric_types or self.constant: + ans += self.constant + if ( + self.multiplier.__class__ not in native_numeric_types + or self.multiplier != 1 + ): + ans *= self.multiplier + return ans + + def append(self, other): + """Append a child result from acceptChildResult + + Notes + ----- + This method assumes that the operator was "+". It is implemented + so that we can directly use a LinearRepn() as a `data` object in + the expression walker (thereby allowing us to use the default + implementation of acceptChildResult [which calls + `data.append()`] and avoid the function call for a custom + callback). + + """ + _type, other = other + if _type is _CONSTANT: + self.constant += other + return + + mult = other.multiplier + try: + _mult = bool(mult) + if not _mult: + return + if mult == 1: + _mult = False + except: + _mult = True + + const = other.constant + try: + _const = bool(const) + except: + _const = True + + if _mult: + if _const: + self.constant += mult * const + if other.linear: + _merge_dict(self.linear, mult, other.linear) + if other.nonlinear is not None: + nl = mult * other.nonlinear + if self.nonlinear is None: + self.nonlinear = nl + else: + self.nonlinear += nl + else: + if _const: + self.constant += const + if other.linear: + _merge_dict(self.linear, 1, other.linear) + if other.nonlinear is not None: + nl = other.nonlinear + if self.nonlinear is None: + self.nonlinear = nl + else: + self.nonlinear += nl class MultiLevelLinearBeforeChildDispatcher(LinearBeforeChildDispatcher): @@ -50,7 +164,7 @@ def _before_var(visitor, child): if _id not in visitor.var_map: if child.fixed: return False, ( - ExprType.CONSTANT, + _CONSTANT, visitor.check_constant(child.value, child), ) MultiLevelLinearBeforeChildDispatcher._record_var(visitor, child) @@ -59,14 +173,32 @@ def _before_var(visitor, child): return False, (ExprType.LINEAR, ans) else: # We aren't treating this Var as a Var for the purposes of this walker - return False, (ExprType.CONSTANT, child) + return False, (_CONSTANT, child) _before_child_dispatcher = MultiLevelLinearBeforeChildDispatcher() +_exit_node_handlers = copy.deepcopy(linear._exit_node_handlers) +def _handle_product_constant_constant(visitor, node, arg1, arg2): + # ESJ: Can I do this? Just let the potential nans go through? + return _CONSTANT, arg1[1] * arg2[1] + +_exit_node_handlers[ProductExpression].update( + { + (_CONSTANT, _CONSTANT): _handle_product_constant_constant, + } +) + + # LinearSubsystemRepnVisitor class MultilevelLinearRepnVisitor(LinearRepnVisitor): + Result = LinearSubsystemRepn + exit_node_handlers = _exit_node_handlers + exit_node_dispatcher = ExitNodeDispatcher( + _initialize_exit_node_dispatcher(_exit_node_handlers) + ) + def __init__(self, subexpression_cache, var_map, var_order, sorter, wrt): super().__init__(subexpression_cache, var_map, var_order, sorter) self.wrt = ComponentSet(_flattened(wrt)) @@ -74,6 +206,22 @@ def __init__(self, subexpression_cache, var_map, var_order, sorter, wrt): def beforeChild(self, node, child, child_idx): return _before_child_dispatcher[child.__class__](self, child) + def _factor_multiplier_into_linear_terms(self, ans, mult): + linear = ans.linear + zeros = [] + for vid, coef in linear.items(): + if coef.__class__ not in native_numeric_types or coef: + linear[vid] = mult * coef + else: + zeros.append(vid) + for vid in zeros: + del linear[vid] + if ans.nonlinear is not None: + ans.nonlinear *= mult + if ans.constant.__class__ not in native_numeric_types or ans.constant: + ans.constant *= mult + ans.multiplier = 1 + def finalizeResult(self, result): ans = result[1] if ans.__class__ is self.Result: @@ -112,6 +260,6 @@ def finalizeResult(self, result): return ans ans = self.Result() - assert result[0] is ExprType.CONSTANT + assert result[0] is _CONSTANT ans.constant = result[1] return ans From e09953a615370ec0f1c97c35d2ace2781377db0a Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Wed, 24 Apr 2024 09:54:11 -0600 Subject: [PATCH 13/66] Correcting nan propogation test assertion --- pyomo/repn/tests/test_linear_wrt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyomo/repn/tests/test_linear_wrt.py b/pyomo/repn/tests/test_linear_wrt.py index d00668d5c00..43bc0a51914 100644 --- a/pyomo/repn/tests/test_linear_wrt.py +++ b/pyomo/repn/tests/test_linear_wrt.py @@ -241,15 +241,15 @@ def test_errors_propogate_nan(self): repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(expr) self.assertEqual( LOG.getvalue(), - "Exception encountered evaluating expression 'div(3, 0)'\n" + "Exception encountered evaluating expression 'div(3*z, 0)'\n" "\tmessage: division by zero\n" - "\texpression: 3/p\n", + "\texpression: 3*z*x/p\n", ) self.assertEqual(repn.multiplier, 1) assertExpressionsEqual( self, repn.constant, - 1 + m.z + m.y + m.z ) self.assertEqual(len(repn.linear), 1) self.assertEqual(str(repn.linear[id(m.x)]), 'InvalidNumber(nan)') From d51bb99711b20ce3661c2e112fd804f9fd45d9b3 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Wed, 24 Apr 2024 11:40:35 -0600 Subject: [PATCH 14/66] Adding handling of nan in assertExpressionsEqual --- pyomo/core/expr/compare.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/pyomo/core/expr/compare.py b/pyomo/core/expr/compare.py index 790bc30aaee..e57e65a08f0 100644 --- a/pyomo/core/expr/compare.py +++ b/pyomo/core/expr/compare.py @@ -230,10 +230,14 @@ def assertExpressionsEqual(test, a, b, include_named_exprs=True, places=None): test.assertEqual(len(prefix_a), len(prefix_b)) for _a, _b in zip(prefix_a, prefix_b): test.assertIs(_a.__class__, _b.__class__) - if places is None: - test.assertEqual(_a, _b) + # If _a is nan, check _b is nan + if _a != _a: + test.assertTrue(_b != _b) else: - test.assertAlmostEqual(_a, _b, places=places) + if places is None: + test.assertEqual(_a, _b) + else: + test.assertAlmostEqual(_a, _b, places=places) except (PyomoException, AssertionError): test.fail( f"Expressions not equal:\n\t" @@ -292,10 +296,13 @@ def assertExpressionsStructurallyEqual( for _a, _b in zip(prefix_a, prefix_b): if _a.__class__ not in native_types and _b.__class__ not in native_types: test.assertIs(_a.__class__, _b.__class__) - if places is None: - test.assertEqual(_a, _b) + if _a != _a: + test.assertTrue(_b != _b) else: - test.assertAlmostEqual(_a, _b, places=places) + if places is None: + test.assertEqual(_a, _b) + else: + test.assertAlmostEqual(_a, _b, places=places) except (PyomoException, AssertionError): test.fail( f"Expressions not structurally equal:\n\t" From ec5879b5191aa1fbc525867d3ed48d0e55294e50 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Wed, 24 Apr 2024 11:44:22 -0600 Subject: [PATCH 15/66] nan propogation tests are finally passing --- pyomo/repn/tests/test_linear_wrt.py | 43 +++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/pyomo/repn/tests/test_linear_wrt.py b/pyomo/repn/tests/test_linear_wrt.py index 43bc0a51914..ef86aed47f8 100644 --- a/pyomo/repn/tests/test_linear_wrt.py +++ b/pyomo/repn/tests/test_linear_wrt.py @@ -15,6 +15,7 @@ from pyomo.environ import Any, Binary, ConcreteModel, log, Param, Var from pyomo.repn.linear_wrt import MultilevelLinearRepnVisitor from pyomo.repn.tests.test_linear import VisitorConfig +from pyomo.repn.util import InvalidNumber class TestMultilevelLinearRepnVisitor(unittest.TestCase): @@ -227,6 +228,34 @@ def test_finalize(self): ) assertExpressionsEqual(self, repn.nonlinear, (m.z**2 + 3 * m.w * m.y**3) * 5) + def test_ANY_over_constant_division(self): + m = ConcreteModel() + m.p = Param(mutable=True, initialize=2, domain=Any) + m.x = Var() + m.y = Var() + m.z = Var() + # We aren't treating this as a Var, so we don't really care that it's fixed. + m.y.fix(1) + + expr = m.y + m.x + m.z + ((3 * m.z * m.x) / m.p) / m.y + cfg = VisitorConfig() + repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(expr) + + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual( + self, + repn.constant, + m.y + m.z + ) + self.assertEqual(len(repn.linear), 1) + print(repn.linear[id(m.x)]) + assertExpressionsEqual( + self, + repn.linear[id(m.x)], + 1 + 1.5 * m.z / m.y + ) + self.assertEqual(repn.nonlinear, None) + def test_errors_propogate_nan(self): m = ConcreteModel() m.p = Param(mutable=True, initialize=0, domain=Any) @@ -252,13 +281,23 @@ def test_errors_propogate_nan(self): m.y + m.z ) self.assertEqual(len(repn.linear), 1) - self.assertEqual(str(repn.linear[id(m.x)]), 'InvalidNumber(nan)') + self.assertIsInstance(repn.linear[id(m.x)], InvalidNumber) + assertExpressionsEqual( + self, + repn.linear[id(m.x)].value, + 1 + float('nan')/m.y + ) self.assertEqual(repn.nonlinear, None) m.y.fix(None) expr = m.z * log(m.y) + 3 repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(expr) self.assertEqual(repn.multiplier, 1) - self.assertEqual(str(repn.constant), 'InvalidNumber(nan)') + self.assertIsInstance(repn.constant, InvalidNumber) + assertExpressionsEqual( + self, + repn.constant.value, + float('nan')*m.z + 3 + ) self.assertEqual(repn.linear, {}) self.assertEqual(repn.nonlinear, None) From 4f5f50e64f6cddbc4ae9dd227deba5225575e320 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 29 Apr 2024 13:59:50 -0600 Subject: [PATCH 16/66] More sum tests --- pyomo/repn/tests/test_linear_wrt.py | 33 +++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/pyomo/repn/tests/test_linear_wrt.py b/pyomo/repn/tests/test_linear_wrt.py index ef86aed47f8..6853c0df7f6 100644 --- a/pyomo/repn/tests/test_linear_wrt.py +++ b/pyomo/repn/tests/test_linear_wrt.py @@ -62,6 +62,39 @@ def test_walk_triple_sum(self): self.assertEqual(repn.multiplier, 1) assertExpressionsEqual(self, repn.to_expression(visitor), m.x + m.z * m.y + m.z) + def test_sum_two_of_the_same(self): + # This hits the mult == 1 and vid in dest_dict case in _merge_dict + m = self.make_model() + e = m.x + m.x + cfg = VisitorConfig() + visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x, m.y]) + + repn = visitor.walk_expression(e) + + self.assertIsNone(repn.nonlinear) + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.x), repn.linear) + self.assertEqual(repn.linear[id(m.x)], 2) + self.assertEqual(repn.constant, 0) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), 2*m.x) + + def test_sum_with_mult_0(self): + m = self.make_model() + e = 0*m.x + m.x + m.y + + cfg = VisitorConfig() + visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + + repn = visitor.walk_expression(e) + self.assertIsNone(repn.nonlinear) + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.x), repn.linear) + self.assertEqual(repn.linear[id(m.x)], 1) + self.assertIs(repn.constant, m.y) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), m.x + m.y) + def test_bilinear_term(self): m = self.make_model() e = m.x * m.y From c9659c8c5a243959f7752f1d75a4f79ee53d6e4c Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 29 Apr 2024 14:24:51 -0600 Subject: [PATCH 17/66] Tests for everything in append --- pyomo/repn/tests/test_linear_wrt.py | 65 +++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/pyomo/repn/tests/test_linear_wrt.py b/pyomo/repn/tests/test_linear_wrt.py index 6853c0df7f6..a9a31ce8232 100644 --- a/pyomo/repn/tests/test_linear_wrt.py +++ b/pyomo/repn/tests/test_linear_wrt.py @@ -95,6 +95,71 @@ def test_sum_with_mult_0(self): self.assertEqual(repn.multiplier, 1) assertExpressionsEqual(self, repn.to_expression(visitor), m.x + m.y) + def test_sum_nonlinear_to_linear(self): + m = self.make_model() + e = m.y * m.x**2 + m.y * m.x + 3 + + cfg = VisitorConfig() + visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + + repn = visitor.walk_expression(e) + assertExpressionsEqual( + self, + repn.nonlinear, + m.y * m.x ** 2 + ) + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.x), repn.linear) + self.assertIs(repn.linear[id(m.x)], m.y) + self.assertEqual(repn.constant, 3) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), m.y * m.x ** 2 + + m.y * m.x + 3) + + def test_sum_nonlinear_to_nonlinear(self): + m = self.make_model() + e = m.x ** 3 + 3 + m.x**2 + + cfg = VisitorConfig() + visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + + repn = visitor.walk_expression(e) + assertExpressionsEqual( + self, + repn.nonlinear, + m.x ** 3 + m.x ** 2 + ) + self.assertEqual(repn.constant, 3) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.to_expression(visitor), m.x ** 3 + + m.x ** 2 + 3) + + def test_sum_to_linear_expr(self): + m = self.make_model() + e = m.x + m.y * (m.x + 5) + + cfg = VisitorConfig() + visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + + repn = visitor.walk_expression(e) + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.x), repn.linear) + assertExpressionsEqual( + self, + repn.linear[id(m.x)], + 1 + m.y + ) + assertExpressionsEqual( + self, + repn.constant, + m.y * 5 + ) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual( + self, + repn.to_expression(visitor), (1 + m.y) * m.x + m.y * 5 + ) + def test_bilinear_term(self): m = self.make_model() e = m.x * m.y From 7f224c8c1f4c0ae79b35d75269441042b8429186 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Tue, 30 Apr 2024 13:43:52 -0600 Subject: [PATCH 18/66] Not blindly evaluating unary functions, but that actually causes me a whole conundrum about fixed variables --- pyomo/repn/linear_wrt.py | 24 ++++++++++++- pyomo/repn/tests/test_linear_wrt.py | 55 +++++++++++++++++++++++------ 2 files changed, 68 insertions(+), 11 deletions(-) diff --git a/pyomo/repn/linear_wrt.py b/pyomo/repn/linear_wrt.py index bbd4c6a7d25..c5cd6c130b0 100644 --- a/pyomo/repn/linear_wrt.py +++ b/pyomo/repn/linear_wrt.py @@ -16,11 +16,13 @@ from pyomo.core import Var from pyomo.core.expr.logical_expr import _flattened from pyomo.core.expr.numeric_expr import ( + AbsExpression, LinearExpression, MonomialTermExpression, mutable_expression, ProductExpression, SumExpression, + UnaryFunctionExpression, ) from pyomo.repn.linear import ( ExitNodeDispatcher, @@ -183,13 +185,33 @@ def _before_var(visitor, child): def _handle_product_constant_constant(visitor, node, arg1, arg2): # ESJ: Can I do this? Just let the potential nans go through? return _CONSTANT, arg1[1] * arg2[1] + _exit_node_handlers[ProductExpression].update( { (_CONSTANT, _CONSTANT): _handle_product_constant_constant, } ) - + +def _handle_unary_constant(visitor, node, arg): + # We override this because we can't blindly use apply_node_operation in this case + if arg.__class__ not in native_numeric_types: + return _CONSTANT, node.create_node_with_local_data( + (linear.to_expression(visitor, arg),)) + # otherwise do the usual: + ans = apply_node_operation(node, (arg[1],)) + # Unary includes sqrt() which can return complex numbers + if ans.__class__ in native_complex_types: + ans = complex_number_error(ans, visitor, node) + return _CONSTANT, ans + +_exit_node_handlers[UnaryFunctionExpression].update( + { + (_CONSTANT,): _handle_unary_constant + } +) +_exit_node_handlers[AbsExpression] = _exit_node_handlers[UnaryFunctionExpression] + # LinearSubsystemRepnVisitor class MultilevelLinearRepnVisitor(LinearRepnVisitor): diff --git a/pyomo/repn/tests/test_linear_wrt.py b/pyomo/repn/tests/test_linear_wrt.py index a9a31ce8232..fa7aaf7799b 100644 --- a/pyomo/repn/tests/test_linear_wrt.py +++ b/pyomo/repn/tests/test_linear_wrt.py @@ -23,6 +23,7 @@ def make_model(self): m = ConcreteModel() m.x = Var(bounds=(0, 45)) m.y = Var(domain=Binary) + m.z = Var() return m @@ -44,7 +45,6 @@ def test_walk_sum(self): def test_walk_triple_sum(self): m = self.make_model() - m.z = Var() e = m.x + m.z * m.y + m.z cfg = VisitorConfig() @@ -81,7 +81,7 @@ def test_sum_two_of_the_same(self): def test_sum_with_mult_0(self): m = self.make_model() - e = 0*m.x + m.x + m.y + e = 0*m.x + m.x - m.y cfg = VisitorConfig() visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) @@ -91,13 +91,17 @@ def test_sum_with_mult_0(self): self.assertEqual(len(repn.linear), 1) self.assertIn(id(m.x), repn.linear) self.assertEqual(repn.linear[id(m.x)], 1) - self.assertIs(repn.constant, m.y) + assertExpressionsEqual( + self, + repn.constant, + - m.y + ) self.assertEqual(repn.multiplier, 1) - assertExpressionsEqual(self, repn.to_expression(visitor), m.x + m.y) + assertExpressionsEqual(self, repn.to_expression(visitor), m.x - m.y) def test_sum_nonlinear_to_linear(self): m = self.make_model() - e = m.y * m.x**2 + m.y * m.x + 3 + e = m.y * m.x**2 + m.y * m.x - 3 cfg = VisitorConfig() visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) @@ -111,10 +115,10 @@ def test_sum_nonlinear_to_linear(self): self.assertEqual(len(repn.linear), 1) self.assertIn(id(m.x), repn.linear) self.assertIs(repn.linear[id(m.x)], m.y) - self.assertEqual(repn.constant, 3) + self.assertEqual(repn.constant, -3) self.assertEqual(repn.multiplier, 1) assertExpressionsEqual(self, repn.to_expression(visitor), m.y * m.x ** 2 - + m.y * m.x + 3) + + m.y * m.x - 3) def test_sum_nonlinear_to_nonlinear(self): m = self.make_model() @@ -256,7 +260,6 @@ def test_nonlinear(self): def test_finalize(self): m = self.make_model() - m.z = Var() m.w = Var() e = m.x + 2 * m.w**2 * m.y - m.x - m.w * m.z @@ -330,8 +333,8 @@ def test_ANY_over_constant_division(self): m = ConcreteModel() m.p = Param(mutable=True, initialize=2, domain=Any) m.x = Var() - m.y = Var() m.z = Var() + m.y = Var() # We aren't treating this as a Var, so we don't really care that it's fixed. m.y.fix(1) @@ -358,8 +361,8 @@ def test_errors_propogate_nan(self): m = ConcreteModel() m.p = Param(mutable=True, initialize=0, domain=Any) m.x = Var() - m.y = Var() m.z = Var() + m.y = Var() m.y.fix(1) expr = m.y + m.x + m.z + ((3 * m.z * m.x) / m.p) / m.y @@ -399,3 +402,35 @@ def test_errors_propogate_nan(self): ) self.assertEqual(repn.linear, {}) self.assertEqual(repn.nonlinear, None) + + def test_negation_constant(self): + m = self.make_model() + e = - (m.y * m.z + 17) + + cfg = VisitorConfig() + repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual( + self, + repn.constant, + - 1 * (m.y * m.z + 17) + ) + self.assertIsNone(repn.nonlinear) + + def test_product_nonlinear(self): + m = self.make_model() + e = (m.x ** 2) * (log(m.y) * m.z ** 4) * m.y + cfg = VisitorConfig() + repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x, m.z]).walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + self.assertEqual(repn.constant, 0) + print(repn.nonlinear) + assertExpressionsEqual( + self, + repn.nonlinear, + (m.x ** 2) * (m.z ** 4 * log(m.y)) * m.y + ) From 20dd5aaf73bc279273c145f2ae6979c920bf5855 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Thu, 2 May 2024 15:57:06 -0600 Subject: [PATCH 19/66] Renaming my walker and repn to something sane, but that meant changing the meaning of the 'wrt' argument --- ...{linear_wrt.py => parameterized_linear.py} | 31 +-- ...ar_wrt.py => test_parameterized_linear.py} | 179 ++++++------------ 2 files changed, 77 insertions(+), 133 deletions(-) rename pyomo/repn/{linear_wrt.py => parameterized_linear.py} (94%) rename pyomo/repn/tests/{test_linear_wrt.py => test_parameterized_linear.py} (73%) diff --git a/pyomo/repn/linear_wrt.py b/pyomo/repn/parameterized_linear.py similarity index 94% rename from pyomo/repn/linear_wrt.py rename to pyomo/repn/parameterized_linear.py index c5cd6c130b0..9df0ea458db 100644 --- a/pyomo/repn/linear_wrt.py +++ b/pyomo/repn/parameterized_linear.py @@ -52,7 +52,7 @@ def _merge_dict(dest_dict, mult, src_dict): dest_dict[vid] = coef -class LinearSubsystemRepn(LinearRepn): +class ParameterizedLinearRepn(LinearRepn): def to_expression(self, visitor): if self.nonlinear is not None: # We want to start with the nonlinear term (and use @@ -160,7 +160,7 @@ def _before_general_expression(visitor, child): @staticmethod def _before_var(visitor, child): - if child in visitor.wrt: + if child not in visitor.wrt: # This is a normal situation _id = id(child) if _id not in visitor.var_map: @@ -185,19 +185,19 @@ def _before_var(visitor, child): def _handle_product_constant_constant(visitor, node, arg1, arg2): # ESJ: Can I do this? Just let the potential nans go through? return _CONSTANT, arg1[1] * arg2[1] - + _exit_node_handlers[ProductExpression].update( - { - (_CONSTANT, _CONSTANT): _handle_product_constant_constant, - } + {(_CONSTANT, _CONSTANT): _handle_product_constant_constant} ) + def _handle_unary_constant(visitor, node, arg): # We override this because we can't blindly use apply_node_operation in this case if arg.__class__ not in native_numeric_types: return _CONSTANT, node.create_node_with_local_data( - (linear.to_expression(visitor, arg),)) + (linear.to_expression(visitor, arg),) + ) # otherwise do the usual: ans = apply_node_operation(node, (arg[1],)) # Unary includes sqrt() which can return complex numbers @@ -205,17 +205,15 @@ def _handle_unary_constant(visitor, node, arg): ans = complex_number_error(ans, visitor, node) return _CONSTANT, ans + _exit_node_handlers[UnaryFunctionExpression].update( - { - (_CONSTANT,): _handle_unary_constant - } + {(_CONSTANT,): _handle_unary_constant} ) _exit_node_handlers[AbsExpression] = _exit_node_handlers[UnaryFunctionExpression] -# LinearSubsystemRepnVisitor -class MultilevelLinearRepnVisitor(LinearRepnVisitor): - Result = LinearSubsystemRepn +class ParameterizedLinearRepnVisitor(LinearRepnVisitor): + Result = ParameterizedLinearRepn exit_node_handlers = _exit_node_handlers exit_node_dispatcher = ExitNodeDispatcher( _initialize_exit_node_dispatcher(_exit_node_handlers) @@ -253,8 +251,11 @@ def finalizeResult(self, result): self._factor_multiplier_into_linear_terms(ans, mult) return ans if mult == 1: - zeros = [(vid, coef) for vid, coef in ans.linear.items() if - coef.__class__ in native_numeric_types and not coef] + zeros = [ + (vid, coef) + for vid, coef in ans.linear.items() + if coef.__class__ in native_numeric_types and not coef + ] for vid, coef in zeros: del ans.linear[vid] elif not mult: diff --git a/pyomo/repn/tests/test_linear_wrt.py b/pyomo/repn/tests/test_parameterized_linear.py similarity index 73% rename from pyomo/repn/tests/test_linear_wrt.py rename to pyomo/repn/tests/test_parameterized_linear.py index fa7aaf7799b..32f58dbfc13 100644 --- a/pyomo/repn/tests/test_linear_wrt.py +++ b/pyomo/repn/tests/test_parameterized_linear.py @@ -13,12 +13,12 @@ import pyomo.common.unittest as unittest from pyomo.core.expr.compare import assertExpressionsEqual from pyomo.environ import Any, Binary, ConcreteModel, log, Param, Var -from pyomo.repn.linear_wrt import MultilevelLinearRepnVisitor +from pyomo.repn.parameterized_linear import ParameterizedLinearRepnVisitor from pyomo.repn.tests.test_linear import VisitorConfig from pyomo.repn.util import InvalidNumber -class TestMultilevelLinearRepnVisitor(unittest.TestCase): +class TestParameterizedLinearRepnVisitor(unittest.TestCase): def make_model(self): m = ConcreteModel() m.x = Var(bounds=(0, 45)) @@ -31,7 +31,7 @@ def test_walk_sum(self): m = self.make_model() e = m.x + m.y cfg = VisitorConfig() - visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) repn = visitor.walk_expression(e) @@ -48,7 +48,7 @@ def test_walk_triple_sum(self): e = m.x + m.z * m.y + m.z cfg = VisitorConfig() - visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x, m.y]) + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.z]) repn = visitor.walk_expression(e) @@ -67,7 +67,7 @@ def test_sum_two_of_the_same(self): m = self.make_model() e = m.x + m.x cfg = VisitorConfig() - visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x, m.y]) + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.z]) repn = visitor.walk_expression(e) @@ -77,25 +77,21 @@ def test_sum_two_of_the_same(self): self.assertEqual(repn.linear[id(m.x)], 2) self.assertEqual(repn.constant, 0) self.assertEqual(repn.multiplier, 1) - assertExpressionsEqual(self, repn.to_expression(visitor), 2*m.x) + assertExpressionsEqual(self, repn.to_expression(visitor), 2 * m.x) def test_sum_with_mult_0(self): m = self.make_model() - e = 0*m.x + m.x - m.y - + e = 0 * m.x + m.x - m.y + cfg = VisitorConfig() - visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) repn = visitor.walk_expression(e) self.assertIsNone(repn.nonlinear) self.assertEqual(len(repn.linear), 1) self.assertIn(id(m.x), repn.linear) self.assertEqual(repn.linear[id(m.x)], 1) - assertExpressionsEqual( - self, - repn.constant, - - m.y - ) + assertExpressionsEqual(self, repn.constant, -m.y) self.assertEqual(repn.multiplier, 1) assertExpressionsEqual(self, repn.to_expression(visitor), m.x - m.y) @@ -104,71 +100,56 @@ def test_sum_nonlinear_to_linear(self): e = m.y * m.x**2 + m.y * m.x - 3 cfg = VisitorConfig() - visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) repn = visitor.walk_expression(e) - assertExpressionsEqual( - self, - repn.nonlinear, - m.y * m.x ** 2 - ) + assertExpressionsEqual(self, repn.nonlinear, m.y * m.x**2) self.assertEqual(len(repn.linear), 1) self.assertIn(id(m.x), repn.linear) self.assertIs(repn.linear[id(m.x)], m.y) self.assertEqual(repn.constant, -3) self.assertEqual(repn.multiplier, 1) - assertExpressionsEqual(self, repn.to_expression(visitor), m.y * m.x ** 2 - + m.y * m.x - 3) + assertExpressionsEqual( + self, repn.to_expression(visitor), m.y * m.x**2 + m.y * m.x - 3 + ) def test_sum_nonlinear_to_nonlinear(self): m = self.make_model() - e = m.x ** 3 + 3 + m.x**2 + e = m.x**3 + 3 + m.x**2 cfg = VisitorConfig() - visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) repn = visitor.walk_expression(e) - assertExpressionsEqual( - self, - repn.nonlinear, - m.x ** 3 + m.x ** 2 - ) + assertExpressionsEqual(self, repn.nonlinear, m.x**3 + m.x**2) self.assertEqual(repn.constant, 3) self.assertEqual(repn.multiplier, 1) - assertExpressionsEqual(self, repn.to_expression(visitor), m.x ** 3 - + m.x ** 2 + 3) + assertExpressionsEqual( + self, repn.to_expression(visitor), m.x**3 + m.x**2 + 3 + ) def test_sum_to_linear_expr(self): m = self.make_model() e = m.x + m.y * (m.x + 5) cfg = VisitorConfig() - visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) repn = visitor.walk_expression(e) self.assertEqual(len(repn.linear), 1) self.assertIn(id(m.x), repn.linear) - assertExpressionsEqual( - self, - repn.linear[id(m.x)], - 1 + m.y - ) - assertExpressionsEqual( - self, - repn.constant, - m.y * 5 - ) + assertExpressionsEqual(self, repn.linear[id(m.x)], 1 + m.y) + assertExpressionsEqual(self, repn.constant, m.y * 5) self.assertEqual(repn.multiplier, 1) assertExpressionsEqual( - self, - repn.to_expression(visitor), (1 + m.y) * m.x + m.y * 5 + self, repn.to_expression(visitor), (1 + m.y) * m.x + m.y * 5 ) def test_bilinear_term(self): m = self.make_model() e = m.x * m.y cfg = VisitorConfig() - visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) repn = visitor.walk_expression(e) @@ -184,7 +165,7 @@ def test_distributed_bilinear_term(self): m = self.make_model() e = m.y * (m.x + 7) cfg = VisitorConfig() - visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) repn = visitor.walk_expression(e) @@ -200,7 +181,7 @@ def test_monomial(self): m = self.make_model() e = 45 * m.y cfg = VisitorConfig() - visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.y]) + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.x, m.z]) repn = visitor.walk_expression(e) @@ -216,7 +197,7 @@ def test_constant(self): m = self.make_model() e = 45 * m.y cfg = VisitorConfig() - visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) repn = visitor.walk_expression(e) @@ -232,7 +213,7 @@ def test_fixed_var(self): e = (m.y**2) * (m.x + m.x**2) cfg = VisitorConfig() - visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) repn = visitor.walk_expression(e) @@ -247,7 +228,7 @@ def test_nonlinear(self): e = (m.y * log(m.x)) * (m.y + 2) / m.x cfg = VisitorConfig() - visitor = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]) + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]) repn = visitor.walk_expression(e) @@ -265,7 +246,7 @@ def test_finalize(self): e = m.x + 2 * m.w**2 * m.y - m.x - m.w * m.z cfg = VisitorConfig() - repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x, m.y, m.z]).walk_expression(e) + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.w]).walk_expression(e) self.assertEqual(cfg.subexpr, {}) self.assertEqual(cfg.var_map, {id(m.x): m.x, id(m.y): m.y, id(m.z): m.z}) self.assertEqual(cfg.var_order, {id(m.x): 0, id(m.y): 1, id(m.z): 2}) @@ -273,23 +254,15 @@ def test_finalize(self): self.assertEqual(repn.constant, 0) self.assertEqual(len(repn.linear), 2) self.assertIn(id(m.y), repn.linear) - assertExpressionsEqual( - self, - repn.linear[id(m.y)], - 2 * m.w ** 2 - ) + assertExpressionsEqual(self, repn.linear[id(m.y)], 2 * m.w**2) self.assertIn(id(m.z), repn.linear) - assertExpressionsEqual( - self, - repn.linear[id(m.z)], - -m.w - ) + assertExpressionsEqual(self, repn.linear[id(m.z)], -m.w) self.assertEqual(repn.nonlinear, None) e *= 5 cfg = VisitorConfig() - repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x, m.y, m.z]).walk_expression(e) + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.w]).walk_expression(e) self.assertEqual(cfg.subexpr, {}) self.assertEqual(cfg.var_map, {id(m.x): m.x, id(m.y): m.y, id(m.z): m.z}) self.assertEqual(cfg.var_order, {id(m.x): 0, id(m.y): 1, id(m.z): 2}) @@ -298,23 +271,15 @@ def test_finalize(self): self.assertEqual(len(repn.linear), 2) self.assertIn(id(m.y), repn.linear) print(repn.linear[id(m.y)]) - assertExpressionsEqual( - self, - repn.linear[id(m.y)], - 5 * (2 * m.w ** 2) - ) + assertExpressionsEqual(self, repn.linear[id(m.y)], 5 * (2 * m.w**2)) self.assertIn(id(m.z), repn.linear) - assertExpressionsEqual( - self, - repn.linear[id(m.z)], - -5 * m.w - ) + assertExpressionsEqual(self, repn.linear[id(m.z)], -5 * m.w) self.assertEqual(repn.nonlinear, None) e = 5 * (m.w * m.y + m.z**2 + 3 * m.w * m.y**3) cfg = VisitorConfig() - repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x, m.y, m.z]).walk_expression(e) + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.w]).walk_expression(e) self.assertEqual(cfg.subexpr, {}) self.assertEqual(cfg.var_map, {id(m.y): m.y, id(m.z): m.z}) self.assertEqual(cfg.var_order, {id(m.y): 0, id(m.z): 1}) @@ -322,12 +287,10 @@ def test_finalize(self): self.assertEqual(repn.constant, 0) self.assertEqual(len(repn.linear), 1) self.assertIn(id(m.y), repn.linear) + assertExpressionsEqual(self, repn.linear[id(m.y)], 5 * m.w) assertExpressionsEqual( - self, - repn.linear[id(m.y)], - 5 * m.w + self, repn.nonlinear, (m.z**2 + 3 * m.w * m.y**3) * 5 ) - assertExpressionsEqual(self, repn.nonlinear, (m.z**2 + 3 * m.w * m.y**3) * 5) def test_ANY_over_constant_division(self): m = ConcreteModel() @@ -340,21 +303,15 @@ def test_ANY_over_constant_division(self): expr = m.y + m.x + m.z + ((3 * m.z * m.x) / m.p) / m.y cfg = VisitorConfig() - repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(expr) + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]).walk_expression( + expr + ) self.assertEqual(repn.multiplier, 1) - assertExpressionsEqual( - self, - repn.constant, - m.y + m.z - ) + assertExpressionsEqual(self, repn.constant, m.y + m.z) self.assertEqual(len(repn.linear), 1) print(repn.linear[id(m.x)]) - assertExpressionsEqual( - self, - repn.linear[id(m.x)], - 1 + 1.5 * m.z / m.y - ) + assertExpressionsEqual(self, repn.linear[id(m.x)], 1 + 1.5 * m.z / m.y) self.assertEqual(repn.nonlinear, None) def test_errors_propogate_nan(self): @@ -368,7 +325,9 @@ def test_errors_propogate_nan(self): expr = m.y + m.x + m.z + ((3 * m.z * m.x) / m.p) / m.y cfg = VisitorConfig() with LoggingIntercept() as LOG: - repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(expr) + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]).walk_expression( + expr + ) self.assertEqual( LOG.getvalue(), "Exception encountered evaluating expression 'div(3*z, 0)'\n" @@ -376,61 +335,45 @@ def test_errors_propogate_nan(self): "\texpression: 3*z*x/p\n", ) self.assertEqual(repn.multiplier, 1) - assertExpressionsEqual( - self, - repn.constant, - m.y + m.z - ) + assertExpressionsEqual(self, repn.constant, m.y + m.z) self.assertEqual(len(repn.linear), 1) self.assertIsInstance(repn.linear[id(m.x)], InvalidNumber) - assertExpressionsEqual( - self, - repn.linear[id(m.x)].value, - 1 + float('nan')/m.y - ) + assertExpressionsEqual(self, repn.linear[id(m.x)].value, 1 + float('nan') / m.y) self.assertEqual(repn.nonlinear, None) m.y.fix(None) expr = m.z * log(m.y) + 3 - repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(expr) + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]).walk_expression( + expr + ) self.assertEqual(repn.multiplier, 1) self.assertIsInstance(repn.constant, InvalidNumber) - assertExpressionsEqual( - self, - repn.constant.value, - float('nan')*m.z + 3 - ) + assertExpressionsEqual(self, repn.constant.value, float('nan') * m.z + 3) self.assertEqual(repn.linear, {}) self.assertEqual(repn.nonlinear, None) def test_negation_constant(self): m = self.make_model() - e = - (m.y * m.z + 17) + e = -(m.y * m.z + 17) cfg = VisitorConfig() - repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(e) + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y, m.z]).walk_expression(e) self.assertEqual(len(repn.linear), 0) self.assertEqual(repn.multiplier, 1) - assertExpressionsEqual( - self, - repn.constant, - - 1 * (m.y * m.z + 17) - ) + assertExpressionsEqual(self, repn.constant, -1 * (m.y * m.z + 17)) self.assertIsNone(repn.nonlinear) - + def test_product_nonlinear(self): m = self.make_model() - e = (m.x ** 2) * (log(m.y) * m.z ** 4) * m.y + e = (m.x**2) * (log(m.y) * m.z**4) * m.y cfg = VisitorConfig() - repn = MultilevelLinearRepnVisitor(*cfg, wrt=[m.x, m.z]).walk_expression(e) + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y]).walk_expression(e) self.assertEqual(len(repn.linear), 0) self.assertEqual(repn.multiplier, 1) self.assertEqual(repn.constant, 0) print(repn.nonlinear) assertExpressionsEqual( - self, - repn.nonlinear, - (m.x ** 2) * (m.z ** 4 * log(m.y)) * m.y + self, repn.nonlinear, (m.x**2) * (m.z**4 * log(m.y)) * m.y ) From b8d91c86473fd8709fc6b430fecd4bcd2fbddff6 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Sun, 12 May 2024 11:54:04 -0600 Subject: [PATCH 20/66] Fixing before_var handler so that we always use the values of fixed Vars regardless of if they are parameters or Vars from the perspective of the walker --- pyomo/repn/parameterized_linear.py | 33 ++++++++++--------- pyomo/repn/tests/test_parameterized_linear.py | 13 +++++--- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index 9df0ea458db..b180ee91862 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -160,22 +160,23 @@ def _before_general_expression(visitor, child): @staticmethod def _before_var(visitor, child): - if child not in visitor.wrt: + _id = id(child) + if _id not in visitor.var_map: + if child.fixed: + return False, ( + _CONSTANT, + visitor.check_constant(child.value, child), + ) + if child in visitor.wrt: + # psueudo-constant + # We aren't treating this Var as a Var for the purposes of this walker + return False, (_CONSTANT, child) # This is a normal situation - _id = id(child) - if _id not in visitor.var_map: - if child.fixed: - return False, ( - _CONSTANT, - visitor.check_constant(child.value, child), - ) - MultiLevelLinearBeforeChildDispatcher._record_var(visitor, child) - ans = visitor.Result() - ans.linear[_id] = 1 - return False, (ExprType.LINEAR, ans) - else: - # We aren't treating this Var as a Var for the purposes of this walker - return False, (_CONSTANT, child) + # TODO: override record var to not record things in wrt + MultiLevelLinearBeforeChildDispatcher._record_var(visitor, child) + ans = visitor.Result() + ans.linear[_id] = 1 + return False, (ExprType.LINEAR, ans) _before_child_dispatcher = MultiLevelLinearBeforeChildDispatcher() @@ -259,7 +260,7 @@ def finalizeResult(self, result): for vid, coef in zeros: del ans.linear[vid] elif not mult: - # the mulltiplier has cleared out the entire expression. + # the multiplier has cleared out the entire expression. # Warn if this is suppressing a NaN (unusual, and # non-standard, but we will wait to remove this behavior # for the time being) diff --git a/pyomo/repn/tests/test_parameterized_linear.py b/pyomo/repn/tests/test_parameterized_linear.py index 32f58dbfc13..b5e0a1a0348 100644 --- a/pyomo/repn/tests/test_parameterized_linear.py +++ b/pyomo/repn/tests/test_parameterized_linear.py @@ -298,7 +298,8 @@ def test_ANY_over_constant_division(self): m.x = Var() m.z = Var() m.y = Var() - # We aren't treating this as a Var, so we don't really care that it's fixed. + # We will use the fixed value regardless of the fact that we aren't + # treating this as a Var. m.y.fix(1) expr = m.y + m.x + m.z + ((3 * m.z * m.x) / m.p) / m.y @@ -308,10 +309,10 @@ def test_ANY_over_constant_division(self): ) self.assertEqual(repn.multiplier, 1) - assertExpressionsEqual(self, repn.constant, m.y + m.z) + assertExpressionsEqual(self, repn.constant, 1 + m.z) self.assertEqual(len(repn.linear), 1) print(repn.linear[id(m.x)]) - assertExpressionsEqual(self, repn.linear[id(m.x)], 1 + 1.5 * m.z / m.y) + assertExpressionsEqual(self, repn.linear[id(m.x)], 1 + 1.5 * m.z) self.assertEqual(repn.nonlinear, None) def test_errors_propogate_nan(self): @@ -335,10 +336,10 @@ def test_errors_propogate_nan(self): "\texpression: 3*z*x/p\n", ) self.assertEqual(repn.multiplier, 1) - assertExpressionsEqual(self, repn.constant, m.y + m.z) + assertExpressionsEqual(self, repn.constant, 1 + m.z) self.assertEqual(len(repn.linear), 1) self.assertIsInstance(repn.linear[id(m.x)], InvalidNumber) - assertExpressionsEqual(self, repn.linear[id(m.x)].value, 1 + float('nan') / m.y) + assertExpressionsEqual(self, repn.linear[id(m.x)].value, 1 + float('nan')) self.assertEqual(repn.nonlinear, None) m.y.fix(None) @@ -347,6 +348,8 @@ def test_errors_propogate_nan(self): expr ) self.assertEqual(repn.multiplier, 1) + # TODO: Is this expected to just wrap up into a single InvalidNumber? + print(repn.constant) self.assertIsInstance(repn.constant, InvalidNumber) assertExpressionsEqual(self, repn.constant.value, float('nan') * m.z + 3) self.assertEqual(repn.linear, {}) From b4101b7bf7eb88fde08133839aa080056deab853 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Sun, 12 May 2024 12:01:14 -0600 Subject: [PATCH 21/66] Extending the ExprType enum to include 'pseudo constant', bwahahaha --- pyomo/repn/parameterized_linear.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index b180ee91862..2163b544184 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -10,8 +10,10 @@ # ___________________________________________________________________________ import copy +import enum from pyomo.common.collections import ComponentSet +from pyomo.common.enums import ExtendedEnumType from pyomo.common.numeric_types import native_numeric_types from pyomo.core import Var from pyomo.core.expr.logical_expr import _flattened @@ -34,7 +36,12 @@ from pyomo.repn.util import ExprType from . import linear -_CONSTANT = ExprType.CONSTANT + +class ParameterizedExprType(enum.IntEnum, metaclass=ExtendedEnumType): + __base_enum__ = ExprType + PSUEDO_CONSTANT = 50 + +_CONSTANT = ParameterizedExprType.CONSTANT def _merge_dict(dest_dict, mult, src_dict): From c4c3de08b8db83a211b5395f54faf842243d4ac3 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 13 May 2024 14:26:11 -0600 Subject: [PATCH 22/66] starting to add pseudo-constant handlers but they don't work yet --- pyomo/repn/parameterized_linear.py | 95 ++++++++++++++++++++++++++++-- 1 file changed, 90 insertions(+), 5 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index 2163b544184..29e6583d19d 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -19,9 +19,12 @@ from pyomo.core.expr.logical_expr import _flattened from pyomo.core.expr.numeric_expr import ( AbsExpression, + DivisionExpression, LinearExpression, MonomialTermExpression, + NegationExpression, mutable_expression, + PowExpression, ProductExpression, SumExpression, UnaryFunctionExpression, @@ -41,7 +44,10 @@ class ParameterizedExprType(enum.IntEnum, metaclass=ExtendedEnumType): __base_enum__ = ExprType PSUEDO_CONSTANT = 50 +_PSEUDO_CONSTANT = ParameterizedExprType.PSUEDO_CONSTANT _CONSTANT = ParameterizedExprType.CONSTANT +_LINEAR = ParameterizedExprType.LINEAR +_GENERAL = ParameterizedExprType.GENERAL def _merge_dict(dest_dict, mult, src_dict): @@ -177,7 +183,7 @@ def _before_var(visitor, child): if child in visitor.wrt: # psueudo-constant # We aren't treating this Var as a Var for the purposes of this walker - return False, (_CONSTANT, child) + return False, (_PSEUDO_CONSTANT, child) # This is a normal situation # TODO: override record var to not record things in wrt MultiLevelLinearBeforeChildDispatcher._record_var(visitor, child) @@ -189,14 +195,93 @@ def _before_var(visitor, child): _before_child_dispatcher = MultiLevelLinearBeforeChildDispatcher() _exit_node_handlers = copy.deepcopy(linear._exit_node_handlers) +# +# NEGATION handler +# + +def _handle_negation_pseudo_constant(visitor, node, arg): + return (_PSEUDO_CONSTANT, -1 * arg[1]) -def _handle_product_constant_constant(visitor, node, arg1, arg2): - # ESJ: Can I do this? Just let the potential nans go through? - return _CONSTANT, arg1[1] * arg2[1] + +_exit_node_handlers[NegationExpression].update( + {(_PSEUDO_CONSTANT,): _handle_negation_pseudo_constant,} +) + + +# +# PRODUCT handler +# + + +def _handle_product_pseudo_constant_pseudo_constant(visitor, node, arg1, arg2): + return _PSEUDO_CONSTANT, arg1[1] * arg2[1] + + +def _handle_product_pseudo_constant_constant(visitor, node, arg1, arg2): + return _PSEUDO_CONSTANT, arg1[1] * arg2[1] _exit_node_handlers[ProductExpression].update( - {(_CONSTANT, _CONSTANT): _handle_product_constant_constant} + { + (_PSEUDO_CONSTANT, _PSEUDO_CONSTANT): _handle_product_pseudo_constant_pseudo_constant, + (_PSEUDO_CONSTANT, _CONSTANT): _handle_product_pseudo_constant_constant, + (_CONSTANT, _PSEUDO_CONSTANT): _handle_product_pseudo_constant_constant, + (_PSEUDO_CONSTANT, _LINEAR): linear._handle_product_constant_ANY, + (_LINEAR, _PSEUDO_CONSTANT): linear._handle_product_ANY_constant, + (_PSEUDO_CONSTANT, _GENERAL): linear._handle_product_constant_ANY, + (_GENERAL, _PSEUDO_CONSTANT): linear._handle_product_ANY_constant, + } +) +_exit_node_handlers[MonomialTermExpression].update(_exit_node_handlers[ProductExpression]) + +# +# DIVISION handlers +# + +def _handle_division_pseudo_constant_pseudo_constant(visitor, node, arg1, arg2): + return _PSEUDO_CONSTANT, arg1[1] / arg2[1] + + +def _handle_division_pseudo_constant_constant(visitor, node, arg1, arg2): + return _PSEUDO_CONSTANT, arg[1] / arg2[1] + + +def _handle_division_ANY_pseudo_constant(visitor, node, arg1, arg2): + arg1[1].multiplier = arg1[1].multiplier / arg2[1] + return arg1 + + +_exit_node_handlers[DivisionExpression].update( + { + (_PSEUDO_CONSTANT, _PSEUDO_CONSTANT): _handle_division_pseudo_constant_pseudo_constant, + (_PSEUDO_CONSTANT, _CONSTANT): _handle_division_pseudo_constant_constant, + (_CONSTANT, _PSEUDO_CONSTANT): _handle_division_pseudo_constant_constant, + (_LINEAR, _PSEUDO_CONSTANT): _handle_division_ANY_pseudo_constant, + (_GENERAL, _PSEUDO_CONSTANT): _handle_division_ANY_pseudo_constant, + } +) + +# +# EXPONENTIATION handlers +# + +def _handle_pow_pseudo_constant_pseudo_constant(visitor, node, arg1, arg2): + return _PSEUDO_CONSTANT, node.create_node_with_local_data( + linear.to_expression(visitor, arg1), linear.to_expression(visitor, arg2)) + + +def _handle_pow_ANY_pseudo_constant(visitor, node, arg1, arg2): + # TODO + pass + + +_exit_node_handlers[PowExpression].update( + { + (_PSEUDO_CONSTANT, _PSEUDO_CONSTANT): _handle_pow_pseudo_constant_pseudo_constant, + (_PSEUDO_CONSTANT, _CONSTANT): _handle_pow_pseudo_constant_pseudo_constant, + (_CONSTANT, _PSEUDO_CONSTANT): _handle_pow_pseudo_constant_pseudo_constant, + (_LINEAR, _PSEUDO_CONSTANT): _handle_pow_ANY_pseudo_constant, + } ) From 2a9abeae16467245d5fbc1bc2246488f6d25958c Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 20 May 2024 10:29:43 -0600 Subject: [PATCH 23/66] Full draft of new exit node handlers for psuedo constant expressions, all tests passing --- pyomo/repn/parameterized_linear.py | 83 ++++++++++--------- pyomo/repn/tests/test_parameterized_linear.py | 8 +- 2 files changed, 45 insertions(+), 46 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index 29e6583d19d..9556cae1cf7 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -44,6 +44,7 @@ class ParameterizedExprType(enum.IntEnum, metaclass=ExtendedEnumType): __base_enum__ = ExprType PSUEDO_CONSTANT = 50 + _PSEUDO_CONSTANT = ParameterizedExprType.PSUEDO_CONSTANT _CONSTANT = ParameterizedExprType.CONSTANT _LINEAR = ParameterizedExprType.LINEAR @@ -65,6 +66,13 @@ def _merge_dict(dest_dict, mult, src_dict): dest_dict[vid] = coef +def to_expression(visitor, arg): + if arg[0] in (_CONSTANT, _PSEUDO_CONSTANT): + return arg[1] + else: + return arg[1].to_expression(visitor) + + class ParameterizedLinearRepn(LinearRepn): def to_expression(self, visitor): if self.nonlinear is not None: @@ -107,7 +115,7 @@ def append(self, other): """ _type, other = other - if _type is _CONSTANT: + if _type in (_CONSTANT, _PSEUDO_CONSTANT): self.constant += other return @@ -176,10 +184,7 @@ def _before_var(visitor, child): _id = id(child) if _id not in visitor.var_map: if child.fixed: - return False, ( - _CONSTANT, - visitor.check_constant(child.value, child), - ) + return False, (_CONSTANT, visitor.check_constant(child.value, child)) if child in visitor.wrt: # psueudo-constant # We aren't treating this Var as a Var for the purposes of this walker @@ -199,12 +204,13 @@ def _before_var(visitor, child): # NEGATION handler # + def _handle_negation_pseudo_constant(visitor, node, arg): return (_PSEUDO_CONSTANT, -1 * arg[1]) _exit_node_handlers[NegationExpression].update( - {(_PSEUDO_CONSTANT,): _handle_negation_pseudo_constant,} + {(_PSEUDO_CONSTANT,): _handle_negation_pseudo_constant} ) @@ -213,17 +219,13 @@ def _handle_negation_pseudo_constant(visitor, node, arg): # -def _handle_product_pseudo_constant_pseudo_constant(visitor, node, arg1, arg2): - return _PSEUDO_CONSTANT, arg1[1] * arg2[1] - - def _handle_product_pseudo_constant_constant(visitor, node, arg1, arg2): return _PSEUDO_CONSTANT, arg1[1] * arg2[1] _exit_node_handlers[ProductExpression].update( { - (_PSEUDO_CONSTANT, _PSEUDO_CONSTANT): _handle_product_pseudo_constant_pseudo_constant, + (_PSEUDO_CONSTANT, _PSEUDO_CONSTANT): _handle_product_pseudo_constant_constant, (_PSEUDO_CONSTANT, _CONSTANT): _handle_product_pseudo_constant_constant, (_CONSTANT, _PSEUDO_CONSTANT): _handle_product_pseudo_constant_constant, (_PSEUDO_CONSTANT, _LINEAR): linear._handle_product_constant_ANY, @@ -232,18 +234,17 @@ def _handle_product_pseudo_constant_constant(visitor, node, arg1, arg2): (_GENERAL, _PSEUDO_CONSTANT): linear._handle_product_ANY_constant, } ) -_exit_node_handlers[MonomialTermExpression].update(_exit_node_handlers[ProductExpression]) +_exit_node_handlers[MonomialTermExpression].update( + _exit_node_handlers[ProductExpression] +) # # DIVISION handlers # -def _handle_division_pseudo_constant_pseudo_constant(visitor, node, arg1, arg2): - return _PSEUDO_CONSTANT, arg1[1] / arg2[1] - def _handle_division_pseudo_constant_constant(visitor, node, arg1, arg2): - return _PSEUDO_CONSTANT, arg[1] / arg2[1] + return _PSEUDO_CONSTANT, arg1[1] / arg2[1] def _handle_division_ANY_pseudo_constant(visitor, node, arg1, arg2): @@ -253,7 +254,7 @@ def _handle_division_ANY_pseudo_constant(visitor, node, arg1, arg2): _exit_node_handlers[DivisionExpression].update( { - (_PSEUDO_CONSTANT, _PSEUDO_CONSTANT): _handle_division_pseudo_constant_pseudo_constant, + (_PSEUDO_CONSTANT, _PSEUDO_CONSTANT): _handle_division_pseudo_constant_constant, (_PSEUDO_CONSTANT, _CONSTANT): _handle_division_pseudo_constant_constant, (_CONSTANT, _PSEUDO_CONSTANT): _handle_division_pseudo_constant_constant, (_LINEAR, _PSEUDO_CONSTANT): _handle_division_ANY_pseudo_constant, @@ -265,42 +266,44 @@ def _handle_division_ANY_pseudo_constant(visitor, node, arg1, arg2): # EXPONENTIATION handlers # -def _handle_pow_pseudo_constant_pseudo_constant(visitor, node, arg1, arg2): - return _PSEUDO_CONSTANT, node.create_node_with_local_data( - linear.to_expression(visitor, arg1), linear.to_expression(visitor, arg2)) + +def _handle_pow_pseudo_constant_constant(visitor, node, arg1, arg2): + print("creating node") + print(to_expression(visitor, arg1)) + print(to_expression(visitor, arg2)) + return _PSEUDO_CONSTANT, to_expression(visitor, arg1) ** to_expression( + visitor, arg2 + ) -def _handle_pow_ANY_pseudo_constant(visitor, node, arg1, arg2): - # TODO - pass +def _handle_pow_ANY_psuedo_constant(visitor, node, arg1, arg2): + return linear._handle_pow_nonlinear(visitor, node, arg1, arg2) _exit_node_handlers[PowExpression].update( { - (_PSEUDO_CONSTANT, _PSEUDO_CONSTANT): _handle_pow_pseudo_constant_pseudo_constant, - (_PSEUDO_CONSTANT, _CONSTANT): _handle_pow_pseudo_constant_pseudo_constant, - (_CONSTANT, _PSEUDO_CONSTANT): _handle_pow_pseudo_constant_pseudo_constant, - (_LINEAR, _PSEUDO_CONSTANT): _handle_pow_ANY_pseudo_constant, + (_PSEUDO_CONSTANT, _PSEUDO_CONSTANT): _handle_pow_pseudo_constant_constant, + (_PSEUDO_CONSTANT, _CONSTANT): _handle_pow_pseudo_constant_constant, + (_CONSTANT, _PSEUDO_CONSTANT): _handle_pow_pseudo_constant_constant, + (_LINEAR, _PSEUDO_CONSTANT): _handle_pow_pseudo_constant_constant, + (_GENERAL, _PSEUDO_CONSTANT): _handle_pow_ANY_psuedo_constant, } ) +# +# ABS and UNARY handlers +# + -def _handle_unary_constant(visitor, node, arg): +def _handle_unary_pseudo_constant(visitor, node, arg): # We override this because we can't blindly use apply_node_operation in this case - if arg.__class__ not in native_numeric_types: - return _CONSTANT, node.create_node_with_local_data( - (linear.to_expression(visitor, arg),) - ) - # otherwise do the usual: - ans = apply_node_operation(node, (arg[1],)) - # Unary includes sqrt() which can return complex numbers - if ans.__class__ in native_complex_types: - ans = complex_number_error(ans, visitor, node) - return _CONSTANT, ans + return _PSEUDO_CONSTANT, node.create_node_with_local_data( + (to_expression(visitor, arg),) + ) _exit_node_handlers[UnaryFunctionExpression].update( - {(_CONSTANT,): _handle_unary_constant} + {(_PSEUDO_CONSTANT,): _handle_unary_pseudo_constant} ) _exit_node_handlers[AbsExpression] = _exit_node_handlers[UnaryFunctionExpression] @@ -376,6 +379,6 @@ def finalizeResult(self, result): return ans ans = self.Result() - assert result[0] is _CONSTANT + assert result[0] in (_CONSTANT, _PSEUDO_CONSTANT) ans.constant = result[1] return ans diff --git a/pyomo/repn/tests/test_parameterized_linear.py b/pyomo/repn/tests/test_parameterized_linear.py index b5e0a1a0348..068afe16929 100644 --- a/pyomo/repn/tests/test_parameterized_linear.py +++ b/pyomo/repn/tests/test_parameterized_linear.py @@ -124,9 +124,7 @@ def test_sum_nonlinear_to_nonlinear(self): assertExpressionsEqual(self, repn.nonlinear, m.x**3 + m.x**2) self.assertEqual(repn.constant, 3) self.assertEqual(repn.multiplier, 1) - assertExpressionsEqual( - self, repn.to_expression(visitor), m.x**3 + m.x**2 + 3 - ) + assertExpressionsEqual(self, repn.to_expression(visitor), m.x**3 + m.x**2 + 3) def test_sum_to_linear_expr(self): m = self.make_model() @@ -288,9 +286,7 @@ def test_finalize(self): self.assertEqual(len(repn.linear), 1) self.assertIn(id(m.y), repn.linear) assertExpressionsEqual(self, repn.linear[id(m.y)], 5 * m.w) - assertExpressionsEqual( - self, repn.nonlinear, (m.z**2 + 3 * m.w * m.y**3) * 5 - ) + assertExpressionsEqual(self, repn.nonlinear, (m.z**2 + 3 * m.w * m.y**3) * 5) def test_ANY_over_constant_division(self): m = ConcreteModel() From 0d99834c446045efbf58a615cb2c4ae81f23f06a Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 20 May 2024 13:59:38 -0600 Subject: [PATCH 24/66] Fixing a bug where psuedo constants were getting labeled as constants in walker_exitNode --- pyomo/repn/parameterized_linear.py | 27 +++-- pyomo/repn/tests/test_parameterized_linear.py | 105 +++++++++++++++++- 2 files changed, 120 insertions(+), 12 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index 9556cae1cf7..67fb4a7421e 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -74,6 +74,16 @@ def to_expression(visitor, arg): class ParameterizedLinearRepn(LinearRepn): + def walker_exitNode(self): + if self.nonlinear is not None: + return _GENERAL, self + elif self.linear: + return _LINEAR, self + elif self.constant.__class__ in native_numeric_types: + return _CONSTANT, self.multiplier * self.constant + else: + return _PSEUDO_CONSTANT, self.multiplier * self.constant + def to_expression(self, visitor): if self.nonlinear is not None: # We want to start with the nonlinear term (and use @@ -268,16 +278,17 @@ def _handle_division_ANY_pseudo_constant(visitor, node, arg1, arg2): def _handle_pow_pseudo_constant_constant(visitor, node, arg1, arg2): - print("creating node") - print(to_expression(visitor, arg1)) - print(to_expression(visitor, arg2)) return _PSEUDO_CONSTANT, to_expression(visitor, arg1) ** to_expression( visitor, arg2 ) -def _handle_pow_ANY_psuedo_constant(visitor, node, arg1, arg2): - return linear._handle_pow_nonlinear(visitor, node, arg1, arg2) +def _handle_pow_nonlinear(visitor, node, arg1, arg2): + # ESJ: We override this because we need our own to_expression implementation + # if pseudo constants are involved. + ans = visitor.Result() + ans.nonlinear = to_expression(visitor, arg1) ** to_expression(visitor, arg2) + return _GENERAL, ans _exit_node_handlers[PowExpression].update( @@ -285,8 +296,10 @@ def _handle_pow_ANY_psuedo_constant(visitor, node, arg1, arg2): (_PSEUDO_CONSTANT, _PSEUDO_CONSTANT): _handle_pow_pseudo_constant_constant, (_PSEUDO_CONSTANT, _CONSTANT): _handle_pow_pseudo_constant_constant, (_CONSTANT, _PSEUDO_CONSTANT): _handle_pow_pseudo_constant_constant, - (_LINEAR, _PSEUDO_CONSTANT): _handle_pow_pseudo_constant_constant, - (_GENERAL, _PSEUDO_CONSTANT): _handle_pow_ANY_psuedo_constant, + (_LINEAR, _PSEUDO_CONSTANT): _handle_pow_nonlinear, + (_PSEUDO_CONSTANT, _LINEAR): _handle_pow_nonlinear, + (_GENERAL, _PSEUDO_CONSTANT): _handle_pow_nonlinear, + (_PSEUDO_CONSTANT, _GENERAL): _handle_pow_nonlinear, } ) diff --git a/pyomo/repn/tests/test_parameterized_linear.py b/pyomo/repn/tests/test_parameterized_linear.py index 068afe16929..de4de301a83 100644 --- a/pyomo/repn/tests/test_parameterized_linear.py +++ b/pyomo/repn/tests/test_parameterized_linear.py @@ -268,7 +268,6 @@ def test_finalize(self): self.assertEqual(repn.constant, 0) self.assertEqual(len(repn.linear), 2) self.assertIn(id(m.y), repn.linear) - print(repn.linear[id(m.y)]) assertExpressionsEqual(self, repn.linear[id(m.y)], 5 * (2 * m.w**2)) self.assertIn(id(m.z), repn.linear) assertExpressionsEqual(self, repn.linear[id(m.z)], -5 * m.w) @@ -307,7 +306,6 @@ def test_ANY_over_constant_division(self): self.assertEqual(repn.multiplier, 1) assertExpressionsEqual(self, repn.constant, 1 + m.z) self.assertEqual(len(repn.linear), 1) - print(repn.linear[id(m.x)]) assertExpressionsEqual(self, repn.linear[id(m.x)], 1 + 1.5 * m.z) self.assertEqual(repn.nonlinear, None) @@ -344,8 +342,6 @@ def test_errors_propogate_nan(self): expr ) self.assertEqual(repn.multiplier, 1) - # TODO: Is this expected to just wrap up into a single InvalidNumber? - print(repn.constant) self.assertIsInstance(repn.constant, InvalidNumber) assertExpressionsEqual(self, repn.constant.value, float('nan') * m.z + 3) self.assertEqual(repn.linear, {}) @@ -372,7 +368,106 @@ def test_product_nonlinear(self): self.assertEqual(len(repn.linear), 0) self.assertEqual(repn.multiplier, 1) self.assertEqual(repn.constant, 0) - print(repn.nonlinear) assertExpressionsEqual( self, repn.nonlinear, (m.x**2) * (m.z**4 * log(m.y)) * m.y ) + + def test_division_pseudo_constant_constant(self): + m = self.make_model() + e = m.x / 4 + m.y + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(e) + + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.y), repn.linear) + self.assertEqual(repn.linear[id(m.y)], 1) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.constant, m.x / 4) + self.assertIsNone(repn.nonlinear) + + e = 4 / m.x + m.y + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(e) + + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.y), repn.linear) + self.assertEqual(repn.linear[id(m.y)], 1) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.constant, 4 / m.x) + self.assertIsNone(repn.nonlinear) + + e = m.z / m.x + m.y + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.x, m.z]).walk_expression(e) + + self.assertEqual(len(repn.linear), 1) + self.assertIn(id(m.y), repn.linear) + self.assertEqual(repn.linear[id(m.y)], 1) + self.assertEqual(repn.multiplier, 1) + assertExpressionsEqual(self, repn.constant, m.z / m.x) + self.assertIsNone(repn.nonlinear) + + def test_division_ANY_psuedo_constant(self): + m = self.make_model() + e = (m.x + 3 * m.z) / m.y + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y]).walk_expression(e) + + self.assertEqual(len(repn.linear), 2) + self.assertIn(id(m.x), repn.linear) + assertExpressionsEqual(self, repn.linear[id(m.x)], 1 / m.y) + self.assertIn(id(m.z), repn.linear) + assertExpressionsEqual(self, repn.linear[id(m.z)], (1 / m.y) * 3) + self.assertEqual(repn.multiplier, 1) + self.assertEqual(repn.constant, 0) + self.assertIsNone(repn.nonlinear) + + def test_pow_ANY_psuedo_constant(self): + m = self.make_model() + e = (m.x**2 + 3 * m.z) ** m.y + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y]).walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + self.assertEqual(repn.constant, 0) + assertExpressionsEqual(self, repn.nonlinear, (m.x**2 + 3 * m.z) ** m.y) + + def test_pow_psuedo_constant_ANY(self): + m = self.make_model() + e = m.y ** (m.x**2 + 3 * m.z) + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y]).walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + self.assertEqual(repn.constant, 0) + assertExpressionsEqual(self, repn.nonlinear, m.y ** (m.x**2 + 3 * m.z)) + + def test_pow_linear_pseudo_constant(self): + m = self.make_model() + e = (m.x + 3 * m.z) ** m.y + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y]).walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + self.assertEqual(repn.constant, 0) + assertExpressionsEqual(self, repn.nonlinear, (m.x + 3 * m.z) ** m.y) + + def test_pow_pseudo_constant_linear(self): + m = self.make_model() + e = m.y ** (m.x + 3 * m.z) + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y]).walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + self.assertEqual(repn.constant, 0) + assertExpressionsEqual(self, repn.nonlinear, m.y ** (m.x + 3 * m.z)) From f13d3ab5905f9bba4c67af6b9ea6cd9a68b38d52 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 20 May 2024 16:17:53 -0600 Subject: [PATCH 25/66] Making the new walker compliant with IEEE 754 when multiplying 0 and nan --- pyomo/repn/parameterized_linear.py | 15 ++---- pyomo/repn/tests/test_parameterized_linear.py | 53 +++++++++++++++++++ 2 files changed, 58 insertions(+), 10 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index 67fb4a7421e..6748c406bb7 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -200,7 +200,6 @@ def _before_var(visitor, child): # We aren't treating this Var as a Var for the purposes of this walker return False, (_PSEUDO_CONSTANT, child) # This is a normal situation - # TODO: override record var to not record things in wrt MultiLevelLinearBeforeChildDispatcher._record_var(visitor, child) ans = visitor.Result() ans.linear[_id] = 1 @@ -228,6 +227,8 @@ def _handle_negation_pseudo_constant(visitor, node, arg): # PRODUCT handler # +def _handle_product_constant_constant(visitor, node, arg1, arg2): + return _CONSTANT, arg1[1] * arg2[1] def _handle_product_pseudo_constant_constant(visitor, node, arg1, arg2): return _PSEUDO_CONSTANT, arg1[1] * arg2[1] @@ -235,6 +236,7 @@ def _handle_product_pseudo_constant_constant(visitor, node, arg1, arg2): _exit_node_handlers[ProductExpression].update( { + (_CONSTANT, _CONSTANT): _handle_product_constant_constant, (_PSEUDO_CONSTANT, _PSEUDO_CONSTANT): _handle_product_pseudo_constant_constant, (_PSEUDO_CONSTANT, _CONSTANT): _handle_product_pseudo_constant_constant, (_CONSTANT, _PSEUDO_CONSTANT): _handle_product_pseudo_constant_constant, @@ -372,18 +374,11 @@ def finalizeResult(self, result): # Warn if this is suppressing a NaN (unusual, and # non-standard, but we will wait to remove this behavior # for the time being) - # ESJ TODO: This won't work either actually... - # I'm not sure how to do it. if ans.constant != ans.constant or any( c != c for c in ans.linear.values() ): - deprecation_warning( - f"Encountered {str(mult)}*nan in expression tree. " - "Mapping the NaN result to 0 for compatibility " - "with the lp_v1 writer. In the future, this NaN " - "will be preserved/emitted to comply with IEEE-754.", - version='6.6.0', - ) + # There's a nan in here, so we keep it + self._factor_multiplier_into_linear_terms(ans, mult) return self.Result() else: # mult not in {0, 1}: factor it into the constant, diff --git a/pyomo/repn/tests/test_parameterized_linear.py b/pyomo/repn/tests/test_parameterized_linear.py index de4de301a83..2148e3053f4 100644 --- a/pyomo/repn/tests/test_parameterized_linear.py +++ b/pyomo/repn/tests/test_parameterized_linear.py @@ -471,3 +471,56 @@ def test_pow_pseudo_constant_linear(self): self.assertEqual(repn.multiplier, 1) self.assertEqual(repn.constant, 0) assertExpressionsEqual(self, repn.nonlinear, m.y ** (m.x + 3 * m.z)) + + def test_0_mult(self): + m = self.make_model() + m.p = Var() + m.p.fix(0) + e = m.p * (m.y ** 2 + m.z) + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.z]).walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + self.assertIsNone(repn.nonlinear) + self.assertEqual(repn.constant, 0) + + def test_0_mult_nan(self): + m = self.make_model() + m.p = Param(initialize=0, mutable=True) + m.y.domain = Any + m.y.fix(float('nan')) + e = m.p * (m.y ** 2 + m.x) + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + self.assertIsNone(repn.nonlinear) + self.assertIsInstance(repn.constant, InvalidNumber) + assertExpressionsEqual( + self, + repn.constant.value, + 0 * (float('nan') + m.x) + ) + + def test_0_mult_nan_param(self): + m = self.make_model() + m.p = Param(initialize=0, mutable=True) + m.y.fix(float('nan')) + e = m.p * (m.y ** 2) + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y]).walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + self.assertIsNone(repn.nonlinear) + self.assertIsInstance(repn.constant, InvalidNumber) + assertExpressionsEqual( + self, + repn.constant.value, + 0 * float('nan') + ) From 42ff2932f5c88ad5025971593ced3e48345240b7 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 20 May 2024 16:18:21 -0600 Subject: [PATCH 26/66] Black --- pyomo/repn/parameterized_linear.py | 2 ++ pyomo/repn/tests/test_parameterized_linear.py | 20 ++++++------------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index 6748c406bb7..5928d1f18d9 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -227,9 +227,11 @@ def _handle_negation_pseudo_constant(visitor, node, arg): # PRODUCT handler # + def _handle_product_constant_constant(visitor, node, arg1, arg2): return _CONSTANT, arg1[1] * arg2[1] + def _handle_product_pseudo_constant_constant(visitor, node, arg1, arg2): return _PSEUDO_CONSTANT, arg1[1] * arg2[1] diff --git a/pyomo/repn/tests/test_parameterized_linear.py b/pyomo/repn/tests/test_parameterized_linear.py index 2148e3053f4..1b0ab630462 100644 --- a/pyomo/repn/tests/test_parameterized_linear.py +++ b/pyomo/repn/tests/test_parameterized_linear.py @@ -476,7 +476,7 @@ def test_0_mult(self): m = self.make_model() m.p = Var() m.p.fix(0) - e = m.p * (m.y ** 2 + m.z) + e = m.p * (m.y**2 + m.z) cfg = VisitorConfig() repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.z]).walk_expression(e) @@ -491,7 +491,7 @@ def test_0_mult_nan(self): m.p = Param(initialize=0, mutable=True) m.y.domain = Any m.y.fix(float('nan')) - e = m.p * (m.y ** 2 + m.x) + e = m.p * (m.y**2 + m.x) cfg = VisitorConfig() repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(e) @@ -500,17 +500,13 @@ def test_0_mult_nan(self): self.assertEqual(repn.multiplier, 1) self.assertIsNone(repn.nonlinear) self.assertIsInstance(repn.constant, InvalidNumber) - assertExpressionsEqual( - self, - repn.constant.value, - 0 * (float('nan') + m.x) - ) - + assertExpressionsEqual(self, repn.constant.value, 0 * (float('nan') + m.x)) + def test_0_mult_nan_param(self): m = self.make_model() m.p = Param(initialize=0, mutable=True) m.y.fix(float('nan')) - e = m.p * (m.y ** 2) + e = m.p * (m.y**2) cfg = VisitorConfig() repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y]).walk_expression(e) @@ -519,8 +515,4 @@ def test_0_mult_nan_param(self): self.assertEqual(repn.multiplier, 1) self.assertIsNone(repn.nonlinear) self.assertIsInstance(repn.constant, InvalidNumber) - assertExpressionsEqual( - self, - repn.constant.value, - 0 * float('nan') - ) + assertExpressionsEqual(self, repn.constant.value, 0 * float('nan')) From fb78c898b9f1c8fe1aa3e45fa1bb43b923fbe518 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 20 May 2024 16:21:49 -0600 Subject: [PATCH 27/66] Fixing some typos --- pyomo/repn/parameterized_linear.py | 6 +++--- pyomo/repn/tests/test_parameterized_linear.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index 5928d1f18d9..8200b4cf01e 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -42,10 +42,10 @@ class ParameterizedExprType(enum.IntEnum, metaclass=ExtendedEnumType): __base_enum__ = ExprType - PSUEDO_CONSTANT = 50 + PSEUDO_CONSTANT = 50 -_PSEUDO_CONSTANT = ParameterizedExprType.PSUEDO_CONSTANT +_PSEUDO_CONSTANT = ParameterizedExprType.PSEUDO_CONSTANT _CONSTANT = ParameterizedExprType.CONSTANT _LINEAR = ParameterizedExprType.LINEAR _GENERAL = ParameterizedExprType.GENERAL @@ -196,7 +196,7 @@ def _before_var(visitor, child): if child.fixed: return False, (_CONSTANT, visitor.check_constant(child.value, child)) if child in visitor.wrt: - # psueudo-constant + # pseudo-constant # We aren't treating this Var as a Var for the purposes of this walker return False, (_PSEUDO_CONSTANT, child) # This is a normal situation diff --git a/pyomo/repn/tests/test_parameterized_linear.py b/pyomo/repn/tests/test_parameterized_linear.py index 1b0ab630462..7d99acf8bb8 100644 --- a/pyomo/repn/tests/test_parameterized_linear.py +++ b/pyomo/repn/tests/test_parameterized_linear.py @@ -309,7 +309,7 @@ def test_ANY_over_constant_division(self): assertExpressionsEqual(self, repn.linear[id(m.x)], 1 + 1.5 * m.z) self.assertEqual(repn.nonlinear, None) - def test_errors_propogate_nan(self): + def test_errors_propagate_nan(self): m = ConcreteModel() m.p = Param(mutable=True, initialize=0, domain=Any) m.x = Var() From 95fa5245de86ebef4ad45f0ebff122a28ebf7f2e Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Tue, 21 May 2024 09:21:07 -0600 Subject: [PATCH 28/66] Fixing more typos --- pyomo/repn/tests/test_parameterized_linear.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyomo/repn/tests/test_parameterized_linear.py b/pyomo/repn/tests/test_parameterized_linear.py index 7d99acf8bb8..4e92c5f11f2 100644 --- a/pyomo/repn/tests/test_parameterized_linear.py +++ b/pyomo/repn/tests/test_parameterized_linear.py @@ -408,7 +408,7 @@ def test_division_pseudo_constant_constant(self): assertExpressionsEqual(self, repn.constant, m.z / m.x) self.assertIsNone(repn.nonlinear) - def test_division_ANY_psuedo_constant(self): + def test_division_ANY_pseudo_constant(self): m = self.make_model() e = (m.x + 3 * m.z) / m.y @@ -424,7 +424,7 @@ def test_division_ANY_psuedo_constant(self): self.assertEqual(repn.constant, 0) self.assertIsNone(repn.nonlinear) - def test_pow_ANY_psuedo_constant(self): + def test_pow_ANY_pseudo_constant(self): m = self.make_model() e = (m.x**2 + 3 * m.z) ** m.y @@ -436,7 +436,7 @@ def test_pow_ANY_psuedo_constant(self): self.assertEqual(repn.constant, 0) assertExpressionsEqual(self, repn.nonlinear, (m.x**2 + 3 * m.z) ** m.y) - def test_pow_psuedo_constant_ANY(self): + def test_pow_pseudo_constant_ANY(self): m = self.make_model() e = m.y ** (m.x**2 + 3 * m.z) From 794a3bf2ccaa5561c8def704b6c7b1111c0746e4 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Tue, 21 May 2024 10:59:12 -0600 Subject: [PATCH 29/66] Distributing 0 if there are nans present during finalizeResult --- pyomo/repn/parameterized_linear.py | 10 +++++----- pyomo/repn/tests/test_parameterized_linear.py | 20 +++++++++++++++++++ 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index 8200b4cf01e..c1647e44732 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -372,15 +372,15 @@ def finalizeResult(self, result): for vid, coef in zeros: del ans.linear[vid] elif not mult: - # the multiplier has cleared out the entire expression. - # Warn if this is suppressing a NaN (unusual, and - # non-standard, but we will wait to remove this behavior - # for the time being) + # the multiplier has cleared out the entire expression. Check + # if this is suppressing a NaN because we can't clear everything + # out if it is if ans.constant != ans.constant or any( c != c for c in ans.linear.values() ): - # There's a nan in here, so we keep it + # There's a nan in here, so we distribute the 0 self._factor_multiplier_into_linear_terms(ans, mult) + return ans return self.Result() else: # mult not in {0, 1}: factor it into the constant, diff --git a/pyomo/repn/tests/test_parameterized_linear.py b/pyomo/repn/tests/test_parameterized_linear.py index 4e92c5f11f2..fd2f2aaec68 100644 --- a/pyomo/repn/tests/test_parameterized_linear.py +++ b/pyomo/repn/tests/test_parameterized_linear.py @@ -516,3 +516,23 @@ def test_0_mult_nan_param(self): self.assertIsNone(repn.nonlinear) self.assertIsInstance(repn.constant, InvalidNumber) assertExpressionsEqual(self, repn.constant.value, 0 * float('nan')) + + def test_0_mult_linear_with_nan(self): + m = self.make_model() + m.p = Param(initialize=0, mutable=True) + m.x.domain = Any + m.x.fix(float('nan')) + e = m.p * (3 * m.x * m.y + m.z) + + cfg = VisitorConfig() + repn = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.x]).walk_expression(e) + + self.assertEqual(len(repn.linear), 2) + self.assertIn(id(m.y), repn.linear) + self.assertIsInstance(repn.linear[id(m.y)], InvalidNumber) + assertExpressionsEqual(self, repn.linear[id(m.y)].value, 0 * 3 * float('nan')) + self.assertIn(id(m.z), repn.linear) + self.assertEqual(repn.linear[id(m.z)], 0) + self.assertEqual(repn.multiplier, 1) + self.assertIsNone(repn.nonlinear) + self.assertEqual(repn.constant, 0) From 2c4a1cbd9cc309e1925a484444a468e09fd4fed6 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Tue, 21 May 2024 11:23:03 -0600 Subject: [PATCH 30/66] Testing duplicate in ParameterizedLinearRepn, changing the string representation to not be the same as LinearRepn --- pyomo/repn/parameterized_linear.py | 6 ++++++ pyomo/repn/tests/test_parameterized_linear.py | 14 ++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index c1647e44732..eb6dd619168 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -74,6 +74,12 @@ def to_expression(visitor, arg): class ParameterizedLinearRepn(LinearRepn): + def __str__(self): + return ( + f"ParameterizedLinearRepn(mult={self.multiplier}, const={self.constant}, " + f"linear={self.linear}, nonlinear={self.nonlinear})" + ) + def walker_exitNode(self): if self.nonlinear is not None: return _GENERAL, self diff --git a/pyomo/repn/tests/test_parameterized_linear.py b/pyomo/repn/tests/test_parameterized_linear.py index fd2f2aaec68..624f8390d16 100644 --- a/pyomo/repn/tests/test_parameterized_linear.py +++ b/pyomo/repn/tests/test_parameterized_linear.py @@ -424,6 +424,20 @@ def test_division_ANY_pseudo_constant(self): self.assertEqual(repn.constant, 0) self.assertIsNone(repn.nonlinear) + def test_duplicate(self): + m = self.make_model() + e = (1 + m.x) ** 2 + m.y + + cfg = VisitorConfig() + visitor = ParameterizedLinearRepnVisitor(*cfg, wrt=[m.y]) + visitor.max_exponential_expansion = 2 + repn = visitor.walk_expression(e) + + self.assertEqual(len(repn.linear), 0) + self.assertEqual(repn.multiplier, 1) + self.assertIs(repn.constant, m.y) + assertExpressionsEqual(self, repn.nonlinear, (m.x + 1) * (m.x + 1)) + def test_pow_ANY_pseudo_constant(self): m = self.make_model() e = (m.x**2 + 3 * m.z) ** m.y From 7d4270108a46eca83d288d1d78a4593f4a8f1ad4 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Tue, 21 May 2024 11:28:56 -0600 Subject: [PATCH 31/66] NFC: fixing some comment typos --- pyomo/repn/parameterized_linear.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index eb6dd619168..f976523a37f 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -216,7 +216,7 @@ def _before_var(visitor, child): _exit_node_handlers = copy.deepcopy(linear._exit_node_handlers) # -# NEGATION handler +# NEGATION handlers # @@ -230,7 +230,7 @@ def _handle_negation_pseudo_constant(visitor, node, arg): # -# PRODUCT handler +# PRODUCT handlers # From 71d8801f277d7b40510aeff23e0f46c3839121bd Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Tue, 21 May 2024 11:40:18 -0600 Subject: [PATCH 32/66] Whoops, one last class name change I missed --- pyomo/repn/parameterized_linear.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index f976523a37f..0633c285155 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -175,7 +175,7 @@ def append(self, other): self.nonlinear += nl -class MultiLevelLinearBeforeChildDispatcher(LinearBeforeChildDispatcher): +class ParameterizedLinearBeforeChildDispatcher(LinearBeforeChildDispatcher): def __init__(self): super().__init__() self[Var] = self._before_var @@ -206,13 +206,13 @@ def _before_var(visitor, child): # We aren't treating this Var as a Var for the purposes of this walker return False, (_PSEUDO_CONSTANT, child) # This is a normal situation - MultiLevelLinearBeforeChildDispatcher._record_var(visitor, child) + ParameterizedLinearBeforeChildDispatcher._record_var(visitor, child) ans = visitor.Result() ans.linear[_id] = 1 return False, (ExprType.LINEAR, ans) -_before_child_dispatcher = MultiLevelLinearBeforeChildDispatcher() +_before_child_dispatcher = ParameterizedLinearBeforeChildDispatcher() _exit_node_handlers = copy.deepcopy(linear._exit_node_handlers) # From 91f0aaa7fe7e8e14940fddd8a037981b2245d7cc Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Wed, 22 May 2024 12:54:54 -0600 Subject: [PATCH 33/66] Addressing John's comments --- pyomo/repn/parameterized_linear.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index 0633c285155..892edd5643d 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -37,12 +37,12 @@ LinearRepnVisitor, ) from pyomo.repn.util import ExprType -from . import linear +import pyomo.repn.linear as linear class ParameterizedExprType(enum.IntEnum, metaclass=ExtendedEnumType): __base_enum__ = ExprType - PSEUDO_CONSTANT = 50 + PSEUDO_CONSTANT = 5 _PSEUDO_CONSTANT = ParameterizedExprType.PSEUDO_CONSTANT @@ -123,7 +123,7 @@ def append(self, other): Notes ----- This method assumes that the operator was "+". It is implemented - so that we can directly use a LinearRepn() as a `data` object in + so that we can directly use a ParameterizedLinearRepn() as a `data` object in the expression walker (thereby allowing us to use the default implementation of acceptChildResult [which calls `data.append()`] and avoid the function call for a custom @@ -131,7 +131,7 @@ def append(self, other): """ _type, other = other - if _type in (_CONSTANT, _PSEUDO_CONSTANT): + if _type is _CONSTANT or type is _PSEUDO_CONSTANT: self.constant += other return @@ -235,6 +235,8 @@ def _handle_negation_pseudo_constant(visitor, node, arg): def _handle_product_constant_constant(visitor, node, arg1, arg2): + # [ESJ 5/22/24]: Overriding this handler to exclude the deprecation path for + # 0 * nan. It doesn't need overridden when that deprecation path goes away. return _CONSTANT, arg1[1] * arg2[1] From 5aebb02fce467f10bfc4cacc653e8dbce3894cfa Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Wed, 22 May 2024 13:46:29 -0600 Subject: [PATCH 34/66] Whoops, bad typo --- pyomo/repn/parameterized_linear.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index 892edd5643d..ae0856cfe76 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -131,7 +131,7 @@ def append(self, other): """ _type, other = other - if _type is _CONSTANT or type is _PSEUDO_CONSTANT: + if _type is _CONSTANT or _type is _PSEUDO_CONSTANT: self.constant += other return From 67fbe0bb958c7ea1f47bba71a38e4bf094f511a7 Mon Sep 17 00:00:00 2001 From: jasherma Date: Wed, 29 May 2024 19:18:56 -0400 Subject: [PATCH 35/66] Make PyROS temporarily adjust Pyomo NL writer tol --- pyomo/contrib/pyros/util.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pyomo/contrib/pyros/util.py b/pyomo/contrib/pyros/util.py index 3b0187af7dd..8f86f0cc179 100644 --- a/pyomo/contrib/pyros/util.py +++ b/pyomo/contrib/pyros/util.py @@ -38,6 +38,7 @@ from pyomo.core.expr import value from pyomo.core.expr.numeric_expr import NPV_MaxExpression, NPV_MinExpression from pyomo.repn.standard_repn import generate_standard_repn +from pyomo.repn.plugins import nl_writer as pyomo_nl_writer from pyomo.core.expr.visitor import ( identify_variables, identify_mutable_parameters, @@ -1809,6 +1810,16 @@ def call_solver(model, solver, config, timing_obj, timer_name, err_msg): timing_obj.start_timer(timer_name) tt_timer.tic(msg=None) + # tentative: reduce risk of InfeasibleConstraintException + # occurring due to discrepancies between Pyomo NL writer + # tolerance and (default) subordinate solver (e.g. IPOPT) + # feasibility tolerances. + # e.g., a Var fixed outside bounds beyond the Pyomo NL writer + # tolerance, but still within the default IPOPT feasibility + # tolerance + current_nl_writer_tol = pyomo_nl_writer.TOL + pyomo_nl_writer.TOL = 1e-4 + try: results = solver.solve( model, @@ -1827,6 +1838,8 @@ def call_solver(model, solver, config, timing_obj, timer_name, err_msg): results.solver, TIC_TOC_SOLVE_TIME_ATTR, tt_timer.toc(msg=None, delta=True) ) finally: + pyomo_nl_writer.TOL = current_nl_writer_tol + timing_obj.stop_timer(timer_name) revert_solver_max_time_adjustment( solver, orig_setting, custom_setting_present, config From 7460625becccdcd5adf2f3e8b228d76884d28c01 Mon Sep 17 00:00:00 2001 From: jasherma Date: Wed, 29 May 2024 19:40:20 -0400 Subject: [PATCH 36/66] Add test for adjustment of NL writer tolerance --- pyomo/contrib/pyros/tests/test_grcs.py | 72 +++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/pyros/tests/test_grcs.py b/pyomo/contrib/pyros/tests/test_grcs.py index f7efec4d6e7..5e323ad7a78 100644 --- a/pyomo/contrib/pyros/tests/test_grcs.py +++ b/pyomo/contrib/pyros/tests/test_grcs.py @@ -42,6 +42,7 @@ from pyomo.contrib.pyros.util import get_vars_from_component from pyomo.contrib.pyros.util import identify_objective_functions from pyomo.common.collections import Bunch +from pyomo.repn.plugins import nl_writer as pyomo_nl_writer import time import math from pyomo.contrib.pyros.util import time_code @@ -68,7 +69,7 @@ from pyomo.common.dependencies import numpy as np, numpy_available from pyomo.common.dependencies import scipy as sp, scipy_available from pyomo.environ import maximize as pyo_max -from pyomo.common.errors import ApplicationError +from pyomo.common.errors import ApplicationError, InfeasibleConstraintException from pyomo.opt import ( SolverResults, SolverStatus, @@ -4616,6 +4617,75 @@ def test_discrete_separation_subsolver_error(self): ), ) + def test_pyros_nl_writer_tol(self): + """ + Test PyROS subsolver call routine behavior + with respect to the NL writer tolerance is as + expected. + """ + m = ConcreteModel() + m.q = Param(initialize=1, mutable=True) + m.x1 = Var(initialize=1, bounds=(0, 1)) + m.x2 = Var(initialize=2, bounds=(0, m.q)) + m.obj = Objective(expr=m.x1 + m.x2) + + # fixed just inside the PyROS-specified NL writer tolerance. + m.x1.fix(m.x1.upper + 9.9e-5) + + current_nl_writer_tol = pyomo_nl_writer.TOL + ipopt_solver = SolverFactory("ipopt") + pyros_solver = SolverFactory("pyros") + + pyros_solver.solve( + model=m, + first_stage_variables=[m.x1], + second_stage_variables=[m.x2], + uncertain_params=[m.q], + uncertainty_set=BoxSet([[0, 1]]), + local_solver=ipopt_solver, + global_solver=ipopt_solver, + decision_rule_order=0, + solve_master_globally=False, + bypass_global_separation=True, + ) + + self.assertEqual( + pyomo_nl_writer.TOL, + current_nl_writer_tol, + msg="Pyomo NL writer tolerance not restored as expected.", + ) + + # fixed just outside the PyROS-specified NL writer tolerance. + # this should be exceptional. + m.x1.fix(m.x1.upper + 1.01e-4) + + err_msg = ( + "model contains a trivially infeasible variable.*x1" + ".*fixed.*outside bounds" + ) + with self.assertRaisesRegex(InfeasibleConstraintException, err_msg): + pyros_solver.solve( + model=m, + first_stage_variables=[m.x1], + second_stage_variables=[m.x2], + uncertain_params=[m.q], + uncertainty_set=BoxSet([[0, 1]]), + local_solver=ipopt_solver, + global_solver=ipopt_solver, + decision_rule_order=0, + solve_master_globally=False, + bypass_global_separation=True, + ) + + self.assertEqual( + pyomo_nl_writer.TOL, + current_nl_writer_tol, + msg=( + "Pyomo NL writer tolerance not restored as expected " + "after exceptional test." + ), + ) + @unittest.skipUnless( baron_license_is_valid, "Global NLP solver is not available and licensed." ) From 4f2cda6bf374a21d8bb16098140edfcd41154112 Mon Sep 17 00:00:00 2001 From: jasherma Date: Wed, 29 May 2024 20:07:08 -0400 Subject: [PATCH 37/66] Fix IPOPT subsolver wall time limit restoration --- pyomo/contrib/pyros/util.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/pyros/util.py b/pyomo/contrib/pyros/util.py index 8f86f0cc179..ecabca8f115 100644 --- a/pyomo/contrib/pyros/util.py +++ b/pyomo/contrib/pyros/util.py @@ -378,7 +378,14 @@ def revert_solver_max_time_adjustment( elif isinstance(solver, SolverFactory.get_class("baron")): options_key = "MaxTime" elif isinstance(solver, SolverFactory.get_class("ipopt")): - options_key = "max_cpu_time" + options_key = ( + # IPOPT 3.14.0+ added support for specifying + # wall time limit explicitly; this is preferred + # over CPU time limit + "max_wall_time" + if solver.version() >= (3, 14, 0, 0) + else "max_cpu_time" + ) elif isinstance(solver, SolverFactory.get_class("scip")): options_key = "limits/time" else: From edeaf1d2c75ae999596072ef130253024f14b3dd Mon Sep 17 00:00:00 2001 From: jasherma Date: Wed, 29 May 2024 22:07:01 -0400 Subject: [PATCH 38/66] Add IPOPT availability check to new test --- pyomo/contrib/pyros/tests/test_grcs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyomo/contrib/pyros/tests/test_grcs.py b/pyomo/contrib/pyros/tests/test_grcs.py index 5e323ad7a78..f2954750a16 100644 --- a/pyomo/contrib/pyros/tests/test_grcs.py +++ b/pyomo/contrib/pyros/tests/test_grcs.py @@ -4617,6 +4617,7 @@ def test_discrete_separation_subsolver_error(self): ), ) + @unittest.skipUnless(ipopt_available, "IPOPT is not available.") def test_pyros_nl_writer_tol(self): """ Test PyROS subsolver call routine behavior From 96355df4552fb1ba8b96231b8320c2876707801f Mon Sep 17 00:00:00 2001 From: Atalay Kutlay Date: Sat, 1 Jun 2024 23:50:59 -0400 Subject: [PATCH 39/66] bug: Sort indices before sending to solver --- pyomo/contrib/appsi/solvers/highs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/appsi/solvers/highs.py b/pyomo/contrib/appsi/solvers/highs.py index c948444839d..57a7b1eac72 100644 --- a/pyomo/contrib/appsi/solvers/highs.py +++ b/pyomo/contrib/appsi/solvers/highs.py @@ -481,7 +481,7 @@ def _remove_constraints(self, cons: List[ConstraintData]): indices_to_remove.append(con_ndx) self._mutable_helpers.pop(con, None) self._solver_model.deleteRows( - len(indices_to_remove), np.array(indices_to_remove) + len(indices_to_remove), np.sort(np.array(indices_to_remove)) ) con_ndx = 0 new_con_map = dict() From 2bfabd5523ec53539da2a91cd86fc2e97cefed01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20Magalh=C3=A3es?= Date: Thu, 13 Jun 2024 12:35:33 +0200 Subject: [PATCH 40/66] Fixed bug. Added tests to ensure the functionality is kept in the future. --- examples/pyomo/tutorials/set.dat | 3 + examples/pyomo/tutorials/set.out | 8 +- examples/pyomo/tutorials/set.py | 7 + pyomo/core/base/set.py | 5 +- pyomo/core/tests/unit/test_set.py | 247 ++++++++++++++++++++++++++++++ 5 files changed, 266 insertions(+), 4 deletions(-) diff --git a/examples/pyomo/tutorials/set.dat b/examples/pyomo/tutorials/set.dat index ab0d00b43cc..d136397f54e 100644 --- a/examples/pyomo/tutorials/set.dat +++ b/examples/pyomo/tutorials/set.dat @@ -14,5 +14,8 @@ set M := 1 3; set S[2] := 1 3; set S[5] := 2 3; +set X[2] := 1; +set X[5] := 2 3; + set T[2] := 1 3; set T[5] := 2 3; diff --git a/examples/pyomo/tutorials/set.out b/examples/pyomo/tutorials/set.out index 818977f6155..dd1ef2d4335 100644 --- a/examples/pyomo/tutorials/set.out +++ b/examples/pyomo/tutorials/set.out @@ -1,4 +1,4 @@ -23 Set Declarations +24 Set Declarations A : Size=1, Index=None, Ordered=Insertion Key : Dimen : Domain : Size : Members None : 1 : Any : 3 : {1, 2, 3} @@ -89,5 +89,9 @@ 2 : 1 : Any : 5 : {1, 3, 5, 7, 9} 3 : 1 : Any : 5 : {1, 4, 7, 10, 13} 4 : 1 : Any : 5 : {1, 5, 9, 13, 17} + X : Size=2, Index=B, Ordered=Insertion + Key : Dimen : Domain : Size : Members + 2 : 1 : S[2] : 1 : {1,} + 5 : 1 : S[5] : 2 : {2, 3} -23 Declarations: A B C D E F G H Hsub I J K K_2 L M N O P R S T U V +24 Declarations: A B C D E F G H Hsub I J K K_2 L M N O P R S X T U V diff --git a/examples/pyomo/tutorials/set.py b/examples/pyomo/tutorials/set.py index a14301484c9..c1ea60b48ad 100644 --- a/examples/pyomo/tutorials/set.py +++ b/examples/pyomo/tutorials/set.py @@ -171,6 +171,13 @@ def P_init(model, i, j): # model.S = Set(model.B, within=model.A) +# +# Validation of a set array can also be linked to another set array. If so, the +# elements under each index must also be found under the corresponding index in +# the validation set array: +# +model.X = Set(model.B, within=model.S) + # # Validation of set arrays can also be performed with the _validate_ option. diff --git a/pyomo/core/base/set.py b/pyomo/core/base/set.py index 8b7c2a246d6..3f50d032cad 100644 --- a/pyomo/core/base/set.py +++ b/pyomo/core/base/set.py @@ -1932,7 +1932,8 @@ class Set(IndexedComponent): within : initialiser(set), optional A set that defines the valid values that can be contained - in this set + in this set. If the latter is indexed, the former can be indexed or + non-indexed, in which case it applies to all indices. domain : initializer(set), optional A set that defines the valid values that can be contained in this set @@ -2217,7 +2218,7 @@ def _getitem_when_not_present(self, index): _d = None domain = self._init_domain(_block, index, self) - if domain is not None: + if domain is not None and hasattr(domain, "construct"): domain.construct() if _d is UnknownSetDimen and domain is not None and domain.dimen is not None: _d = domain.dimen diff --git a/pyomo/core/tests/unit/test_set.py b/pyomo/core/tests/unit/test_set.py index f62589a6873..90e629958d8 100644 --- a/pyomo/core/tests/unit/test_set.py +++ b/pyomo/core/tests/unit/test_set.py @@ -4543,9 +4543,11 @@ def test_construction(self): m.I = Set(initialize=[1, 2, 3]) m.J = Set(initialize=[4, 5, 6]) m.K = Set(initialize=[(1, 4), (2, 6), (3, 5)], within=m.I * m.J) + m.L = Set(initialize=[1, 3], within=m.I) m.II = Set([1, 2, 3], initialize={1: [0], 2: [1, 2], 3: range(3)}) m.JJ = Set([1, 2, 3], initialize={1: [0], 2: [1, 2], 3: range(3)}) m.KK = Set([1, 2], initialize=[], dimen=lambda m, i: i) + m.LL = Set([2, 3], within=m.II, initialize={2: [1, 2], 3: [1]}) output = StringIO() m.I.pprint(ostream=output) @@ -4569,6 +4571,8 @@ def test_construction(self): 'I': [-1, 0], 'II': {1: [10, 11], 3: [30]}, 'K': [-1, 4, -1, 6, 0, 5], + 'L': [-1], + 'LL': {3: [30]}, } } ) @@ -4576,6 +4580,7 @@ def test_construction(self): self.assertEqual(list(i.I), [-1, 0]) self.assertEqual(list(i.J), [4, 5, 6]) self.assertEqual(list(i.K), [(-1, 4), (-1, 6), (0, 5)]) + self.assertEqual(list(i.L), [-1]) self.assertEqual(list(i.II[1]), [10, 11]) self.assertEqual(list(i.II[3]), [30]) self.assertEqual(list(i.JJ[1]), [0]) @@ -4583,9 +4588,11 @@ def test_construction(self): self.assertEqual(list(i.JJ[3]), [0, 1, 2]) self.assertEqual(list(i.KK[1]), []) self.assertEqual(list(i.KK[2]), []) + self.assertEqual(list(i.LL[3]), [30]) # Implicitly-constructed set should fall back on initialize! self.assertEqual(list(i.II[2]), [1, 2]) + self.assertEqual(list(i.LL[2]), [1, 2]) # Additional tests for tuplize: i = m.create_instance(data={None: {'K': [(1, 4), (2, 6)], 'KK': [1, 4, 2, 6]}}) @@ -6388,3 +6395,243 @@ def test_issue_1112(self): self.assertEqual(len(vals), 1) self.assertIsInstance(vals[0], SetProduct_OrderedSet) self.assertIsNot(vals[0], cross) + + def test_issue_3284(self): + # test creating (indexed and non-indexed) sets using the within argument + # using concrete model and initialization + problem = ConcreteModel() + # non-indexed sets not using the within argument + problem.A = Set(initialize=[1,2,3]) + problem.B = Set(dimen=2, initialize=[(1,2),(3,4),(5,6)]) + # non-indexed sets using within argument + problem.subset_A = Set(within=problem.A, initialize=[2,3]) + problem.subset_B = Set(within=problem.B, dimen=2, initialize=[(1,2),(5,6)]) + # indexed sets not using the within argument + problem.C = Set(problem.A, initialize={1:[-1,3], 2:[4,7], 3:[3, 8]}) + problem.D = Set(problem.B, initialize={(1,2): [1,5], (3,4): [3], (5,6): [6,8,9]}) + # indexed sets using an indexed set for the within argument + problem.subset_C = Set(problem.A, within=problem.C, initialize={1:[-1], 2:[4], 3:[3, 8]}) + problem.subset_D = Set(problem.B, within=problem.D, initialize={(1,2): [1,5], (3,4): [], (5,6): [6]}) + # indexed sets using a non-indexed set for the within argument + problem.E = Set([0, 1], within=problem.A, initialize={0:[1, 2], 1:[3]}) + problem.F = Set([(1,2,3), (4,5,6)], within=problem.B, initialize={(1,2,3):[(1,2)], (4,5,6):[(3,4)]}) + # check them + self.assertEqual(list(problem.A), [1,2,3]) + self.assertEqual(list(problem.B), [(1,2),(3,4),(5,6)]) + self.assertEqual(list(problem.subset_A), [2, 3]) + self.assertEqual(list(problem.subset_B), [(1,2),(5,6)]) + self.assertEqual(list(problem.C[1]), [-1, 3]) + self.assertEqual(list(problem.C[2]), [4, 7]) + self.assertEqual(list(problem.C[3]), [3, 8]) + self.assertEqual(list(problem.D[(1,2)]), [1,5]) + self.assertEqual(list(problem.D[(3,4)]), [3]) + self.assertEqual(list(problem.D[(5,6)]), [6,8,9]) + self.assertEqual(list(problem.subset_C[1]), [-1]) + self.assertEqual(list(problem.subset_C[2]), [4]) + self.assertEqual(list(problem.subset_C[3]), [3, 8]) + self.assertEqual(list(problem.subset_D[(1,2)]), [1,5]) + self.assertEqual(list(problem.subset_D[(3,4)]), []) + self.assertEqual(list(problem.subset_D[(5,6)]), [6]) + self.assertEqual(list(problem.E[0]), [1,2]) + self.assertEqual(list(problem.E[1]), [3]) + self.assertEqual(list(problem.F[(1,2,3)]), [(1,2)]) + self.assertEqual(list(problem.F[(4,5,6)]), [(3,4)]) + + # try adding elements to test the domains (1 compatible, 1 incompatible) + # set subset_A + problem.subset_A.add(1) + error_raised = False + try: + problem.subset_A.add(4) + except ValueError: + error_raised = True + self.assertEqual(error_raised, True) + # set subset_B + problem.subset_B.add((3,4)) + error_raised = False + try: + problem.subset_B.add((7,8)) + except ValueError: + error_raised = True + self.assertEqual(error_raised, True) + # set subset_C + problem.subset_C[2].add(7) + error_raised = False + try: + problem.subset_C[2].add(8) + except ValueError: + error_raised = True + self.assertEqual(error_raised, True) + # set subset_D + problem.subset_D[(3,4)].add(3) + error_raised = False + try: + problem.subset_D[(3,4)].add(4) + except ValueError: + error_raised = True + self.assertEqual(error_raised, True) + # set E + problem.E[1].add(2) + error_raised = False + try: + problem.E[1].add(4) + except ValueError: + error_raised = True + self.assertEqual(error_raised, True) + # set F + problem.F[(1,2,3)].add((3,4)) + error_raised = False + try: + problem.F[(4,5,6)].add((4,3)) + except ValueError: + error_raised = True + self.assertEqual(error_raised, True) + # check them + self.assertEqual(list(problem.A), [1,2,3]) + self.assertEqual(list(problem.B), [(1,2),(3,4),(5,6)]) + self.assertEqual(list(problem.subset_A), [2, 3, 1]) + self.assertEqual(list(problem.subset_B), [(1,2),(5,6),(3,4)]) + self.assertEqual(list(problem.C[1]), [-1, 3]) + self.assertEqual(list(problem.C[2]), [4, 7]) + self.assertEqual(list(problem.C[3]), [3, 8]) + self.assertEqual(list(problem.D[(1,2)]), [1,5]) + self.assertEqual(list(problem.D[(3,4)]), [3]) + self.assertEqual(list(problem.D[(5,6)]), [6,8,9]) + self.assertEqual(list(problem.subset_C[1]), [-1]) + self.assertEqual(list(problem.subset_C[2]), [4, 7]) + self.assertEqual(list(problem.subset_C[3]), [3, 8]) + self.assertEqual(list(problem.subset_D[(1,2)]), [1,5]) + self.assertEqual(list(problem.subset_D[(3,4)]), [3]) + self.assertEqual(list(problem.subset_D[(5,6)]), [6]) + self.assertEqual(list(problem.E[0]), [1,2]) + self.assertEqual(list(problem.E[1]), [3,2]) + self.assertEqual(list(problem.F[(1,2,3)]), [(1,2),(3,4)]) + self.assertEqual(list(problem.F[(4,5,6)]), [(3,4)]) + + + # using abstract model and no initialization + model = AbstractModel() + # non-indexed sets not using the within argument + model.A = Set() + model.B = Set(dimen=2) + # non-indexed sets using within argument + model.subset_A = Set(within=model.A) + model.subset_B = Set(within=model.B, dimen=2) + # indexed sets not using the within argument + model.C = Set(model.A) + model.D = Set(model.B) + # indexed sets using an indexed set for the within argument + model.subset_C = Set(model.A, within=model.C) + model.subset_D = Set(model.B, within=model.D) + # indexed sets using a non-indexed set for the within argument + model.E_index = Set() + model.F_index = Set() + model.E = Set(model.E_index, within=model.A) + model.F = Set(model.F_index, within=model.B) + problem = model.create_instance( + data={ + None: { + 'A': [3, 4, 5], + 'B': [(1,2),(7,8)], + 'subset_A': [3, 4], + 'subset_B': [(1,2)], + 'C': {3: [3], 4: [4,8], 5: [5,6]}, + 'D': {(1,2): [2], (7,8): [0, 1]}, + 'subset_C': {3: [3], 4: [8], 5: []}, + 'subset_D': {(1,2): [], (7,8): [0, 1]}, + 'E_index': [0, 1], + 'F_index': [(1,2,3), (4,5,6)], + 'E': {0:[3, 4], 1:[5]}, + 'F': {(1,2,3):[(1,2)], (4,5,6):[(7,8)]}, + } + } + ) + + # check them + self.assertEqual(list(problem.A), [3, 4, 5]) + self.assertEqual(list(problem.B), [(1,2),(7,8)]) + self.assertEqual(list(problem.subset_A), [3, 4]) + self.assertEqual(list(problem.subset_B), [(1,2)]) + self.assertEqual(list(problem.C[3]), [3]) + self.assertEqual(list(problem.C[4]), [4, 8]) + self.assertEqual(list(problem.C[5]), [5, 6]) + self.assertEqual(list(problem.D[(1,2)]), [2]) + self.assertEqual(list(problem.D[(7,8)]), [0, 1]) + self.assertEqual(list(problem.subset_C[3]), [3]) + self.assertEqual(list(problem.subset_C[4]), [8]) + self.assertEqual(list(problem.subset_C[5]), []) + self.assertEqual(list(problem.subset_D[(1,2)]), []) + self.assertEqual(list(problem.subset_D[(7,8)]), [0, 1]) + self.assertEqual(list(problem.E[0]), [3,4]) + self.assertEqual(list(problem.E[1]), [5]) + self.assertEqual(list(problem.F[(1,2,3)]), [(1,2)]) + self.assertEqual(list(problem.F[(4,5,6)]), [(7,8)]) + + # try adding elements to test the domains (1 compatible, 1 incompatible) + # set subset_A + problem.subset_A.add(5) + error_raised = False + try: + problem.subset_A.add(6) + except ValueError: + error_raised = True + self.assertEqual(error_raised, True) + # set subset_B + problem.subset_B.add((7,8)) + error_raised = False + try: + problem.subset_B.add((3,4)) + except ValueError: + error_raised = True + self.assertEqual(error_raised, True) + # set subset_C + problem.subset_C[4].add(4) + error_raised = False + try: + problem.subset_C[4].add(9) + except ValueError: + error_raised = True + self.assertEqual(error_raised, True) + # set subset_D + problem.subset_D[(1,2)].add(2) + error_raised = False + try: + problem.subset_D[(1,2)].add(3) + except ValueError: + error_raised = True + self.assertEqual(error_raised, True) + # set E + problem.E[1].add(4) + error_raised = False + try: + problem.E[1].add(1) + except ValueError: + error_raised = True + self.assertEqual(error_raised, True) + # set F + problem.F[(1,2,3)].add((7,8)) + error_raised = False + try: + problem.F[(4,5,6)].add((4,3)) + except ValueError: + error_raised = True + self.assertEqual(error_raised, True) + # check them + self.assertEqual(list(problem.A), [3, 4, 5]) + self.assertEqual(list(problem.B), [(1,2),(7,8)]) + self.assertEqual(list(problem.subset_A), [3, 4, 5]) + self.assertEqual(list(problem.subset_B), [(1,2),(7,8)]) + self.assertEqual(list(problem.C[3]), [3]) + self.assertEqual(list(problem.C[4]), [4, 8]) + self.assertEqual(list(problem.C[5]), [5, 6]) + self.assertEqual(list(problem.D[(1,2)]), [2]) + self.assertEqual(list(problem.D[(7,8)]), [0, 1]) + self.assertEqual(list(problem.subset_C[3]), [3]) + self.assertEqual(list(problem.subset_C[4]), [8, 4]) + self.assertEqual(list(problem.subset_C[5]), []) + self.assertEqual(list(problem.subset_D[(1,2)]), [2]) + self.assertEqual(list(problem.subset_D[(7,8)]), [0, 1]) + self.assertEqual(list(problem.E[0]), [3,4]) + self.assertEqual(list(problem.E[1]), [5,4]) + self.assertEqual(list(problem.F[(1,2,3)]), [(1,2),(7,8)]) + self.assertEqual(list(problem.F[(4,5,6)]), [(7,8)]) From 2fcfd13c2c0c73b09c0bc5ef3b096ca4d92f6f08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20Magalh=C3=A3es?= Date: Sat, 15 Jun 2024 20:26:59 +0200 Subject: [PATCH 41/66] Blackened. --- pyomo/core/base/set.py | 2 +- pyomo/core/tests/unit/test_set.py | 171 ++++++++++++++++-------------- 2 files changed, 92 insertions(+), 81 deletions(-) diff --git a/pyomo/core/base/set.py b/pyomo/core/base/set.py index 3f50d032cad..666332124f1 100644 --- a/pyomo/core/base/set.py +++ b/pyomo/core/base/set.py @@ -1932,7 +1932,7 @@ class Set(IndexedComponent): within : initialiser(set), optional A set that defines the valid values that can be contained - in this set. If the latter is indexed, the former can be indexed or + in this set. If the latter is indexed, the former can be indexed or non-indexed, in which case it applies to all indices. domain : initializer(set), optional A set that defines the valid values that can be contained diff --git a/pyomo/core/tests/unit/test_set.py b/pyomo/core/tests/unit/test_set.py index 90e629958d8..a1c502a86d0 100644 --- a/pyomo/core/tests/unit/test_set.py +++ b/pyomo/core/tests/unit/test_set.py @@ -6395,48 +6395,60 @@ def test_issue_1112(self): self.assertEqual(len(vals), 1) self.assertIsInstance(vals[0], SetProduct_OrderedSet) self.assertIsNot(vals[0], cross) - + def test_issue_3284(self): # test creating (indexed and non-indexed) sets using the within argument # using concrete model and initialization problem = ConcreteModel() # non-indexed sets not using the within argument - problem.A = Set(initialize=[1,2,3]) - problem.B = Set(dimen=2, initialize=[(1,2),(3,4),(5,6)]) + problem.A = Set(initialize=[1, 2, 3]) + problem.B = Set(dimen=2, initialize=[(1, 2), (3, 4), (5, 6)]) # non-indexed sets using within argument - problem.subset_A = Set(within=problem.A, initialize=[2,3]) - problem.subset_B = Set(within=problem.B, dimen=2, initialize=[(1,2),(5,6)]) + problem.subset_A = Set(within=problem.A, initialize=[2, 3]) + problem.subset_B = Set(within=problem.B, dimen=2, initialize=[(1, 2), (5, 6)]) # indexed sets not using the within argument - problem.C = Set(problem.A, initialize={1:[-1,3], 2:[4,7], 3:[3, 8]}) - problem.D = Set(problem.B, initialize={(1,2): [1,5], (3,4): [3], (5,6): [6,8,9]}) + problem.C = Set(problem.A, initialize={1: [-1, 3], 2: [4, 7], 3: [3, 8]}) + problem.D = Set( + problem.B, initialize={(1, 2): [1, 5], (3, 4): [3], (5, 6): [6, 8, 9]} + ) # indexed sets using an indexed set for the within argument - problem.subset_C = Set(problem.A, within=problem.C, initialize={1:[-1], 2:[4], 3:[3, 8]}) - problem.subset_D = Set(problem.B, within=problem.D, initialize={(1,2): [1,5], (3,4): [], (5,6): [6]}) + problem.subset_C = Set( + problem.A, within=problem.C, initialize={1: [-1], 2: [4], 3: [3, 8]} + ) + problem.subset_D = Set( + problem.B, + within=problem.D, + initialize={(1, 2): [1, 5], (3, 4): [], (5, 6): [6]}, + ) # indexed sets using a non-indexed set for the within argument - problem.E = Set([0, 1], within=problem.A, initialize={0:[1, 2], 1:[3]}) - problem.F = Set([(1,2,3), (4,5,6)], within=problem.B, initialize={(1,2,3):[(1,2)], (4,5,6):[(3,4)]}) + problem.E = Set([0, 1], within=problem.A, initialize={0: [1, 2], 1: [3]}) + problem.F = Set( + [(1, 2, 3), (4, 5, 6)], + within=problem.B, + initialize={(1, 2, 3): [(1, 2)], (4, 5, 6): [(3, 4)]}, + ) # check them - self.assertEqual(list(problem.A), [1,2,3]) - self.assertEqual(list(problem.B), [(1,2),(3,4),(5,6)]) + self.assertEqual(list(problem.A), [1, 2, 3]) + self.assertEqual(list(problem.B), [(1, 2), (3, 4), (5, 6)]) self.assertEqual(list(problem.subset_A), [2, 3]) - self.assertEqual(list(problem.subset_B), [(1,2),(5,6)]) + self.assertEqual(list(problem.subset_B), [(1, 2), (5, 6)]) self.assertEqual(list(problem.C[1]), [-1, 3]) self.assertEqual(list(problem.C[2]), [4, 7]) self.assertEqual(list(problem.C[3]), [3, 8]) - self.assertEqual(list(problem.D[(1,2)]), [1,5]) - self.assertEqual(list(problem.D[(3,4)]), [3]) - self.assertEqual(list(problem.D[(5,6)]), [6,8,9]) + self.assertEqual(list(problem.D[(1, 2)]), [1, 5]) + self.assertEqual(list(problem.D[(3, 4)]), [3]) + self.assertEqual(list(problem.D[(5, 6)]), [6, 8, 9]) self.assertEqual(list(problem.subset_C[1]), [-1]) self.assertEqual(list(problem.subset_C[2]), [4]) self.assertEqual(list(problem.subset_C[3]), [3, 8]) - self.assertEqual(list(problem.subset_D[(1,2)]), [1,5]) - self.assertEqual(list(problem.subset_D[(3,4)]), []) - self.assertEqual(list(problem.subset_D[(5,6)]), [6]) - self.assertEqual(list(problem.E[0]), [1,2]) + self.assertEqual(list(problem.subset_D[(1, 2)]), [1, 5]) + self.assertEqual(list(problem.subset_D[(3, 4)]), []) + self.assertEqual(list(problem.subset_D[(5, 6)]), [6]) + self.assertEqual(list(problem.E[0]), [1, 2]) self.assertEqual(list(problem.E[1]), [3]) - self.assertEqual(list(problem.F[(1,2,3)]), [(1,2)]) - self.assertEqual(list(problem.F[(4,5,6)]), [(3,4)]) - + self.assertEqual(list(problem.F[(1, 2, 3)]), [(1, 2)]) + self.assertEqual(list(problem.F[(4, 5, 6)]), [(3, 4)]) + # try adding elements to test the domains (1 compatible, 1 incompatible) # set subset_A problem.subset_A.add(1) @@ -6447,10 +6459,10 @@ def test_issue_3284(self): error_raised = True self.assertEqual(error_raised, True) # set subset_B - problem.subset_B.add((3,4)) + problem.subset_B.add((3, 4)) error_raised = False try: - problem.subset_B.add((7,8)) + problem.subset_B.add((7, 8)) except ValueError: error_raised = True self.assertEqual(error_raised, True) @@ -6463,10 +6475,10 @@ def test_issue_3284(self): error_raised = True self.assertEqual(error_raised, True) # set subset_D - problem.subset_D[(3,4)].add(3) + problem.subset_D[(3, 4)].add(3) error_raised = False try: - problem.subset_D[(3,4)].add(4) + problem.subset_D[(3, 4)].add(4) except ValueError: error_raised = True self.assertEqual(error_raised, True) @@ -6479,36 +6491,35 @@ def test_issue_3284(self): error_raised = True self.assertEqual(error_raised, True) # set F - problem.F[(1,2,3)].add((3,4)) + problem.F[(1, 2, 3)].add((3, 4)) error_raised = False try: - problem.F[(4,5,6)].add((4,3)) + problem.F[(4, 5, 6)].add((4, 3)) except ValueError: error_raised = True self.assertEqual(error_raised, True) # check them - self.assertEqual(list(problem.A), [1,2,3]) - self.assertEqual(list(problem.B), [(1,2),(3,4),(5,6)]) + self.assertEqual(list(problem.A), [1, 2, 3]) + self.assertEqual(list(problem.B), [(1, 2), (3, 4), (5, 6)]) self.assertEqual(list(problem.subset_A), [2, 3, 1]) - self.assertEqual(list(problem.subset_B), [(1,2),(5,6),(3,4)]) + self.assertEqual(list(problem.subset_B), [(1, 2), (5, 6), (3, 4)]) self.assertEqual(list(problem.C[1]), [-1, 3]) self.assertEqual(list(problem.C[2]), [4, 7]) self.assertEqual(list(problem.C[3]), [3, 8]) - self.assertEqual(list(problem.D[(1,2)]), [1,5]) - self.assertEqual(list(problem.D[(3,4)]), [3]) - self.assertEqual(list(problem.D[(5,6)]), [6,8,9]) + self.assertEqual(list(problem.D[(1, 2)]), [1, 5]) + self.assertEqual(list(problem.D[(3, 4)]), [3]) + self.assertEqual(list(problem.D[(5, 6)]), [6, 8, 9]) self.assertEqual(list(problem.subset_C[1]), [-1]) self.assertEqual(list(problem.subset_C[2]), [4, 7]) self.assertEqual(list(problem.subset_C[3]), [3, 8]) - self.assertEqual(list(problem.subset_D[(1,2)]), [1,5]) - self.assertEqual(list(problem.subset_D[(3,4)]), [3]) - self.assertEqual(list(problem.subset_D[(5,6)]), [6]) - self.assertEqual(list(problem.E[0]), [1,2]) - self.assertEqual(list(problem.E[1]), [3,2]) - self.assertEqual(list(problem.F[(1,2,3)]), [(1,2),(3,4)]) - self.assertEqual(list(problem.F[(4,5,6)]), [(3,4)]) - - + self.assertEqual(list(problem.subset_D[(1, 2)]), [1, 5]) + self.assertEqual(list(problem.subset_D[(3, 4)]), [3]) + self.assertEqual(list(problem.subset_D[(5, 6)]), [6]) + self.assertEqual(list(problem.E[0]), [1, 2]) + self.assertEqual(list(problem.E[1]), [3, 2]) + self.assertEqual(list(problem.F[(1, 2, 3)]), [(1, 2), (3, 4)]) + self.assertEqual(list(problem.F[(4, 5, 6)]), [(3, 4)]) + # using abstract model and no initialization model = AbstractModel() # non-indexed sets not using the within argument @@ -6532,41 +6543,41 @@ def test_issue_3284(self): data={ None: { 'A': [3, 4, 5], - 'B': [(1,2),(7,8)], + 'B': [(1, 2), (7, 8)], 'subset_A': [3, 4], - 'subset_B': [(1,2)], - 'C': {3: [3], 4: [4,8], 5: [5,6]}, - 'D': {(1,2): [2], (7,8): [0, 1]}, + 'subset_B': [(1, 2)], + 'C': {3: [3], 4: [4, 8], 5: [5, 6]}, + 'D': {(1, 2): [2], (7, 8): [0, 1]}, 'subset_C': {3: [3], 4: [8], 5: []}, - 'subset_D': {(1,2): [], (7,8): [0, 1]}, + 'subset_D': {(1, 2): [], (7, 8): [0, 1]}, 'E_index': [0, 1], - 'F_index': [(1,2,3), (4,5,6)], - 'E': {0:[3, 4], 1:[5]}, - 'F': {(1,2,3):[(1,2)], (4,5,6):[(7,8)]}, + 'F_index': [(1, 2, 3), (4, 5, 6)], + 'E': {0: [3, 4], 1: [5]}, + 'F': {(1, 2, 3): [(1, 2)], (4, 5, 6): [(7, 8)]}, } } ) - + # check them self.assertEqual(list(problem.A), [3, 4, 5]) - self.assertEqual(list(problem.B), [(1,2),(7,8)]) + self.assertEqual(list(problem.B), [(1, 2), (7, 8)]) self.assertEqual(list(problem.subset_A), [3, 4]) - self.assertEqual(list(problem.subset_B), [(1,2)]) + self.assertEqual(list(problem.subset_B), [(1, 2)]) self.assertEqual(list(problem.C[3]), [3]) self.assertEqual(list(problem.C[4]), [4, 8]) self.assertEqual(list(problem.C[5]), [5, 6]) - self.assertEqual(list(problem.D[(1,2)]), [2]) - self.assertEqual(list(problem.D[(7,8)]), [0, 1]) + self.assertEqual(list(problem.D[(1, 2)]), [2]) + self.assertEqual(list(problem.D[(7, 8)]), [0, 1]) self.assertEqual(list(problem.subset_C[3]), [3]) self.assertEqual(list(problem.subset_C[4]), [8]) self.assertEqual(list(problem.subset_C[5]), []) - self.assertEqual(list(problem.subset_D[(1,2)]), []) - self.assertEqual(list(problem.subset_D[(7,8)]), [0, 1]) - self.assertEqual(list(problem.E[0]), [3,4]) + self.assertEqual(list(problem.subset_D[(1, 2)]), []) + self.assertEqual(list(problem.subset_D[(7, 8)]), [0, 1]) + self.assertEqual(list(problem.E[0]), [3, 4]) self.assertEqual(list(problem.E[1]), [5]) - self.assertEqual(list(problem.F[(1,2,3)]), [(1,2)]) - self.assertEqual(list(problem.F[(4,5,6)]), [(7,8)]) - + self.assertEqual(list(problem.F[(1, 2, 3)]), [(1, 2)]) + self.assertEqual(list(problem.F[(4, 5, 6)]), [(7, 8)]) + # try adding elements to test the domains (1 compatible, 1 incompatible) # set subset_A problem.subset_A.add(5) @@ -6577,10 +6588,10 @@ def test_issue_3284(self): error_raised = True self.assertEqual(error_raised, True) # set subset_B - problem.subset_B.add((7,8)) + problem.subset_B.add((7, 8)) error_raised = False try: - problem.subset_B.add((3,4)) + problem.subset_B.add((3, 4)) except ValueError: error_raised = True self.assertEqual(error_raised, True) @@ -6593,10 +6604,10 @@ def test_issue_3284(self): error_raised = True self.assertEqual(error_raised, True) # set subset_D - problem.subset_D[(1,2)].add(2) + problem.subset_D[(1, 2)].add(2) error_raised = False try: - problem.subset_D[(1,2)].add(3) + problem.subset_D[(1, 2)].add(3) except ValueError: error_raised = True self.assertEqual(error_raised, True) @@ -6609,29 +6620,29 @@ def test_issue_3284(self): error_raised = True self.assertEqual(error_raised, True) # set F - problem.F[(1,2,3)].add((7,8)) + problem.F[(1, 2, 3)].add((7, 8)) error_raised = False try: - problem.F[(4,5,6)].add((4,3)) + problem.F[(4, 5, 6)].add((4, 3)) except ValueError: error_raised = True self.assertEqual(error_raised, True) # check them self.assertEqual(list(problem.A), [3, 4, 5]) - self.assertEqual(list(problem.B), [(1,2),(7,8)]) + self.assertEqual(list(problem.B), [(1, 2), (7, 8)]) self.assertEqual(list(problem.subset_A), [3, 4, 5]) - self.assertEqual(list(problem.subset_B), [(1,2),(7,8)]) + self.assertEqual(list(problem.subset_B), [(1, 2), (7, 8)]) self.assertEqual(list(problem.C[3]), [3]) self.assertEqual(list(problem.C[4]), [4, 8]) self.assertEqual(list(problem.C[5]), [5, 6]) - self.assertEqual(list(problem.D[(1,2)]), [2]) - self.assertEqual(list(problem.D[(7,8)]), [0, 1]) + self.assertEqual(list(problem.D[(1, 2)]), [2]) + self.assertEqual(list(problem.D[(7, 8)]), [0, 1]) self.assertEqual(list(problem.subset_C[3]), [3]) self.assertEqual(list(problem.subset_C[4]), [8, 4]) self.assertEqual(list(problem.subset_C[5]), []) - self.assertEqual(list(problem.subset_D[(1,2)]), [2]) - self.assertEqual(list(problem.subset_D[(7,8)]), [0, 1]) - self.assertEqual(list(problem.E[0]), [3,4]) - self.assertEqual(list(problem.E[1]), [5,4]) - self.assertEqual(list(problem.F[(1,2,3)]), [(1,2),(7,8)]) - self.assertEqual(list(problem.F[(4,5,6)]), [(7,8)]) + self.assertEqual(list(problem.subset_D[(1, 2)]), [2]) + self.assertEqual(list(problem.subset_D[(7, 8)]), [0, 1]) + self.assertEqual(list(problem.E[0]), [3, 4]) + self.assertEqual(list(problem.E[1]), [5, 4]) + self.assertEqual(list(problem.F[(1, 2, 3)]), [(1, 2), (7, 8)]) + self.assertEqual(list(problem.F[(4, 5, 6)]), [(7, 8)]) From 9c73e8f8d0a7a6acc6c0041220c709bf6dae96c6 Mon Sep 17 00:00:00 2001 From: Atalay Kutlay Date: Sat, 1 Jun 2024 23:52:21 -0400 Subject: [PATCH 42/66] Add test for variable fix and unfix --- .../solvers/tests/test_highs_persistent.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/pyomo/contrib/appsi/solvers/tests/test_highs_persistent.py b/pyomo/contrib/appsi/solvers/tests/test_highs_persistent.py index b26f45ff2cc..4d8251e0de9 100644 --- a/pyomo/contrib/appsi/solvers/tests/test_highs_persistent.py +++ b/pyomo/contrib/appsi/solvers/tests/test_highs_persistent.py @@ -80,6 +80,43 @@ def test_mutable_params_with_remove_vars(self): res = opt.solve(m) self.assertAlmostEqual(res.best_feasible_objective, -9) + def test_fix_and_unfix(self): + # Tests issue https://github.com/Pyomo/pyomo/issues/3127 + + m = pe.ConcreteModel() + m.x = pe.Var(domain=pe.Binary) + m.y = pe.Var(domain=pe.Binary) + m.fx = pe.Var(domain=pe.NonNegativeReals) + m.fy = pe.Var(domain=pe.NonNegativeReals) + m.c1 = pe.Constraint(expr=m.fx <= m.x) + m.c2 = pe.Constraint(expr=m.fy <= m.y) + m.c3 = pe.Constraint(expr=m.x + m.y <= 1) + + m.obj = pe.Objective(expr=m.fx * 0.5 + m.fy * 0.4, sense=pe.maximize) + + opt = Highs() + + # solution 1 has m.x == 1 and m.y == 0 + r = opt.solve(m) + self.assertAlmostEqual(m.fx.value, 1, places=5) + self.assertAlmostEqual(m.fy.value, 0, places=5) + self.assertAlmostEqual(r.best_feasible_objective, 0.5, places=5) + + # solution 2 has m.x == 0 and m.y == 1 + m.y.fix(1) + r = opt.solve(m) + self.assertAlmostEqual(m.fx.value, 0, places=5) + self.assertAlmostEqual(m.fy.value, 1, places=5) + self.assertAlmostEqual(r.best_feasible_objective, 0.4, places=5) + + # solution 3 should be equal solution 1 + m.y.unfix() + m.x.fix(1) + r = opt.solve(m) + self.assertAlmostEqual(m.fx.value, 1, places=5) + self.assertAlmostEqual(m.fy.value, 0, places=5) + self.assertAlmostEqual(r.best_feasible_objective, 0.5, places=5) + def test_capture_highs_output(self): # tests issue #3003 # From f3b65cd50686886fa6b06dbb33c6c77c8d6a226f Mon Sep 17 00:00:00 2001 From: Alma Walmsley Date: Mon, 24 Jun 2024 18:01:21 +1200 Subject: [PATCH 43/66] Ignore errors on ASL solver version check --- pyomo/solvers/plugins/solvers/ASL.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyomo/solvers/plugins/solvers/ASL.py b/pyomo/solvers/plugins/solvers/ASL.py index bb8174a013e..a29c64e017f 100644 --- a/pyomo/solvers/plugins/solvers/ASL.py +++ b/pyomo/solvers/plugins/solvers/ASL.py @@ -102,6 +102,7 @@ def _get_version(self): stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, + errors='ignore', ) ver = _extract_version(results.stdout) if ver is None: From 498312e587c424fa58081238895966b03b4b255f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20Magalh=C3=A3es?= Date: Sat, 29 Jun 2024 00:57:14 +0200 Subject: [PATCH 44/66] Edits after review. --- examples/pyomo/tutorials/set.dat | 6 +-- pyomo/core/base/set.py | 4 +- pyomo/core/tests/unit/test_set.py | 85 ++++++++----------------------- 3 files changed, 26 insertions(+), 69 deletions(-) diff --git a/examples/pyomo/tutorials/set.dat b/examples/pyomo/tutorials/set.dat index d136397f54e..e2ad04122d8 100644 --- a/examples/pyomo/tutorials/set.dat +++ b/examples/pyomo/tutorials/set.dat @@ -14,8 +14,8 @@ set M := 1 3; set S[2] := 1 3; set S[5] := 2 3; -set X[2] := 1; -set X[5] := 2 3; - set T[2] := 1 3; set T[5] := 2 3; + +set X[2] := 1; +set X[5] := 2 3; \ No newline at end of file diff --git a/pyomo/core/base/set.py b/pyomo/core/base/set.py index 666332124f1..804c02f7991 100644 --- a/pyomo/core/base/set.py +++ b/pyomo/core/base/set.py @@ -2218,8 +2218,8 @@ def _getitem_when_not_present(self, index): _d = None domain = self._init_domain(_block, index, self) - if domain is not None and hasattr(domain, "construct"): - domain.construct() + if domain is not None: + domain.parent_component().construct() if _d is UnknownSetDimen and domain is not None and domain.dimen is not None: _d = domain.dimen diff --git a/pyomo/core/tests/unit/test_set.py b/pyomo/core/tests/unit/test_set.py index a1c502a86d0..012669a3484 100644 --- a/pyomo/core/tests/unit/test_set.py +++ b/pyomo/core/tests/unit/test_set.py @@ -6452,52 +6452,33 @@ def test_issue_3284(self): # try adding elements to test the domains (1 compatible, 1 incompatible) # set subset_A problem.subset_A.add(1) - error_raised = False - try: + error_message = ( + "Cannot add value %s to Set %s.\n" + "\tThe value is not in the domain %s" + % (4, 'subset_A', 'A') + ) + with self.assertRaisesRegex(ValueError, error_message): problem.subset_A.add(4) - except ValueError: - error_raised = True - self.assertEqual(error_raised, True) # set subset_B problem.subset_B.add((3, 4)) - error_raised = False - try: + with self.assertRaisesRegex(ValueError, ".*Cannot add value "): problem.subset_B.add((7, 8)) - except ValueError: - error_raised = True - self.assertEqual(error_raised, True) # set subset_C problem.subset_C[2].add(7) - error_raised = False - try: + with self.assertRaisesRegex(ValueError, ".*Cannot add value 8 to Set"): problem.subset_C[2].add(8) - except ValueError: - error_raised = True - self.assertEqual(error_raised, True) # set subset_D - problem.subset_D[(3, 4)].add(3) - error_raised = False - try: - problem.subset_D[(3, 4)].add(4) - except ValueError: - error_raised = True - self.assertEqual(error_raised, True) + problem.subset_D[(5, 6)].add(9) + with self.assertRaisesRegex(ValueError, ".*Cannot add value 2 to Set"): + problem.subset_D[(3, 4)].add(2) # set E problem.E[1].add(2) - error_raised = False - try: + with self.assertRaisesRegex(ValueError, ".*Cannot add value 4 to Set"): problem.E[1].add(4) - except ValueError: - error_raised = True - self.assertEqual(error_raised, True) # set F problem.F[(1, 2, 3)].add((3, 4)) - error_raised = False - try: + with self.assertRaisesRegex(ValueError, ".*Cannot add value "): problem.F[(4, 5, 6)].add((4, 3)) - except ValueError: - error_raised = True - self.assertEqual(error_raised, True) # check them self.assertEqual(list(problem.A), [1, 2, 3]) self.assertEqual(list(problem.B), [(1, 2), (3, 4), (5, 6)]) @@ -6513,8 +6494,8 @@ def test_issue_3284(self): self.assertEqual(list(problem.subset_C[2]), [4, 7]) self.assertEqual(list(problem.subset_C[3]), [3, 8]) self.assertEqual(list(problem.subset_D[(1, 2)]), [1, 5]) - self.assertEqual(list(problem.subset_D[(3, 4)]), [3]) - self.assertEqual(list(problem.subset_D[(5, 6)]), [6]) + self.assertEqual(list(problem.subset_D[(3, 4)]), []) + self.assertEqual(list(problem.subset_D[(5, 6)]), [6, 9]) self.assertEqual(list(problem.E[0]), [1, 2]) self.assertEqual(list(problem.E[1]), [3, 2]) self.assertEqual(list(problem.F[(1, 2, 3)]), [(1, 2), (3, 4)]) @@ -6581,52 +6562,28 @@ def test_issue_3284(self): # try adding elements to test the domains (1 compatible, 1 incompatible) # set subset_A problem.subset_A.add(5) - error_raised = False - try: + with self.assertRaisesRegex(ValueError, ".*Cannot add value "): problem.subset_A.add(6) - except ValueError: - error_raised = True - self.assertEqual(error_raised, True) # set subset_B problem.subset_B.add((7, 8)) - error_raised = False - try: + with self.assertRaisesRegex(ValueError, ".*Cannot add value "): problem.subset_B.add((3, 4)) - except ValueError: - error_raised = True - self.assertEqual(error_raised, True) # set subset_C problem.subset_C[4].add(4) - error_raised = False - try: + with self.assertRaisesRegex(ValueError, ".*Cannot add value "): problem.subset_C[4].add(9) - except ValueError: - error_raised = True - self.assertEqual(error_raised, True) # set subset_D problem.subset_D[(1, 2)].add(2) - error_raised = False - try: + with self.assertRaisesRegex(ValueError, ".*Cannot add value "): problem.subset_D[(1, 2)].add(3) - except ValueError: - error_raised = True - self.assertEqual(error_raised, True) # set E problem.E[1].add(4) - error_raised = False - try: + with self.assertRaisesRegex(ValueError, ".*Cannot add value "): problem.E[1].add(1) - except ValueError: - error_raised = True - self.assertEqual(error_raised, True) # set F problem.F[(1, 2, 3)].add((7, 8)) - error_raised = False - try: + with self.assertRaisesRegex(ValueError, ".*Cannot add value "): problem.F[(4, 5, 6)].add((4, 3)) - except ValueError: - error_raised = True - self.assertEqual(error_raised, True) # check them self.assertEqual(list(problem.A), [3, 4, 5]) self.assertEqual(list(problem.B), [(1, 2), (7, 8)]) From c787f3707c60160ccac6606a0b8b87f645d277d9 Mon Sep 17 00:00:00 2001 From: "Philipp Christophel (phchri)" Date: Mon, 8 Jul 2024 06:34:40 -0400 Subject: [PATCH 45/66] fix: Issues in CAS interface --- pyomo/solvers/plugins/solvers/SAS.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index f2cfe279fdc..4e1653ad1b3 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -629,6 +629,7 @@ def _uploadMpsFile(self, s, unique): data=mpscsv_table_name, casOut={"name": mpsdata_table_name, "replace": True}, format="FREE", + maxLength=256 ) # Delete the table we don't need anymore @@ -641,6 +642,7 @@ def _uploadMpsFile(self, s, unique): mpsFileString=mps_file.read(), casout={"name": mpsdata_table_name, "replace": True}, format="FREE", + maxLength=256 ) return mpsdata_table_name @@ -716,7 +718,7 @@ def _apply_solver(self): action = "solveMilp" if self._has_integer_variables() else "solveLp" # Get a unique identifier, always use the same with different prefixes - unique = uuid4().hex[:16] + unique = uuid.uuid4().hex[:16] # Creat the output stream, we want to print to a log string as well as to the console self._log = StringIO() From c2b0be1c56dd5383f1ea4af22c1427ca7091fbb4 Mon Sep 17 00:00:00 2001 From: "Philipp Christophel (phchri)" Date: Mon, 8 Jul 2024 09:42:01 -0400 Subject: [PATCH 46/66] fix: adjust for black issues --- pyomo/solvers/plugins/solvers/SAS.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/SAS.py b/pyomo/solvers/plugins/solvers/SAS.py index 4e1653ad1b3..d7b09e29fde 100644 --- a/pyomo/solvers/plugins/solvers/SAS.py +++ b/pyomo/solvers/plugins/solvers/SAS.py @@ -629,7 +629,7 @@ def _uploadMpsFile(self, s, unique): data=mpscsv_table_name, casOut={"name": mpsdata_table_name, "replace": True}, format="FREE", - maxLength=256 + maxLength=256, ) # Delete the table we don't need anymore @@ -642,7 +642,7 @@ def _uploadMpsFile(self, s, unique): mpsFileString=mps_file.read(), casout={"name": mpsdata_table_name, "replace": True}, format="FREE", - maxLength=256 + maxLength=256, ) return mpsdata_table_name From 6c05a17221923a6ef6d9545974e330600c335259 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 8 Jul 2024 14:17:17 -0600 Subject: [PATCH 47/66] Renaming pseudo-constant to fixed and promoting it to the ExprType enum --- pyomo/repn/parameterized_linear.py | 77 ++++++++++++++---------------- pyomo/repn/util.py | 1 + 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index ae0856cfe76..16e9ee52a7e 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -10,10 +10,8 @@ # ___________________________________________________________________________ import copy -import enum from pyomo.common.collections import ComponentSet -from pyomo.common.enums import ExtendedEnumType from pyomo.common.numeric_types import native_numeric_types from pyomo.core import Var from pyomo.core.expr.logical_expr import _flattened @@ -40,15 +38,10 @@ import pyomo.repn.linear as linear -class ParameterizedExprType(enum.IntEnum, metaclass=ExtendedEnumType): - __base_enum__ = ExprType - PSEUDO_CONSTANT = 5 - - -_PSEUDO_CONSTANT = ParameterizedExprType.PSEUDO_CONSTANT -_CONSTANT = ParameterizedExprType.CONSTANT -_LINEAR = ParameterizedExprType.LINEAR -_GENERAL = ParameterizedExprType.GENERAL +_FIXED = ExprType.FIXED +_CONSTANT = ExprType.CONSTANT +_LINEAR = ExprType.LINEAR +_GENERAL = ExprType.GENERAL def _merge_dict(dest_dict, mult, src_dict): @@ -67,7 +60,7 @@ def _merge_dict(dest_dict, mult, src_dict): def to_expression(visitor, arg): - if arg[0] in (_CONSTANT, _PSEUDO_CONSTANT): + if arg[0] in (_CONSTANT, _FIXED): return arg[1] else: return arg[1].to_expression(visitor) @@ -88,7 +81,7 @@ def walker_exitNode(self): elif self.constant.__class__ in native_numeric_types: return _CONSTANT, self.multiplier * self.constant else: - return _PSEUDO_CONSTANT, self.multiplier * self.constant + return _FIXED, self.multiplier * self.constant def to_expression(self, visitor): if self.nonlinear is not None: @@ -131,7 +124,7 @@ def append(self, other): """ _type, other = other - if _type is _CONSTANT or _type is _PSEUDO_CONSTANT: + if _type is _CONSTANT or _type is _FIXED: self.constant += other return @@ -204,7 +197,7 @@ def _before_var(visitor, child): if child in visitor.wrt: # pseudo-constant # We aren't treating this Var as a Var for the purposes of this walker - return False, (_PSEUDO_CONSTANT, child) + return False, (_FIXED, child) # This is a normal situation ParameterizedLinearBeforeChildDispatcher._record_var(visitor, child) ans = visitor.Result() @@ -221,11 +214,11 @@ def _before_var(visitor, child): def _handle_negation_pseudo_constant(visitor, node, arg): - return (_PSEUDO_CONSTANT, -1 * arg[1]) + return (_FIXED, -1 * arg[1]) _exit_node_handlers[NegationExpression].update( - {(_PSEUDO_CONSTANT,): _handle_negation_pseudo_constant} + {(_FIXED,): _handle_negation_pseudo_constant} ) @@ -241,19 +234,19 @@ def _handle_product_constant_constant(visitor, node, arg1, arg2): def _handle_product_pseudo_constant_constant(visitor, node, arg1, arg2): - return _PSEUDO_CONSTANT, arg1[1] * arg2[1] + return _FIXED, arg1[1] * arg2[1] _exit_node_handlers[ProductExpression].update( { (_CONSTANT, _CONSTANT): _handle_product_constant_constant, - (_PSEUDO_CONSTANT, _PSEUDO_CONSTANT): _handle_product_pseudo_constant_constant, - (_PSEUDO_CONSTANT, _CONSTANT): _handle_product_pseudo_constant_constant, - (_CONSTANT, _PSEUDO_CONSTANT): _handle_product_pseudo_constant_constant, - (_PSEUDO_CONSTANT, _LINEAR): linear._handle_product_constant_ANY, - (_LINEAR, _PSEUDO_CONSTANT): linear._handle_product_ANY_constant, - (_PSEUDO_CONSTANT, _GENERAL): linear._handle_product_constant_ANY, - (_GENERAL, _PSEUDO_CONSTANT): linear._handle_product_ANY_constant, + (_FIXED, _FIXED): _handle_product_pseudo_constant_constant, + (_FIXED, _CONSTANT): _handle_product_pseudo_constant_constant, + (_CONSTANT, _FIXED): _handle_product_pseudo_constant_constant, + (_FIXED, _LINEAR): linear._handle_product_constant_ANY, + (_LINEAR, _FIXED): linear._handle_product_ANY_constant, + (_FIXED, _GENERAL): linear._handle_product_constant_ANY, + (_GENERAL, _FIXED): linear._handle_product_ANY_constant, } ) _exit_node_handlers[MonomialTermExpression].update( @@ -266,7 +259,7 @@ def _handle_product_pseudo_constant_constant(visitor, node, arg1, arg2): def _handle_division_pseudo_constant_constant(visitor, node, arg1, arg2): - return _PSEUDO_CONSTANT, arg1[1] / arg2[1] + return _FIXED, arg1[1] / arg2[1] def _handle_division_ANY_pseudo_constant(visitor, node, arg1, arg2): @@ -276,11 +269,11 @@ def _handle_division_ANY_pseudo_constant(visitor, node, arg1, arg2): _exit_node_handlers[DivisionExpression].update( { - (_PSEUDO_CONSTANT, _PSEUDO_CONSTANT): _handle_division_pseudo_constant_constant, - (_PSEUDO_CONSTANT, _CONSTANT): _handle_division_pseudo_constant_constant, - (_CONSTANT, _PSEUDO_CONSTANT): _handle_division_pseudo_constant_constant, - (_LINEAR, _PSEUDO_CONSTANT): _handle_division_ANY_pseudo_constant, - (_GENERAL, _PSEUDO_CONSTANT): _handle_division_ANY_pseudo_constant, + (_FIXED, _FIXED): _handle_division_pseudo_constant_constant, + (_FIXED, _CONSTANT): _handle_division_pseudo_constant_constant, + (_CONSTANT, _FIXED): _handle_division_pseudo_constant_constant, + (_LINEAR, _FIXED): _handle_division_ANY_pseudo_constant, + (_GENERAL, _FIXED): _handle_division_ANY_pseudo_constant, } ) @@ -290,7 +283,7 @@ def _handle_division_ANY_pseudo_constant(visitor, node, arg1, arg2): def _handle_pow_pseudo_constant_constant(visitor, node, arg1, arg2): - return _PSEUDO_CONSTANT, to_expression(visitor, arg1) ** to_expression( + return _FIXED, to_expression(visitor, arg1) ** to_expression( visitor, arg2 ) @@ -305,13 +298,13 @@ def _handle_pow_nonlinear(visitor, node, arg1, arg2): _exit_node_handlers[PowExpression].update( { - (_PSEUDO_CONSTANT, _PSEUDO_CONSTANT): _handle_pow_pseudo_constant_constant, - (_PSEUDO_CONSTANT, _CONSTANT): _handle_pow_pseudo_constant_constant, - (_CONSTANT, _PSEUDO_CONSTANT): _handle_pow_pseudo_constant_constant, - (_LINEAR, _PSEUDO_CONSTANT): _handle_pow_nonlinear, - (_PSEUDO_CONSTANT, _LINEAR): _handle_pow_nonlinear, - (_GENERAL, _PSEUDO_CONSTANT): _handle_pow_nonlinear, - (_PSEUDO_CONSTANT, _GENERAL): _handle_pow_nonlinear, + (_FIXED, _FIXED): _handle_pow_pseudo_constant_constant, + (_FIXED, _CONSTANT): _handle_pow_pseudo_constant_constant, + (_CONSTANT, _FIXED): _handle_pow_pseudo_constant_constant, + (_LINEAR, _FIXED): _handle_pow_nonlinear, + (_FIXED, _LINEAR): _handle_pow_nonlinear, + (_GENERAL, _FIXED): _handle_pow_nonlinear, + (_FIXED, _GENERAL): _handle_pow_nonlinear, } ) @@ -322,13 +315,13 @@ def _handle_pow_nonlinear(visitor, node, arg1, arg2): def _handle_unary_pseudo_constant(visitor, node, arg): # We override this because we can't blindly use apply_node_operation in this case - return _PSEUDO_CONSTANT, node.create_node_with_local_data( + return _FIXED, node.create_node_with_local_data( (to_expression(visitor, arg),) ) _exit_node_handlers[UnaryFunctionExpression].update( - {(_PSEUDO_CONSTANT,): _handle_unary_pseudo_constant} + {(_FIXED,): _handle_unary_pseudo_constant} ) _exit_node_handlers[AbsExpression] = _exit_node_handlers[UnaryFunctionExpression] @@ -397,6 +390,6 @@ def finalizeResult(self, result): return ans ans = self.Result() - assert result[0] in (_CONSTANT, _PSEUDO_CONSTANT) + assert result[0] in (_CONSTANT, _FIXED) ans.constant = result[1] return ans diff --git a/pyomo/repn/util.py b/pyomo/repn/util.py index 8d902d0f99a..c8fc6212d08 100644 --- a/pyomo/repn/util.py +++ b/pyomo/repn/util.py @@ -67,6 +67,7 @@ class ExprType(enum.IntEnum): CONSTANT = 0 + FIXED = 5 MONOMIAL = 10 LINEAR = 20 QUADRATIC = 30 From dd1d3b2d66a9eeeb6ee743e90298e639e9d2dd61 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 8 Jul 2024 14:17:48 -0600 Subject: [PATCH 48/66] Black has an opinion --- pyomo/repn/parameterized_linear.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index 16e9ee52a7e..45ff2943b11 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -283,9 +283,7 @@ def _handle_division_ANY_pseudo_constant(visitor, node, arg1, arg2): def _handle_pow_pseudo_constant_constant(visitor, node, arg1, arg2): - return _FIXED, to_expression(visitor, arg1) ** to_expression( - visitor, arg2 - ) + return _FIXED, to_expression(visitor, arg1) ** to_expression(visitor, arg2) def _handle_pow_nonlinear(visitor, node, arg1, arg2): @@ -315,9 +313,7 @@ def _handle_pow_nonlinear(visitor, node, arg1, arg2): def _handle_unary_pseudo_constant(visitor, node, arg): # We override this because we can't blindly use apply_node_operation in this case - return _FIXED, node.create_node_with_local_data( - (to_expression(visitor, arg),) - ) + return _FIXED, node.create_node_with_local_data((to_expression(visitor, arg),)) _exit_node_handlers[UnaryFunctionExpression].update( From a49f8187cfabe463e8a9f0029460088b320caa8f Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 8 Jul 2024 14:36:05 -0600 Subject: [PATCH 49/66] Move exit node handler registrations into a helper function --- pyomo/repn/linear.py | 122 +++++++++++++++++------------------- pyomo/repn/quadratic.py | 133 +++++++++++++++++++--------------------- pyomo/repn/util.py | 10 ++- 3 files changed, 124 insertions(+), 141 deletions(-) diff --git a/pyomo/repn/linear.py b/pyomo/repn/linear.py index 6d084067511..029fe892b62 100644 --- a/pyomo/repn/linear.py +++ b/pyomo/repn/linear.py @@ -183,8 +183,6 @@ def to_expression(visitor, arg): return arg[1].to_expression(visitor) -_exit_node_handlers = {} - # # NEGATION handlers # @@ -199,11 +197,6 @@ def _handle_negation_ANY(visitor, node, arg): return arg -_exit_node_handlers[NegationExpression] = { - None: _handle_negation_ANY, - (_CONSTANT,): _handle_negation_constant, -} - # # PRODUCT handlers # @@ -272,16 +265,6 @@ def _handle_product_nonlinear(visitor, node, arg1, arg2): return _GENERAL, ans -_exit_node_handlers[ProductExpression] = { - None: _handle_product_nonlinear, - (_CONSTANT, _CONSTANT): _handle_product_constant_constant, - (_CONSTANT, _LINEAR): _handle_product_constant_ANY, - (_CONSTANT, _GENERAL): _handle_product_constant_ANY, - (_LINEAR, _CONSTANT): _handle_product_ANY_constant, - (_GENERAL, _CONSTANT): _handle_product_ANY_constant, -} -_exit_node_handlers[MonomialTermExpression] = _exit_node_handlers[ProductExpression] - # # DIVISION handlers # @@ -302,13 +285,6 @@ def _handle_division_nonlinear(visitor, node, arg1, arg2): return _GENERAL, ans -_exit_node_handlers[DivisionExpression] = { - None: _handle_division_nonlinear, - (_CONSTANT, _CONSTANT): _handle_division_constant_constant, - (_LINEAR, _CONSTANT): _handle_division_ANY_constant, - (_GENERAL, _CONSTANT): _handle_division_ANY_constant, -} - # # EXPONENTIATION handlers # @@ -345,13 +321,6 @@ def _handle_pow_nonlinear(visitor, node, arg1, arg2): return _GENERAL, ans -_exit_node_handlers[PowExpression] = { - None: _handle_pow_nonlinear, - (_CONSTANT, _CONSTANT): _handle_pow_constant_constant, - (_LINEAR, _CONSTANT): _handle_pow_ANY_constant, - (_GENERAL, _CONSTANT): _handle_pow_ANY_constant, -} - # # ABS and UNARY handlers # @@ -371,12 +340,6 @@ def _handle_unary_nonlinear(visitor, node, arg): return _GENERAL, ans -_exit_node_handlers[UnaryFunctionExpression] = { - None: _handle_unary_nonlinear, - (_CONSTANT,): _handle_unary_constant, -} -_exit_node_handlers[AbsExpression] = _exit_node_handlers[UnaryFunctionExpression] - # # NAMED EXPRESSION handlers # @@ -395,11 +358,6 @@ def _handle_named_ANY(visitor, node, arg1): return _type, arg1.duplicate() -_exit_node_handlers[Expression] = { - None: _handle_named_ANY, - (_CONSTANT,): _handle_named_constant, -} - # # EXPR_IF handlers # @@ -430,11 +388,6 @@ def _handle_expr_if_nonlinear(visitor, node, arg1, arg2, arg3): return _GENERAL, ans -_exit_node_handlers[Expr_ifExpression] = {None: _handle_expr_if_nonlinear} -for j in (_CONSTANT, _LINEAR, _GENERAL): - for k in (_CONSTANT, _LINEAR, _GENERAL): - _exit_node_handlers[Expr_ifExpression][_CONSTANT, j, k] = _handle_expr_if_const - # # Relational expression handlers # @@ -462,12 +415,6 @@ def _handle_equality_general(visitor, node, arg1, arg2): return _GENERAL, ans -_exit_node_handlers[EqualityExpression] = { - None: _handle_equality_general, - (_CONSTANT, _CONSTANT): _handle_equality_const, -} - - def _handle_inequality_const(visitor, node, arg1, arg2): # It is exceptionally likely that if we get here, one of the # arguments is an InvalidNumber @@ -490,12 +437,6 @@ def _handle_inequality_general(visitor, node, arg1, arg2): return _GENERAL, ans -_exit_node_handlers[InequalityExpression] = { - None: _handle_inequality_general, - (_CONSTANT, _CONSTANT): _handle_inequality_const, -} - - def _handle_ranged_const(visitor, node, arg1, arg2, arg3): # It is exceptionally likely that if we get here, one of the # arguments is an InvalidNumber @@ -523,10 +464,62 @@ def _handle_ranged_general(visitor, node, arg1, arg2, arg3): return _GENERAL, ans -_exit_node_handlers[RangedExpression] = { - None: _handle_ranged_general, - (_CONSTANT, _CONSTANT, _CONSTANT): _handle_ranged_const, -} +def define_exit_node_handlers(_exit_node_handlers=None): + if _exit_node_handlers is None: + _exit_node_handlers = {} + _exit_node_handlers[NegationExpression] = { + None: _handle_negation_ANY, + (_CONSTANT,): _handle_negation_constant, + } + _exit_node_handlers[ProductExpression] = { + None: _handle_product_nonlinear, + (_CONSTANT, _CONSTANT): _handle_product_constant_constant, + (_CONSTANT, _LINEAR): _handle_product_constant_ANY, + (_CONSTANT, _GENERAL): _handle_product_constant_ANY, + (_LINEAR, _CONSTANT): _handle_product_ANY_constant, + (_GENERAL, _CONSTANT): _handle_product_ANY_constant, + } + _exit_node_handlers[MonomialTermExpression] = _exit_node_handlers[ProductExpression] + _exit_node_handlers[DivisionExpression] = { + None: _handle_division_nonlinear, + (_CONSTANT, _CONSTANT): _handle_division_constant_constant, + (_LINEAR, _CONSTANT): _handle_division_ANY_constant, + (_GENERAL, _CONSTANT): _handle_division_ANY_constant, + } + _exit_node_handlers[PowExpression] = { + None: _handle_pow_nonlinear, + (_CONSTANT, _CONSTANT): _handle_pow_constant_constant, + (_LINEAR, _CONSTANT): _handle_pow_ANY_constant, + (_GENERAL, _CONSTANT): _handle_pow_ANY_constant, + } + _exit_node_handlers[UnaryFunctionExpression] = { + None: _handle_unary_nonlinear, + (_CONSTANT,): _handle_unary_constant, + } + _exit_node_handlers[AbsExpression] = _exit_node_handlers[UnaryFunctionExpression] + _exit_node_handlers[Expression] = { + None: _handle_named_ANY, + (_CONSTANT,): _handle_named_constant, + } + _exit_node_handlers[Expr_ifExpression] = {None: _handle_expr_if_nonlinear} + for j in (_CONSTANT, _LINEAR, _GENERAL): + for k in (_CONSTANT, _LINEAR, _GENERAL): + _exit_node_handlers[Expr_ifExpression][ + _CONSTANT, j, k + ] = _handle_expr_if_const + _exit_node_handlers[EqualityExpression] = { + None: _handle_equality_general, + (_CONSTANT, _CONSTANT): _handle_equality_const, + } + _exit_node_handlers[InequalityExpression] = { + None: _handle_inequality_general, + (_CONSTANT, _CONSTANT): _handle_inequality_const, + } + _exit_node_handlers[RangedExpression] = { + None: _handle_ranged_general, + (_CONSTANT, _CONSTANT, _CONSTANT): _handle_ranged_const, + } + return _exit_node_handlers class LinearBeforeChildDispatcher(BeforeChildDispatcher): @@ -728,9 +721,8 @@ def _initialize_exit_node_dispatcher(exit_handlers): class LinearRepnVisitor(StreamBasedExpressionVisitor): Result = LinearRepn - exit_node_handlers = _exit_node_handlers exit_node_dispatcher = ExitNodeDispatcher( - _initialize_exit_node_dispatcher(_exit_node_handlers) + _initialize_exit_node_dispatcher(define_exit_node_handlers()) ) expand_nonlinear_products = False max_exponential_expansion = 1 diff --git a/pyomo/repn/quadratic.py b/pyomo/repn/quadratic.py index f6e0a43623d..a9c8b7bf2b5 100644 --- a/pyomo/repn/quadratic.py +++ b/pyomo/repn/quadratic.py @@ -157,17 +157,6 @@ def append(self, other): self.nonlinear += nl -_exit_node_handlers = copy.deepcopy(linear._exit_node_handlers) - -# -# NEGATION -# -_exit_node_handlers[NegationExpression][(_QUADRATIC,)] = linear._handle_negation_ANY - - -# -# PRODUCT -# def _mul_linear_linear(varOrder, linear1, linear2): quadratic = {} for vid1, coef1 in linear1.items(): @@ -275,69 +264,73 @@ def _handle_product_nonlinear(visitor, node, arg1, arg2): return _GENERAL, ans -_exit_node_handlers[ProductExpression].update( - { - None: _handle_product_nonlinear, - (_CONSTANT, _QUADRATIC): linear._handle_product_constant_ANY, - (_QUADRATIC, _CONSTANT): linear._handle_product_ANY_constant, - # Replace handler from the linear walker - (_LINEAR, _LINEAR): _handle_product_linear_linear, - } -) - -# -# DIVISION -# -_exit_node_handlers[DivisionExpression].update( - {(_QUADRATIC, _CONSTANT): linear._handle_division_ANY_constant} -) - - -# -# EXPONENTIATION -# -_exit_node_handlers[PowExpression].update( - {(_QUADRATIC, _CONSTANT): linear._handle_pow_ANY_constant} -) - -# -# ABS and UNARY handlers -# -# (no changes needed) - -# -# NAMED EXPRESSION handlers -# -# (no changes needed) - -# -# EXPR_IF handlers -# -# Note: it is easier to just recreate the entire data structure, rather -# than update it -_exit_node_handlers[Expr_ifExpression].update( - { - (_CONSTANT, i, _QUADRATIC): linear._handle_expr_if_const - for i in (_CONSTANT, _LINEAR, _QUADRATIC, _GENERAL) - } -) -_exit_node_handlers[Expr_ifExpression].update( - { - (_CONSTANT, _QUADRATIC, i): linear._handle_expr_if_const - for i in (_CONSTANT, _LINEAR, _GENERAL) - } -) - -# -# RELATIONAL handlers -# -# (no changes needed) +def define_exit_node_handlers(_exit_node_handlers=None): + if _exit_node_handlers is None: + _exit_node_handlers = {} + linear.define_exit_node_handlers(_exit_node_handlers) + # + # NEGATION + # + _exit_node_handlers[NegationExpression][(_QUADRATIC,)] = linear._handle_negation_ANY + # + # PRODUCT + # + _exit_node_handlers[ProductExpression].update( + { + None: _handle_product_nonlinear, + (_CONSTANT, _QUADRATIC): linear._handle_product_constant_ANY, + (_QUADRATIC, _CONSTANT): linear._handle_product_ANY_constant, + # Replace handler from the linear walker + (_LINEAR, _LINEAR): _handle_product_linear_linear, + } + ) + # + # DIVISION + # + _exit_node_handlers[DivisionExpression].update( + {(_QUADRATIC, _CONSTANT): linear._handle_division_ANY_constant} + ) + # + # EXPONENTIATION + # + _exit_node_handlers[PowExpression].update( + {(_QUADRATIC, _CONSTANT): linear._handle_pow_ANY_constant} + ) + # + # ABS and UNARY handlers + # + # (no changes needed) + # + # NAMED EXPRESSION handlers + # + # (no changes needed) + # + # EXPR_IF handlers + # + # Note: it is easier to just recreate the entire data structure, rather + # than update it + _exit_node_handlers[Expr_ifExpression].update( + { + (_CONSTANT, i, _QUADRATIC): linear._handle_expr_if_const + for i in (_CONSTANT, _LINEAR, _QUADRATIC, _GENERAL) + } + ) + _exit_node_handlers[Expr_ifExpression].update( + { + (_CONSTANT, _QUADRATIC, i): linear._handle_expr_if_const + for i in (_CONSTANT, _LINEAR, _GENERAL) + } + ) + # + # RELATIONAL handlers + # + # (no changes needed) + return _exit_node_handlers class QuadraticRepnVisitor(linear.LinearRepnVisitor): Result = QuadraticRepn - exit_node_handlers = _exit_node_handlers exit_node_dispatcher = linear.ExitNodeDispatcher( - linear._initialize_exit_node_dispatcher(_exit_node_handlers) + linear._initialize_exit_node_dispatcher(define_exit_node_handlers()) ) max_exponential_expansion = 2 diff --git a/pyomo/repn/util.py b/pyomo/repn/util.py index 8d902d0f99a..1c58821ba6c 100644 --- a/pyomo/repn/util.py +++ b/pyomo/repn/util.py @@ -378,18 +378,16 @@ class ExitNodeDispatcher(collections.defaultdict): `exitNode` callback This dispatcher implements a specialization of :py:`defaultdict` - that supports automatic type registration. Any missing types will - return the :py:meth:`register_dispatcher` method, which (when called - as a callback) will interrogate the type, identify the appropriate - callback, add the callback to the dict, and return the result of - calling the callback. As the callback is added to the dict, no type - will incur the overhead of `register_dispatcher` more than once. + that supports automatic type registration. As the identified + callback is added to the dict, no type will incur the overhead of + `register_dispatcher` more than once. Note that in this case, the client is expected to register all non-NPV expression types. The auto-registration is designed to only handle two cases: - Auto-detection of user-defined Named Expression types - Automatic mappimg of NPV expressions to their equivalent non-NPV handlers + - Automatic registration of derived expression types """ From b8ac19a6f19fc7cf27154072665c27ff3f568690 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 8 Jul 2024 14:37:10 -0600 Subject: [PATCH 50/66] Defining define exit node handlers function --- pyomo/repn/parameterized_linear.py | 104 +++++++++++++++-------------- 1 file changed, 55 insertions(+), 49 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index 45ff2943b11..2f83c450ea5 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -208,20 +208,14 @@ def _before_var(visitor, child): _before_child_dispatcher = ParameterizedLinearBeforeChildDispatcher() _exit_node_handlers = copy.deepcopy(linear._exit_node_handlers) + # # NEGATION handlers # - def _handle_negation_pseudo_constant(visitor, node, arg): return (_FIXED, -1 * arg[1]) - -_exit_node_handlers[NegationExpression].update( - {(_FIXED,): _handle_negation_pseudo_constant} -) - - # # PRODUCT handlers # @@ -237,22 +231,6 @@ def _handle_product_pseudo_constant_constant(visitor, node, arg1, arg2): return _FIXED, arg1[1] * arg2[1] -_exit_node_handlers[ProductExpression].update( - { - (_CONSTANT, _CONSTANT): _handle_product_constant_constant, - (_FIXED, _FIXED): _handle_product_pseudo_constant_constant, - (_FIXED, _CONSTANT): _handle_product_pseudo_constant_constant, - (_CONSTANT, _FIXED): _handle_product_pseudo_constant_constant, - (_FIXED, _LINEAR): linear._handle_product_constant_ANY, - (_LINEAR, _FIXED): linear._handle_product_ANY_constant, - (_FIXED, _GENERAL): linear._handle_product_constant_ANY, - (_GENERAL, _FIXED): linear._handle_product_ANY_constant, - } -) -_exit_node_handlers[MonomialTermExpression].update( - _exit_node_handlers[ProductExpression] -) - # # DIVISION handlers # @@ -267,15 +245,6 @@ def _handle_division_ANY_pseudo_constant(visitor, node, arg1, arg2): return arg1 -_exit_node_handlers[DivisionExpression].update( - { - (_FIXED, _FIXED): _handle_division_pseudo_constant_constant, - (_FIXED, _CONSTANT): _handle_division_pseudo_constant_constant, - (_CONSTANT, _FIXED): _handle_division_pseudo_constant_constant, - (_LINEAR, _FIXED): _handle_division_ANY_pseudo_constant, - (_GENERAL, _FIXED): _handle_division_ANY_pseudo_constant, - } -) # # EXPONENTIATION handlers @@ -294,17 +263,6 @@ def _handle_pow_nonlinear(visitor, node, arg1, arg2): return _GENERAL, ans -_exit_node_handlers[PowExpression].update( - { - (_FIXED, _FIXED): _handle_pow_pseudo_constant_constant, - (_FIXED, _CONSTANT): _handle_pow_pseudo_constant_constant, - (_CONSTANT, _FIXED): _handle_pow_pseudo_constant_constant, - (_LINEAR, _FIXED): _handle_pow_nonlinear, - (_FIXED, _LINEAR): _handle_pow_nonlinear, - (_GENERAL, _FIXED): _handle_pow_nonlinear, - (_FIXED, _GENERAL): _handle_pow_nonlinear, - } -) # # ABS and UNARY handlers @@ -316,17 +274,65 @@ def _handle_unary_pseudo_constant(visitor, node, arg): return _FIXED, node.create_node_with_local_data((to_expression(visitor, arg),)) -_exit_node_handlers[UnaryFunctionExpression].update( - {(_FIXED,): _handle_unary_pseudo_constant} -) -_exit_node_handlers[AbsExpression] = _exit_node_handlers[UnaryFunctionExpression] +def define_exit_node_handlers(_exit_node_handlers=None): + if _exit_node_handlers is None: + _exit_node_handlers = {} + + _exit_node_handlers[NegationExpression].update( + {(_FIXED,): _handle_negation_pseudo_constant} + ) + + _exit_node_handlers[ProductExpression].update( + { + (_CONSTANT, _CONSTANT): _handle_product_constant_constant, + (_FIXED, _FIXED): _handle_product_pseudo_constant_constant, + (_FIXED, _CONSTANT): _handle_product_pseudo_constant_constant, + (_CONSTANT, _FIXED): _handle_product_pseudo_constant_constant, + (_FIXED, _LINEAR): linear._handle_product_constant_ANY, + (_LINEAR, _FIXED): linear._handle_product_ANY_constant, + (_FIXED, _GENERAL): linear._handle_product_constant_ANY, + (_GENERAL, _FIXED): linear._handle_product_ANY_constant, + } + ) + + _exit_node_handlers[MonomialTermExpression].update( + _exit_node_handlers[ProductExpression] + ) + + _exit_node_handlers[DivisionExpression].update( + { + (_FIXED, _FIXED): _handle_division_pseudo_constant_constant, + (_FIXED, _CONSTANT): _handle_division_pseudo_constant_constant, + (_CONSTANT, _FIXED): _handle_division_pseudo_constant_constant, + (_LINEAR, _FIXED): _handle_division_ANY_pseudo_constant, + (_GENERAL, _FIXED): _handle_division_ANY_pseudo_constant, + } + ) + + _exit_node_handlers[PowExpression].update( + { + (_FIXED, _FIXED): _handle_pow_pseudo_constant_constant, + (_FIXED, _CONSTANT): _handle_pow_pseudo_constant_constant, + (_CONSTANT, _FIXED): _handle_pow_pseudo_constant_constant, + (_LINEAR, _FIXED): _handle_pow_nonlinear, + (_FIXED, _LINEAR): _handle_pow_nonlinear, + (_GENERAL, _FIXED): _handle_pow_nonlinear, + (_FIXED, _GENERAL): _handle_pow_nonlinear, + } + ) + _exit_node_handlers[UnaryFunctionExpression].update( + {(_FIXED,): _handle_unary_pseudo_constant} + ) + _exit_node_handlers[AbsExpression] = _exit_node_handlers[UnaryFunctionExpression] + + return _exit_node_handlers class ParameterizedLinearRepnVisitor(LinearRepnVisitor): Result = ParameterizedLinearRepn - exit_node_handlers = _exit_node_handlers + exit_node_handlers = define_exit_node_handlers(_exit_node_handlers) exit_node_dispatcher = ExitNodeDispatcher( - _initialize_exit_node_dispatcher(_exit_node_handlers) + _initialize_exit_node_dispatcher(exit_node_handlers) ) def __init__(self, subexpression_cache, var_map, var_order, sorter, wrt): From 0f7390da8319c2e81b0d8c7e556567529032d32f Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 8 Jul 2024 14:46:45 -0600 Subject: [PATCH 51/66] Calling linear define exit node handler --- pyomo/repn/parameterized_linear.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index 2f83c450ea5..47f9445bd7f 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -206,8 +206,6 @@ def _before_var(visitor, child): _before_child_dispatcher = ParameterizedLinearBeforeChildDispatcher() -_exit_node_handlers = copy.deepcopy(linear._exit_node_handlers) - # # NEGATION handlers @@ -274,15 +272,16 @@ def _handle_unary_pseudo_constant(visitor, node, arg): return _FIXED, node.create_node_with_local_data((to_expression(visitor, arg),)) -def define_exit_node_handlers(_exit_node_handlers=None): - if _exit_node_handlers is None: - _exit_node_handlers = {} +def define_exit_node_handlers(exit_node_handlers=None): + if exit_node_handlers is None: + exit_node_handlers = {} + linear.define_exit_node_handlers(exit_node_handlers) - _exit_node_handlers[NegationExpression].update( + exit_node_handlers[NegationExpression].update( {(_FIXED,): _handle_negation_pseudo_constant} ) - _exit_node_handlers[ProductExpression].update( + exit_node_handlers[ProductExpression].update( { (_CONSTANT, _CONSTANT): _handle_product_constant_constant, (_FIXED, _FIXED): _handle_product_pseudo_constant_constant, @@ -295,11 +294,11 @@ def define_exit_node_handlers(_exit_node_handlers=None): } ) - _exit_node_handlers[MonomialTermExpression].update( - _exit_node_handlers[ProductExpression] + exit_node_handlers[MonomialTermExpression].update( + exit_node_handlers[ProductExpression] ) - _exit_node_handlers[DivisionExpression].update( + exit_node_handlers[DivisionExpression].update( { (_FIXED, _FIXED): _handle_division_pseudo_constant_constant, (_FIXED, _CONSTANT): _handle_division_pseudo_constant_constant, @@ -309,7 +308,7 @@ def define_exit_node_handlers(_exit_node_handlers=None): } ) - _exit_node_handlers[PowExpression].update( + exit_node_handlers[PowExpression].update( { (_FIXED, _FIXED): _handle_pow_pseudo_constant_constant, (_FIXED, _CONSTANT): _handle_pow_pseudo_constant_constant, @@ -320,19 +319,18 @@ def define_exit_node_handlers(_exit_node_handlers=None): (_FIXED, _GENERAL): _handle_pow_nonlinear, } ) - _exit_node_handlers[UnaryFunctionExpression].update( + exit_node_handlers[UnaryFunctionExpression].update( {(_FIXED,): _handle_unary_pseudo_constant} ) - _exit_node_handlers[AbsExpression] = _exit_node_handlers[UnaryFunctionExpression] + exit_node_handlers[AbsExpression] = exit_node_handlers[UnaryFunctionExpression] - return _exit_node_handlers + return exit_node_handlers class ParameterizedLinearRepnVisitor(LinearRepnVisitor): Result = ParameterizedLinearRepn - exit_node_handlers = define_exit_node_handlers(_exit_node_handlers) exit_node_dispatcher = ExitNodeDispatcher( - _initialize_exit_node_dispatcher(exit_node_handlers) + _initialize_exit_node_dispatcher(define_exit_node_handlers()) ) def __init__(self, subexpression_cache, var_map, var_order, sorter, wrt): From 3235f928cd73724815236b28bec93ba0b2cde25b Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Mon, 8 Jul 2024 14:47:26 -0600 Subject: [PATCH 52/66] Black: breaking even on whitespace, for once --- pyomo/repn/parameterized_linear.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyomo/repn/parameterized_linear.py b/pyomo/repn/parameterized_linear.py index 47f9445bd7f..d1295b73e14 100644 --- a/pyomo/repn/parameterized_linear.py +++ b/pyomo/repn/parameterized_linear.py @@ -211,9 +211,11 @@ def _before_var(visitor, child): # NEGATION handlers # + def _handle_negation_pseudo_constant(visitor, node, arg): return (_FIXED, -1 * arg[1]) + # # PRODUCT handlers # @@ -243,7 +245,6 @@ def _handle_division_ANY_pseudo_constant(visitor, node, arg1, arg2): return arg1 - # # EXPONENTIATION handlers # @@ -261,7 +262,6 @@ def _handle_pow_nonlinear(visitor, node, arg1, arg2): return _GENERAL, ans - # # ABS and UNARY handlers # @@ -280,7 +280,7 @@ def define_exit_node_handlers(exit_node_handlers=None): exit_node_handlers[NegationExpression].update( {(_FIXED,): _handle_negation_pseudo_constant} ) - + exit_node_handlers[ProductExpression].update( { (_CONSTANT, _CONSTANT): _handle_product_constant_constant, From 205eb3d7caa2507875df784e0d51d970efa3bdf7 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 8 Jul 2024 15:30:31 -0600 Subject: [PATCH 53/66] Set default git SHA for codecov when building main branch --- .jenkins.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.jenkins.sh b/.jenkins.sh index 51dd92ba123..f64c413690c 100644 --- a/.jenkins.sh +++ b/.jenkins.sh @@ -208,7 +208,10 @@ if test -z "$MODE" -o "$MODE" == test; then coverage report -i || exit 1 coverage xml -i || exit 1 export OS=`uname` - if test -n "$CODECOV_TOKEN"; then + if test -z "$PYOMO_SOURCE_SHA"; then + PYOMO_SOURCE_SHA=$GIT_COMMIT + fi + if test -n "$CODECOV_TOKEN" -a -n "$PYOMO_SOURCE_SHA"; then CODECOV_JOB_NAME=`echo ${JOB_NAME} | sed -r 's/^(.*autotest_)?Pyomo_([^\/]+).*/\2/'`.$BUILD_NUMBER.$python if test -z "$CODECOV_REPO_OWNER"; then CODECOV_REPO_OWNER="pyomo" From 7197f966f4060a047094f5a56c96f46df22b76ef Mon Sep 17 00:00:00 2001 From: John Siirola Date: Mon, 8 Jul 2024 16:57:11 -0600 Subject: [PATCH 54/66] Jenkins: include calculation of CODECOV_REPO_OWNER and CODECOV_SOURCE_BRANCH --- .jenkins.sh | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.jenkins.sh b/.jenkins.sh index f64c413690c..defb9879b90 100644 --- a/.jenkins.sh +++ b/.jenkins.sh @@ -212,12 +212,24 @@ if test -z "$MODE" -o "$MODE" == test; then PYOMO_SOURCE_SHA=$GIT_COMMIT fi if test -n "$CODECOV_TOKEN" -a -n "$PYOMO_SOURCE_SHA"; then - CODECOV_JOB_NAME=`echo ${JOB_NAME} | sed -r 's/^(.*autotest_)?Pyomo_([^\/]+).*/\2/'`.$BUILD_NUMBER.$python + CODECOV_JOB_NAME=$(echo ${JOB_NAME} \ + | sed -r 's/^(.*autotest_)?Pyomo_([^\/]+).*/\2/').$BUILD_NUMBER.$python if test -z "$CODECOV_REPO_OWNER"; then - CODECOV_REPO_OWNER="pyomo" + if test -n "$PYOMO_SOURCE_REPO" + CODECOV_REPO_OWNER=$(echo $PYOMO_SOURCE_REPO | cut -d '/' -f 4) + elif test -n "$GIT_URL"; then + CODECOV_REPO_OWNER=$(echo $GIT_URL | cut -d '/' -f 4) + else + CODECOV_REPO_OWNER="" + fi fi - if test -z "CODECOV_SOURCE_BRANCH"; then - CODECOV_SOURCE_BRANCH="main" + if test -z "$CODECOV_SOURCE_BRANCH"; then + CODECOV_SOURCE_BRANCH=$(git branch -av --contains "$PYOMO_SOURCE_SHA" \ + | grep "${PYOMO_SOURCE_SHA:0:7}" | grep "/origin/" \ + | cut -d '/' -f 3 | cut -d' ' -f 1) + if test -z "$CODECOV_SOURCE_BRANCH"; then + CODECOV_SOURCE_BRANCH=main + fi fi i=0 while /bin/true; do From 6edb167805eefc16e7ea45b474483fb416e56bd1 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 9 Jul 2024 09:57:21 -0600 Subject: [PATCH 55/66] Fix typo in Jenkins driver --- .jenkins.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.jenkins.sh b/.jenkins.sh index defb9879b90..8771427805d 100644 --- a/.jenkins.sh +++ b/.jenkins.sh @@ -215,10 +215,10 @@ if test -z "$MODE" -o "$MODE" == test; then CODECOV_JOB_NAME=$(echo ${JOB_NAME} \ | sed -r 's/^(.*autotest_)?Pyomo_([^\/]+).*/\2/').$BUILD_NUMBER.$python if test -z "$CODECOV_REPO_OWNER"; then - if test -n "$PYOMO_SOURCE_REPO" - CODECOV_REPO_OWNER=$(echo $PYOMO_SOURCE_REPO | cut -d '/' -f 4) + if test -n "$PYOMO_SOURCE_REPO"; then + CODECOV_REPO_OWNER=$(echo "$PYOMO_SOURCE_REPO" | cut -d '/' -f 4) elif test -n "$GIT_URL"; then - CODECOV_REPO_OWNER=$(echo $GIT_URL | cut -d '/' -f 4) + CODECOV_REPO_OWNER=$(echo "$GIT_URL" | cut -d '/' -f 4) else CODECOV_REPO_OWNER="" fi From c8bf1312c4c08fb7a35a484fda0d9e651c0822c6 Mon Sep 17 00:00:00 2001 From: Miranda Mundt <55767766+mrmundt@users.noreply.github.com> Date: Tue, 9 Jul 2024 10:08:26 -0600 Subject: [PATCH 56/66] Change universal_newlines -> text --- pyomo/solvers/plugins/solvers/ASL.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/solvers/plugins/solvers/ASL.py b/pyomo/solvers/plugins/solvers/ASL.py index a29c64e017f..7acd59936b1 100644 --- a/pyomo/solvers/plugins/solvers/ASL.py +++ b/pyomo/solvers/plugins/solvers/ASL.py @@ -101,7 +101,7 @@ def _get_version(self): timeout=5, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - universal_newlines=True, + text=True, errors='ignore', ) ver = _extract_version(results.stdout) From 3bb2100b5bc8eb1972f1e62a78c4314ae494fd4d Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 9 Jul 2024 15:37:07 -0600 Subject: [PATCH 57/66] Update test_set.py exception reference strings --- pyomo/core/tests/unit/test_set.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pyomo/core/tests/unit/test_set.py b/pyomo/core/tests/unit/test_set.py index 012669a3484..4a49f6a2fae 100644 --- a/pyomo/core/tests/unit/test_set.py +++ b/pyomo/core/tests/unit/test_set.py @@ -6453,15 +6453,14 @@ def test_issue_3284(self): # set subset_A problem.subset_A.add(1) error_message = ( - "Cannot add value %s to Set %s.\n" - "\tThe value is not in the domain %s" - % (4, 'subset_A', 'A') + "Cannot add value 4 to Set subset_A.\n" + "\tThe value is not in the domain A" ) with self.assertRaisesRegex(ValueError, error_message): problem.subset_A.add(4) # set subset_B problem.subset_B.add((3, 4)) - with self.assertRaisesRegex(ValueError, ".*Cannot add value "): + with self.assertRaisesRegex(ValueError, ".*Cannot add value \(7, 8\)"): problem.subset_B.add((7, 8)) # set subset_C problem.subset_C[2].add(7) @@ -6477,7 +6476,7 @@ def test_issue_3284(self): problem.E[1].add(4) # set F problem.F[(1, 2, 3)].add((3, 4)) - with self.assertRaisesRegex(ValueError, ".*Cannot add value "): + with self.assertRaisesRegex(ValueError, ".*Cannot add value \(4, 3\)"): problem.F[(4, 5, 6)].add((4, 3)) # check them self.assertEqual(list(problem.A), [1, 2, 3]) From ef55217c0229fe67fcf6e07951a9152009e8119e Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 9 Jul 2024 15:38:59 -0600 Subject: [PATCH 58/66] NFC: apply black --- pyomo/core/tests/unit/test_set.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pyomo/core/tests/unit/test_set.py b/pyomo/core/tests/unit/test_set.py index 4a49f6a2fae..1ef1b5d6867 100644 --- a/pyomo/core/tests/unit/test_set.py +++ b/pyomo/core/tests/unit/test_set.py @@ -6453,9 +6453,8 @@ def test_issue_3284(self): # set subset_A problem.subset_A.add(1) error_message = ( - "Cannot add value 4 to Set subset_A.\n" - "\tThe value is not in the domain A" - ) + "Cannot add value 4 to Set subset_A.\n\tThe value is not in the domain A" + ) with self.assertRaisesRegex(ValueError, error_message): problem.subset_A.add(4) # set subset_B From 643c0e55deb54b823ecb0d47da2c0d95501c1bbf Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 9 Jul 2024 16:13:17 -0600 Subject: [PATCH 59/66] Bugfix: regex should be a raw string --- pyomo/core/tests/unit/test_set.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/core/tests/unit/test_set.py b/pyomo/core/tests/unit/test_set.py index 1ef1b5d6867..2b0da8b861d 100644 --- a/pyomo/core/tests/unit/test_set.py +++ b/pyomo/core/tests/unit/test_set.py @@ -6459,7 +6459,7 @@ def test_issue_3284(self): problem.subset_A.add(4) # set subset_B problem.subset_B.add((3, 4)) - with self.assertRaisesRegex(ValueError, ".*Cannot add value \(7, 8\)"): + with self.assertRaisesRegex(ValueError, r".*Cannot add value \(7, 8\)"): problem.subset_B.add((7, 8)) # set subset_C problem.subset_C[2].add(7) @@ -6475,7 +6475,7 @@ def test_issue_3284(self): problem.E[1].add(4) # set F problem.F[(1, 2, 3)].add((3, 4)) - with self.assertRaisesRegex(ValueError, ".*Cannot add value \(4, 3\)"): + with self.assertRaisesRegex(ValueError, r".*Cannot add value \(4, 3\)"): problem.F[(4, 5, 6)].add((4, 3)) # check them self.assertEqual(list(problem.A), [1, 2, 3]) From 663b64982bc9012ec11da627e4aad1b58aca3540 Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Wed, 10 Jul 2024 13:00:13 -0600 Subject: [PATCH 60/66] Fixing a bug in multiple bigm where we were not updating the list of active Disjuncts after we encountered an infeasible one and deactivated it. Nothing was wrong here, but we were doing extra work (and hitting a bug in the baron writer) --- pyomo/gdp/plugins/multiple_bigm.py | 25 +++++++++++++++++++------ pyomo/gdp/tests/test_mbigm.py | 14 ++++++-------- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/pyomo/gdp/plugins/multiple_bigm.py b/pyomo/gdp/plugins/multiple_bigm.py index 4dffd4e9f9a..6914d6937ef 100644 --- a/pyomo/gdp/plugins/multiple_bigm.py +++ b/pyomo/gdp/plugins/multiple_bigm.py @@ -12,7 +12,7 @@ import itertools import logging -from pyomo.common.collections import ComponentMap +from pyomo.common.collections import ComponentMap, ComponentSet from pyomo.common.config import ConfigDict, ConfigValue from pyomo.common.gc_manager import PauseGC from pyomo.common.modeling import unique_component_name @@ -310,9 +310,12 @@ def _transform_disjunctionData(self, obj, index, parent_disjunct, root_disjunct) arg_Ms = self._config.bigM if self._config.bigM is not None else {} + # ESJ: I am relying on the fact that the ComponentSet is going to be + # ordered here, but using a set because I will remove infeasible + # Disjuncts from it if I encounter them calculating M's. + active_disjuncts = ComponentSet(disj for disj in obj.disjuncts if disj.active) # First handle the bound constraints if we are dealing with them # separately - active_disjuncts = [disj for disj in obj.disjuncts if disj.active] transformed_constraints = set() if self._config.reduce_bound_constraints: transformed_constraints = self._transform_bound_constraints( @@ -585,7 +588,7 @@ def _calculate_missing_M_values( ): if disjunct is other_disjunct: continue - if id(other_disjunct) in scratch_blocks: + elif id(other_disjunct) in scratch_blocks: scratch = scratch_blocks[id(other_disjunct)] else: scratch = scratch_blocks[id(other_disjunct)] = Block() @@ -631,7 +634,7 @@ def _calculate_missing_M_values( scratch.obj.expr = constraint.body - constraint.lower scratch.obj.sense = minimize lower_M = self._solve_disjunct_for_M( - other_disjunct, scratch, unsuccessful_solve_msg + other_disjunct, scratch, unsuccessful_solve_msg, active_disjuncts ) if constraint.upper is not None and upper_M is None: # last resort: calculate @@ -639,7 +642,7 @@ def _calculate_missing_M_values( scratch.obj.expr = constraint.body - constraint.upper scratch.obj.sense = maximize upper_M = self._solve_disjunct_for_M( - other_disjunct, scratch, unsuccessful_solve_msg + other_disjunct, scratch, unsuccessful_solve_msg, active_disjuncts ) arg_Ms[constraint, other_disjunct] = (lower_M, upper_M) transBlock._mbm_values[constraint, other_disjunct] = (lower_M, upper_M) @@ -651,9 +654,18 @@ def _calculate_missing_M_values( return arg_Ms def _solve_disjunct_for_M( - self, other_disjunct, scratch_block, unsuccessful_solve_msg + self, other_disjunct, scratch_block, unsuccessful_solve_msg, active_disjuncts ): + if not other_disjunct.active: + # If a Disjunct is infeasible, we will discover that and deactivate + # it when we are calculating the M values. We remove that disjunct + # from active_disjuncts inside of the loop in + # _calculate_missing_M_values. So that means that we might have + # deactivated Disjuncts here that we should skip over. + return 0 + solver = self._config.solver + results = solver.solve(other_disjunct, load_solutions=False) if results.solver.termination_condition is TerminationCondition.infeasible: # [2/18/24]: TODO: After the solver rewrite is complete, we will not @@ -669,6 +681,7 @@ def _solve_disjunct_for_M( "Disjunct '%s' is infeasible, deactivating." % other_disjunct.name ) other_disjunct.deactivate() + active_disjuncts.remove(other_disjunct) M = 0 else: # This is a solver that might report diff --git a/pyomo/gdp/tests/test_mbigm.py b/pyomo/gdp/tests/test_mbigm.py index 9e82b1010f9..1516681f5a4 100644 --- a/pyomo/gdp/tests/test_mbigm.py +++ b/pyomo/gdp/tests/test_mbigm.py @@ -1019,11 +1019,14 @@ def test_calculate_Ms_infeasible_Disjunct(self): out.getvalue().strip(), ) - # We just fixed the infeasible by to False + # We just fixed the infeasible disjunct to False self.assertFalse(m.disjunction.disjuncts[0].active) self.assertTrue(m.disjunction.disjuncts[0].indicator_var.fixed) self.assertFalse(value(m.disjunction.disjuncts[0].indicator_var)) + # We didn't actually transform the infeasible disjunct + self.assertIsNone(m.disjunction.disjuncts[0].transformation_block) + # the remaining constraints are transformed correctly. cons = mbm.get_transformed_constraints(m.disjunction.disjuncts[1].constraint[1]) self.assertEqual(len(cons), 1) @@ -1031,18 +1034,14 @@ def test_calculate_Ms_infeasible_Disjunct(self): self, cons[0].expr, 21 + m.x - m.y - <= 0 * m.disjunction.disjuncts[0].binary_indicator_var - + 12.0 * m.disjunction.disjuncts[2].binary_indicator_var, + <= 12.0 * m.disjunction.disjuncts[2].binary_indicator_var, ) cons = mbm.get_transformed_constraints(m.disjunction.disjuncts[2].constraint[1]) self.assertEqual(len(cons), 2) - print(cons[0].expr) - print(cons[1].expr) assertExpressionsEqual( self, cons[0].expr, - 0.0 * m.disjunction_disjuncts[0].binary_indicator_var - 12.0 * m.disjunction_disjuncts[1].binary_indicator_var <= m.x - (m.y - 9), ) @@ -1050,8 +1049,7 @@ def test_calculate_Ms_infeasible_Disjunct(self): self, cons[1].expr, m.x - (m.y - 9) - <= 0.0 * m.disjunction_disjuncts[0].binary_indicator_var - - 12.0 * m.disjunction_disjuncts[1].binary_indicator_var, + <= - 12.0 * m.disjunction_disjuncts[1].binary_indicator_var, ) @unittest.skipUnless( From 6f0d95461f2805f8cba0ff1f38dbc3190eb4ac5b Mon Sep 17 00:00:00 2001 From: Emma Johnson Date: Wed, 10 Jul 2024 13:00:53 -0600 Subject: [PATCH 61/66] Blackify --- pyomo/gdp/plugins/multiple_bigm.py | 10 ++++++++-- pyomo/gdp/tests/test_mbigm.py | 9 +++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/pyomo/gdp/plugins/multiple_bigm.py b/pyomo/gdp/plugins/multiple_bigm.py index 6914d6937ef..3362276246b 100644 --- a/pyomo/gdp/plugins/multiple_bigm.py +++ b/pyomo/gdp/plugins/multiple_bigm.py @@ -634,7 +634,10 @@ def _calculate_missing_M_values( scratch.obj.expr = constraint.body - constraint.lower scratch.obj.sense = minimize lower_M = self._solve_disjunct_for_M( - other_disjunct, scratch, unsuccessful_solve_msg, active_disjuncts + other_disjunct, + scratch, + unsuccessful_solve_msg, + active_disjuncts, ) if constraint.upper is not None and upper_M is None: # last resort: calculate @@ -642,7 +645,10 @@ def _calculate_missing_M_values( scratch.obj.expr = constraint.body - constraint.upper scratch.obj.sense = maximize upper_M = self._solve_disjunct_for_M( - other_disjunct, scratch, unsuccessful_solve_msg, active_disjuncts + other_disjunct, + scratch, + unsuccessful_solve_msg, + active_disjuncts, ) arg_Ms[constraint, other_disjunct] = (lower_M, upper_M) transBlock._mbm_values[constraint, other_disjunct] = (lower_M, upper_M) diff --git a/pyomo/gdp/tests/test_mbigm.py b/pyomo/gdp/tests/test_mbigm.py index 1516681f5a4..14a23160574 100644 --- a/pyomo/gdp/tests/test_mbigm.py +++ b/pyomo/gdp/tests/test_mbigm.py @@ -1033,8 +1033,7 @@ def test_calculate_Ms_infeasible_Disjunct(self): assertExpressionsEqual( self, cons[0].expr, - 21 + m.x - m.y - <= 12.0 * m.disjunction.disjuncts[2].binary_indicator_var, + 21 + m.x - m.y <= 12.0 * m.disjunction.disjuncts[2].binary_indicator_var, ) cons = mbm.get_transformed_constraints(m.disjunction.disjuncts[2].constraint[1]) @@ -1042,14 +1041,12 @@ def test_calculate_Ms_infeasible_Disjunct(self): assertExpressionsEqual( self, cons[0].expr, - - 12.0 * m.disjunction_disjuncts[1].binary_indicator_var - <= m.x - (m.y - 9), + -12.0 * m.disjunction_disjuncts[1].binary_indicator_var <= m.x - (m.y - 9), ) assertExpressionsEqual( self, cons[1].expr, - m.x - (m.y - 9) - <= - 12.0 * m.disjunction_disjuncts[1].binary_indicator_var, + m.x - (m.y - 9) <= -12.0 * m.disjunction_disjuncts[1].binary_indicator_var, ) @unittest.skipUnless( From e720fd837f4cae5aa1c17587da9503b3cf17890e Mon Sep 17 00:00:00 2001 From: John Siirola Date: Sun, 14 Jul 2024 15:50:51 -0600 Subject: [PATCH 62/66] NLv2: support models with expressions with nested external functions --- pyomo/repn/plugins/nl_writer.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/pyomo/repn/plugins/nl_writer.py b/pyomo/repn/plugins/nl_writer.py index a8966e44f71..8fc82d21d30 100644 --- a/pyomo/repn/plugins/nl_writer.py +++ b/pyomo/repn/plugins/nl_writer.py @@ -1988,6 +1988,18 @@ def _record_named_expression_usage(self, named_exprs, src, comp_type): elif info[comp_type] != src: info[comp_type] = 0 + def _resolve_subexpression_args(self, nl, args): + final_args = [] + for arg in args: + if arg in self.var_id_to_nl_map: + final_args.append(self.var_id_to_nl_map[arg]) + else: + _nl, _ids, _ = self.subexpression_cache[arg][1].compile_repn( + self.visitor + ) + final_args.append(self._resolve_subexpression_args(_nl, _ids)) + return nl % tuple(final_args) + def _write_nl_expression(self, repn, include_const): # Note that repn.mult should always be 1 (the AMPLRepn was # compiled before this point). Omitting the assertion for @@ -2007,18 +2019,7 @@ def _write_nl_expression(self, repn, include_const): nl % tuple(map(self.var_id_to_nl_map.__getitem__, args)) ) except KeyError: - final_args = [] - for arg in args: - if arg in self.var_id_to_nl_map: - final_args.append(self.var_id_to_nl_map[arg]) - else: - _nl, _ids, _ = self.subexpression_cache[arg][1].compile_repn( - self.visitor - ) - final_args.append( - _nl % tuple(map(self.var_id_to_nl_map.__getitem__, _ids)) - ) - self.ostream.write(nl % tuple(final_args)) + self.ostream.write(self._resolve_subexpression_args(nl, args)) elif include_const: self.ostream.write(self.template.const % repn.const) From 4464b99c969660be7309097f59a05c6aee8b60e9 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Sun, 14 Jul 2024 15:51:32 -0600 Subject: [PATCH 63/66] Add test --- pyomo/repn/tests/ampl/test_nlv2.py | 69 ++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/pyomo/repn/tests/ampl/test_nlv2.py b/pyomo/repn/tests/ampl/test_nlv2.py index b6bb5f6c074..4d7b5d9ab6c 100644 --- a/pyomo/repn/tests/ampl/test_nlv2.py +++ b/pyomo/repn/tests/ampl/test_nlv2.py @@ -2703,3 +2703,72 @@ def test_presolve_check_invalid_monomial_constraints(self): r"\(fixed body value 5.0 outside bounds \[10, None\]\)\.", ): nl_writer.NLWriter().write(m, OUT, linear_presolve=True) + + def test_nested_external_expressions(self): + # This tests nested external functions in a single expression + DLL = find_GSL() + if not DLL: + self.skipTest("Could not find the amplgsl.dll library") + + m = ConcreteModel() + m.hypot = ExternalFunction(library=DLL, function="gsl_hypot") + m.p = Param(initialize=1, mutable=True) + m.x = Var(bounds=(None, 3)) + m.y = Var(bounds=(3, None)) + m.z = Var(initialize=1) + m.o = Objective(expr=m.z**2 * m.hypot(m.z, m.hypot(m.x, m.y)) ** 2) + m.c = Constraint(expr=m.x == m.y) + + OUT = io.StringIO() + nl_writer.NLWriter().write( + m, OUT, symbolic_solver_labels=True, linear_presolve=False + ) + self.assertEqual( + *nl_diff( + """g3 1 1 0 #problem unknown + 3 1 1 0 1 #vars, constraints, objectives, ranges, eqns + 0 1 0 0 0 0 #nonlinear constrs, objs; ccons: lin, nonlin, nd, nzlb + 0 0 #network constraints: nonlinear, linear + 0 3 0 #nonlinear vars in constraints, objectives, both + 0 1 0 1 #linear network variables; functions; arith, flags + 0 0 0 0 0 #discrete variables: binary, integer, nonlinear (b,c,o) + 2 3 #nonzeros in Jacobian, obj. gradient + 1 1 #max name lengths: constraints, variables + 0 0 0 0 0 #common exprs: b,c,o,c1,o1 +F0 1 -1 gsl_hypot +C0 #c +n0 +O0 0 #o +o2 #* +o5 #^ +v0 #z +n2 +o5 #^ +f0 2 #hypot +v0 #z +f0 2 #hypot +v1 #x +v2 #y +n2 +x1 #initial guess +0 1 #z +r #1 ranges (rhs's) +4 0 #c +b #3 bounds (on variables) +3 #z +1 3 #x +2 3 #y +k2 #intermediate Jacobian column lengths +0 +1 +J0 2 #c +1 1 +2 -1 +G0 3 #o +0 0 +1 0 +2 0 +""", + OUT.getvalue(), + ) + ) From 6d25c370393d34454d79711b4290c64aca6472ed Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 16 Jul 2024 08:01:29 -0600 Subject: [PATCH 64/66] Disable interface/testing for NEOS/octeract --- pyomo/neos/__init__.py | 2 +- pyomo/neos/tests/test_neos.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pyomo/neos/__init__.py b/pyomo/neos/__init__.py index 7d18535e753..9f910f4a302 100644 --- a/pyomo/neos/__init__.py +++ b/pyomo/neos/__init__.py @@ -30,7 +30,7 @@ 'minos': 'SLC NLP solver', 'minto': 'MILP solver', 'mosek': 'Interior point NLP solver', - 'octeract': 'Deterministic global MINLP solver', + #'octeract': 'Deterministic global MINLP solver', 'ooqp': 'Convex QP solver', 'path': 'Nonlinear MCP solver', 'snopt': 'SQP NLP solver', diff --git a/pyomo/neos/tests/test_neos.py b/pyomo/neos/tests/test_neos.py index a4c4e9e6367..01b19a76b15 100644 --- a/pyomo/neos/tests/test_neos.py +++ b/pyomo/neos/tests/test_neos.py @@ -149,8 +149,11 @@ def test_minto(self): def test_mosek(self): self._run('mosek') - def test_octeract(self): - self._run('octeract') + # [16 Jul 24] Octeract is erroring. We will disable the interface + # (and testing) until we have time to resolve #3321 + # + # def test_octeract(self): + # self._run('octeract') def test_ooqp(self): if self.sense == pyo.maximize: From 5a8ea16c9965d6dbaca5a93fbea81ed2d10335b5 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 16 Jul 2024 08:31:01 -0600 Subject: [PATCH 65/66] Remove octeract as an expected solver interface --- pyomo/neos/tests/test_neos.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyomo/neos/tests/test_neos.py b/pyomo/neos/tests/test_neos.py index 01b19a76b15..363368cd616 100644 --- a/pyomo/neos/tests/test_neos.py +++ b/pyomo/neos/tests/test_neos.py @@ -79,6 +79,9 @@ def test_doc(self): doc = pyomo.neos.doc dockeys = set(doc.keys()) + # Octeract interface is disabled, see #3321 + amplsolvers.pop('octeract') + self.assertEqual(amplsolvers, dockeys) # gamssolvers = set(v[0].lower() for v in tmp if v[1]=='GAMS') From e7a3711475bc905f6a4a608b4cf8cbcfb178b59e Mon Sep 17 00:00:00 2001 From: John Siirola Date: Tue, 16 Jul 2024 09:05:52 -0600 Subject: [PATCH 66/66] bugfix: use correct set api --- pyomo/neos/tests/test_neos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/neos/tests/test_neos.py b/pyomo/neos/tests/test_neos.py index 363368cd616..681856781be 100644 --- a/pyomo/neos/tests/test_neos.py +++ b/pyomo/neos/tests/test_neos.py @@ -80,7 +80,7 @@ def test_doc(self): dockeys = set(doc.keys()) # Octeract interface is disabled, see #3321 - amplsolvers.pop('octeract') + amplsolvers.remove('octeract') self.assertEqual(amplsolvers, dockeys)