From e5524392a9c99208448fe863952586e545776927 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Mon, 18 Mar 2024 15:14:12 +0100 Subject: [PATCH 001/107] Add first draft of SCIP persistent solving --- pyomo/solvers/plugins/solvers/__init__.py | 2 + pyomo/solvers/plugins/solvers/scip_direct.py | 838 ++++++++++++++++++ .../plugins/solvers/scip_persistent.py | 185 ++++ pyomo/solvers/tests/checks/test_SCIPDirect.py | 335 +++++++ .../tests/checks/test_SCIPPersistent.py | 318 +++++++ pyomo/solvers/tests/solvers.py | 21 + 6 files changed, 1699 insertions(+) create mode 100644 pyomo/solvers/plugins/solvers/scip_direct.py create mode 100644 pyomo/solvers/plugins/solvers/scip_persistent.py create mode 100644 pyomo/solvers/tests/checks/test_SCIPDirect.py create mode 100644 pyomo/solvers/tests/checks/test_SCIPPersistent.py diff --git a/pyomo/solvers/plugins/solvers/__init__.py b/pyomo/solvers/plugins/solvers/__init__.py index 9b2507d876c..e8f4e00e31a 100644 --- a/pyomo/solvers/plugins/solvers/__init__.py +++ b/pyomo/solvers/plugins/solvers/__init__.py @@ -30,3 +30,5 @@ import pyomo.solvers.plugins.solvers.mosek_persistent import pyomo.solvers.plugins.solvers.xpress_direct import pyomo.solvers.plugins.solvers.xpress_persistent +import pyomo.solvers.plugins.solvers.scip_direct +import pyomo.solvers.plugins.solvers.scip_persistent diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py new file mode 100644 index 00000000000..0aafb596007 --- /dev/null +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -0,0 +1,838 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2024 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import logging +import re +import sys + +from pyomo.common.collections import ComponentSet, ComponentMap, Bunch +from pyomo.common.tempfiles import TempfileManager +from pyomo.core import Var +from pyomo.core.expr.numeric_expr import ( + SumExpression, + ProductExpression, + UnaryFunctionExpression, + PowExpression, + DivisionExpression, +) +from pyomo.core.expr.numvalue import is_fixed +from pyomo.core.expr.numvalue import value +from pyomo.core.staleflag import StaleFlagManager +from pyomo.repn import generate_standard_repn +from pyomo.solvers.plugins.solvers.direct_solver import DirectSolver +from pyomo.solvers.plugins.solvers.direct_or_persistent_solver import ( + DirectOrPersistentSolver, +) +from pyomo.core.kernel.objective import minimize, maximize +from pyomo.opt.results.results_ import SolverResults +from pyomo.opt.results.solution import Solution, SolutionStatus +from pyomo.opt.results.solver import TerminationCondition, SolverStatus +from pyomo.opt.base import SolverFactory +from pyomo.core.base.suffix import Suffix + + +logger = logging.getLogger("pyomo.solvers") + + +class DegreeError(ValueError): + pass + + +def _is_numeric(x): + try: + float(x) + except ValueError: + return False + return True + + +@SolverFactory.register("scip_direct", doc="Direct python interface to SCIP") +class SCIPDirect(DirectSolver): + + def __init__(self, **kwds): + kwds["type"] = "scipdirect" + DirectSolver.__init__(self, **kwds) + self._init() + self._solver_model = None + + def _init(self): + try: + import pyscipopt + + self._scip = pyscipopt + self._python_api_exists = True + self._version = str(self._scip.Model().version()) + self._version_major = self._version.split(".")[0] + except ImportError: + self._python_api_exists = False + except Exception as e: + print("Import of pyscipopt failed - SCIP message=" + str(e) + "\n") + self._python_api_exists = False + + # Note: Undefined capabilities default to None + self._max_constraint_degree = None + self._max_obj_degree = 1 + self._capabilities.linear = True + self._capabilities.quadratic_objective = False + self._capabilities.quadratic_constraint = True + self._capabilities.integer = True + self._capabilities.sos1 = True + self._capabilities.sos2 = True + + # Dictionary used exclusively for SCIP, as we want the constraint expressions + self._pyomo_var_to_solver_var_expr_map = ComponentMap() + self._pyomo_con_to_solver_con_expr_map = dict() + + def _apply_solver(self): + StaleFlagManager.mark_all_as_stale() + + # Supress solver output if requested + if self._tee: + self._solver_model.hideOutput(quiet=False) + else: + self._solver_model.hideOutput(quiet=True) + + # Redirect solver output to a logfile if requested + if self._keepfiles: + # Only save log file when the user wants to keep it. + self._solver_model.setLogfile(self._log_file) + print("Solver log file: " + self._log_file) + + # Set user specified parameters + for key, option in self.options.items(): + try: + key_type = type(self._solver_model.getParam(key)) + except KeyError: + raise ValueError(f"Key {key} is an invalid parameter for SCIP") + + if key_type == str: + self._solver_model.setParam(key, option) + else: + if not _is_numeric(option): + raise ValueError( + f"Value {option} for parameter {key} is not a string and can't be converted to float" + ) + self._solver_model.setParam(key, float(option)) + + self._solver_model.optimize() + + # TODO: Check if this is even needed, or if it is sufficient to close the open file + # if self._keepfiles: + # self._solver_model.setLogfile(None) + + # FIXME: can we get a return code indicating if SCIP had a significant failure? + return Bunch(rc=None, log=None) + + def _get_expr_from_pyomo_repn(self, repn, max_degree=None): + referenced_vars = ComponentSet() + + new_expr = repn.constant + + if len(repn.linear_vars) > 0: + referenced_vars.update(repn.linear_vars) + new_expr += sum( + repn.linear_coefs[i] * self._pyomo_var_to_solver_var_expr_map[var] + for i, var in enumerate(repn.linear_vars) + ) + + for i, v in enumerate(repn.quadratic_vars): + x, y = v + new_expr += ( + repn.quadratic_coefs[i] + * self._pyomo_var_to_solver_var_expr_map[x] + * self._pyomo_var_to_solver_var_expr_map[y] + ) + referenced_vars.add(x) + referenced_vars.add(y) + + # TODO: Introduce handling on non-linear expressions + if repn.nonlinear_expr is not None: + + def get_nl_expr_recursively(pyomo_expr): + if not hasattr(pyomo_expr, "args"): + if not isinstance(pyomo_expr, Var): + return float(pyomo_expr) + else: + referenced_vars.add(pyomo_expr) + return self._pyomo_var_to_solver_var_expr_map[pyomo_expr] + scip_expr_list = [0 for i in range(pyomo_expr.nargs())] + for i in range(pyomo_expr.nargs()): + scip_expr_list[i] = get_nl_expr_recursively(pyomo_expr.args[i]) + if isinstance(pyomo_expr, PowExpression): + if len(scip_expr_list) != 2: + raise ValueError( + f"PowExpression has {len(scip_expr_list)} many terms instead of two!" + ) + return scip_expr_list[0] ** (scip_expr_list[1]) + elif isinstance(pyomo_expr, ProductExpression): + return self._scip.quickprod(scip_expr_list) + elif isinstance(pyomo_expr, SumExpression): + return self._scip.quicksum(scip_expr_list) + elif isinstance(pyomo_expr, DivisionExpression): + if len(scip_expr_list) != 2: + raise ValueError( + f"DivisonExpression has {len(scip_expr_list)} many terms instead of two!" + ) + return scip_expr_list[0] / scip_expr_list[1] + elif isinstance(pyomo_expr, UnaryFunctionExpression): + if len(scip_expr_list) != 1: + raise ValueError( + f"UnaryExpression has {len(scip_expr_list)} many terms instead of one!" + ) + if pyomo_expr.name == "sin": + return self._scip.sin(scip_expr_list[0]) + elif pyomo_expr.name == "cos": + return self._scip.cos(scip_expr_list[0]) + elif pyomo_expr.name == "exp": + return self._scip.exp(scip_expr_list[0]) + elif pyomo_expr.name == "log": + return self._scip.log(scip_expr_list[0]) + else: + raise NotImplementedError( + f"PySCIPOpt through Pyomo does not support the unary function {pyomo_expr.name}" + ) + else: + raise NotImplementedError( + f"PySCIPOpt through Pyomo does not yet support expression type {type(pyomo_expr)}" + ) + + new_expr += get_nl_expr_recursively(repn.nonlinear_expr) + + return new_expr, referenced_vars + + def _get_expr_from_pyomo_expr(self, expr, max_degree=None): + if max_degree is None or max_degree >= 2: + repn = generate_standard_repn(expr, quadratic=True) + else: + repn = generate_standard_repn(expr, quadratic=False) + + scip_expr, referenced_vars = self._get_expr_from_pyomo_repn(repn, max_degree) + + return scip_expr, referenced_vars + + def _scip_lb_ub_from_var(self, var): + if var.is_fixed(): + val = var.value + return val, val + if var.has_lb(): + lb = value(var.lb) + else: + lb = -self._solver_model.infinity() + if var.has_ub(): + ub = value(var.ub) + else: + ub = self._solver_model.infinity() + return lb, ub + + def _add_var(self, var): + varname = self._symbol_map.getSymbol(var, self._labeler) + vtype = self._scip_vtype_from_var(var) + lb, ub = self._scip_lb_ub_from_var(var) + + scip_var = self._solver_model.addVar(lb=lb, ub=ub, vtype=vtype, name=varname) + + self._pyomo_var_to_solver_var_expr_map[var] = scip_var + self._pyomo_var_to_solver_var_map[var] = scip_var.name + self._solver_var_to_pyomo_var_map[varname] = var + self._referenced_variables[var] = 0 + + def close(self): + """Frees SCIP resources used by this solver instance.""" + + if self._solver_model is not None: + self._solver_model.freeProb() + self._solver_model = None + + def __exit__(self, t, v, traceback): + super().__exit__(t, v, traceback) + self.close() + + def _set_instance(self, model, kwds={}): + DirectOrPersistentSolver._set_instance(self, model, kwds) + try: + self._solver_model = self._scip.Model() + except Exception: + e = sys.exc_info()[1] + msg = ( + "Unable to create SCIP model. " + "Have you installed PySCIPOpt correctly?\n\n\t" + + "Error message: {0}".format(e) + ) + raise Exception(msg) + + self._add_block(model) + + for var, n_ref in self._referenced_variables.items(): + if n_ref != 0: + if var.fixed: + if not self._output_fixed_variable_bounds: + raise ValueError( + "Encountered a fixed variable (%s) inside " + "an active objective or constraint " + "expression on model %s, which is usually " + "indicative of a preprocessing error. Use " + "the IO-option 'output_fixed_variable_bounds=True' " + "to suppress this error and fix the variable " + "by overwriting its bounds in the SCIP instance." + % (var.name, self._pyomo_model.name) + ) + + def _add_block(self, block): + DirectOrPersistentSolver._add_block(self, block) + + def _add_constraint(self, con): + if not con.active: + return None + + if is_fixed(con.body) and self._skip_trivial_constraints: + return None + + conname = self._symbol_map.getSymbol(con, self._labeler) + + if con._linear_canonical_form: + scip_expr, referenced_vars = self._get_expr_from_pyomo_repn( + con.canonical_form(), self._max_constraint_degree + ) + else: + scip_expr, referenced_vars = self._get_expr_from_pyomo_expr( + con.body, self._max_constraint_degree + ) + + if con.has_lb(): + if not is_fixed(con.lower): + raise ValueError( + "Lower bound of constraint {0} is not constant.".format(con) + ) + if con.has_ub(): + if not is_fixed(con.upper): + raise ValueError( + "Upper bound of constraint {0} is not constant.".format(con) + ) + + if con.equality: + scip_cons = self._solver_model.addCons( + scip_expr == value(con.lower), name=conname + ) + elif con.has_lb() and con.has_ub(): + scip_cons = self._solver_model.addCons( + value(con.lower) <= (scip_expr <= value(con.upper)), name=conname + ) + elif con.has_lb(): + scip_cons = self._solver_model.addCons( + value(con.lower) <= scip_expr, name=conname + ) + elif con.has_ub(): + scip_cons = self._solver_model.addCons( + scip_expr <= value(con.upper), name=conname + ) + else: + raise ValueError( + "Constraint does not have a lower " + "or an upper bound: {0} \n".format(con) + ) + + for var in referenced_vars: + self._referenced_variables[var] += 1 + self._vars_referenced_by_con[con] = referenced_vars + self._pyomo_con_to_solver_con_expr_map[con] = scip_cons + self._pyomo_con_to_solver_con_map[con] = scip_cons.name + self._solver_con_to_pyomo_con_map[conname] = con + + def _add_sos_constraint(self, con): + if not con.active: + return None + + conname = self._symbol_map.getSymbol(con, self._labeler) + level = con.level + if level not in [1, 2]: + raise ValueError(f"Solver does not support SOS level {level} constraints") + + scip_vars = [] + weights = [] + + self._vars_referenced_by_con[con] = ComponentSet() + + if hasattr(con, "get_items"): + # aml sos constraint + sos_items = list(con.get_items()) + else: + # kernel sos constraint + sos_items = list(con.items()) + + for v, w in sos_items: + self._vars_referenced_by_con[con].add(v) + scip_vars.append(self._pyomo_var_to_solver_var_expr_map[v]) + self._referenced_variables[v] += 1 + weights.append(w) + + if level == 1: + scip_cons = self._solver_model.addConsSOS1( + scip_vars, weights=weights, name=conname + ) + else: + scip_cons = self._solver_model.addConsSOS2( + scip_vars, weights=weights, name=conname + ) + self._pyomo_con_to_solver_con_expr_map[con] = scip_cons + self._pyomo_con_to_solver_con_map[con] = scip_cons.name + self._solver_con_to_pyomo_con_map[conname] = con + + def _scip_vtype_from_var(self, var): + """ + This function takes a pyomo variable and returns the appropriate SCIP variable type + :param var: pyomo.core.base.var.Var + :return: B, I, or C + """ + if var.is_binary(): + vtype = "B" + elif var.is_integer(): + vtype = "I" + elif var.is_continuous(): + vtype = "C" + else: + raise ValueError( + "Variable domain type is not recognized for {0}".format(var.domain) + ) + return vtype + + def _set_objective(self, obj): + if self._objective is not None: + for var in self._vars_referenced_by_obj: + self._referenced_variables[var] -= 1 + self._vars_referenced_by_obj = ComponentSet() + self._objective = None + + if obj.active is False: + raise ValueError("Cannot add inactive objective to solver.") + + if obj.sense == minimize: + sense = "minimize" + elif obj.sense == maximize: + sense = "maximize" + else: + raise ValueError("Objective sense is not recognized: {0}".format(obj.sense)) + + scip_expr, referenced_vars = self._get_expr_from_pyomo_expr( + obj.expr, self._max_obj_degree + ) + + for var in referenced_vars: + self._referenced_variables[var] += 1 + + self._solver_model.setObjective(scip_expr, sense=sense) + self._objective = obj + self._vars_referenced_by_obj = referenced_vars + + self._needs_updated = True + + def _postsolve(self): + # the only suffixes that we extract from SCIP are + # constraint duals, constraint slacks, and variable + # reduced-costs. scan through the solver suffix list + # and throw an exception if the user has specified + # any others. + extract_duals = False + extract_slacks = False + extract_reduced_costs = False + for suffix in self._suffixes: + flag = False + if re.match(suffix, "dual"): + extract_duals = True + flag = True + if re.match(suffix, "slack"): + extract_slacks = True + flag = True + if re.match(suffix, "rc"): + extract_reduced_costs = True + flag = True + if not flag: + raise RuntimeError( + "***The scip_direct solver plugin cannot extract solution suffix=" + + suffix + ) + + scip = self._solver_model + status = scip.getStatus() + scip_vars = scip.getVars() + n_bin_vars = sum([scip_var.vtype() == "BINARY" for scip_var in scip_vars]) + n_int_vars = sum([scip_var.vtype() == "INTEGER" for scip_var in scip_vars]) + n_con_vars = sum([scip_var.vtype() == "CONTINUOUS" for scip_var in scip_vars]) + + if n_bin_vars + n_int_vars > 0: + if extract_reduced_costs: + logger.warning("Cannot get reduced costs for MIP.") + if extract_duals: + logger.warning("Cannot get duals for MIP.") + extract_reduced_costs = False + extract_duals = False + + self.results = SolverResults() + soln = Solution() + + self.results.solver.name = f"SCIP{self._version}" + self.results.solver.wallclock_time = scip.getSolvingTime() + + if scip.getStage() == 1: # SCIP Model is created but not yet optimized + self.results.solver.status = SolverStatus.aborted + self.results.solver.termination_message = ( + "Model is loaded, but no solution information is available." + ) + self.results.solver.termination_condition = TerminationCondition.error + soln.status = SolutionStatus.unknown + elif status == "optimal": # optimal + self.results.solver.status = SolverStatus.ok + self.results.solver.termination_message = ( + "Model was solved to optimality (subject to tolerances), " + "and an optimal solution is available." + ) + self.results.solver.termination_condition = TerminationCondition.optimal + soln.status = SolutionStatus.optimal + elif status == "infeasible": + self.results.solver.status = SolverStatus.warning + self.results.solver.termination_message = ( + "Model was proven to be infeasible" + ) + self.results.solver.termination_condition = TerminationCondition.infeasible + soln.status = SolutionStatus.infeasible + elif status == "inforunbd": + self.results.solver.status = SolverStatus.warning + self.results.solver.termination_message = ( + "Problem proven to be infeasible or unbounded." + ) + self.results.solver.termination_condition = ( + TerminationCondition.infeasibleOrUnbounded + ) + soln.status = SolutionStatus.unsure + elif status == "unbounded": + self.results.solver.status = SolverStatus.warning + self.results.solver.termination_message = ( + "Model was proven to be unbounded." + ) + self.results.solver.termination_condition = TerminationCondition.unbounded + soln.status = SolutionStatus.unbounded + elif status == "gaplimit": + self.results.solver.status = SolverStatus.aborted + self.results.solver.termination_message = ( + "Optimization terminated because the gap dropped below " + "the value specified in the " + "limits/gap parameter." + ) + self.results.solver.termination_condition = TerminationCondition.unknown + soln.status = SolutionStatus.stoppedByLimit + elif status == "stallnodelimit": + self.results.solver.status = SolverStatus.aborted + self.results.solver.termination_message = ( + "Optimization terminated because the stalling node limit " + "exceeded the value specified in the " + "limits/stallnodes parameter." + ) + self.results.solver.termination_condition = TerminationCondition.unknown + soln.status = SolutionStatus.stoppedByLimit + elif status == "restartlimit": + self.results.solver.status = SolverStatus.aborted + self.results.solver.termination_message = ( + "Optimization terminated because the total number of restarts " + "exceeded the value specified in the " + "limits/restarts parameter." + ) + self.results.solver.termination_condition = TerminationCondition.unknown + soln.status = SolutionStatus.stoppedByLimit + elif status == "nodelimit" or status == "totalnodelimit": + self.results.solver.status = SolverStatus.aborted + self.results.solver.termination_message = ( + "Optimization terminated because the number of " + "branch-and-cut nodes explored exceeded the limits specified " + "in the limits/nodes or limits/totalnodes parameter" + ) + self.results.solver.termination_condition = ( + TerminationCondition.maxEvaluations + ) + soln.status = SolutionStatus.stoppedByLimit + elif status == "timelimit": + self.results.solver.status = SolverStatus.aborted + self.results.solver.termination_message = ( + "Optimization terminated because the time expended exceeded " + "the value specified in the limits/time parameter." + ) + self.results.solver.termination_condition = ( + TerminationCondition.maxTimeLimit + ) + soln.status = SolutionStatus.stoppedByLimit + elif status == "sollimit" or status == "bestsollimit": + self.results.solver.status = SolverStatus.aborted + self.results.solver.termination_message = ( + "Optimization terminated because the number of solutions found " + "reached the value specified in the limits/solutions or" + "limits/bestsol parameter." + ) + self.results.solver.termination_condition = TerminationCondition.unknown + soln.status = SolutionStatus.stoppedByLimit + elif status == "memlimit": + self.results.solver.status = SolverStatus.aborted + self.results.solver.termination_message = ( + "Optimization terminated because the memory used exceeded " + "the value specified in the limits/memory parameter." + ) + self.results.solver.termination_condition = TerminationCondition.unknown + soln.status = SolutionStatus.stoppedByLimit + elif status == "userinterrupt": + self.results.solver.status = SolverStatus.aborted + self.results.solver.termination_message = ( + "Optimization was terminated by the user." + ) + self.results.solver.termination_condition = TerminationCondition.error + soln.status = SolutionStatus.error + else: + self.results.solver.status = SolverStatus.error + self.results.solver.termination_message = ( + "Unhandled SCIP status (" + str(status) + ")" + ) + self.results.solver.termination_condition = TerminationCondition.error + soln.status = SolutionStatus.error + + self.results.problem.name = scip.getProbName() + + if scip.getObjectiveSense() == "minimize": + self.results.problem.sense = minimize + elif scip.getObjectiveSense() == "maximize": + self.results.problem.sense = maximize + else: + raise RuntimeError( + f"Unrecognized SCIP objective sense: {scip.getObjectiveSense()}" + ) + + self.results.problem.upper_bound = None + self.results.problem.lower_bound = None + if scip.getNSols() > 0: + scip_has_sol = True + else: + scip_has_sol = False + if not scip_has_sol and (status == "inforunbd" or status == "infeasible"): + pass + else: + if n_bin_vars + n_int_vars == 0: + self.results.problem.upper_bound = scip.getObjVal() + self.results.problem.lower_bound = scip.getObjVal() + elif scip.getObjectiveSense() == "minimize": # minimizing + if scip_has_sol: + self.results.problem.upper_bound = scip.getObjVal() + else: + self.results.problem.upper_bound = scip.infinity() + self.results.problem.lower_bound = scip.getDualbound() + else: # maximizing + self.results.problem.upper_bound = scip.getDualbound() + if scip_has_sol: + self.results.problem.lower_bound = scip.getObjVal() + else: + self.results.problem.lower_bound = -scip.infinity() + + try: + soln.gap = ( + self.results.problem.upper_bound - self.results.problem.lower_bound + ) + except TypeError: + soln.gap = None + + # TODO: Should these values be of the transformed or the original problem? + self.results.problem.number_of_constraints = scip.getNConss() + # self.results.problem.number_of_nonzeros = None + self.results.problem.number_of_variables = scip.getNVars() + self.results.problem.number_of_binary_variables = n_bin_vars + self.results.problem.number_of_integer_variables = n_int_vars + self.results.problem.number_of_continuous_variables = n_con_vars + self.results.problem.number_of_objectives = 1 + self.results.problem.number_of_solutions = scip.getNSols() + + # if a solve was stopped by a limit, we still need to check to + # see if there is a solution available - this may not always + # be the case, both in LP and MIP contexts. + if self._save_results: + """ + This code in this if statement is only needed for backwards compatibility. It is more efficient to set + _save_results to False and use load_vars, load_duals, etc. + """ + if scip.getNSols() > 0: + soln_variables = soln.variable + soln_constraints = soln.constraint + scip_sol = scip.getBestSol() + + scip_vars = scip.getVars() + scip_var_names = [scip_var.name for scip_var in scip_vars] + var_names = set(self._solver_var_to_pyomo_var_map.keys()) + assert set(scip_var_names) == var_names + var_vals = [scip.getVal(scip_var) for scip_var in scip_vars] + + for scip_var, val, name in zip(scip_vars, var_vals, scip_var_names): + pyomo_var = self._solver_var_to_pyomo_var_map[name] + if self._referenced_variables[pyomo_var] > 0: + soln_variables[name] = {"Value": val} + + if extract_reduced_costs: + vals = [scip.getVarRedcost(scip_var) for scip_var in scip_vars] + for scip_var, val, name in zip(scip_vars, vals, scip_var_names): + pyomo_var = self._solver_var_to_pyomo_var_map[name] + if self._referenced_variables[pyomo_var] > 0: + soln_variables[name]["Rc"] = val + + if extract_duals or extract_slacks: + scip_cons = scip.getConss() + con_names = [cons.name for cons in scip_cons] + assert set(self._solver_con_to_pyomo_con_map.keys()) == set( + con_names + ) + for name in con_names: + soln_constraints[name] = {} + + if extract_duals: + vals = [scip.getDualSolVal(con) for con in scip_cons] + for val, name in zip(vals, con_names): + soln_constraints[name]["Dual"] = val + + if extract_slacks: + vals = [scip.getSlack(con, scip_sol) for con in scip_cons] + for val, name in zip(vals, con_names): + soln_constraints[name]["Slack"] = val + + elif self._load_solutions: + if scip.getNSols() > 0: + self.load_vars() + + if extract_reduced_costs: + self._load_rc() + + if extract_duals: + self._load_duals() + + if extract_slacks: + self._load_slacks() + + self.results.solution.insert(soln) + + # finally, clean any temporary files registered with the temp file + # manager, created populated *directly* by this plugin. + TempfileManager.pop(remove=not self._keepfiles) + + return DirectOrPersistentSolver._postsolve(self) + + def warm_start_capable(self): + return True + + def _warm_start(self): + scip_sol = self._solver_model.createSol() + for pyomo_var, scip_var in self._pyomo_var_to_solver_var_expr_map.items(): + if pyomo_var.value is not None: + scip_sol[scip_var] = value(pyomo_var) + self._solver_model.trySol(scip_sol, free=True) + + def _load_vars(self, vars_to_load=None): + var_map = self._pyomo_var_to_solver_var_expr_map + ref_vars = self._referenced_variables + if vars_to_load is None: + vars_to_load = var_map.keys() + + scip_vars_to_load = [var_map[pyomo_var] for pyomo_var in vars_to_load] + vals = [self._solver_model.getVal(scip_var) for scip_var in scip_vars_to_load] + + for var, val in zip(vars_to_load, vals): + if ref_vars[var] > 0: + var.set_value(val, skip_validation=True) + + def _load_rc(self, vars_to_load=None): + if not hasattr(self._pyomo_model, "rc"): + self._pyomo_model.rc = Suffix(direction=Suffix.IMPORT) + var_map = self._pyomo_var_to_solver_var_expr_map + ref_vars = self._referenced_variables + rc = self._pyomo_model.rc + if vars_to_load is None: + vars_to_load = var_map.keys() + + scip_vars_to_load = [var_map[pyomo_var] for pyomo_var in vars_to_load] + vals = [ + self._solver_model.getVarRedcost(scip_var) for scip_var in scip_vars_to_load + ] + + for var, val in zip(vars_to_load, vals): + if ref_vars[var] > 0: + rc[var] = val + + def _load_duals(self, cons_to_load=None): + if not hasattr(self._pyomo_model, "dual"): + self._pyomo_model.dual = Suffix(direction=Suffix.IMPORT) + con_map = self._pyomo_con_to_solver_con_map + reverse_con_map = self._solver_con_to_pyomo_con_map + dual = self._pyomo_model.dual + scip_cons = self._solver_model.getConss() + + if cons_to_load is None: + con_names = [con.name for con in scip_cons] + vals = [self._solver_model.getDualSolVal(con) for con in scip_cons] + else: + con_names = set([con_map[pyomo_con] for pyomo_con in cons_to_load]) + scip_cons_to_load = [con for con in scip_cons if con.name in con_names] + vals = [self._solver_model.getDualSolVal(con) for con in scip_cons_to_load] + + for i, con_name in enumerate(con_names): + pyomo_con = reverse_con_map[con_name] + dual[pyomo_con] = vals[i] + + def _load_slacks(self, cons_to_load=None): + if not hasattr(self._pyomo_model, "slack"): + self._pyomo_model.slack = Suffix(direction=Suffix.IMPORT) + con_map = self._pyomo_con_to_solver_con_map + reverse_con_map = self._solver_con_to_pyomo_con_map + slack = self._pyomo_model.slack + scip_cons = self._solver_model.getConss() + scip_sol = self._solver_model.getBestSol() + + if cons_to_load is None: + con_names = [con.name for con in scip_cons] + vals = [self._solver_model.getSlack(con, scip_sol) for con in scip_cons] + else: + con_names = set([con_map[pyomo_con] for pyomo_con in cons_to_load]) + scip_cons_to_load = [con for con in scip_cons if con.name in con_names] + vals = [ + self._solver_model.getSlack(con, scip_sol) for con in scip_cons_to_load + ] + + for i, con_name in enumerate(con_names): + pyomo_con = reverse_con_map[con_name] + slack[pyomo_con] = vals[i] + + def load_duals(self, cons_to_load=None): + """ + Load the duals into the 'dual' suffix. The 'dual' suffix must live on the parent model. + + Parameters + ---------- + cons_to_load: list of Constraint + """ + self._load_duals(cons_to_load) + + def load_rc(self, vars_to_load): + """ + Load the reduced costs into the 'rc' suffix. The 'rc' suffix must live on the parent model. + + Parameters + ---------- + vars_to_load: list of Var + """ + self._load_rc(vars_to_load) + + def load_slacks(self, cons_to_load=None): + """ + Load the values of the slack variables into the 'slack' suffix. The 'slack' suffix must live on the parent + model. + + Parameters + ---------- + cons_to_load: list of Constraint + """ + self._load_slacks(cons_to_load) diff --git a/pyomo/solvers/plugins/solvers/scip_persistent.py b/pyomo/solvers/plugins/solvers/scip_persistent.py new file mode 100644 index 00000000000..408aa84633f --- /dev/null +++ b/pyomo/solvers/plugins/solvers/scip_persistent.py @@ -0,0 +1,185 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2024 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +from pyomo.solvers.plugins.solvers.scip_direct import SCIPDirect +from pyomo.solvers.plugins.solvers.persistent_solver import PersistentSolver +from pyomo.opt.base import SolverFactory + + +@SolverFactory.register("scip_persistent", doc="Persistent python interface to SCIP") +class SCIPPersistent(PersistentSolver, SCIPDirect): + """ + A class that provides a persistent interface to SCIP. Direct solver interfaces do not use any file io. + Rather, they interface directly with the python bindings for the specific solver. Persistent solver interfaces + are similar except that they "remember" their model. Thus, persistent solver interfaces allow incremental changes + to the solver model (e.g., the gurobi python model or the cplex python model). Note that users are responsible + for notifying the persistent solver interfaces when changes are made to the corresponding pyomo model. + + Keyword Arguments + ----------------- + model: ConcreteModel + Passing a model to the constructor is equivalent to calling the set_instance method. + type: str + String indicating the class type of the solver instance. + name: str + String representing either the class type of the solver instance or an assigned name. + doc: str + Documentation for the solver + options: dict + Dictionary of solver options + """ + + def __init__(self, **kwds): + kwds["type"] = "scip_persistent" + PersistentSolver.__init__(self, **kwds) + SCIPDirect._init(self) + + self._pyomo_model = kwds.pop("model", None) + if self._pyomo_model is not None: + self.set_instance(self._pyomo_model, **kwds) + + def _remove_constraint(self, solver_conname): + con = self._solver_con_to_pyomo_con_map[solver_conname] + scip_con = self._pyomo_con_to_solver_con_expr_map[con] + self._solver_model.delCons(scip_con) + + def _remove_sos_constraint(self, solver_sos_conname): + con = self._solver_con_to_pyomo_con_map[solver_sos_conname] + scip_con = self._pyomo_con_to_solver_con_expr_map[con] + self._solver_model.delCons(scip_con) + + def _remove_var(self, solver_varname): + var = self._solver_var_to_pyomo_var_map[solver_varname] + scip_var = self._pyomo_var_to_solver_var_expr_map[var] + self._solver_model.delVar(scip_var) + + def _warm_start(self): + SCIPDirect._warm_start(self) + + def update_var(self, var): + """Update a single variable in the solver's model. + + This will update bounds, fix/unfix the variable as needed, and + update the variable type. + + Parameters + ---------- + var: Var (scalar Var or single _VarData) + + """ + # see PR #366 for discussion about handling indexed + # objects and keeping compatibility with the + # pyomo.kernel objects + # if var.is_indexed(): + # for child_var in var.values(): + # self.compile_var(child_var) + # return + if var not in self._pyomo_var_to_solver_var_map: + raise ValueError( + "The Var provided to compile_var needs to be added first: {0}".format( + var + ) + ) + scip_var = self._pyomo_var_to_solver_var_map[var] + vtype = self._scip_vtype_from_var(var) + lb, ub = self._scip_lb_ub_from_var(var) + + self._solver_model.chgVarLb(scip_var, lb) + self._solver_model.chgVarUb(scip_var, ub) + self._solver_model.chgVarType(scip_var, vtype) + + def write(self, filename, filetype=""): + """ + Write the model to a file (e.g., and lp file). + + Parameters + ---------- + filename: str + Name of the file to which the model should be written. + filetype: str + The file type (e.g., lp). + """ + self._solver_model.writeProblem(filename + filetype) + + def set_scip_param(self, param, val): + """ + Set a SCIP parameter. + + Parameters + ---------- + param: str + The SCIP parameter to set. Options include any SCIP parameter. + Please see the SCIP documentation for options. + val: any + The value to set the parameter to. See SCIP documentation for possible values. + """ + self._solver_model.setParam(param, val) + + def get_scip_param(self, param): + """ + Get the value of the SCIP parameter. + + Parameters + ---------- + param: str or int or float + The SCIP parameter to get the value of. See SCIP documentation for possible options. + """ + return self._solver_model.getParam(param) + + def _add_column(self, var, obj_coef, constraints, coefficients): + """Add a column to the solver's model + + This will add the Pyomo variable var to the solver's + model, and put the coefficients on the associated + constraints in the solver model. If the obj_coef is + not zero, it will add obj_coef*var to the objective + of the solver's model. + + Parameters + ---------- + var: Var (scalar Var or single _VarData) + obj_coef: float + constraints: list of solver constraints + coefficients: list of coefficients to put on var in the associated constraint + """ + + # Set-up add var + varname = self._symbol_map.getSymbol(var, self._labeler) + vtype = self._scip_vtype_from_var(var) + lb, ub = self._scip_lb_ub_from_var(var) + + # Add the variable to the model and then to all the constraints + scip_var = self._solver_model.addVar(lb=lb, ub=ub, vtype=vtype, name=varname) + self._pyomo_var_to_solver_var_expr_map[var] = scip_var + self._solver_var_to_pyomo_var_map[varname] = var + self._referenced_variables[var] = len(coefficients) + + # Get the SCIP cons by passing through two dictionaries + pyomo_cons = [self._solver_con_to_pyomo_con_map[con] for con in constraints] + scip_cons = [ + self._pyomo_con_to_solver_con_expr_map[pyomo_con] + for pyomo_con in pyomo_cons + ] + + for i, scip_con in enumerate(scip_cons): + if not scip_con.isLinear(): + raise ValueError( + "_add_column functionality not supported for non-linear constraints" + ) + self._solver_model.addConsCoeff(scip_con, scip_var, coefficients[i]) + con = self._solver_con_to_pyomo_con_map[scip_con.name] + self._vars_referenced_by_con[con].add(var) + + sense = self._solver_model.getObjectiveSense() + self._solver_model.setObjective(obj_coef * scip_var, sense=sense, clear=False) + + def reset(self): + self._solver_model.freeTransform() diff --git a/pyomo/solvers/tests/checks/test_SCIPDirect.py b/pyomo/solvers/tests/checks/test_SCIPDirect.py new file mode 100644 index 00000000000..ee37f5ddcc8 --- /dev/null +++ b/pyomo/solvers/tests/checks/test_SCIPDirect.py @@ -0,0 +1,335 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2024 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import sys + +import pyomo.common.unittest as unittest + +from pyomo.environ import ( + ConcreteModel, + AbstractModel, + Var, + Objective, + Block, + Constraint, + Suffix, + NonNegativeIntegers, + NonNegativeReals, + Integers, + Binary, + value, +) +from pyomo.opt import SolverFactory, TerminationCondition, SolutionStatus + +try: + import pyscipopt + + scip_available = True +except ImportError: + scip_available = False + + +class SCIPDirectTests(unittest.TestCase): + def setUp(self): + self.stderr = sys.stderr + sys.stderr = None + + def tearDown(self): + sys.stderr = self.stderr + + @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") + def test_infeasible_lp(self): + with SolverFactory("scip_direct", solver_io="python") as opt: + model = ConcreteModel() + model.X = Var(within=NonNegativeReals) + model.C1 = Constraint(expr=model.X == 1) + model.C2 = Constraint(expr=model.X == 2) + model.O = Objective(expr=model.X) + + results = opt.solve(model) + + self.assertEqual( + results.solver.termination_condition, TerminationCondition.infeasible + ) + + @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") + def test_unbounded_lp(self): + with SolverFactory("scip_direct", solver_io="python") as opt: + model = ConcreteModel() + model.X = Var() + model.O = Objective(expr=model.X) + + results = opt.solve(model) + + self.assertIn( + results.solver.termination_condition, + ( + TerminationCondition.unbounded, + TerminationCondition.infeasibleOrUnbounded, + ), + ) + + @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") + def test_optimal_lp(self): + with SolverFactory("scip_direct", solver_io="python") as opt: + model = ConcreteModel() + model.X = Var(within=NonNegativeReals) + model.O = Objective(expr=model.X) + + results = opt.solve(model, load_solutions=False) + + self.assertEqual(results.solution.status, SolutionStatus.optimal) + + @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") + def test_get_duals_lp(self): + with SolverFactory("scip_direct", solver_io="python") as opt: + model = ConcreteModel() + model.X = Var(within=NonNegativeReals) + model.Y = Var(within=NonNegativeReals) + + model.C1 = Constraint(expr=2 * model.X + model.Y >= 8) + model.C2 = Constraint(expr=model.X + 3 * model.Y >= 6) + + model.O = Objective(expr=model.X + model.Y) + + results = opt.solve(model, suffixes=["dual"], load_solutions=False) + + model.dual = Suffix(direction=Suffix.IMPORT) + model.solutions.load_from(results) + + self.assertAlmostEqual(model.dual[model.C1], 0.4) + self.assertAlmostEqual(model.dual[model.C2], 0.2) + + @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") + def test_infeasible_mip(self): + with SolverFactory("scip_direct", solver_io="python") as opt: + model = ConcreteModel() + model.X = Var(within=NonNegativeIntegers) + model.C1 = Constraint(expr=model.X == 1) + model.C2 = Constraint(expr=model.X == 2) + model.O = Objective(expr=model.X) + + results = opt.solve(model) + + self.assertEqual( + results.solver.termination_condition, TerminationCondition.infeasible + ) + + @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") + def test_unbounded_mip(self): + with SolverFactory("scip_direct", solver_io="python") as opt: + model = AbstractModel() + model.X = Var(within=Integers) + model.O = Objective(expr=model.X) + + instance = model.create_instance() + results = opt.solve(instance) + + self.assertIn( + results.solver.termination_condition, + ( + TerminationCondition.unbounded, + TerminationCondition.infeasibleOrUnbounded, + ), + ) + + @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") + def test_optimal_mip(self): + with SolverFactory("scip_direct", solver_io="python") as opt: + model = ConcreteModel() + model.X = Var(within=NonNegativeIntegers) + model.O = Objective(expr=model.X) + + results = opt.solve(model, load_solutions=False) + + self.assertEqual(results.solution.status, SolutionStatus.optimal) + + +@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") +class TestAddVar(unittest.TestCase): + def test_add_single_variable(self): + """Test that the variable is added correctly to `solver_model`.""" + model = ConcreteModel() + + opt = SolverFactory("scip_direct", solver_io="python") + opt._set_instance(model) + + self.assertEqual(opt._solver_model.getNVars(), 0) + + model.X = Var(within=Binary) + + opt._add_var(model.X) + + self.assertEqual(opt._solver_model.getNVars(), 1) + self.assertEqual(opt._solver_model.getVars()[0].vtype(), "BINARY") + + def test_add_block_containing_single_variable(self): + """Test that the variable is added correctly to `solver_model`.""" + model = ConcreteModel() + + opt = SolverFactory("scip_direct", solver_io="python") + opt._set_instance(model) + + self.assertEqual(opt._solver_model.getNVars(), 0) + + model.X = Var(within=Binary) + + opt._add_block(model) + + self.assertEqual(opt._solver_model.getNVars(), 1) + self.assertEqual(opt._solver_model.getVars()[0].vtype(), "BINARY") + + def test_add_block_containing_multiple_variables(self): + """Test that: + - The variable is added correctly to `solver_model` + - Fixed variable bounds are set correctly + """ + model = ConcreteModel() + + opt = SolverFactory("scip_direct", solver_io="python") + opt._set_instance(model) + + self.assertEqual(opt._solver_model.getNVars(), 0) + + model.X1 = Var(within=Binary) + model.X2 = Var(within=NonNegativeReals) + model.X3 = Var(within=NonNegativeIntegers) + + model.X3.fix(5) + + opt._add_block(model) + + self.assertEqual(opt._solver_model.getNVars(), 3) + scip_vars = opt._solver_model.getVars() + vtypes = [scip_var.vtype() for scip_var in scip_vars] + assert "BINARY" in vtypes and "CONTINUOUS" in vtypes and "INTEGER" in vtypes + lbs = [scip_var.getLbGlobal() for scip_var in scip_vars] + ubs = [scip_var.getUbGlobal() for scip_var in scip_vars] + assert 0 in lbs and 5 in lbs + assert ( + 1 in ubs + and 5 in ubs + and any([opt._solver_model.isInfinity(ub) for ub in ubs]) + ) + + +@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") +class TestAddCon(unittest.TestCase): + def test_add_single_constraint(self): + model = ConcreteModel() + model.X = Var(within=Binary) + + opt = SolverFactory("scip_direct", solver_io="python") + opt._set_instance(model) + + self.assertEqual(opt._solver_model.getNConss(), 0) + + model.C = Constraint(expr=model.X == 1) + + opt._add_constraint(model.C) + + self.assertEqual(opt._solver_model.getNConss(), 1) + con = opt._solver_model.getConss()[0] + self.assertEqual(con.isLinear(), 1) + self.assertEqual(opt._solver_model.getRhs(con), 1) + + def test_add_block_containing_single_constraint(self): + model = ConcreteModel() + model.X = Var(within=Binary) + + opt = SolverFactory("scip_direct", solver_io="python") + opt._set_instance(model) + + self.assertEqual(opt._solver_model.getNConss(), 0) + + model.B = Block() + model.B.C = Constraint(expr=model.X == 1) + + opt._add_block(model.B) + + self.assertEqual(opt._solver_model.getNConss(), 1) + con = opt._solver_model.getConss()[0] + self.assertEqual(con.isLinear(), 1) + self.assertEqual(opt._solver_model.getRhs(con), 1) + + def test_add_block_containing_multiple_constraints(self): + model = ConcreteModel() + model.X = Var(within=Binary) + + opt = SolverFactory("scip_direct", solver_io="python") + opt._set_instance(model) + + self.assertEqual(opt._solver_model.getNConss(), 0) + + model.B = Block() + model.B.C1 = Constraint(expr=model.X == 1) + model.B.C2 = Constraint(expr=model.X <= 1) + model.B.C3 = Constraint(expr=model.X >= 1) + + opt._add_block(model.B) + + self.assertEqual(opt._solver_model.getNConss(), 3) + + +@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") +class TestLoadVars(unittest.TestCase): + def setUp(self): + opt = SolverFactory("scip_direct", solver_io="python") + model = ConcreteModel() + model.X = Var(within=NonNegativeReals, initialize=0) + model.Y = Var(within=NonNegativeReals, initialize=0) + + model.C1 = Constraint(expr=2 * model.X + model.Y >= 8) + model.C2 = Constraint(expr=model.X + 3 * model.Y >= 6) + + model.O = Objective(expr=model.X + model.Y) + + opt.solve(model, load_solutions=False, save_results=False) + + self._model = model + self._opt = opt + + def test_all_vars_are_loaded(self): + self.assertTrue(self._model.X.stale) + self.assertTrue(self._model.Y.stale) + self.assertEqual(value(self._model.X), 0) + self.assertEqual(value(self._model.Y), 0) + + self._opt.load_vars() + + self.assertFalse(self._model.X.stale) + self.assertFalse(self._model.Y.stale) + self.assertAlmostEqual(value(self._model.X), 3.6) + self.assertAlmostEqual(value(self._model.Y), 0.8) + + def test_only_specified_vars_are_loaded(self): + self.assertTrue(self._model.X.stale) + self.assertTrue(self._model.Y.stale) + self.assertEqual(value(self._model.X), 0) + self.assertEqual(value(self._model.Y), 0) + + self._opt.load_vars([self._model.X]) + + self.assertFalse(self._model.X.stale) + self.assertTrue(self._model.Y.stale) + self.assertAlmostEqual(value(self._model.X), 3.6) + self.assertEqual(value(self._model.Y), 0) + + self._opt.load_vars([self._model.Y]) + + self.assertFalse(self._model.X.stale) + self.assertFalse(self._model.Y.stale) + self.assertAlmostEqual(value(self._model.X), 3.6) + self.assertAlmostEqual(value(self._model.Y), 0.8) + + +if __name__ == "__main__": + unittest.main() diff --git a/pyomo/solvers/tests/checks/test_SCIPPersistent.py b/pyomo/solvers/tests/checks/test_SCIPPersistent.py new file mode 100644 index 00000000000..0cf1aab65f6 --- /dev/null +++ b/pyomo/solvers/tests/checks/test_SCIPPersistent.py @@ -0,0 +1,318 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2024 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import pyomo.environ +import pyomo.common.unittest as unittest + +from pyomo.core import ( + ConcreteModel, + Var, + Objective, + Constraint, + NonNegativeReals, + NonNegativeIntegers, + Reals, + Binary, + SOSConstraint, + Set, + sin, + cos, + exp, + log, +) +from pyomo.opt import SolverFactory + +try: + import pyscipopt + + scip_available = True +except ImportError: + scip_available = False + + +@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") +class TestQuadraticObjective(unittest.TestCase): + def test_quadratic_objective_linear_surrogate_is_set(self): + m = ConcreteModel() + m.X = Var(bounds=(-2, 2)) + m.Y = Var(bounds=(-2, 2)) + m.Z = Var(within=Reals) + m.O = Objective(expr=m.Z) + m.C1 = Constraint(expr=m.Y >= 2 * m.X - 1) + m.C2 = Constraint(expr=m.Y >= -m.X + 2) + m.C3 = Constraint(expr=m.Z >= m.X**2 + m.Y**2) + opt = SolverFactory("scip_persistent") + opt.set_instance(m) + opt.solve() + + self.assertAlmostEqual(m.X.value, 1, places=3) + self.assertAlmostEqual(m.Y.value, 1, places=3) + + opt.reset() + + opt.remove_constraint(m.C3) + del m.C3 + m.C3 = Constraint(expr=m.Z >= m.X**2) + opt.add_constraint(m.C3) + opt.solve() + self.assertAlmostEqual(m.X.value, 0, places=3) + self.assertAlmostEqual(m.Y.value, 2, places=3) + + def test_add_and_remove_sos(self): + m = ConcreteModel() + m.I = Set(initialize=[1, 2, 3]) + m.X = Var(m.I, bounds=(-2, 2)) + + m.C = SOSConstraint(var=m.X, sos=1) + + m.O = Objective(expr=m.X[1] + m.X[2]) + + opt = SolverFactory("scip_persistent") + + opt.set_instance(m) + opt.solve() + + zero_val_var = 0 + for i in range(1, 4): + if -0.001 < m.X[i].value < 0.001: + zero_val_var += 1 + assert zero_val_var == 2 + + opt.reset() + + opt.remove_sos_constraint(m.C) + del m.C + + m.C = SOSConstraint(var=m.X, sos=2) + opt.add_sos_constraint(m.C) + + opt.solve() + + zero_val_var = 0 + for i in range(1, 4): + if -0.001 < m.X[i].value < 0.001: + zero_val_var += 1 + assert zero_val_var == 1 + + def test_get_and_set_param(self): + m = ConcreteModel() + m.X = Var(bounds=(-2, 2)) + m.O = Objective(expr=m.X) + m.C3 = Constraint(expr=m.X <= 2) + opt = SolverFactory("scip_persistent") + opt.set_instance(m) + + opt.set_scip_param("limits/time", 60) + + assert opt.get_scip_param("limits/time") == 60 + + def test_non_linear(self): + + PI = 3.141592653589793238462643 + NWIRES = 11 + DIAMETERS = [ + 0.207, + 0.225, + 0.244, + 0.263, + 0.283, + 0.307, + 0.331, + 0.362, + 0.394, + 0.4375, + 0.500, + ] + PRELOAD = 300.0 + MAXWORKLOAD = 1000.0 + MAXDEFLECT = 6.0 + DEFLECTPRELOAD = 1.25 + MAXFREELEN = 14.0 + MAXCOILDIAM = 3.0 + MAXSHEARSTRESS = 189000.0 + SHEARMOD = 11500000.0 + + m = ConcreteModel() + m.coil = Var(within=NonNegativeReals) + m.wire = Var(within=NonNegativeReals) + m.defl = Var( + bounds=(DEFLECTPRELOAD / (MAXWORKLOAD - PRELOAD), MAXDEFLECT / PRELOAD) + ) + m.ncoils = Var(within=NonNegativeIntegers) + m.const1 = Var(within=NonNegativeReals) + m.const2 = Var(within=NonNegativeReals) + m.volume = Var(within=NonNegativeReals) + m.I = Set(initialize=[i for i in range(NWIRES)]) + m.y = Var(m.I, within=Binary) + + m.O = Objective(expr=m.volume) + + m.c1 = Constraint( + expr=PI / 2 * (m.ncoils + 2) * m.coil * m.wire**2 - m.volume == 0 + ) + + m.c2 = Constraint(expr=m.coil / m.wire - m.const1 == 0) + + m.c3 = Constraint( + expr=(4 * m.const1 - 1) / (4 * m.const1 - 4) + 0.615 / m.const1 - m.const2 + == 0 + ) + + m.c4 = Constraint( + expr=8.0 * MAXWORKLOAD / PI * m.const1 * m.const2 + - MAXSHEARSTRESS * m.wire**2 + <= 0 + ) + + m.c5 = Constraint( + expr=8 / SHEARMOD * m.ncoils * m.const1**3 / m.wire - m.defl == 0 + ) + + m.c6 = Constraint( + expr=MAXWORKLOAD * m.defl + 1.05 * m.ncoils * m.wire + 2.1 * m.wire + <= MAXFREELEN + ) + + m.c7 = Constraint(expr=m.coil + m.wire <= MAXCOILDIAM) + + m.c8 = Constraint( + expr=sum(m.y[i] * DIAMETERS[i] for i in range(NWIRES)) - m.wire == 0 + ) + + m.c9 = Constraint(expr=sum(m.y[i] for i in range(NWIRES)) == 1) + + opt = SolverFactory("scip_persistent") + opt.set_instance(m) + + opt.solve() + + self.assertAlmostEqual(m.volume.value, 1.6924910128, places=2) + + def test_non_linear_unary_expressions(self): + + m = ConcreteModel() + m.X = Var(bounds=(1, 2)) + m.Y = Var(within=Reals) + + m.O = Objective(expr=m.Y) + + m.C = Constraint(expr=exp(m.X) == m.Y) + + opt = SolverFactory("scip_persistent") + opt.set_instance(m) + + opt.solve() + self.assertAlmostEqual(m.X.value, 1, places=3) + self.assertAlmostEqual(m.Y.value, exp(1), places=3) + + opt.reset() + opt.remove_constraint(m.C) + del m.C + + m.C = Constraint(expr=log(m.X) == m.Y) + opt.add_constraint(m.C) + opt.solve() + self.assertAlmostEqual(m.X.value, 1, places=3) + self.assertAlmostEqual(m.Y.value, 0, places=3) + + opt.reset() + opt.remove_constraint(m.C) + del m.C + + m.C = Constraint(expr=sin(m.X) == m.Y) + opt.add_constraint(m.C) + opt.solve() + self.assertAlmostEqual(m.X.value, 1, places=3) + self.assertAlmostEqual(m.Y.value, sin(1), places=3) + + opt.reset() + opt.remove_constraint(m.C) + del m.C + + m.C = Constraint(expr=cos(m.X) == m.Y) + opt.add_constraint(m.C) + opt.solve() + self.assertAlmostEqual(m.X.value, 2, places=3) + self.assertAlmostEqual(m.Y.value, cos(2), places=3) + + def test_add_column(self): + m = ConcreteModel() + m.x = Var(within=NonNegativeReals) + m.c = Constraint(expr=(0, m.x, 1)) + m.obj = Objective(expr=-m.x) + + opt = SolverFactory("scip_persistent") + opt.set_instance(m) + opt.solve() + self.assertAlmostEqual(m.x.value, 1) + + m.y = Var(within=NonNegativeReals) + + opt.reset() + + opt.add_column(m, m.y, -3, [m.c], [2]) + opt.solve() + + self.assertAlmostEqual(m.x.value, 0) + self.assertAlmostEqual(m.y.value, 0.5) + + def test_add_column_exceptions(self): + m = ConcreteModel() + m.x = Var() + m.c = Constraint(expr=(0, m.x, 1)) + m.ci = Constraint([1, 2], rule=lambda m, i: (0, m.x, i + 1)) + m.cd = Constraint(expr=(0, -m.x, 1)) + m.cd.deactivate() + m.obj = Objective(expr=-m.x) + + opt = SolverFactory("scip_persistent") + + # set_instance not called + self.assertRaises(RuntimeError, opt.add_column, m, m.x, 0, [m.c], [1]) + + opt.set_instance(m) + + m2 = ConcreteModel() + m2.y = Var() + m2.c = Constraint(expr=(0, m.x, 1)) + + # different model than attached to opt + self.assertRaises(RuntimeError, opt.add_column, m2, m2.y, 0, [], []) + # pyomo var attached to different model + self.assertRaises(RuntimeError, opt.add_column, m, m2.y, 0, [], []) + + z = Var() + # pyomo var floating + self.assertRaises(RuntimeError, opt.add_column, m, z, -2, [m.c, z], [1]) + + m.y = Var() + # len(coefficients) == len(constraints) + self.assertRaises(RuntimeError, opt.add_column, m, m.y, -2, [m.c], [1, 2]) + self.assertRaises(RuntimeError, opt.add_column, m, m.y, -2, [m.c, z], [1]) + + # add indexed constraint + self.assertRaises(AttributeError, opt.add_column, m, m.y, -2, [m.ci], [1]) + # add something not a _ConstraintData + self.assertRaises(AttributeError, opt.add_column, m, m.y, -2, [m.x], [1]) + + # constraint not on solver model + self.assertRaises(KeyError, opt.add_column, m, m.y, -2, [m2.c], [1]) + + # inactive constraint + self.assertRaises(KeyError, opt.add_column, m, m.y, -2, [m.cd], [1]) + + opt.add_var(m.y) + # var already in solver model + self.assertRaises(RuntimeError, opt.add_column, m, m.y, -2, [m.c], [1]) + + +if __name__ == "__main__": + unittest.main() diff --git a/pyomo/solvers/tests/solvers.py b/pyomo/solvers/tests/solvers.py index 918a801ae37..3ad944de8d1 100644 --- a/pyomo/solvers/tests/solvers.py +++ b/pyomo/solvers/tests/solvers.py @@ -376,6 +376,27 @@ def test_solver_cases(*args): name='scip', io='nl', capabilities=_scip_capabilities, import_suffixes=[] ) + # + # SCIP PERSISTENT + # + + _scip_persistent_capabilities = set( + [ + "linear", + "integer", + "quadratic_constraint", + "sos1", + "sos2", + ] + ) + + _test_solver_cases["scip_persistent", "python"] = initialize( + name="scip_persistent", + io="python", + capabilities=_scip_persistent_capabilities, + import_suffixes=["slack", "dual", "rc"], + ) + # # CONOPT # From 6a14f108636dc9afb4e854b2fb27512aeb719ad0 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Mon, 18 Mar 2024 17:20:01 +0100 Subject: [PATCH 002/107] Add SCIPPersistent to docs --- doc/OnlineDocs/library_reference/solvers/index.rst | 1 + .../library_reference/solvers/scip_persistent.rst | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 doc/OnlineDocs/library_reference/solvers/scip_persistent.rst diff --git a/doc/OnlineDocs/library_reference/solvers/index.rst b/doc/OnlineDocs/library_reference/solvers/index.rst index 400032df076..628f9cfdab0 100644 --- a/doc/OnlineDocs/library_reference/solvers/index.rst +++ b/doc/OnlineDocs/library_reference/solvers/index.rst @@ -9,3 +9,4 @@ Solver Interfaces gurobi_direct.rst gurobi_persistent.rst xpress_persistent.rst + scip_persistent.rst diff --git a/doc/OnlineDocs/library_reference/solvers/scip_persistent.rst b/doc/OnlineDocs/library_reference/solvers/scip_persistent.rst new file mode 100644 index 00000000000..63ed55b74e3 --- /dev/null +++ b/doc/OnlineDocs/library_reference/solvers/scip_persistent.rst @@ -0,0 +1,7 @@ +SCIPPersistent +================ + +.. autoclass:: pyomo.solvers.plugins.solvers.scip_persistent.SCIPPersistent + :members: + :inherited-members: + :show-inheritance: \ No newline at end of file From c1079090567bbe95b290402a3918c936a0ded576 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 11:48:19 +0100 Subject: [PATCH 003/107] Add SCIp to Github action scripts --- .github/workflows/test_branches.yml | 6 ++++++ .github/workflows/test_pr_and_main.yml | 6 ++++++ pyomo/solvers/plugins/solvers/scip_persistent.py | 5 +++++ 3 files changed, 17 insertions(+) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 55f903a37f9..89e789db5ba 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -268,6 +268,12 @@ jobs: || echo "WARNING: Gurobi is not available" python -m pip install --cache-dir cache/pip xpress \ || echo "WARNING: Xpress Community Edition is not available" + if [[ ${{matrix.python}} == pypy* ]]; then + echo "skipping SCIP for pypy" + else + python -m pip install --cache-dir cache/pip pyscipopt==5.0.0 \ + || echo "WARNING: SCIP is not available" + fi if [[ ${{matrix.python}} == pypy* ]]; then echo "skipping wntr for pypy" else diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index 76ec6de951a..a6cf6ef7eec 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -298,6 +298,12 @@ jobs: || echo "WARNING: Gurobi is not available" python -m pip install --cache-dir cache/pip xpress \ || echo "WARNING: Xpress Community Edition is not available" + if [[ ${{matrix.python}} == pypy* ]]; then + echo "skipping SCIP for pypy" + else + python -m pip install --cache-dir cache/pip pyscipopt==5.0.0 \ + || echo "WARNING: SCIP is not available" + fi if [[ ${{matrix.python}} == pypy* ]]; then echo "skipping wntr for pypy" else diff --git a/pyomo/solvers/plugins/solvers/scip_persistent.py b/pyomo/solvers/plugins/solvers/scip_persistent.py index 408aa84633f..e28c91073ab 100644 --- a/pyomo/solvers/plugins/solvers/scip_persistent.py +++ b/pyomo/solvers/plugins/solvers/scip_persistent.py @@ -182,4 +182,9 @@ def _add_column(self, var, obj_coef, constraints, coefficients): self._solver_model.setObjective(obj_coef * scip_var, sense=sense, clear=False) def reset(self): + """ This function is necessary to call before making any changes to the + SCIP model after optimizing. It frees solution run specific information + that is not automatically done when changes to an already solved model + are made. Making changes to an already optimized model, e.g. adding additional + constraints will raise an error unless this function is called. """ self._solver_model.freeTransform() From e00ded8e33e823b8a9146facff85166291e17d71 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 14:55:52 +0100 Subject: [PATCH 004/107] Remove 5.0.0 specific version. Add conda to workflow --- .github/workflows/test_branches.yml | 4 ++-- .github/workflows/test_pr_and_main.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 89e789db5ba..1d61aaf2d77 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -271,7 +271,7 @@ jobs: if [[ ${{matrix.python}} == pypy* ]]; then echo "skipping SCIP for pypy" else - python -m pip install --cache-dir cache/pip pyscipopt==5.0.0 \ + python -m pip install --cache-dir cache/pip pyscipopt \ || echo "WARNING: SCIP is not available" fi if [[ ${{matrix.python}} == pypy* ]]; then @@ -347,7 +347,7 @@ jobs: if test -z "${{matrix.slim}}"; then PYVER=$(echo "py${{matrix.python}}" | sed 's/\.//g') echo "Installing for $PYVER" - for PKG in 'cplex>=12.10' docplex 'gurobi=10.0.3' xpress cyipopt pymumps scip; do + for PKG in 'cplex>=12.10' docplex 'gurobi=10.0.3' xpress cyipopt pymumps scip pyscipopt; do echo "" echo "*** Install $PKG ***" # conda can literally take an hour to determine that a diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index a6cf6ef7eec..89fd90c41d0 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -301,7 +301,7 @@ jobs: if [[ ${{matrix.python}} == pypy* ]]; then echo "skipping SCIP for pypy" else - python -m pip install --cache-dir cache/pip pyscipopt==5.0.0 \ + python -m pip install --cache-dir cache/pip pyscipopt \ || echo "WARNING: SCIP is not available" fi if [[ ${{matrix.python}} == pypy* ]]; then @@ -376,7 +376,7 @@ jobs: if test -z "${{matrix.slim}}"; then PYVER=$(echo "py${{matrix.python}}" | sed 's/\.//g') echo "Installing for $PYVER" - for PKG in 'cplex>=12.10' docplex 'gurobi=10.0.3' xpress cyipopt pymumps scip; do + for PKG in 'cplex>=12.10' docplex 'gurobi=10.0.3' xpress cyipopt pymumps scip pyscipopt; do echo "" echo "*** Install $PKG ***" # conda can literally take an hour to determine that a From a0b625060217e04aa126dc9f2cb9a410e0968078 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 15:14:54 +0100 Subject: [PATCH 005/107] Standardise string formatting to fstring --- pyomo/solvers/plugins/solvers/scip_direct.py | 35 +++++++------------ .../plugins/solvers/scip_persistent.py | 8 ++--- 2 files changed, 16 insertions(+), 27 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 0aafb596007..e93e5579f26 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -74,7 +74,7 @@ def _init(self): except ImportError: self._python_api_exists = False except Exception as e: - print("Import of pyscipopt failed - SCIP message=" + str(e) + "\n") + print(f"Import of pyscipopt failed - SCIP message={str(e)}\n") self._python_api_exists = False # Note: Undefined capabilities default to None @@ -104,7 +104,7 @@ def _apply_solver(self): if self._keepfiles: # Only save log file when the user wants to keep it. self._solver_model.setLogfile(self._log_file) - print("Solver log file: " + self._log_file) + print(f"Solver log file: {self._log_file}") # Set user specified parameters for key, option in self.options.items(): @@ -257,14 +257,14 @@ def __exit__(self, t, v, traceback): def _set_instance(self, model, kwds={}): DirectOrPersistentSolver._set_instance(self, model, kwds) + self.available() try: self._solver_model = self._scip.Model() except Exception: e = sys.exc_info()[1] msg = ( "Unable to create SCIP model. " - "Have you installed PySCIPOpt correctly?\n\n\t" - + "Error message: {0}".format(e) + f"Have you installed PySCIPOpt correctly?\n\n\t Error message: {e}" ) raise Exception(msg) @@ -275,14 +275,13 @@ def _set_instance(self, model, kwds={}): if var.fixed: if not self._output_fixed_variable_bounds: raise ValueError( - "Encountered a fixed variable (%s) inside " + f"Encountered a fixed variable {var.name} inside " "an active objective or constraint " - "expression on model %s, which is usually " + f"expression on model {self._pyomo_model.name}, which is usually " "indicative of a preprocessing error. Use " "the IO-option 'output_fixed_variable_bounds=True' " "to suppress this error and fix the variable " "by overwriting its bounds in the SCIP instance." - % (var.name, self._pyomo_model.name) ) def _add_block(self, block): @@ -308,14 +307,10 @@ def _add_constraint(self, con): if con.has_lb(): if not is_fixed(con.lower): - raise ValueError( - "Lower bound of constraint {0} is not constant.".format(con) - ) + raise ValueError(f"Lower bound of constraint {con} is not constant.") if con.has_ub(): if not is_fixed(con.upper): - raise ValueError( - "Upper bound of constraint {0} is not constant.".format(con) - ) + raise ValueError(f"Upper bound of constraint {con} is not constant.") if con.equality: scip_cons = self._solver_model.addCons( @@ -335,8 +330,7 @@ def _add_constraint(self, con): ) else: raise ValueError( - "Constraint does not have a lower " - "or an upper bound: {0} \n".format(con) + f"Constraint does not have a lower or an upper bound: {con} \n" ) for var in referenced_vars: @@ -398,9 +392,7 @@ def _scip_vtype_from_var(self, var): elif var.is_continuous(): vtype = "C" else: - raise ValueError( - "Variable domain type is not recognized for {0}".format(var.domain) - ) + raise ValueError(f"Variable domain type is not recognized for {var.domain}") return vtype def _set_objective(self, obj): @@ -418,7 +410,7 @@ def _set_objective(self, obj): elif obj.sense == maximize: sense = "maximize" else: - raise ValueError("Objective sense is not recognized: {0}".format(obj.sense)) + raise ValueError(f"Objective sense is not recognized: {obj.sense}") scip_expr, referenced_vars = self._get_expr_from_pyomo_expr( obj.expr, self._max_obj_degree @@ -455,8 +447,7 @@ def _postsolve(self): flag = True if not flag: raise RuntimeError( - "***The scip_direct solver plugin cannot extract solution suffix=" - + suffix + f"***The scip_direct solver plugin cannot extract solution suffix={suffix}" ) scip = self._solver_model @@ -593,7 +584,7 @@ def _postsolve(self): else: self.results.solver.status = SolverStatus.error self.results.solver.termination_message = ( - "Unhandled SCIP status (" + str(status) + ")" + f"Unhandled SCIP status ({str(status)})" ) self.results.solver.termination_condition = TerminationCondition.error soln.status = SolutionStatus.error diff --git a/pyomo/solvers/plugins/solvers/scip_persistent.py b/pyomo/solvers/plugins/solvers/scip_persistent.py index e28c91073ab..abb85b8dbca 100644 --- a/pyomo/solvers/plugins/solvers/scip_persistent.py +++ b/pyomo/solvers/plugins/solvers/scip_persistent.py @@ -84,9 +84,7 @@ def update_var(self, var): # return if var not in self._pyomo_var_to_solver_var_map: raise ValueError( - "The Var provided to compile_var needs to be added first: {0}".format( - var - ) + f"The Var provided to compile_var needs to be added first: {var}" ) scip_var = self._pyomo_var_to_solver_var_map[var] vtype = self._scip_vtype_from_var(var) @@ -182,9 +180,9 @@ def _add_column(self, var, obj_coef, constraints, coefficients): self._solver_model.setObjective(obj_coef * scip_var, sense=sense, clear=False) def reset(self): - """ This function is necessary to call before making any changes to the + """This function is necessary to call before making any changes to the SCIP model after optimizing. It frees solution run specific information that is not automatically done when changes to an already solved model are made. Making changes to an already optimized model, e.g. adding additional - constraints will raise an error unless this function is called. """ + constraints will raise an error unless this function is called.""" self._solver_model.freeTransform() From d0816eb008bae43eff5bff54390873c98e5b7a1b Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 15:17:29 +0100 Subject: [PATCH 006/107] Add parameter link to docstring --- pyomo/solvers/plugins/solvers/scip_persistent.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyomo/solvers/plugins/solvers/scip_persistent.py b/pyomo/solvers/plugins/solvers/scip_persistent.py index abb85b8dbca..49fe224e72a 100644 --- a/pyomo/solvers/plugins/solvers/scip_persistent.py +++ b/pyomo/solvers/plugins/solvers/scip_persistent.py @@ -116,6 +116,7 @@ def set_scip_param(self, param, val): param: str The SCIP parameter to set. Options include any SCIP parameter. Please see the SCIP documentation for options. + Link at: https://www.scipopt.org/doc/html/PARAMETERS.php val: any The value to set the parameter to. See SCIP documentation for possible values. """ @@ -129,6 +130,7 @@ def get_scip_param(self, param): ---------- param: str or int or float The SCIP parameter to get the value of. See SCIP documentation for possible options. + Link at: https://www.scipopt.org/doc/html/PARAMETERS.php """ return self._solver_model.getParam(param) From 0e11f112161b6947ddf30d6530d6167ef174cf44 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 15:20:07 +0100 Subject: [PATCH 007/107] Remove redundant second objective sense check --- pyomo/solvers/plugins/solvers/scip_direct.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index e93e5579f26..c6285ff53cb 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -591,15 +591,6 @@ def _postsolve(self): self.results.problem.name = scip.getProbName() - if scip.getObjectiveSense() == "minimize": - self.results.problem.sense = minimize - elif scip.getObjectiveSense() == "maximize": - self.results.problem.sense = maximize - else: - raise RuntimeError( - f"Unrecognized SCIP objective sense: {scip.getObjectiveSense()}" - ) - self.results.problem.upper_bound = None self.results.problem.lower_bound = None if scip.getNSols() > 0: From 068ec99277321743b611c5323be3858a5b505ce8 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 15:35:54 +0100 Subject: [PATCH 008/107] Clean up _post_solve with a helper function for status handling --- pyomo/solvers/plugins/solvers/scip_direct.py | 109 +++++++++++-------- 1 file changed, 63 insertions(+), 46 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index c6285ff53cb..9074d40870f 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -382,8 +382,16 @@ def _add_sos_constraint(self, con): def _scip_vtype_from_var(self, var): """ This function takes a pyomo variable and returns the appropriate SCIP variable type - :param var: pyomo.core.base.var.Var - :return: B, I, or C + + Parameters + ---------- + var: pyomo.core.base.var.Var + The pyomo variable that we want to retrieve the SCIP vtype of + + Returns + ------- + vtype: str + B for Binary, I for Integer, or C for Continuous """ if var.is_binary(): vtype = "B" @@ -425,52 +433,12 @@ def _set_objective(self, obj): self._needs_updated = True - def _postsolve(self): - # the only suffixes that we extract from SCIP are - # constraint duals, constraint slacks, and variable - # reduced-costs. scan through the solver suffix list - # and throw an exception if the user has specified - # any others. - extract_duals = False - extract_slacks = False - extract_reduced_costs = False - for suffix in self._suffixes: - flag = False - if re.match(suffix, "dual"): - extract_duals = True - flag = True - if re.match(suffix, "slack"): - extract_slacks = True - flag = True - if re.match(suffix, "rc"): - extract_reduced_costs = True - flag = True - if not flag: - raise RuntimeError( - f"***The scip_direct solver plugin cannot extract solution suffix={suffix}" - ) - - scip = self._solver_model + def _get_solver_solution_status(self, scip, soln): + """ """ + # Get the status of the SCIP Model currently status = scip.getStatus() - scip_vars = scip.getVars() - n_bin_vars = sum([scip_var.vtype() == "BINARY" for scip_var in scip_vars]) - n_int_vars = sum([scip_var.vtype() == "INTEGER" for scip_var in scip_vars]) - n_con_vars = sum([scip_var.vtype() == "CONTINUOUS" for scip_var in scip_vars]) - - if n_bin_vars + n_int_vars > 0: - if extract_reduced_costs: - logger.warning("Cannot get reduced costs for MIP.") - if extract_duals: - logger.warning("Cannot get duals for MIP.") - extract_reduced_costs = False - extract_duals = False - - self.results = SolverResults() - soln = Solution() - - self.results.solver.name = f"SCIP{self._version}" - self.results.solver.wallclock_time = scip.getSolvingTime() + # Go through each potential case and update appropriately if scip.getStage() == 1: # SCIP Model is created but not yet optimized self.results.solver.status = SolverStatus.aborted self.results.solver.termination_message = ( @@ -588,6 +556,55 @@ def _postsolve(self): ) self.results.solver.termination_condition = TerminationCondition.error soln.status = SolutionStatus.error + return soln + + def _postsolve(self): + # the only suffixes that we extract from SCIP are + # constraint duals, constraint slacks, and variable + # reduced-costs. scan through the solver suffix list + # and throw an exception if the user has specified + # any others. + extract_duals = False + extract_slacks = False + extract_reduced_costs = False + for suffix in self._suffixes: + flag = False + if re.match(suffix, "dual"): + extract_duals = True + flag = True + if re.match(suffix, "slack"): + extract_slacks = True + flag = True + if re.match(suffix, "rc"): + extract_reduced_costs = True + flag = True + if not flag: + raise RuntimeError( + f"***The scip_direct solver plugin cannot extract solution suffix={suffix}" + ) + + scip = self._solver_model + status = scip.getStatus() + scip_vars = scip.getVars() + n_bin_vars = sum([scip_var.vtype() == "BINARY" for scip_var in scip_vars]) + n_int_vars = sum([scip_var.vtype() == "INTEGER" for scip_var in scip_vars]) + n_con_vars = sum([scip_var.vtype() == "CONTINUOUS" for scip_var in scip_vars]) + + if n_bin_vars + n_int_vars > 0: + if extract_reduced_costs: + logger.warning("Cannot get reduced costs for MIP.") + if extract_duals: + logger.warning("Cannot get duals for MIP.") + extract_reduced_costs = False + extract_duals = False + + self.results = SolverResults() + soln = Solution() + + self.results.solver.name = f"SCIP{self._version}" + self.results.solver.wallclock_time = scip.getSolvingTime() + + soln = self._get_solver_solution_status(scip, soln) self.results.problem.name = scip.getProbName() From 63af6d8ce13a28b50a59c3d9a027d57db64d8ca6 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 15:38:45 +0100 Subject: [PATCH 009/107] Remove individual skip_test option --- pyomo/solvers/tests/checks/test_SCIPDirect.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pyomo/solvers/tests/checks/test_SCIPDirect.py b/pyomo/solvers/tests/checks/test_SCIPDirect.py index ee37f5ddcc8..cc9e114fed1 100644 --- a/pyomo/solvers/tests/checks/test_SCIPDirect.py +++ b/pyomo/solvers/tests/checks/test_SCIPDirect.py @@ -37,6 +37,7 @@ scip_available = False +@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") class SCIPDirectTests(unittest.TestCase): def setUp(self): self.stderr = sys.stderr @@ -45,7 +46,6 @@ def setUp(self): def tearDown(self): sys.stderr = self.stderr - @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") def test_infeasible_lp(self): with SolverFactory("scip_direct", solver_io="python") as opt: model = ConcreteModel() @@ -60,7 +60,6 @@ def test_infeasible_lp(self): results.solver.termination_condition, TerminationCondition.infeasible ) - @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") def test_unbounded_lp(self): with SolverFactory("scip_direct", solver_io="python") as opt: model = ConcreteModel() @@ -77,7 +76,6 @@ def test_unbounded_lp(self): ), ) - @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") def test_optimal_lp(self): with SolverFactory("scip_direct", solver_io="python") as opt: model = ConcreteModel() @@ -88,7 +86,6 @@ def test_optimal_lp(self): self.assertEqual(results.solution.status, SolutionStatus.optimal) - @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") def test_get_duals_lp(self): with SolverFactory("scip_direct", solver_io="python") as opt: model = ConcreteModel() @@ -108,7 +105,6 @@ def test_get_duals_lp(self): self.assertAlmostEqual(model.dual[model.C1], 0.4) self.assertAlmostEqual(model.dual[model.C2], 0.2) - @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") def test_infeasible_mip(self): with SolverFactory("scip_direct", solver_io="python") as opt: model = ConcreteModel() @@ -123,7 +119,6 @@ def test_infeasible_mip(self): results.solver.termination_condition, TerminationCondition.infeasible ) - @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") def test_unbounded_mip(self): with SolverFactory("scip_direct", solver_io="python") as opt: model = AbstractModel() @@ -141,7 +136,6 @@ def test_unbounded_mip(self): ), ) - @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") def test_optimal_mip(self): with SolverFactory("scip_direct", solver_io="python") as opt: model = ConcreteModel() From 4075d9ad8ed1200c3d06e74ca5d3a678dd4e7239 Mon Sep 17 00:00:00 2001 From: Mark Turner <64978342+Opt-Mucca@users.noreply.github.com> Date: Tue, 19 Mar 2024 15:48:02 +0100 Subject: [PATCH 010/107] Update pyomo/solvers/plugins/solvers/scip_persistent.py Co-authored-by: Miranda Mundt <55767766+mrmundt@users.noreply.github.com> --- pyomo/solvers/plugins/solvers/scip_persistent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/solvers/plugins/solvers/scip_persistent.py b/pyomo/solvers/plugins/solvers/scip_persistent.py index 49fe224e72a..572a1b638e0 100644 --- a/pyomo/solvers/plugins/solvers/scip_persistent.py +++ b/pyomo/solvers/plugins/solvers/scip_persistent.py @@ -96,7 +96,7 @@ def update_var(self, var): def write(self, filename, filetype=""): """ - Write the model to a file (e.g., and lp file). + Write the model to a file (e.g., an lp file). Parameters ---------- From f55fcc5aaa329049a6ac521738537fb8498f1a36 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 15:52:10 +0100 Subject: [PATCH 011/107] Update from the black command --- pyomo/solvers/tests/solvers.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pyomo/solvers/tests/solvers.py b/pyomo/solvers/tests/solvers.py index 3ad944de8d1..1a5c1671f19 100644 --- a/pyomo/solvers/tests/solvers.py +++ b/pyomo/solvers/tests/solvers.py @@ -381,13 +381,7 @@ def test_solver_cases(*args): # _scip_persistent_capabilities = set( - [ - "linear", - "integer", - "quadratic_constraint", - "sos1", - "sos2", - ] + ["linear", "integer", "quadratic_constraint", "sos1", "sos2"] ) _test_solver_cases["scip_persistent", "python"] = initialize( From 91eae7b573624fe35e5fb579debde19efbed740a Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 16:37:13 +0100 Subject: [PATCH 012/107] Fix typos --- pyomo/solvers/plugins/solvers/scip_direct.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 9074d40870f..5ba3395d1d2 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -94,7 +94,7 @@ def _init(self): def _apply_solver(self): StaleFlagManager.mark_all_as_stale() - # Supress solver output if requested + # Suppress solver output if requested if self._tee: self._solver_model.hideOutput(quiet=False) else: @@ -179,7 +179,7 @@ def get_nl_expr_recursively(pyomo_expr): elif isinstance(pyomo_expr, DivisionExpression): if len(scip_expr_list) != 2: raise ValueError( - f"DivisonExpression has {len(scip_expr_list)} many terms instead of two!" + f"DivisionExpression has {len(scip_expr_list)} many terms instead of two!" ) return scip_expr_list[0] / scip_expr_list[1] elif isinstance(pyomo_expr, UnaryFunctionExpression): From 5ee2007a05921ef90a5ef9b31e76a877c9899007 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 17:28:07 +0100 Subject: [PATCH 013/107] Replace trySol via more safe checkSol --- pyomo/solvers/plugins/solvers/scip_direct.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 5ba3395d1d2..25c668a0a06 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -729,7 +729,14 @@ def _warm_start(self): for pyomo_var, scip_var in self._pyomo_var_to_solver_var_expr_map.items(): if pyomo_var.value is not None: scip_sol[scip_var] = value(pyomo_var) - self._solver_model.trySol(scip_sol, free=True) + feasible = self._solver_model.checkSol(scip_sol) + if feasible: + self._solver_model.addSol(scip_sol) + del scip_sol + else: + logger.warning("Warm start solution was not accepted by SCIP") + self._solver_model.freeSol(scip_sol) + del scip_sol def _load_vars(self, vars_to_load=None): var_map = self._pyomo_var_to_solver_var_expr_map From f6ff0923ba9dbbc2bfd04990846c1446ca9e9ed8 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Wed, 20 Mar 2024 10:01:01 +0100 Subject: [PATCH 014/107] Adds support for partial solution loading --- pyomo/solvers/plugins/solvers/scip_direct.py | 24 +++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 25c668a0a06..0d4ad722459 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -725,18 +725,30 @@ def warm_start_capable(self): return True def _warm_start(self): - scip_sol = self._solver_model.createSol() + partial_sol = False + for pyomo_var in self._pyomo_var_to_solver_var_expr_map: + if pyomo_var.value is None: + partial_sol = True + break + if partial_sol: + scip_sol = self._solver_model.createPartialSol() + else: + scip_sol = self._solver_model.createSol() for pyomo_var, scip_var in self._pyomo_var_to_solver_var_expr_map.items(): if pyomo_var.value is not None: scip_sol[scip_var] = value(pyomo_var) - feasible = self._solver_model.checkSol(scip_sol) - if feasible: + if partial_sol: self._solver_model.addSol(scip_sol) del scip_sol else: - logger.warning("Warm start solution was not accepted by SCIP") - self._solver_model.freeSol(scip_sol) - del scip_sol + feasible = self._solver_model.checkSol(scip_sol) + if feasible: + self._solver_model.addSol(scip_sol) + del scip_sol + else: + logger.warning("Warm start solution was not accepted by SCIP") + self._solver_model.freeSol(scip_sol) + del scip_sol def _load_vars(self, vars_to_load=None): var_map = self._pyomo_var_to_solver_var_expr_map From e7ac980a6e58a9b1035bafc38cf3ee55420900b8 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Thu, 21 Mar 2024 09:58:24 +0100 Subject: [PATCH 015/107] Add error handling for setting non-linear objective --- pyomo/solvers/plugins/solvers/scip_direct.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 0d4ad722459..456b370eff1 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -134,6 +134,15 @@ def _apply_solver(self): def _get_expr_from_pyomo_repn(self, repn, max_degree=None): referenced_vars = ComponentSet() + degree = repn.polynomial_degree() + if (max_degree is not None) and (degree > max_degree): + raise DegreeError( + "While SCIP supports general non-linear constraints, the objective must be linear. " + "Please reformulate the objective by introducing a new variable. " + "For min problems: min z s.t z >= f(x). For max problems: max z s.t z <= f(x). " + "f(x) is the original non-linear objective." + ) + new_expr = repn.constant if len(repn.linear_vars) > 0: From 2540f650df319bcca59cf26d0bc524a7fab7de8c Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Fri, 19 Apr 2024 18:44:29 +0200 Subject: [PATCH 016/107] Remove dual and rc loading for SCIP. Fix bug of ranged rows --- pyomo/solvers/plugins/solvers/scip_direct.py | 151 ++++++------------ .../plugins/solvers/scip_persistent.py | 21 ++- pyomo/solvers/tests/solvers.py | 2 +- 3 files changed, 73 insertions(+), 101 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 456b370eff1..04440b59f9b 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -86,10 +86,12 @@ def _init(self): self._capabilities.integer = True self._capabilities.sos1 = True self._capabilities.sos2 = True + self._skip_trivial_constraints = True # Dictionary used exclusively for SCIP, as we want the constraint expressions self._pyomo_var_to_solver_var_expr_map = ComponentMap() self._pyomo_con_to_solver_con_expr_map = dict() + self._pyomo_con_to_solver_expr_map = dict() def _apply_solver(self): StaleFlagManager.mark_all_as_stale() @@ -239,6 +241,7 @@ def _scip_lb_ub_from_var(self, var): ub = value(var.ub) else: ub = self._solver_model.infinity() + return lb, ub def _add_var(self, var): @@ -327,7 +330,10 @@ def _add_constraint(self, con): ) elif con.has_lb() and con.has_ub(): scip_cons = self._solver_model.addCons( - value(con.lower) <= (scip_expr <= value(con.upper)), name=conname + value(con.lower) <= scip_expr, name=conname + ) + self._solver_model.chgRhs( + scip_cons, value(con.upper) - value(con.body.constant) ) elif con.has_lb(): scip_cons = self._solver_model.addCons( @@ -346,6 +352,7 @@ def _add_constraint(self, con): self._referenced_variables[var] += 1 self._vars_referenced_by_con[con] = referenced_vars self._pyomo_con_to_solver_con_expr_map[con] = scip_cons + self._pyomo_con_to_solver_expr_map[con] = scip_expr self._pyomo_con_to_solver_con_map[con] = scip_cons.name self._solver_con_to_pyomo_con_map[conname] = con @@ -440,8 +447,6 @@ def _set_objective(self, obj): self._objective = obj self._vars_referenced_by_obj = referenced_vars - self._needs_updated = True - def _get_solver_solution_status(self, scip, soln): """ """ # Get the status of the SCIP Model currently @@ -569,24 +574,17 @@ def _get_solver_solution_status(self, scip, soln): def _postsolve(self): # the only suffixes that we extract from SCIP are - # constraint duals, constraint slacks, and variable - # reduced-costs. scan through the solver suffix list + # constraint slacks. constraint duals and variable + # reduced-costs were removed as in SCIP they contain + # too many caveats. scan through the solver suffix list # and throw an exception if the user has specified # any others. - extract_duals = False extract_slacks = False - extract_reduced_costs = False for suffix in self._suffixes: flag = False - if re.match(suffix, "dual"): - extract_duals = True - flag = True if re.match(suffix, "slack"): extract_slacks = True flag = True - if re.match(suffix, "rc"): - extract_reduced_costs = True - flag = True if not flag: raise RuntimeError( f"***The scip_direct solver plugin cannot extract solution suffix={suffix}" @@ -599,14 +597,6 @@ def _postsolve(self): n_int_vars = sum([scip_var.vtype() == "INTEGER" for scip_var in scip_vars]) n_con_vars = sum([scip_var.vtype() == "CONTINUOUS" for scip_var in scip_vars]) - if n_bin_vars + n_int_vars > 0: - if extract_reduced_costs: - logger.warning("Cannot get reduced costs for MIP.") - if extract_duals: - logger.warning("Cannot get duals for MIP.") - extract_reduced_costs = False - extract_duals = False - self.results = SolverResults() soln = Solution() @@ -667,6 +657,7 @@ def _postsolve(self): This code in this if statement is only needed for backwards compatibility. It is more efficient to set _save_results to False and use load_vars, load_duals, etc. """ + if scip.getNSols() > 0: soln_variables = soln.variable soln_constraints = soln.constraint @@ -683,42 +674,35 @@ def _postsolve(self): if self._referenced_variables[pyomo_var] > 0: soln_variables[name] = {"Value": val} - if extract_reduced_costs: - vals = [scip.getVarRedcost(scip_var) for scip_var in scip_vars] - for scip_var, val, name in zip(scip_vars, vals, scip_var_names): - pyomo_var = self._solver_var_to_pyomo_var_map[name] - if self._referenced_variables[pyomo_var] > 0: - soln_variables[name]["Rc"] = val - - if extract_duals or extract_slacks: - scip_cons = scip.getConss() - con_names = [cons.name for cons in scip_cons] - assert set(self._solver_con_to_pyomo_con_map.keys()) == set( - con_names - ) - for name in con_names: - soln_constraints[name] = {} - - if extract_duals: - vals = [scip.getDualSolVal(con) for con in scip_cons] - for val, name in zip(vals, con_names): - soln_constraints[name]["Dual"] = val - if extract_slacks: - vals = [scip.getSlack(con, scip_sol) for con in scip_cons] - for val, name in zip(vals, con_names): - soln_constraints[name]["Slack"] = val + scip_cons = list(self._pyomo_con_to_solver_con_expr_map.values()) + con_names = [cons.name for cons in scip_cons] + if set(self._solver_con_to_pyomo_con_map.keys()) != set(con_names): + raise AssertionError( + f"{set(self._solver_con_to_pyomo_con_map.keys())}, {set(con_names)}" + ) + for cons in scip_cons: + if cons.getConshdlrName() in ["linear", "nonlinear"]: + soln_constraints[cons.name] = {} + pyomo_con = self._solver_con_to_pyomo_con_map[cons.name] + scip_expr = self._pyomo_con_to_solver_expr_map[pyomo_con] + activity = scip_sol[scip_expr] + if pyomo_con.has_lb(): + lhs = value(pyomo_con.lower) + else: + lhs = -1e20 + if pyomo_con.has_ub(): + rhs = value(pyomo_con.upper) + else: + rhs = 1e20 + soln_constraints[cons.name]["Slack"] = min( + activity - lhs, rhs - activity + ) elif self._load_solutions: if scip.getNSols() > 0: self.load_vars() - if extract_reduced_costs: - self._load_rc() - - if extract_duals: - self._load_duals() - if extract_slacks: self._load_slacks() @@ -773,65 +757,36 @@ def _load_vars(self, vars_to_load=None): var.set_value(val, skip_validation=True) def _load_rc(self, vars_to_load=None): - if not hasattr(self._pyomo_model, "rc"): - self._pyomo_model.rc = Suffix(direction=Suffix.IMPORT) - var_map = self._pyomo_var_to_solver_var_expr_map - ref_vars = self._referenced_variables - rc = self._pyomo_model.rc - if vars_to_load is None: - vars_to_load = var_map.keys() - - scip_vars_to_load = [var_map[pyomo_var] for pyomo_var in vars_to_load] - vals = [ - self._solver_model.getVarRedcost(scip_var) for scip_var in scip_vars_to_load - ] - - for var, val in zip(vars_to_load, vals): - if ref_vars[var] > 0: - rc[var] = val + raise NotImplementedError( + "SCIP via Pyomo does not support reduced cost loading." + ) def _load_duals(self, cons_to_load=None): - if not hasattr(self._pyomo_model, "dual"): - self._pyomo_model.dual = Suffix(direction=Suffix.IMPORT) - con_map = self._pyomo_con_to_solver_con_map - reverse_con_map = self._solver_con_to_pyomo_con_map - dual = self._pyomo_model.dual - scip_cons = self._solver_model.getConss() - - if cons_to_load is None: - con_names = [con.name for con in scip_cons] - vals = [self._solver_model.getDualSolVal(con) for con in scip_cons] - else: - con_names = set([con_map[pyomo_con] for pyomo_con in cons_to_load]) - scip_cons_to_load = [con for con in scip_cons if con.name in con_names] - vals = [self._solver_model.getDualSolVal(con) for con in scip_cons_to_load] - - for i, con_name in enumerate(con_names): - pyomo_con = reverse_con_map[con_name] - dual[pyomo_con] = vals[i] + raise NotImplementedError( + "SCIP via Pyomo does not support dual solution loading" + ) def _load_slacks(self, cons_to_load=None): if not hasattr(self._pyomo_model, "slack"): self._pyomo_model.slack = Suffix(direction=Suffix.IMPORT) - con_map = self._pyomo_con_to_solver_con_map - reverse_con_map = self._solver_con_to_pyomo_con_map slack = self._pyomo_model.slack - scip_cons = self._solver_model.getConss() scip_sol = self._solver_model.getBestSol() if cons_to_load is None: - con_names = [con.name for con in scip_cons] - vals = [self._solver_model.getSlack(con, scip_sol) for con in scip_cons] + scip_cons = list(self._pyomo_con_to_solver_con_expr_map.values()) else: - con_names = set([con_map[pyomo_con] for pyomo_con in cons_to_load]) - scip_cons_to_load = [con for con in scip_cons if con.name in con_names] - vals = [ - self._solver_model.getSlack(con, scip_sol) for con in scip_cons_to_load + scip_cons = [ + self._pyomo_con_to_solver_con_expr_map[pyomo_cons] + for pyomo_cons in cons_to_load ] - - for i, con_name in enumerate(con_names): - pyomo_con = reverse_con_map[con_name] - slack[pyomo_con] = vals[i] + for cons in scip_cons: + if cons.getConshdlrName() in ["linear", "nonlinear"]: + pyomo_con = self._solver_con_to_pyomo_con_map[cons.name] + scip_expr = self._pyomo_con_to_solver_expr_map[pyomo_con] + activity = scip_sol[scip_expr] + rhs = self._solver_model.getRhs(cons) + lhs = self._solver_model.getLhs(cons) + slack[pyomo_con] = min(activity - lhs, rhs - activity) def load_duals(self, cons_to_load=None): """ diff --git a/pyomo/solvers/plugins/solvers/scip_persistent.py b/pyomo/solvers/plugins/solvers/scip_persistent.py index 572a1b638e0..880380ced1f 100644 --- a/pyomo/solvers/plugins/solvers/scip_persistent.py +++ b/pyomo/solvers/plugins/solvers/scip_persistent.py @@ -8,7 +8,6 @@ # rights in this software. # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ - from pyomo.solvers.plugins.solvers.scip_direct import SCIPDirect from pyomo.solvers.plugins.solvers.persistent_solver import PersistentSolver from pyomo.opt.base import SolverFactory @@ -50,16 +49,34 @@ def _remove_constraint(self, solver_conname): con = self._solver_con_to_pyomo_con_map[solver_conname] scip_con = self._pyomo_con_to_solver_con_expr_map[con] self._solver_model.delCons(scip_con) - + for var in self._vars_reference_by_con[con]: + self._references_vars[var] -= 1 + del self._vars_reference_by_con[con] + del self._pyomo_con_to_solver_con_map[con] + del self._pyomo_con_to_solver_con_expr_map[con] + del self._pyomo_con_to_solver_expr_map[con] + del self._solver_con_to_pyomo_con_map[solver_conname] + + def _remove_sos_constraint(self, solver_sos_conname): con = self._solver_con_to_pyomo_con_map[solver_sos_conname] scip_con = self._pyomo_con_to_solver_con_expr_map[con] self._solver_model.delCons(scip_con) + for var in self._vars_reference_by_con[con]: + self._references_vars[var] -= 1 + del self._vars_reference_by_con[con] + del self._pyomo_con_to_solver_con_map[con] + del self._pyomo_con_to_solver_con_expr_map[con] + del self._solver_con_to_pyomo_con_map[solver_conname] def _remove_var(self, solver_varname): var = self._solver_var_to_pyomo_var_map[solver_varname] scip_var = self._pyomo_var_to_solver_var_expr_map[var] self._solver_model.delVar(scip_var) + del self._pyomo_var_to_solver_var_expr_map[var] + del self._pyomo_var_to_solver_var_map[var] + del self._solver_var_to_pyomo_var_map[scip_var.name] + del self._referenced_variables[var] def _warm_start(self): SCIPDirect._warm_start(self) diff --git a/pyomo/solvers/tests/solvers.py b/pyomo/solvers/tests/solvers.py index 1a5c1671f19..b66c1ca5af5 100644 --- a/pyomo/solvers/tests/solvers.py +++ b/pyomo/solvers/tests/solvers.py @@ -388,7 +388,7 @@ def test_solver_cases(*args): name="scip_persistent", io="python", capabilities=_scip_persistent_capabilities, - import_suffixes=["slack", "dual", "rc"], + import_suffixes=["slack"], ) # From 9e5d9442ea0900d36629a3f0677eb6c6ce8d7f19 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Fri, 19 Apr 2024 18:48:34 +0200 Subject: [PATCH 017/107] Add safe con.body.constant check --- pyomo/solvers/plugins/solvers/scip_direct.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 04440b59f9b..1b5e81db302 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -332,9 +332,10 @@ def _add_constraint(self, con): scip_cons = self._solver_model.addCons( value(con.lower) <= scip_expr, name=conname ) - self._solver_model.chgRhs( - scip_cons, value(con.upper) - value(con.body.constant) - ) + rhs = value(con.upper) + if hasattr(con.body, "constant"): + rhs -= value(con.body.constant) + self._solver_model.chgRhs(scip_cons, rhs) elif con.has_lb(): scip_cons = self._solver_model.addCons( value(con.lower) <= scip_expr, name=conname From f90dfade88dafd2d150409efd0a216c34578c89d Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Mon, 29 Apr 2024 10:33:26 +0200 Subject: [PATCH 018/107] Remove slack loading for SCIP --- pyomo/solvers/plugins/solvers/scip_direct.py | 72 +++---------------- .../plugins/solvers/scip_persistent.py | 15 ---- 2 files changed, 8 insertions(+), 79 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 1b5e81db302..57cfc213f3d 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -91,7 +91,6 @@ def _init(self): # Dictionary used exclusively for SCIP, as we want the constraint expressions self._pyomo_var_to_solver_var_expr_map = ComponentMap() self._pyomo_con_to_solver_con_expr_map = dict() - self._pyomo_con_to_solver_expr_map = dict() def _apply_solver(self): StaleFlagManager.mark_all_as_stale() @@ -353,7 +352,6 @@ def _add_constraint(self, con): self._referenced_variables[var] += 1 self._vars_referenced_by_con[con] = referenced_vars self._pyomo_con_to_solver_con_expr_map[con] = scip_cons - self._pyomo_con_to_solver_expr_map[con] = scip_expr self._pyomo_con_to_solver_con_map[con] = scip_cons.name self._solver_con_to_pyomo_con_map[conname] = con @@ -574,22 +572,17 @@ def _get_solver_solution_status(self, scip, soln): return soln def _postsolve(self): - # the only suffixes that we extract from SCIP are - # constraint slacks. constraint duals and variable + # Constraint duals and variable # reduced-costs were removed as in SCIP they contain - # too many caveats. scan through the solver suffix list + # too many caveats. Slacks were removed as later + # planned interfaces do not intend to support. + # Scan through the solver suffix list # and throw an exception if the user has specified # any others. - extract_slacks = False for suffix in self._suffixes: - flag = False - if re.match(suffix, "slack"): - extract_slacks = True - flag = True - if not flag: - raise RuntimeError( - f"***The scip_direct solver plugin cannot extract solution suffix={suffix}" - ) + raise RuntimeError( + f"***The scip_direct solver plugin cannot extract solution suffix={suffix}" + ) scip = self._solver_model status = scip.getStatus() @@ -661,8 +654,6 @@ def _postsolve(self): if scip.getNSols() > 0: soln_variables = soln.variable - soln_constraints = soln.constraint - scip_sol = scip.getBestSol() scip_vars = scip.getVars() scip_var_names = [scip_var.name for scip_var in scip_vars] @@ -675,38 +666,10 @@ def _postsolve(self): if self._referenced_variables[pyomo_var] > 0: soln_variables[name] = {"Value": val} - if extract_slacks: - scip_cons = list(self._pyomo_con_to_solver_con_expr_map.values()) - con_names = [cons.name for cons in scip_cons] - if set(self._solver_con_to_pyomo_con_map.keys()) != set(con_names): - raise AssertionError( - f"{set(self._solver_con_to_pyomo_con_map.keys())}, {set(con_names)}" - ) - for cons in scip_cons: - if cons.getConshdlrName() in ["linear", "nonlinear"]: - soln_constraints[cons.name] = {} - pyomo_con = self._solver_con_to_pyomo_con_map[cons.name] - scip_expr = self._pyomo_con_to_solver_expr_map[pyomo_con] - activity = scip_sol[scip_expr] - if pyomo_con.has_lb(): - lhs = value(pyomo_con.lower) - else: - lhs = -1e20 - if pyomo_con.has_ub(): - rhs = value(pyomo_con.upper) - else: - rhs = 1e20 - soln_constraints[cons.name]["Slack"] = min( - activity - lhs, rhs - activity - ) - elif self._load_solutions: if scip.getNSols() > 0: self.load_vars() - if extract_slacks: - self._load_slacks() - self.results.solution.insert(soln) # finally, clean any temporary files registered with the temp file @@ -768,26 +731,7 @@ def _load_duals(self, cons_to_load=None): ) def _load_slacks(self, cons_to_load=None): - if not hasattr(self._pyomo_model, "slack"): - self._pyomo_model.slack = Suffix(direction=Suffix.IMPORT) - slack = self._pyomo_model.slack - scip_sol = self._solver_model.getBestSol() - - if cons_to_load is None: - scip_cons = list(self._pyomo_con_to_solver_con_expr_map.values()) - else: - scip_cons = [ - self._pyomo_con_to_solver_con_expr_map[pyomo_cons] - for pyomo_cons in cons_to_load - ] - for cons in scip_cons: - if cons.getConshdlrName() in ["linear", "nonlinear"]: - pyomo_con = self._solver_con_to_pyomo_con_map[cons.name] - scip_expr = self._pyomo_con_to_solver_expr_map[pyomo_con] - activity = scip_sol[scip_expr] - rhs = self._solver_model.getRhs(cons) - lhs = self._solver_model.getLhs(cons) - slack[pyomo_con] = min(activity - lhs, rhs - activity) + raise NotImplementedError("SCIP via Pyomo does not support slack loading") def load_duals(self, cons_to_load=None): """ diff --git a/pyomo/solvers/plugins/solvers/scip_persistent.py b/pyomo/solvers/plugins/solvers/scip_persistent.py index 880380ced1f..e3fe9e37b5d 100644 --- a/pyomo/solvers/plugins/solvers/scip_persistent.py +++ b/pyomo/solvers/plugins/solvers/scip_persistent.py @@ -49,34 +49,19 @@ def _remove_constraint(self, solver_conname): con = self._solver_con_to_pyomo_con_map[solver_conname] scip_con = self._pyomo_con_to_solver_con_expr_map[con] self._solver_model.delCons(scip_con) - for var in self._vars_reference_by_con[con]: - self._references_vars[var] -= 1 - del self._vars_reference_by_con[con] - del self._pyomo_con_to_solver_con_map[con] del self._pyomo_con_to_solver_con_expr_map[con] - del self._pyomo_con_to_solver_expr_map[con] - del self._solver_con_to_pyomo_con_map[solver_conname] - def _remove_sos_constraint(self, solver_sos_conname): con = self._solver_con_to_pyomo_con_map[solver_sos_conname] scip_con = self._pyomo_con_to_solver_con_expr_map[con] self._solver_model.delCons(scip_con) - for var in self._vars_reference_by_con[con]: - self._references_vars[var] -= 1 - del self._vars_reference_by_con[con] - del self._pyomo_con_to_solver_con_map[con] del self._pyomo_con_to_solver_con_expr_map[con] - del self._solver_con_to_pyomo_con_map[solver_conname] def _remove_var(self, solver_varname): var = self._solver_var_to_pyomo_var_map[solver_varname] scip_var = self._pyomo_var_to_solver_var_expr_map[var] self._solver_model.delVar(scip_var) del self._pyomo_var_to_solver_var_expr_map[var] - del self._pyomo_var_to_solver_var_map[var] - del self._solver_var_to_pyomo_var_map[scip_var.name] - del self._referenced_variables[var] def _warm_start(self): SCIPDirect._warm_start(self) From f703d1f71128a95d509aa9ea0b08d12de2dcb41a Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Mon, 29 Apr 2024 10:38:10 +0200 Subject: [PATCH 019/107] Remove dual loading test for SCIP --- pyomo/solvers/tests/checks/test_SCIPDirect.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/pyomo/solvers/tests/checks/test_SCIPDirect.py b/pyomo/solvers/tests/checks/test_SCIPDirect.py index cc9e114fed1..5863a54bdcb 100644 --- a/pyomo/solvers/tests/checks/test_SCIPDirect.py +++ b/pyomo/solvers/tests/checks/test_SCIPDirect.py @@ -86,25 +86,6 @@ def test_optimal_lp(self): self.assertEqual(results.solution.status, SolutionStatus.optimal) - def test_get_duals_lp(self): - with SolverFactory("scip_direct", solver_io="python") as opt: - model = ConcreteModel() - model.X = Var(within=NonNegativeReals) - model.Y = Var(within=NonNegativeReals) - - model.C1 = Constraint(expr=2 * model.X + model.Y >= 8) - model.C2 = Constraint(expr=model.X + 3 * model.Y >= 6) - - model.O = Objective(expr=model.X + model.Y) - - results = opt.solve(model, suffixes=["dual"], load_solutions=False) - - model.dual = Suffix(direction=Suffix.IMPORT) - model.solutions.load_from(results) - - self.assertAlmostEqual(model.dual[model.C1], 0.4) - self.assertAlmostEqual(model.dual[model.C2], 0.2) - def test_infeasible_mip(self): with SolverFactory("scip_direct", solver_io="python") as opt: model = ConcreteModel() From 5c02d32009990b8054440f0a6049bdf934247a79 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Mon, 29 Apr 2024 10:42:00 +0200 Subject: [PATCH 020/107] Remove slack for suffix in tests --- pyomo/solvers/plugins/solvers/scip_direct.py | 2 -- pyomo/solvers/tests/solvers.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 57cfc213f3d..a965e66362e 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -10,7 +10,6 @@ # ___________________________________________________________________________ import logging -import re import sys from pyomo.common.collections import ComponentSet, ComponentMap, Bunch @@ -36,7 +35,6 @@ from pyomo.opt.results.solution import Solution, SolutionStatus from pyomo.opt.results.solver import TerminationCondition, SolverStatus from pyomo.opt.base import SolverFactory -from pyomo.core.base.suffix import Suffix logger = logging.getLogger("pyomo.solvers") diff --git a/pyomo/solvers/tests/solvers.py b/pyomo/solvers/tests/solvers.py index b66c1ca5af5..ba1530c67cc 100644 --- a/pyomo/solvers/tests/solvers.py +++ b/pyomo/solvers/tests/solvers.py @@ -388,7 +388,7 @@ def test_solver_cases(*args): name="scip_persistent", io="python", capabilities=_scip_persistent_capabilities, - import_suffixes=["slack"], + import_suffixes=[], ) # From 8ebcf88365267e28a5b820eedce12a0d1bf5473c Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Mon, 29 Apr 2024 11:13:28 +0200 Subject: [PATCH 021/107] Remove TODO for nonlinear handling --- pyomo/solvers/plugins/solvers/scip_direct.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index a965e66362e..9061deac6ad 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -161,7 +161,6 @@ def _get_expr_from_pyomo_repn(self, repn, max_degree=None): referenced_vars.add(x) referenced_vars.add(y) - # TODO: Introduce handling on non-linear expressions if repn.nonlinear_expr is not None: def get_nl_expr_recursively(pyomo_expr): From 30d8cc62d5903e9e8dce4f0cabe79f22a5aba495 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Thu, 13 Jun 2024 14:48:43 +0200 Subject: [PATCH 022/107] Skip LP_trivial_constraints for SCIP persistent --- pyomo/solvers/plugins/solvers/scip_direct.py | 6 ++++-- pyomo/solvers/tests/testcases.py | 9 +++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 9061deac6ad..39c3a4fd996 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -67,8 +67,10 @@ def _init(self): self._scip = pyscipopt self._python_api_exists = True - self._version = str(self._scip.Model().version()) - self._version_major = self._version.split(".")[0] + self._version = tuple( + int(k) for k in str(self._scip.Model().version()).split(".") + ) + self._version_major = self._version[0] except ImportError: self._python_api_exists = False except Exception as e: diff --git a/pyomo/solvers/tests/testcases.py b/pyomo/solvers/tests/testcases.py index 6bef40818d9..f586e22b1e1 100644 --- a/pyomo/solvers/tests/testcases.py +++ b/pyomo/solvers/tests/testcases.py @@ -248,6 +248,15 @@ "inside NL files. A ticket has been filed.", ) +# +# SCIP Persistent +# + +ExpectedFailures["scip_persistent", "python", "LP_trivial_constraints"] = ( + lambda v: v <= _trunk_version, + "SCIP does not allow empty constraints with no variables to be added to the Model.", +) + # # BARON # From 30e5e65bfd063b049f950b9f74c0d238187077ca Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Thu, 13 Jun 2024 16:12:24 +0200 Subject: [PATCH 023/107] Add transformation for add_cons with non float/int rhs e.g. np.int --- pyomo/solvers/plugins/solvers/scip_direct.py | 21 ++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 39c3a4fd996..7c26670c2b4 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -318,29 +318,38 @@ def _add_constraint(self, con): if con.has_lb(): if not is_fixed(con.lower): raise ValueError(f"Lower bound of constraint {con} is not constant.") + con_lower = value(con.lower) + if not isinstance(con_lower, (float, int)): + con_lower = float(con_lower) if con.has_ub(): if not is_fixed(con.upper): raise ValueError(f"Upper bound of constraint {con} is not constant.") + con_upper = value(con.upper) + if not isinstance(con_upper, (float, int)): + con_upper = float(con_upper) if con.equality: scip_cons = self._solver_model.addCons( - scip_expr == value(con.lower), name=conname + scip_expr == con_lower, name=conname ) elif con.has_lb() and con.has_ub(): scip_cons = self._solver_model.addCons( - value(con.lower) <= scip_expr, name=conname + con_lower <= scip_expr, name=conname ) - rhs = value(con.upper) + rhs = con_upper if hasattr(con.body, "constant"): - rhs -= value(con.body.constant) + con_constant = value(con.body.constant) + if not isinstance(con_constant, (float, int)): + con_body = float(con_constant) + rhs -= con_constant self._solver_model.chgRhs(scip_cons, rhs) elif con.has_lb(): scip_cons = self._solver_model.addCons( - value(con.lower) <= scip_expr, name=conname + con_lower <= scip_expr, name=conname ) elif con.has_ub(): scip_cons = self._solver_model.addCons( - scip_expr <= value(con.upper), name=conname + scip_expr <= con_upper, name=conname ) else: raise ValueError( From 9104a921c55f9cd170bd1a7e93e1628869de2360 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Thu, 13 Jun 2024 16:35:41 +0200 Subject: [PATCH 024/107] Add warning if type is converted. Tidy up logic --- pyomo/solvers/plugins/solvers/scip_direct.py | 30 ++++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 7c26670c2b4..6ce98d80e27 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -319,38 +319,38 @@ def _add_constraint(self, con): if not is_fixed(con.lower): raise ValueError(f"Lower bound of constraint {con} is not constant.") con_lower = value(con.lower) - if not isinstance(con_lower, (float, int)): + if type(con_lower) != float and type(con_lower) != int: + logger.warning( + f"Constraint {conname} has LHS type {type(value(con.lower))}. " + f"Converting to float as type is not allowed for SCIP." + ) con_lower = float(con_lower) if con.has_ub(): if not is_fixed(con.upper): raise ValueError(f"Upper bound of constraint {con} is not constant.") con_upper = value(con.upper) - if not isinstance(con_upper, (float, int)): + if type(con_upper) != float and type(con_upper) != int: + logger.warning( + f"Constraint {conname} has RHS type {type(value(con.upper))}. " + f"Converting to float as type is not allowed for SCIP." + ) con_upper = float(con_upper) if con.equality: - scip_cons = self._solver_model.addCons( - scip_expr == con_lower, name=conname - ) + scip_cons = self._solver_model.addCons(scip_expr == con_lower, name=conname) elif con.has_lb() and con.has_ub(): - scip_cons = self._solver_model.addCons( - con_lower <= scip_expr, name=conname - ) + scip_cons = self._solver_model.addCons(con_lower <= scip_expr, name=conname) rhs = con_upper if hasattr(con.body, "constant"): con_constant = value(con.body.constant) if not isinstance(con_constant, (float, int)): - con_body = float(con_constant) + con_constant = float(con_constant) rhs -= con_constant self._solver_model.chgRhs(scip_cons, rhs) elif con.has_lb(): - scip_cons = self._solver_model.addCons( - con_lower <= scip_expr, name=conname - ) + scip_cons = self._solver_model.addCons(con_lower <= scip_expr, name=conname) elif con.has_ub(): - scip_cons = self._solver_model.addCons( - scip_expr <= con_upper, name=conname - ) + scip_cons = self._solver_model.addCons(scip_expr <= con_upper, name=conname) else: raise ValueError( f"Constraint does not have a lower or an upper bound: {con} \n" From f3f2d7c0334afbea37e9d8d24d60227f333ee4c1 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Wed, 20 Nov 2024 17:13:03 +0100 Subject: [PATCH 025/107] Fix num. vars and cons from transformed. Silent warm start fail --- pyomo/solvers/plugins/solvers/scip_direct.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 6ce98d80e27..89dd25b86ee 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -641,10 +641,9 @@ def _postsolve(self): except TypeError: soln.gap = None - # TODO: Should these values be of the transformed or the original problem? - self.results.problem.number_of_constraints = scip.getNConss() + self.results.problem.number_of_constraints = scip.getNConss(transformed=False) # self.results.problem.number_of_nonzeros = None - self.results.problem.number_of_variables = scip.getNVars() + self.results.problem.number_of_variables = scip.getNVars(transformed=False) self.results.problem.number_of_binary_variables = n_bin_vars self.results.problem.number_of_integer_variables = n_int_vars self.results.problem.number_of_continuous_variables = n_con_vars @@ -704,16 +703,13 @@ def _warm_start(self): scip_sol[scip_var] = value(pyomo_var) if partial_sol: self._solver_model.addSol(scip_sol) - del scip_sol else: - feasible = self._solver_model.checkSol(scip_sol) + feasible = self._solver_model.checkSol(scip_sol, printreason=not self._tee) if feasible: self._solver_model.addSol(scip_sol) - del scip_sol else: logger.warning("Warm start solution was not accepted by SCIP") self._solver_model.freeSol(scip_sol) - del scip_sol def _load_vars(self, vars_to_load=None): var_map = self._pyomo_var_to_solver_var_expr_map From 7b18354386df3954cf9fc59e5f0ec9b587e316ed Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Wed, 19 Feb 2025 16:38:06 +0100 Subject: [PATCH 026/107] Add minor changes --- pyomo/solvers/plugins/solvers/scip_direct.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 89dd25b86ee..314fce40da5 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -322,19 +322,13 @@ def _add_constraint(self, con): if type(con_lower) != float and type(con_lower) != int: logger.warning( f"Constraint {conname} has LHS type {type(value(con.lower))}. " - f"Converting to float as type is not allowed for SCIP." + f"Converting to float as SCIP fails otherwise." ) con_lower = float(con_lower) if con.has_ub(): if not is_fixed(con.upper): raise ValueError(f"Upper bound of constraint {con} is not constant.") con_upper = value(con.upper) - if type(con_upper) != float and type(con_upper) != int: - logger.warning( - f"Constraint {conname} has RHS type {type(value(con.upper))}. " - f"Converting to float as type is not allowed for SCIP." - ) - con_upper = float(con_upper) if con.equality: scip_cons = self._solver_model.addCons(scip_expr == con_lower, name=conname) @@ -642,7 +636,6 @@ def _postsolve(self): soln.gap = None self.results.problem.number_of_constraints = scip.getNConss(transformed=False) - # self.results.problem.number_of_nonzeros = None self.results.problem.number_of_variables = scip.getNVars(transformed=False) self.results.problem.number_of_binary_variables = n_bin_vars self.results.problem.number_of_integer_variables = n_int_vars From 27e3d108662f554966819c4ea5db52df1ca38ffc Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Fri, 21 Mar 2025 11:26:23 +0100 Subject: [PATCH 027/107] Change copyright 2024 to 2025 --- pyomo/solvers/plugins/solvers/scip_direct.py | 2 +- pyomo/solvers/plugins/solvers/scip_persistent.py | 2 +- pyomo/solvers/tests/checks/test_SCIPDirect.py | 2 +- pyomo/solvers/tests/checks/test_SCIPPersistent.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 314fce40da5..c862d9047c1 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -1,7 +1,7 @@ # ___________________________________________________________________________ # # Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2024 +# Copyright (c) 2008-2025 # National Technology and Engineering Solutions of Sandia, LLC # Under the terms of Contract DE-NA0003525 with National Technology and # Engineering Solutions of Sandia, LLC, the U.S. Government retains certain diff --git a/pyomo/solvers/plugins/solvers/scip_persistent.py b/pyomo/solvers/plugins/solvers/scip_persistent.py index e3fe9e37b5d..bc64edc28a8 100644 --- a/pyomo/solvers/plugins/solvers/scip_persistent.py +++ b/pyomo/solvers/plugins/solvers/scip_persistent.py @@ -1,7 +1,7 @@ # ___________________________________________________________________________ # # Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2024 +# Copyright (c) 2008-2025 # National Technology and Engineering Solutions of Sandia, LLC # Under the terms of Contract DE-NA0003525 with National Technology and # Engineering Solutions of Sandia, LLC, the U.S. Government retains certain diff --git a/pyomo/solvers/tests/checks/test_SCIPDirect.py b/pyomo/solvers/tests/checks/test_SCIPDirect.py index 5863a54bdcb..186de0eaf58 100644 --- a/pyomo/solvers/tests/checks/test_SCIPDirect.py +++ b/pyomo/solvers/tests/checks/test_SCIPDirect.py @@ -1,7 +1,7 @@ # ___________________________________________________________________________ # # Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2024 +# Copyright (c) 2008-2025 # National Technology and Engineering Solutions of Sandia, LLC # Under the terms of Contract DE-NA0003525 with National Technology and # Engineering Solutions of Sandia, LLC, the U.S. Government retains certain diff --git a/pyomo/solvers/tests/checks/test_SCIPPersistent.py b/pyomo/solvers/tests/checks/test_SCIPPersistent.py index 0cf1aab65f6..61cf7385352 100644 --- a/pyomo/solvers/tests/checks/test_SCIPPersistent.py +++ b/pyomo/solvers/tests/checks/test_SCIPPersistent.py @@ -1,7 +1,7 @@ # ___________________________________________________________________________ # # Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2024 +# Copyright (c) 2008-2025 # National Technology and Engineering Solutions of Sandia, LLC # Under the terms of Contract DE-NA0003525 with National Technology and # Engineering Solutions of Sandia, LLC, the U.S. Government retains certain From d16bee5fd1f27cc7c3b55ff858ab28b7478a6975 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 18:00:06 -0600 Subject: [PATCH 028/107] adding tests for trivial constraints and fixing bugs --- pyomo/contrib/observer/model_observer.py | 2 +- .../contrib/solver/common/solution_loader.py | 30 +++++++++++ .../solvers/gurobi/gurobi_direct_base.py | 23 +++++++- .../solvers/gurobi/gurobi_persistent.py | 6 ++- .../solver/tests/solvers/test_solvers.py | 53 +++++++++++++++++++ pyomo/repn/plugins/standard_form.py | 3 +- 6 files changed, 113 insertions(+), 4 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 4ab52100376..325e7a8aee6 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -614,7 +614,7 @@ def _check_for_var_changes(self): vars_to_update.append(v) elif _domain_interval != v.domain.get_interval(): vars_to_update.append(v) - elif v.value != _value: + elif v.fixed and v.value != _value: vars_to_update.append(v) cons_to_update = list(cons_to_update.keys()) return vars_to_update, cons_to_update, update_obj diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index e399d6bea55..be0ea7ad00c 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -17,6 +17,7 @@ from pyomo.core.base.var import VarData from pyomo.core.staleflag import StaleFlagManager from pyomo.core.base.suffix import Suffix +from .util import NoSolutionError def load_import_suffixes(pyomo_model, solution_loader: SolutionLoaderBase, solution_id=None): @@ -194,6 +195,35 @@ def load_import_suffixes(self, solution_id=None): return NotImplemented +class NoSolutionSolutionLoader(SolutionLoaderBase): + def __init__(self) -> None: + pass + + def get_solution_ids(self) -> List[Any]: + return [] + + def get_number_of_solutions(self) -> int: + return 0 + + def load_solution(self, solution_id=None): + raise NoSolutionError() + + def load_vars(self, vars_to_load: Sequence[VarData] | None = None, solution_id=None) -> None: + raise NoSolutionError() + + def get_vars(self, vars_to_load: Sequence[VarData] | None = None, solution_id=None) -> Mapping[VarData, float]: + raise NoSolutionError() + + def get_duals(self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=None) -> Dict[ConstraintData, float]: + raise NoSolutionError() + + def get_reduced_costs(self, vars_to_load: Sequence[VarData] | None = None, solution_id=None) -> Mapping[VarData, float]: + raise NoSolutionError() + + def load_import_suffixes(self, solution_id=None): + raise NoSolutionError() + + class PersistentSolutionLoader(SolutionLoaderBase): """ Loader for persistent solvers diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index e99d24025d5..ce77c31c6f7 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -18,7 +18,7 @@ from pyomo.common.config import ConfigValue from pyomo.common.dependencies import attempt_import from pyomo.common.enums import ObjectiveSense -from pyomo.common.errors import ApplicationError +from pyomo.common.errors import ApplicationError, InfeasibleConstraintException from pyomo.common.shutdown import python_is_shutting_down from pyomo.common.tee import capture_output, TeeStream from pyomo.common.timing import HierarchicalTimer @@ -33,6 +33,7 @@ NoReducedCostsError, NoSolutionError, ) +from pyomo.contrib.solver.common.solution_loader import NoSolutionSolutionLoader from pyomo.contrib.solver.common.results import ( Results, SolutionStatus, @@ -353,6 +354,8 @@ def solve(self, model, **kwds) -> Results: res = self._postsolve( grb_model=gurobi_model, solution_loader=solution_loader, has_obj=has_obj ) + except InfeasibleConstraintException: + res = self._get_infeasible_results() finally: os.chdir(orig_cwd) @@ -390,6 +393,24 @@ def _get_tc_map(self): } return GurobiDirectBase._tc_map + def _get_infeasible_results(self): + res = Results() + res.solution_loader = NoSolutionSolutionLoader() + res.solution_status = SolutionStatus.noSolution + res.termination_condition = TerminationCondition.provenInfeasible + res.incumbent_objective = None + res.objective_bound = None + res.iteration_count = None + res.timing_info.gurobi_time = None + res.solver_config = self.config + res.solver_name = self.name + res.solver_version = self.version() + if self.config.raise_exception_on_nonoptimal_result: + raise NoOptimalSolutionError() + if self.config.load_solutions: + raise NoFeasibleSolutionError() + return res + def _postsolve(self, grb_model, solution_loader, has_obj): status = grb_model.Status diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 9f19bae307f..27a61e27916 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -17,6 +17,7 @@ from pyomo.common.collections import ComponentSet, OrderedSet from pyomo.common.shutdown import python_is_shutting_down from pyomo.common.timing import HierarchicalTimer +from pyomo.common.errors import InfeasibleConstraintException from pyomo.core.base.objective import ObjectiveData from pyomo.core.kernel.objective import minimize, maximize from pyomo.core.base.var import VarData @@ -464,7 +465,9 @@ def _get_expr_from_pyomo_repn(self, repn): vlist = [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars] new_expr = gurobipy.LinExpr(coef_list, vlist) else: - new_expr = 0.0 + # this can't just be zero in case the constraint is a + # trivial one + new_expr = gurobipy.LinExpr() if len(repn.quadratic_vars) > 0: missing_vars = {} @@ -714,6 +717,7 @@ def _add_constraints(self, cons: List[ConstraintData]): for ndx, con in enumerate(cons): lb, body, ub = con.to_bounded_expression(evaluate_bounds=False) repn = generate_standard_repn(body, quadratic=True, compute_values=False) + if len(repn.quadratic_vars) > 0: self._quadratic_cons.add(con) else: diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 9c67ed7b1e5..d154253475c 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -35,6 +35,7 @@ GurobiDirectQuadratic, GurobiPersistent, ) +from pyomo.contrib.solver.common.util import NoSolutionError, NoFeasibleSolutionError, NoOptimalSolutionError from pyomo.contrib.solver.solvers.highs import Highs from pyomo.core.expr.numeric_expr import LinearExpression from pyomo.core.expr.compare import assertExpressionsEqual @@ -1053,6 +1054,58 @@ def test_results_infeasible( ): res.solution_loader.get_reduced_costs() + @parameterized.expand(input=_load_tests(all_solvers)) + def test_trivial_constraints( + self, name: str, opt_class: Type[SolverBase], use_presolve: bool + ): + opt: SolverBase = opt_class() + if not opt.available(): + raise unittest.SkipTest(f'Solver {opt.name} not available.') + if any(name.startswith(i) for i in nl_solvers_set): + if use_presolve: + opt.config.writer_config.linear_presolve = True + else: + opt.config.writer_config.linear_presolve = False + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.obj = pyo.Objective(expr=m.y) + m.c1 = pyo.Constraint(expr=m.y >= m.x) + m.c2 = pyo.Constraint(expr=m.y >= -m.x) + m.c3 = pyo.Constraint(expr=m.x >= 0) + + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, 0) + self.assertAlmostEqual(m.y.value, 0) + + m.x.fix(1) + opt.config.tee = True + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, 1) + self.assertAlmostEqual(m.y.value, 1) + + m.x.fix(-1) + with self.assertRaises(NoOptimalSolutionError): + res = opt.solve(m) + + opt.config.load_solutions = False + opt.config.raise_exception_on_nonoptimal_result = False + res = opt.solve(m) + self.assertNotEqual(res.solution_status, SolutionStatus.optimal) + if isinstance(opt, Ipopt): + acceptable_termination_conditions = { + TerminationCondition.locallyInfeasible, + TerminationCondition.unbounded, + TerminationCondition.provenInfeasible, + } + else: + acceptable_termination_conditions = { + TerminationCondition.provenInfeasible, + TerminationCondition.infeasibleOrUnbounded, + } + self.assertIn(res.termination_condition, acceptable_termination_conditions) + self.assertIsNone(res.incumbent_objective) + @parameterized.expand(input=_load_tests(all_solvers)) def test_duals(self, name: str, opt_class: Type[SolverBase], use_presolve: bool): opt: SolverBase = opt_class() diff --git a/pyomo/repn/plugins/standard_form.py b/pyomo/repn/plugins/standard_form.py index 314a1822e09..59c15910350 100644 --- a/pyomo/repn/plugins/standard_form.py +++ b/pyomo/repn/plugins/standard_form.py @@ -20,6 +20,7 @@ InEnum, document_kwargs_from_configdict, ) +from pyomo.common.errors import InfeasibleConstraintException from pyomo.common.dependencies import scipy, numpy as np from pyomo.common.enums import ObjectiveSense from pyomo.common.gc_manager import PauseGC @@ -462,7 +463,7 @@ def write(self, model): # TODO: add a (configurable) feasibility tolerance if (lb is None or lb <= offset) and (ub is None or ub >= offset): continue - raise InfeasibleError( + raise InfeasibleConstraintException( f"model contains a trivially infeasible constraint, '{con.name}'" ) From d25e7215f4f728ff4541060f84f9b6cd92d09229 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 18:06:51 -0600 Subject: [PATCH 029/107] run black --- .../contrib/solver/common/solution_loader.py | 30 ++++++++++++------- .../solvers/gurobi/gurobi_persistent.py | 2 +- .../solver/tests/solvers/test_solvers.py | 10 +++++-- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index b0bddeb56ba..666ea66e1e9 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -194,25 +194,33 @@ def __init__(self) -> None: def get_solution_ids(self) -> List[Any]: return [] - + def get_number_of_solutions(self) -> int: return 0 - + def load_solution(self, solution_id=None): raise NoSolutionError() - - def load_vars(self, vars_to_load: Sequence[VarData] | None = None, solution_id=None) -> None: + + def load_vars( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None + ) -> None: raise NoSolutionError() - - def get_vars(self, vars_to_load: Sequence[VarData] | None = None, solution_id=None) -> Mapping[VarData, float]: + + def get_vars( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None + ) -> Mapping[VarData, float]: raise NoSolutionError() - - def get_duals(self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=None) -> Dict[ConstraintData, float]: + + def get_duals( + self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=None + ) -> Dict[ConstraintData, float]: raise NoSolutionError() - - def get_reduced_costs(self, vars_to_load: Sequence[VarData] | None = None, solution_id=None) -> Mapping[VarData, float]: + + def get_reduced_costs( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None + ) -> Mapping[VarData, float]: raise NoSolutionError() - + def load_import_suffixes(self, solution_id=None): raise NoSolutionError() diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index c0211604324..b9ea9a6c8e8 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -488,7 +488,7 @@ def _get_expr_from_pyomo_repn(self, repn): vlist = [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars] new_expr = gurobipy.LinExpr(coef_list, vlist) else: - # this can't just be zero in case the constraint is a + # this can't just be zero in case the constraint is a # trivial one new_expr = gurobipy.LinExpr() diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index d154253475c..189b0373780 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -35,7 +35,11 @@ GurobiDirectQuadratic, GurobiPersistent, ) -from pyomo.contrib.solver.common.util import NoSolutionError, NoFeasibleSolutionError, NoOptimalSolutionError +from pyomo.contrib.solver.common.util import ( + NoSolutionError, + NoFeasibleSolutionError, + NoOptimalSolutionError, +) from pyomo.contrib.solver.solvers.highs import Highs from pyomo.core.expr.numeric_expr import LinearExpression from pyomo.core.expr.compare import assertExpressionsEqual @@ -1073,11 +1077,11 @@ def test_trivial_constraints( m.c1 = pyo.Constraint(expr=m.y >= m.x) m.c2 = pyo.Constraint(expr=m.y >= -m.x) m.c3 = pyo.Constraint(expr=m.x >= 0) - + res = opt.solve(m) self.assertAlmostEqual(m.x.value, 0) self.assertAlmostEqual(m.y.value, 0) - + m.x.fix(1) opt.config.tee = True res = opt.solve(m) From 9e7cd0dc9f4679a7b19b4706b2ffcf963bb7dc3f Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 14 Aug 2025 07:15:39 -0600 Subject: [PATCH 030/107] moving scip to contrib solvers --- .../solvers => contrib/solver/solvers/scip}/scip_direct.py | 0 .../solvers => contrib/solver/solvers/scip}/scip_persistent.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename pyomo/{solvers/plugins/solvers => contrib/solver/solvers/scip}/scip_direct.py (100%) rename pyomo/{solvers/plugins/solvers => contrib/solver/solvers/scip}/scip_persistent.py (100%) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py similarity index 100% rename from pyomo/solvers/plugins/solvers/scip_direct.py rename to pyomo/contrib/solver/solvers/scip/scip_direct.py diff --git a/pyomo/solvers/plugins/solvers/scip_persistent.py b/pyomo/contrib/solver/solvers/scip/scip_persistent.py similarity index 100% rename from pyomo/solvers/plugins/solvers/scip_persistent.py rename to pyomo/contrib/solver/solvers/scip/scip_persistent.py From bf204bb2b6d21644fa5dd8f0d83521075541cb9f Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 14 Aug 2025 07:15:57 -0600 Subject: [PATCH 031/107] moving scip to contrib solvers --- pyomo/contrib/solver/solvers/scip/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 pyomo/contrib/solver/solvers/scip/__init__.py diff --git a/pyomo/contrib/solver/solvers/scip/__init__.py b/pyomo/contrib/solver/solvers/scip/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From dabf031e34490ed6dcf00e25eaba68abafc7a022 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 14 Aug 2025 23:36:24 -0600 Subject: [PATCH 032/107] porting scip interface --- .../contrib/solver/common/solution_loader.py | 20 +- .../solvers/gurobi/gurobi_direct_base.py | 1 - .../solver/solvers/scip/scip_direct.py | 1228 ++++++++--------- 3 files changed, 604 insertions(+), 645 deletions(-) diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index 666ea66e1e9..f8723b6e0f4 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -18,6 +18,10 @@ from pyomo.core.staleflag import StaleFlagManager from pyomo.core.base.suffix import Suffix from .util import NoSolutionError +import logging + + +logger = logging.getLogger(__name__) def load_import_suffixes( @@ -33,11 +37,19 @@ def load_import_suffixes( elif suffix.local_name == 'rc': rc_suffix = suffix if dual_suffix is not None: - for k, v in solution_loader.get_duals(solution_id=solution_id).items(): - dual_suffix[k] = v + duals = solution_loader.get_duals(solution_id=solution_id) + if duals is NotImplemented: + logger.warning(f'Cannot load duals into suffix') + else: + for k, v in duals.items(): + dual_suffix[k] = v if rc_suffix is not None: - for k, v in solution_loader.get_reduced_costs(solution_id=solution_id).items(): - rc_suffix[k] = v + rc = solution_loader.get_reduced_costs(solution_id=solution_id) + if rc is NotImplemented: + logger.warning(f'cannot load duals into suffix') + else: + for k, v in rc.items(): + rc_suffix[k] = v class SolutionLoaderBase: diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index ce77c31c6f7..8989fc5047a 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -362,7 +362,6 @@ def solve(self, model, **kwds) -> Results: # hack to work around legacy solver wrapper __setattr__ # otherwise, this would just be self.config = orig_config object.__setattr__(self, 'config', orig_config) - self.config = orig_config res.solver_log = ostreams[0].getvalue() end_timestamp = datetime.datetime.now(datetime.timezone.utc) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index c862d9047c1..b8d4d14a6c1 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -9,392 +9,578 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ +import datetime +import io import logging -import sys - -from pyomo.common.collections import ComponentSet, ComponentMap, Bunch -from pyomo.common.tempfiles import TempfileManager -from pyomo.core import Var +from typing import Tuple, List, Optional, Sequence, Mapping, Dict + +from pyomo.common.collections import ComponentMap +from pyomo.common.numeric_types import native_numeric_types +from pyomo.common.errors import InfeasibleConstraintException, ApplicationError +from pyomo.common.timing import HierarchicalTimer +from pyomo.core.base.block import BlockData +from pyomo.core.base.var import VarData, ScalarVar +from pyomo.core.base.param import ParamData, ScalarParam +from pyomo.core.base.constraint import Constraint, ConstraintData +from pyomo.core.base.sos import SOSConstraint, SOSConstraintData +from pyomo.core.kernel.objective import minimize, maximize from pyomo.core.expr.numeric_expr import ( - SumExpression, - ProductExpression, - UnaryFunctionExpression, + NegationExpression, PowExpression, + ProductExpression, + MonomialTermExpression, DivisionExpression, + SumExpression, + LinearExpression, + UnaryFunctionExpression, + NPV_NegationExpression, + NPV_PowExpression, + NPV_ProductExpression, + NPV_DivisionExpression, + NPV_SumExpression, + NPV_UnaryFunctionExpression, ) -from pyomo.core.expr.numvalue import is_fixed -from pyomo.core.expr.numvalue import value +from pyomo.core.base.expression import ExpressionData, ScalarExpression +from pyomo.core.expr.relational_expr import EqualityExpression, InequalityExpression, RangedExpression from pyomo.core.staleflag import StaleFlagManager -from pyomo.repn import generate_standard_repn -from pyomo.solvers.plugins.solvers.direct_solver import DirectSolver -from pyomo.solvers.plugins.solvers.direct_or_persistent_solver import ( - DirectOrPersistentSolver, +from pyomo.core.expr.visitor import StreamBasedExpressionVisitor +from pyomo.common.dependencies import attempt_import +from pyomo.contrib.solver.common.base import SolverBase, Availability +from pyomo.contrib.solver.common.config import BranchAndBoundConfig +from pyomo.contrib.solver.common.util import ( + NoFeasibleSolutionError, + NoOptimalSolutionError, ) -from pyomo.core.kernel.objective import minimize, maximize -from pyomo.opt.results.results_ import SolverResults -from pyomo.opt.results.solution import Solution, SolutionStatus -from pyomo.opt.results.solver import TerminationCondition, SolverStatus -from pyomo.opt.base import SolverFactory +from pyomo.contrib.solver.common.util import get_objective +from pyomo.contrib.solver.common.solution_loader import NoSolutionSolutionLoader +from pyomo.contrib.solver.common.results import ( + Results, + SolutionStatus, + TerminationCondition, +) +from pyomo.contrib.solver.common.solution_loader import ( + SolutionLoaderBase, + load_import_suffixes, +) +from pyomo.common.config import ConfigValue +from pyomo.common.tee import capture_output, TeeStream -logger = logging.getLogger("pyomo.solvers") +logger = logging.getLogger(__name__) -class DegreeError(ValueError): - pass +scip, scip_available = attempt_import('pyscipyopt') -def _is_numeric(x): - try: - float(x) - except ValueError: - return False - return True +class ScipConfig(BranchAndBoundConfig): + def __init__( + self, + description=None, + doc=None, + implicit=False, + implicit_domain=None, + visibility=0, + ): + BranchAndBoundConfig.__init__( + self, + description=description, + doc=doc, + implicit=implicit, + implicit_domain=implicit_domain, + visibility=visibility, + ) + self.use_mipstart: bool = self.declare( + 'use_mipstart', + ConfigValue( + default=False, + domain=bool, + description="If True, the current values of the integer variables " + "will be passed to Scip.", + ), + ) -@SolverFactory.register("scip_direct", doc="Direct python interface to SCIP") -class SCIPDirect(DirectSolver): +def _handle_var(node, data, opt): + if id(node) not in opt._pyomo_var_to_solver_var_map: + scip_var = opt._add_var(node) + else: + scip_var = opt._pyomo_var_to_solver_var_map[id(node)] + return scip_var - def __init__(self, **kwds): - kwds["type"] = "scipdirect" - DirectSolver.__init__(self, **kwds) - self._init() - self._solver_model = None - def _init(self): - try: - import pyscipopt +def _handle_param(node, data, opt): + if not node.mutable: + return node.value + if id(node) not in opt._pyomo_param_to_solver_param_map: + scip_param = opt._add_param(node) + else: + scip_param = opt._pyomo_param_to_solver_param_map[id(node)] + return scip_param - self._scip = pyscipopt - self._python_api_exists = True - self._version = tuple( - int(k) for k in str(self._scip.Model().version()).split(".") - ) - self._version_major = self._version[0] - except ImportError: - self._python_api_exists = False - except Exception as e: - print(f"Import of pyscipopt failed - SCIP message={str(e)}\n") - self._python_api_exists = False - - # Note: Undefined capabilities default to None - self._max_constraint_degree = None - self._max_obj_degree = 1 - self._capabilities.linear = True - self._capabilities.quadratic_objective = False - self._capabilities.quadratic_constraint = True - self._capabilities.integer = True - self._capabilities.sos1 = True - self._capabilities.sos2 = True - self._skip_trivial_constraints = True - - # Dictionary used exclusively for SCIP, as we want the constraint expressions - self._pyomo_var_to_solver_var_expr_map = ComponentMap() - self._pyomo_con_to_solver_con_expr_map = dict() - - def _apply_solver(self): - StaleFlagManager.mark_all_as_stale() - - # Suppress solver output if requested - if self._tee: - self._solver_model.hideOutput(quiet=False) - else: - self._solver_model.hideOutput(quiet=True) - # Redirect solver output to a logfile if requested - if self._keepfiles: - # Only save log file when the user wants to keep it. - self._solver_model.setLogfile(self._log_file) - print(f"Solver log file: {self._log_file}") +def _handle_float(node, data, opt): + return float(node) - # Set user specified parameters - for key, option in self.options.items(): - try: - key_type = type(self._solver_model.getParam(key)) - except KeyError: - raise ValueError(f"Key {key} is an invalid parameter for SCIP") - if key_type == str: - self._solver_model.setParam(key, option) - else: - if not _is_numeric(option): - raise ValueError( - f"Value {option} for parameter {key} is not a string and can't be converted to float" - ) - self._solver_model.setParam(key, float(option)) - - self._solver_model.optimize() - - # TODO: Check if this is even needed, or if it is sufficient to close the open file - # if self._keepfiles: - # self._solver_model.setLogfile(None) - - # FIXME: can we get a return code indicating if SCIP had a significant failure? - return Bunch(rc=None, log=None) - - def _get_expr_from_pyomo_repn(self, repn, max_degree=None): - referenced_vars = ComponentSet() - - degree = repn.polynomial_degree() - if (max_degree is not None) and (degree > max_degree): - raise DegreeError( - "While SCIP supports general non-linear constraints, the objective must be linear. " - "Please reformulate the objective by introducing a new variable. " - "For min problems: min z s.t z >= f(x). For max problems: max z s.t z <= f(x). " - "f(x) is the original non-linear objective." - ) +def _handle_negation(node, data, opt): + return -data[0] - new_expr = repn.constant - if len(repn.linear_vars) > 0: - referenced_vars.update(repn.linear_vars) - new_expr += sum( - repn.linear_coefs[i] * self._pyomo_var_to_solver_var_expr_map[var] - for i, var in enumerate(repn.linear_vars) - ) +def _handle_pow(node, data, opt): + return data[0] ** data[1] - for i, v in enumerate(repn.quadratic_vars): - x, y = v - new_expr += ( - repn.quadratic_coefs[i] - * self._pyomo_var_to_solver_var_expr_map[x] - * self._pyomo_var_to_solver_var_expr_map[y] - ) - referenced_vars.add(x) - referenced_vars.add(y) - - if repn.nonlinear_expr is not None: - - def get_nl_expr_recursively(pyomo_expr): - if not hasattr(pyomo_expr, "args"): - if not isinstance(pyomo_expr, Var): - return float(pyomo_expr) - else: - referenced_vars.add(pyomo_expr) - return self._pyomo_var_to_solver_var_expr_map[pyomo_expr] - scip_expr_list = [0 for i in range(pyomo_expr.nargs())] - for i in range(pyomo_expr.nargs()): - scip_expr_list[i] = get_nl_expr_recursively(pyomo_expr.args[i]) - if isinstance(pyomo_expr, PowExpression): - if len(scip_expr_list) != 2: - raise ValueError( - f"PowExpression has {len(scip_expr_list)} many terms instead of two!" - ) - return scip_expr_list[0] ** (scip_expr_list[1]) - elif isinstance(pyomo_expr, ProductExpression): - return self._scip.quickprod(scip_expr_list) - elif isinstance(pyomo_expr, SumExpression): - return self._scip.quicksum(scip_expr_list) - elif isinstance(pyomo_expr, DivisionExpression): - if len(scip_expr_list) != 2: - raise ValueError( - f"DivisionExpression has {len(scip_expr_list)} many terms instead of two!" - ) - return scip_expr_list[0] / scip_expr_list[1] - elif isinstance(pyomo_expr, UnaryFunctionExpression): - if len(scip_expr_list) != 1: - raise ValueError( - f"UnaryExpression has {len(scip_expr_list)} many terms instead of one!" - ) - if pyomo_expr.name == "sin": - return self._scip.sin(scip_expr_list[0]) - elif pyomo_expr.name == "cos": - return self._scip.cos(scip_expr_list[0]) - elif pyomo_expr.name == "exp": - return self._scip.exp(scip_expr_list[0]) - elif pyomo_expr.name == "log": - return self._scip.log(scip_expr_list[0]) - else: - raise NotImplementedError( - f"PySCIPOpt through Pyomo does not support the unary function {pyomo_expr.name}" - ) - else: - raise NotImplementedError( - f"PySCIPOpt through Pyomo does not yet support expression type {type(pyomo_expr)}" - ) - new_expr += get_nl_expr_recursively(repn.nonlinear_expr) +def _handle_product(node, data, opt): + assert len(data) == 2 + return data[0] * data[1] + + +def _handle_division(node, data, opt): + return data[0] / data[1] + + +def _handle_sum(node, data, opt): + return sum(data) + + +def _handle_exp(node, data, opt): + return scip.exp(data[0]) + + +def _handle_log(node, data, opt): + return scip.log(data[0]) + + +def _handle_sin(node, data, opt): + return scip.sin(data[0]) + + +def _handle_cos(node, data, opt): + return scip.cos(data[0]) + + +def _handle_sqrt(node, data, opt): + return scip.sqrt(data[0]) + + +def _handle_abs(node, data, opt): + return abs(data[0]) + - return new_expr, referenced_vars +def _handle_tan(node, data, opt): + return scip.sin(data[0]) / scip.cos(data[0]) - def _get_expr_from_pyomo_expr(self, expr, max_degree=None): - if max_degree is None or max_degree >= 2: - repn = generate_standard_repn(expr, quadratic=True) + +_unary_map = { + 'exp': _handle_exp, + 'log': _handle_log, + 'sin': _handle_sin, + 'cos': _handle_cos, + 'sqrt': _handle_sqrt, + 'abs': _handle_abs, + 'tan': _handle_tan, +} + + +def _handle_unary(node, data, opt): + if node.getname() in _unary_map: + return _unary_map[node.getname()](node, data, opt) + else: + raise NotImplementedError(f'unable to handle unary expression: {str(node)}') + + +def _handle_equality(node, data, opt): + return data[0] == data[1] + + +def _handle_ranged(node, data, opt): + return data[0] <= (data[1] <= data[2]) + + +def _handle_inequality(node, data, opt): + return data[0] <= data[1] + + +def _handle_named_expression(node, data, opt): + return data[0] + + +_operator_map = { + NegationExpression: _handle_negation, + PowExpression: _handle_pow, + ProductExpression: _handle_product, + MonomialTermExpression: _handle_product, + DivisionExpression: _handle_division, + SumExpression: _handle_sum, + LinearExpression: _handle_sum, + UnaryFunctionExpression: _handle_unary, + NPV_NegationExpression: _handle_negation, + NPV_PowExpression: _handle_pow, + NPV_ProductExpression: _handle_product, + NPV_DivisionExpression: _handle_division, + NPV_SumExpression: _handle_sum, + NPV_UnaryFunctionExpression: _handle_unary, + EqualityExpression: _handle_equality, + RangedExpression: _handle_ranged, + InequalityExpression: _handle_inequality, + ScalarExpression: _handle_named_expression, + ExpressionData: _handle_named_expression, + VarData: _handle_var, + ScalarVar: _handle_var, + ParamData: _handle_param, + ScalarParam: _handle_param, + float: _handle_float, + int: _handle_float, +} + + +class _PyomoToScipVisitor(StreamBasedExpressionVisitor): + def __init__(self, solver, **kwds): + super().__init__(**kwds) + self.solver = solver + + def exitNode(self, node, data): + nt = type(node) + if nt in _operator_map: + return _operator_map[nt](node, data, self.solver) + elif nt in native_numeric_types: + _operator_map[nt] = _handle_float + return _handle_float(node, data, self.solver) else: - repn = generate_standard_repn(expr, quadratic=False) + raise NotImplementedError(f'unrecognized expression type: {nt}') + + +logger = logging.getLogger("pyomo.solvers") + + +class ScipDirectSolutionLoader(SolutionLoaderBase): + def __init__( + self, + solver_model, + var_id_map, + var_map, + con_map, + pyomo_model, + opt, + ) -> None: + super().__init__() + self._solver_model = solver_model + self._vars = var_id_map + self._var_map = var_map + self._con_map = con_map + self._pyomo_model = pyomo_model + # make sure the scip model does not get freed until the solution loader is garbage collected + self._opt = opt + + def get_number_of_solutions(self) -> int: + return self._solver_model.getNSols() + + def get_solution_ids(self) -> List: + return list(range(self.get_number_of_solutions())) + + def load_vars( + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None + ) -> None: + for v, val in self.get_vars(vars_to_load=vars_to_load, solution_id=solution_id).items(): + v.value = val + + def get_vars( + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None + ) -> Mapping[VarData, float]: + if vars_to_load is None: + vars_to_load = list(self._vars.values()) + if solution_id is None: + solution_id = 0 + sol = self._solver_model.getSols()[solution_id] + res = ComponentMap() + for v in vars_to_load: + sv = self._var_map[id(v)] + res[v] = sol[sv] + return res + + def get_reduced_costs( + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None + ) -> Mapping[VarData, float]: + return NotImplemented + + def get_duals( + self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=None + ) -> Dict[ConstraintData, float]: + return NotImplemented + + def load_import_suffixes(self, solution_id=None): + load_import_suffixes(self._pyomo_model, self, solution_id=solution_id) + + +class SCIPDirect(SolverBase): - scip_expr, referenced_vars = self._get_expr_from_pyomo_repn(repn, max_degree) + _available = None + _tc_map = None + _minimum_version = (5, 5, 0) # this is probably conservative - return scip_expr, referenced_vars + CONFIG = ScipConfig() + + def __init__(self, **kwds): + super().__init__(**kwds) + self._solver_model = None + self._vars = {} # var id to var + self._params = {} # param id to param + self._pyomo_var_to_solver_var_map = {} # var id to scip var + self._pyomo_con_to_solver_con_map = {} + self._pyomo_param_to_solver_param_map = {} # param id to scip var with equal bounds + self._pyomo_sos_to_solver_sos_map = {} + self._expr_visitor = _PyomoToScipVisitor(self) + self._objective = None # pyomo objective + self._obj_var = None # a scip variable because the objective cannot be nonlinear + self._obj_con = None # a scip constraint (obj_var >= obj_expr) + + def _clear(self): + self._solver_model = None + self._vars = {} + self._params = {} + self._pyomo_var_to_solver_var_map = {} + self._pyomo_con_to_solver_con_map = {} + self._pyomo_param_to_solver_param_map = {} + self._pyomo_sos_to_solver_sos_map = {} + self._objective = None + self._obj_var = None + self._obj_con = None + + def available(self) -> Availability: + if self._available is not None: + return self._available + + if not scip_available: + SCIPDirect._available = Availability.NotFound + elif self.version() < self._minimum_version: + SCIPDirect._available = Availability.BadVersion + else: + SCIPDirect._available = Availability.FullLicense + + return self._available + + def version(self) -> Tuple: + return tuple(int(i) for i in scip.__version__) + + def solve(self, model: BlockData, **kwargs) -> Results: + start_timestamp = datetime.datetime.now(datetime.timezone.utc) + orig_config = self.config + if not self.available(): + raise ApplicationError( + f'{self.name} is not available: {self.available()}' + ) + try: + config = self.config(value=kwds, preserve_implicit=True) + + # hack to work around legacy solver wrapper __setattr__ + # otherwise, this would just be self.config = config + object.__setattr__(self, 'config', config) + + StaleFlagManager.mark_all_as_stale() + + if config.timer is None: + config.timer = HierarchicalTimer() + timer = config.timer + + ostreams = [io.StringIO()] + config.tee + + scip_model, solution_loader, has_obj = self._create_solver_model(model) + + scip_model.hideOutput(quiet=False) + if config.threads is not None: + scip_model.setParam('lp/threads', config.threads) + if config.time_limit is not None: + scip_model.setParam('limits/time', config.time_limit) + if config.rel_gap is not None: + scip_model.setParam('limits/gap', config.rel_gap) + if config.abs_gap is not None: + scip_model.setParam('limits/absgap', config.abs_gap) + + if config.use_mipstart: + self._mipstart() + + for key, option in config.solver_options.items(): + scip_model.setParam(key, option) + + timer.start('optimize') + with capture_output(TeeStream(*ostreams), capture_fd=False): + scip_model.optimize() + timer.stop('optimize') + + results = self._postsolve(scip_model, solution_loader, has_obj) + except InfeasibleConstraintException: + results = self._get_infeasible_results() + finally: + # hack to work around legacy solver wrapper __setattr__ + # otherwise, this would just be self.config = orig_config + object.__setattr__(self, 'config', orig_config) + + results.solver_log = ostreams[0].getvalue() + end_timestamp = datetime.datetime.now(datetime.timezone.utc) + results.timing_info.start_timestamp = start_timestamp + results.timing_info.wall_time = (end_timestamp - start_timestamp).total_seconds() + results.timing_info.timer = timer + return results + + def _get_tc_map(self): + if SCIPDirect._tc_map is None: + tc = TerminationCondition + SCIPDirect._tc_map = { + "unknown": tc.unknown, + "userinterrupt": tc.interrupted, + "nodelimit": tc.iterationLimit, + "totalnodelimit": tc.iterationLimit, + "stallnodelimit": tc.iterationLimit, + "timelimit": tc.maxTimeLimit, + "memlimit": tc.unknown, + "gaplimit": tc.convergenceCriteriaSatisfied, # TODO: check this + "primallimit": tc.objectiveLimit, + "duallimit": tc.objectiveLimit, + "sollimit": tc.unknown, + "bestsollimit": tc.unknown, + "restartlimit": tc.unknown, + "optimal": tc.convergenceCriteriaSatisfied, + "infeasible": tc.provenInfeasible, + "unbounded": tc.unbounded, + "inforunbd": tc.infeasibleOrUnbounded, + "terminate": tc.unknown, + } + return SCIPDirect._tc_map + + def _get_infeasible_results(self): + res = Results() + res.solution_loader = NoSolutionSolutionLoader() + res.solution_status = SolutionStatus.noSolution + res.termination_condition = TerminationCondition.provenInfeasible + res.incumbent_objective = None + res.objective_bound = None + res.iteration_count = None + res.timing_info.scip_time = None + res.solver_config = self.config + res.solver_name = self.name + res.solver_version = self.version() + if self.config.raise_exception_on_nonoptimal_result: + raise NoOptimalSolutionError() + if self.config.load_solutions: + raise NoFeasibleSolutionError() + return res def _scip_lb_ub_from_var(self, var): if var.is_fixed(): val = var.value return val, val - if var.has_lb(): - lb = value(var.lb) - else: + + lb, ub = var.bounds() + + if lb is None: lb = -self._solver_model.infinity() - if var.has_ub(): - ub = value(var.ub) - else: + if ub is None: ub = self._solver_model.infinity() return lb, ub def _add_var(self, var): - varname = self._symbol_map.getSymbol(var, self._labeler) vtype = self._scip_vtype_from_var(var) lb, ub = self._scip_lb_ub_from_var(var) - scip_var = self._solver_model.addVar(lb=lb, ub=ub, vtype=vtype, name=varname) - - self._pyomo_var_to_solver_var_expr_map[var] = scip_var - self._pyomo_var_to_solver_var_map[var] = scip_var.name - self._solver_var_to_pyomo_var_map[varname] = var - self._referenced_variables[var] = 0 - - def close(self): + scip_var = self._solver_model.addVar(lb=lb, ub=ub, vtype=vtype) + + self._vars[id(var)] = var + self._pyomo_var_to_solver_var_map[id(var)] = scip_var + return scip_var + + def _add_param(self, p): + vtype = "C" + lb = ub = p.value + scip_var = self._solver_model.addVar(lb=lb, ub=ub, vtype=vtype) + self._params[id(p)] = p + self._pyomo_param_to_solver_param_map[id(p)] = scip_var + return scip_var + + def __del__(self): """Frees SCIP resources used by this solver instance.""" - if self._solver_model is not None: self._solver_model.freeProb() self._solver_model = None - def __exit__(self, t, v, traceback): - super().__exit__(t, v, traceback) - self.close() - - def _set_instance(self, model, kwds={}): - DirectOrPersistentSolver._set_instance(self, model, kwds) - self.available() - try: - self._solver_model = self._scip.Model() - except Exception: - e = sys.exc_info()[1] - msg = ( - "Unable to create SCIP model. " - f"Have you installed PySCIPOpt correctly?\n\n\t Error message: {e}" + def _add_constraints(self, cons: List[ConstraintData]): + for con in cons: + self._add_constraint(con) + + def _add_sos_constraints(self, cons: List[SOSConstraintData]): + for on in cons: + self._add_sos_constraint(con) + + def _create_solver_model(self, model): + timer = self.config.timer + timer.start('create scip model') + self._clear() + self._solver_model = scip.Model() + timer.start('collect constraints') + cons = list( + model.component_data_objects( + Constraint, descend_into=True, active=True ) - raise Exception(msg) - - self._add_block(model) - - for var, n_ref in self._referenced_variables.items(): - if n_ref != 0: - if var.fixed: - if not self._output_fixed_variable_bounds: - raise ValueError( - f"Encountered a fixed variable {var.name} inside " - "an active objective or constraint " - f"expression on model {self._pyomo_model.name}, which is usually " - "indicative of a preprocessing error. Use " - "the IO-option 'output_fixed_variable_bounds=True' " - "to suppress this error and fix the variable " - "by overwriting its bounds in the SCIP instance." - ) - - def _add_block(self, block): - DirectOrPersistentSolver._add_block(self, block) - - def _add_constraint(self, con): - if not con.active: - return None - - if is_fixed(con.body) and self._skip_trivial_constraints: - return None - - conname = self._symbol_map.getSymbol(con, self._labeler) - - if con._linear_canonical_form: - scip_expr, referenced_vars = self._get_expr_from_pyomo_repn( - con.canonical_form(), self._max_constraint_degree - ) - else: - scip_expr, referenced_vars = self._get_expr_from_pyomo_expr( - con.body, self._max_constraint_degree - ) - - if con.has_lb(): - if not is_fixed(con.lower): - raise ValueError(f"Lower bound of constraint {con} is not constant.") - con_lower = value(con.lower) - if type(con_lower) != float and type(con_lower) != int: - logger.warning( - f"Constraint {conname} has LHS type {type(value(con.lower))}. " - f"Converting to float as SCIP fails otherwise." - ) - con_lower = float(con_lower) - if con.has_ub(): - if not is_fixed(con.upper): - raise ValueError(f"Upper bound of constraint {con} is not constant.") - con_upper = value(con.upper) - - if con.equality: - scip_cons = self._solver_model.addCons(scip_expr == con_lower, name=conname) - elif con.has_lb() and con.has_ub(): - scip_cons = self._solver_model.addCons(con_lower <= scip_expr, name=conname) - rhs = con_upper - if hasattr(con.body, "constant"): - con_constant = value(con.body.constant) - if not isinstance(con_constant, (float, int)): - con_constant = float(con_constant) - rhs -= con_constant - self._solver_model.chgRhs(scip_cons, rhs) - elif con.has_lb(): - scip_cons = self._solver_model.addCons(con_lower <= scip_expr, name=conname) - elif con.has_ub(): - scip_cons = self._solver_model.addCons(scip_expr <= con_upper, name=conname) - else: - raise ValueError( - f"Constraint does not have a lower or an upper bound: {con} \n" + ) + timer.stop('collect constraints') + timer.start('translate constraints') + self._add_constraints(cons) + timer.stop('translate constraints') + timer.start('sos') + sos = list( + model.component_data_objects( + SOSConstraint, descend_into=True, active=True ) + ) + self._add_sos_constraints(sos) + timer.stop('sos') + timer.start('get objective') + obj = get_objective(model) + timer.stop('get objective') + timer.start('translate objective') + self._set_objective(obj) + timer.stop('translate objective') + has_obj = obj is not None + solution_loader = ScipDirectSolutionLoader( + solver_model=self._solver_model, + var_id_map=self._vars, + var_map=self._pyomo_var_to_solver_var_map, + con_map=self._pyomo_con_to_solver_con_map, + pyomo_model=model, + opt=self, + ) + timer.stop('create scip model') + return self._solver_model, solution_loader, has_obj - for var in referenced_vars: - self._referenced_variables[var] += 1 - self._vars_referenced_by_con[con] = referenced_vars - self._pyomo_con_to_solver_con_expr_map[con] = scip_cons - self._pyomo_con_to_solver_con_map[con] = scip_cons.name - self._solver_con_to_pyomo_con_map[conname] = con + def _add_constraint(self, con): + scip_expr = self._expr_visitor.walk_expression(con.expr) + scip_con = self._solver_model.addCons(scip_expr) + self._pyomo_con_to_solver_con_map[con] = scip_con def _add_sos_constraint(self, con): - if not con.active: - return None - - conname = self._symbol_map.getSymbol(con, self._labeler) level = con.level if level not in [1, 2]: - raise ValueError(f"Solver does not support SOS level {level} constraints") + raise ValueError(f"{self.name} does not support SOS level {level} constraints") scip_vars = [] weights = [] - self._vars_referenced_by_con[con] = ComponentSet() - - if hasattr(con, "get_items"): - # aml sos constraint - sos_items = list(con.get_items()) - else: - # kernel sos constraint - sos_items = list(con.items()) - - for v, w in sos_items: - self._vars_referenced_by_con[con].add(v) - scip_vars.append(self._pyomo_var_to_solver_var_expr_map[v]) - self._referenced_variables[v] += 1 + for v, w in con.get_items(): + vid = id(v) + if vid not in self._pyomo_var_to_solver_var_map: + self._add_var(v) + scip_vars.append(self._pyomo_var_to_solver_var_map[vid]) weights.append(w) if level == 1: scip_cons = self._solver_model.addConsSOS1( - scip_vars, weights=weights, name=conname + scip_vars, weights=weights ) else: scip_cons = self._solver_model.addConsSOS2( - scip_vars, weights=weights, name=conname + scip_vars, weights=weights ) - self._pyomo_con_to_solver_con_expr_map[con] = scip_cons - self._pyomo_con_to_solver_con_map[con] = scip_cons.name - self._solver_con_to_pyomo_con_map[conname] = con + self._pyomo_con_to_solver_con_map[con] = scip_cons def _scip_vtype_from_var(self, var): """ @@ -421,342 +607,104 @@ def _scip_vtype_from_var(self, var): return vtype def _set_objective(self, obj): + if self._obj_var is None: + self._obj_var = self._solver_model.addVar( + lb=-self._solver_model.infinity(), + ub=self._solver_model.infinity(), + vtype="C" + ) + if self._objective is not None: - for var in self._vars_referenced_by_obj: - self._referenced_variables[var] -= 1 - self._vars_referenced_by_obj = ComponentSet() - self._objective = None + self._solver_model.delCons(self._obj_con) - if obj.active is False: - raise ValueError("Cannot add inactive objective to solver.") + if obj is None: + scip_expr = 0 + else: + scip_expr = self._expr_visitor.walk_expression(obj.expr) if obj.sense == minimize: sense = "minimize" + self._obj_con = self._solver_model.addCons(self._obj_var >= scip_expr) elif obj.sense == maximize: sense = "maximize" + self._obj_con = self._solver_model.addCons(self._obj_var <= scip_expr) else: raise ValueError(f"Objective sense is not recognized: {obj.sense}") - scip_expr, referenced_vars = self._get_expr_from_pyomo_expr( - obj.expr, self._max_obj_degree - ) - - for var in referenced_vars: - self._referenced_variables[var] += 1 - - self._solver_model.setObjective(scip_expr, sense=sense) + self._solver_model.setObjective(self._obj_var, sense=sense) self._objective = obj - self._vars_referenced_by_obj = referenced_vars - - def _get_solver_solution_status(self, scip, soln): - """ """ - # Get the status of the SCIP Model currently - status = scip.getStatus() - - # Go through each potential case and update appropriately - if scip.getStage() == 1: # SCIP Model is created but not yet optimized - self.results.solver.status = SolverStatus.aborted - self.results.solver.termination_message = ( - "Model is loaded, but no solution information is available." - ) - self.results.solver.termination_condition = TerminationCondition.error - soln.status = SolutionStatus.unknown - elif status == "optimal": # optimal - self.results.solver.status = SolverStatus.ok - self.results.solver.termination_message = ( - "Model was solved to optimality (subject to tolerances), " - "and an optimal solution is available." - ) - self.results.solver.termination_condition = TerminationCondition.optimal - soln.status = SolutionStatus.optimal - elif status == "infeasible": - self.results.solver.status = SolverStatus.warning - self.results.solver.termination_message = ( - "Model was proven to be infeasible" - ) - self.results.solver.termination_condition = TerminationCondition.infeasible - soln.status = SolutionStatus.infeasible - elif status == "inforunbd": - self.results.solver.status = SolverStatus.warning - self.results.solver.termination_message = ( - "Problem proven to be infeasible or unbounded." - ) - self.results.solver.termination_condition = ( - TerminationCondition.infeasibleOrUnbounded - ) - soln.status = SolutionStatus.unsure - elif status == "unbounded": - self.results.solver.status = SolverStatus.warning - self.results.solver.termination_message = ( - "Model was proven to be unbounded." - ) - self.results.solver.termination_condition = TerminationCondition.unbounded - soln.status = SolutionStatus.unbounded - elif status == "gaplimit": - self.results.solver.status = SolverStatus.aborted - self.results.solver.termination_message = ( - "Optimization terminated because the gap dropped below " - "the value specified in the " - "limits/gap parameter." - ) - self.results.solver.termination_condition = TerminationCondition.unknown - soln.status = SolutionStatus.stoppedByLimit - elif status == "stallnodelimit": - self.results.solver.status = SolverStatus.aborted - self.results.solver.termination_message = ( - "Optimization terminated because the stalling node limit " - "exceeded the value specified in the " - "limits/stallnodes parameter." - ) - self.results.solver.termination_condition = TerminationCondition.unknown - soln.status = SolutionStatus.stoppedByLimit - elif status == "restartlimit": - self.results.solver.status = SolverStatus.aborted - self.results.solver.termination_message = ( - "Optimization terminated because the total number of restarts " - "exceeded the value specified in the " - "limits/restarts parameter." - ) - self.results.solver.termination_condition = TerminationCondition.unknown - soln.status = SolutionStatus.stoppedByLimit - elif status == "nodelimit" or status == "totalnodelimit": - self.results.solver.status = SolverStatus.aborted - self.results.solver.termination_message = ( - "Optimization terminated because the number of " - "branch-and-cut nodes explored exceeded the limits specified " - "in the limits/nodes or limits/totalnodes parameter" - ) - self.results.solver.termination_condition = ( - TerminationCondition.maxEvaluations - ) - soln.status = SolutionStatus.stoppedByLimit - elif status == "timelimit": - self.results.solver.status = SolverStatus.aborted - self.results.solver.termination_message = ( - "Optimization terminated because the time expended exceeded " - "the value specified in the limits/time parameter." - ) - self.results.solver.termination_condition = ( - TerminationCondition.maxTimeLimit - ) - soln.status = SolutionStatus.stoppedByLimit - elif status == "sollimit" or status == "bestsollimit": - self.results.solver.status = SolverStatus.aborted - self.results.solver.termination_message = ( - "Optimization terminated because the number of solutions found " - "reached the value specified in the limits/solutions or" - "limits/bestsol parameter." - ) - self.results.solver.termination_condition = TerminationCondition.unknown - soln.status = SolutionStatus.stoppedByLimit - elif status == "memlimit": - self.results.solver.status = SolverStatus.aborted - self.results.solver.termination_message = ( - "Optimization terminated because the memory used exceeded " - "the value specified in the limits/memory parameter." - ) - self.results.solver.termination_condition = TerminationCondition.unknown - soln.status = SolutionStatus.stoppedByLimit - elif status == "userinterrupt": - self.results.solver.status = SolverStatus.aborted - self.results.solver.termination_message = ( - "Optimization was terminated by the user." - ) - self.results.solver.termination_condition = TerminationCondition.error - soln.status = SolutionStatus.error - else: - self.results.solver.status = SolverStatus.error - self.results.solver.termination_message = ( - f"Unhandled SCIP status ({str(status)})" - ) - self.results.solver.termination_condition = TerminationCondition.error - soln.status = SolutionStatus.error - return soln - - def _postsolve(self): - # Constraint duals and variable - # reduced-costs were removed as in SCIP they contain - # too many caveats. Slacks were removed as later - # planned interfaces do not intend to support. - # Scan through the solver suffix list - # and throw an exception if the user has specified - # any others. - for suffix in self._suffixes: - raise RuntimeError( - f"***The scip_direct solver plugin cannot extract solution suffix={suffix}" - ) - - scip = self._solver_model - status = scip.getStatus() - scip_vars = scip.getVars() - n_bin_vars = sum([scip_var.vtype() == "BINARY" for scip_var in scip_vars]) - n_int_vars = sum([scip_var.vtype() == "INTEGER" for scip_var in scip_vars]) - n_con_vars = sum([scip_var.vtype() == "CONTINUOUS" for scip_var in scip_vars]) - - self.results = SolverResults() - soln = Solution() - - self.results.solver.name = f"SCIP{self._version}" - self.results.solver.wallclock_time = scip.getSolvingTime() - - soln = self._get_solver_solution_status(scip, soln) - self.results.problem.name = scip.getProbName() - - self.results.problem.upper_bound = None - self.results.problem.lower_bound = None - if scip.getNSols() > 0: - scip_has_sol = True - else: - scip_has_sol = False - if not scip_has_sol and (status == "inforunbd" or status == "infeasible"): - pass + def _postsolve( + self, + scip_model, + solution_loader: ScipDirectSolutionLoader, + has_obj + ): + + results = Results() + results.solution_loader = solution_loader + results.timing_info.scip_time = scip_model.getSolvingTime() + results.termination_condition = self._get_tc_map().get(scip_model.getStatus(), TerminationCondition.unknown) + + if solution_loader.get_number_of_solutions() > 0: + if results.termination_condition == TerminationCondition.convergenceCriteriaSatisfied: + results.solution_status = SolutionStatus.optimal + else: + results.solution_status = SolutionStatus.feasible else: - if n_bin_vars + n_int_vars == 0: - self.results.problem.upper_bound = scip.getObjVal() - self.results.problem.lower_bound = scip.getObjVal() - elif scip.getObjectiveSense() == "minimize": # minimizing - if scip_has_sol: - self.results.problem.upper_bound = scip.getObjVal() - else: - self.results.problem.upper_bound = scip.infinity() - self.results.problem.lower_bound = scip.getDualbound() - else: # maximizing - self.results.problem.upper_bound = scip.getDualbound() - if scip_has_sol: - self.results.problem.lower_bound = scip.getObjVal() + results.solution_status = SolutionStatus.noSolution + + if ( + results.termination_condition + != TerminationCondition.convergenceCriteriaSatisfied + and self.config.raise_exception_on_nonoptimal_result + ): + raise NoOptimalSolutionError() + + if has_obj: + try: + if scip_model.getObjVal() < scip_model.infinity(): + results.incumbent_objective = scip_model.getObjVal() else: - self.results.problem.lower_bound = -scip.infinity() - + results.incumbent_objective = None + except: + results.incumbent_objective = None try: - soln.gap = ( - self.results.problem.upper_bound - self.results.problem.lower_bound - ) - except TypeError: - soln.gap = None - - self.results.problem.number_of_constraints = scip.getNConss(transformed=False) - self.results.problem.number_of_variables = scip.getNVars(transformed=False) - self.results.problem.number_of_binary_variables = n_bin_vars - self.results.problem.number_of_integer_variables = n_int_vars - self.results.problem.number_of_continuous_variables = n_con_vars - self.results.problem.number_of_objectives = 1 - self.results.problem.number_of_solutions = scip.getNSols() - - # if a solve was stopped by a limit, we still need to check to - # see if there is a solution available - this may not always - # be the case, both in LP and MIP contexts. - if self._save_results: - """ - This code in this if statement is only needed for backwards compatibility. It is more efficient to set - _save_results to False and use load_vars, load_duals, etc. - """ - - if scip.getNSols() > 0: - soln_variables = soln.variable - - scip_vars = scip.getVars() - scip_var_names = [scip_var.name for scip_var in scip_vars] - var_names = set(self._solver_var_to_pyomo_var_map.keys()) - assert set(scip_var_names) == var_names - var_vals = [scip.getVal(scip_var) for scip_var in scip_vars] - - for scip_var, val, name in zip(scip_vars, var_vals, scip_var_names): - pyomo_var = self._solver_var_to_pyomo_var_map[name] - if self._referenced_variables[pyomo_var] > 0: - soln_variables[name] = {"Value": val} - - elif self._load_solutions: - if scip.getNSols() > 0: - self.load_vars() - - self.results.solution.insert(soln) - - # finally, clean any temporary files registered with the temp file - # manager, created populated *directly* by this plugin. - TempfileManager.pop(remove=not self._keepfiles) - - return DirectOrPersistentSolver._postsolve(self) - - def warm_start_capable(self): - return True - - def _warm_start(self): - partial_sol = False - for pyomo_var in self._pyomo_var_to_solver_var_expr_map: - if pyomo_var.value is None: - partial_sol = True - break - if partial_sol: - scip_sol = self._solver_model.createPartialSol() - else: - scip_sol = self._solver_model.createSol() - for pyomo_var, scip_var in self._pyomo_var_to_solver_var_expr_map.items(): - if pyomo_var.value is not None: - scip_sol[scip_var] = value(pyomo_var) - if partial_sol: - self._solver_model.addSol(scip_sol) + results.objective_bound = scip_model.getDualbound() + if results.objective_bound <= -scip_model.infinity(): + results.objective_bound = -math.inf + if results.objective_bound >= scip_model.infinity(): + results.objective_bound = math.inf + except: + if self._objective.sense == minimize: + results.objective_bound = -math.inf + else: + results.objective_bound = math.inf else: - feasible = self._solver_model.checkSol(scip_sol, printreason=not self._tee) - if feasible: - self._solver_model.addSol(scip_sol) - else: - logger.warning("Warm start solution was not accepted by SCIP") - self._solver_model.freeSol(scip_sol) - - def _load_vars(self, vars_to_load=None): - var_map = self._pyomo_var_to_solver_var_expr_map - ref_vars = self._referenced_variables - if vars_to_load is None: - vars_to_load = var_map.keys() - - scip_vars_to_load = [var_map[pyomo_var] for pyomo_var in vars_to_load] - vals = [self._solver_model.getVal(scip_var) for scip_var in scip_vars_to_load] - - for var, val in zip(vars_to_load, vals): - if ref_vars[var] > 0: - var.set_value(val, skip_validation=True) - - def _load_rc(self, vars_to_load=None): - raise NotImplementedError( - "SCIP via Pyomo does not support reduced cost loading." - ) + results.incumbent_objective = None + results.objective_bound = None - def _load_duals(self, cons_to_load=None): - raise NotImplementedError( - "SCIP via Pyomo does not support dual solution loading" - ) - - def _load_slacks(self, cons_to_load=None): - raise NotImplementedError("SCIP via Pyomo does not support slack loading") - - def load_duals(self, cons_to_load=None): - """ - Load the duals into the 'dual' suffix. The 'dual' suffix must live on the parent model. - - Parameters - ---------- - cons_to_load: list of Constraint - """ - self._load_duals(cons_to_load) - - def load_rc(self, vars_to_load): - """ - Load the reduced costs into the 'rc' suffix. The 'rc' suffix must live on the parent model. - - Parameters - ---------- - vars_to_load: list of Var - """ - self._load_rc(vars_to_load) - - def load_slacks(self, cons_to_load=None): - """ - Load the values of the slack variables into the 'slack' suffix. The 'slack' suffix must live on the parent - model. - - Parameters - ---------- - cons_to_load: list of Constraint - """ - self._load_slacks(cons_to_load) + self.config.timer.start('load solution') + if self.config.load_solutions: + if solution_loader.get_number_of_solutions() > 0: + solution_loader.load_solution() + else: + raise NoFeasibleSolutionError() + self.config.timer.stop('load solution') + + results.iteration_count = scip_model.getNNodes() + results.solver_config = self.config + results.solver_name = self.name + results.solver_version = self.version() + + return results + + def _mipstart(self): + # TODO: it is also possible to specify continuous variables, but + # I think we should have a differnt option for that + sol = self._solver_model.createPartialSol() + for vid, scip_var in self._pyomo_var_to_solver_var_map.items(): + pyomo_var = self._vars[vid] + if pyomo_var.is_integer(): + sol[scip_var] = pyomo_var.value + self._solver_model.addSol(sol) From 96017680cd9329626ae18bb2bdf7993f97940f74 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 14 Aug 2025 23:47:44 -0600 Subject: [PATCH 033/107] porting scip interface --- pyomo/contrib/solver/solvers/scip/scip_direct.py | 8 ++++---- pyomo/solvers/plugins/solvers/__init__.py | 2 -- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index b8d4d14a6c1..5ea3391eecd 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -69,7 +69,7 @@ logger = logging.getLogger(__name__) -scip, scip_available = attempt_import('pyscipyopt') +scip, scip_available = attempt_import('pyscipopt') class ScipConfig(BranchAndBoundConfig): @@ -360,9 +360,9 @@ def available(self) -> Availability: return self._available def version(self) -> Tuple: - return tuple(int(i) for i in scip.__version__) + return tuple(int(i) for i in scip.__version__.split('.')) - def solve(self, model: BlockData, **kwargs) -> Results: + def solve(self, model: BlockData, **kwds) -> Results: start_timestamp = datetime.datetime.now(datetime.timezone.utc) orig_config = self.config if not self.available(): @@ -470,7 +470,7 @@ def _scip_lb_ub_from_var(self, var): val = var.value return val, val - lb, ub = var.bounds() + lb, ub = var.bounds if lb is None: lb = -self._solver_model.infinity() diff --git a/pyomo/solvers/plugins/solvers/__init__.py b/pyomo/solvers/plugins/solvers/__init__.py index 55baaab9de8..cf10af15186 100644 --- a/pyomo/solvers/plugins/solvers/__init__.py +++ b/pyomo/solvers/plugins/solvers/__init__.py @@ -31,8 +31,6 @@ mosek_persistent, xpress_direct, xpress_persistent, - scip_direct, - scip_persistent, SAS, KNITROAMPL, ) From 9d2f22ab5ba29157c7765c82b0fd0e67d34a8e22 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 14 Aug 2025 23:55:03 -0600 Subject: [PATCH 034/107] porting scip interface --- pyomo/contrib/solver/plugins.py | 6 ++++++ pyomo/contrib/solver/tests/solvers/test_solvers.py | 10 +++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index fed739232ad..a4d6f5f9004 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -15,6 +15,7 @@ from .solvers.gurobi.gurobi_direct import GurobiDirect from .solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic, GurobiPersistent from .solvers.highs import Highs +from .solvers.scip.scip_direct import SCIPDirect def load(): @@ -39,3 +40,8 @@ def load(): SolverFactory.register( name='highs', legacy_name='highs_v2', doc='Persistent interface to HiGHS' )(Highs) + SolverFactory.register( + name='scip_direct', + legacy_name='scip_direct_v2', + doc='Direct interface pyscipopt', + ) diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 189b0373780..f4988ca5c8b 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -35,6 +35,7 @@ GurobiDirectQuadratic, GurobiPersistent, ) +from pyomo.contrib.solver.solvers.scip.scip_direct import SCIPDirect from pyomo.contrib.solver.common.util import ( NoSolutionError, NoFeasibleSolutionError, @@ -60,23 +61,30 @@ ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), ('highs', Highs), + ('scip_direct', SCIPDirect), ] mip_solvers = [ ('gurobi_persistent', GurobiPersistent), ('gurobi_direct', GurobiDirect), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('highs', Highs), + ('scip_direct', SCIPDirect), +] +nlp_solvers = [ + ('ipopt', Ipopt), + ('scip_direct', SCIPDirect), ] -nlp_solvers = [('ipopt', Ipopt)] qcp_solvers = [ ('gurobi_persistent', GurobiPersistent), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), + ('scip_direct', SCIPDirect), ] qp_solvers = qcp_solvers + [("highs", Highs)] miqcqp_solvers = [ ('gurobi_persistent', GurobiPersistent), ('gurobi_direct_quadratic', GurobiDirectQuadratic), + ('scip_direct', SCIPDirect), ] nl_solvers = [('ipopt', Ipopt)] nl_solvers_set = {i[0] for i in nl_solvers} From b4837295c57ee4005643a19f783f25dff3530be9 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Fri, 15 Aug 2025 00:49:53 -0600 Subject: [PATCH 035/107] bugs and tests --- .../solver/solvers/scip/scip_direct.py | 25 ++- .../solver/tests/solvers/test_solvers.py | 180 ++++++++++-------- 2 files changed, 120 insertions(+), 85 deletions(-) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 5ea3391eecd..1ff470223bd 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -12,6 +12,7 @@ import datetime import io import logging +import math from typing import Tuple, List, Optional, Sequence, Mapping, Dict from pyomo.common.collections import ComponentMap @@ -40,6 +41,7 @@ NPV_SumExpression, NPV_UnaryFunctionExpression, ) +from pyomo.gdp.disjunct import AutoLinkedBinaryVar from pyomo.core.base.expression import ExpressionData, ScalarExpression from pyomo.core.expr.relational_expr import EqualityExpression, InequalityExpression, RangedExpression from pyomo.core.staleflag import StaleFlagManager @@ -50,6 +52,7 @@ from pyomo.contrib.solver.common.util import ( NoFeasibleSolutionError, NoOptimalSolutionError, + NoSolutionError, ) from pyomo.contrib.solver.common.util import get_objective from pyomo.contrib.solver.common.solution_loader import NoSolutionSolutionLoader @@ -109,6 +112,8 @@ def _handle_var(node, data, opt): def _handle_param(node, data, opt): + if not opt.is_persistent(): + return node.value if not node.mutable: return node.value if id(node) not in opt._pyomo_param_to_solver_param_map: @@ -231,6 +236,7 @@ def _handle_named_expression(node, data, opt): ScalarParam: _handle_param, float: _handle_float, int: _handle_float, + AutoLinkedBinaryVar: _handle_var, } @@ -287,6 +293,8 @@ def load_vars( def get_vars( self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: + if self.get_number_of_solutions() == 0: + raise NoSolutionError() if vars_to_load is None: vars_to_load = list(self._vars.values()) if solution_id is None: @@ -403,7 +411,7 @@ def solve(self, model: BlockData, **kwds) -> Results: scip_model.setParam(key, option) timer.start('optimize') - with capture_output(TeeStream(*ostreams), capture_fd=False): + with capture_output(TeeStream(*ostreams), capture_fd=True): scip_model.optimize() timer.stop('optimize') @@ -619,17 +627,20 @@ def _set_objective(self, obj): if obj is None: scip_expr = 0 + sense = "minimize" else: scip_expr = self._expr_visitor.walk_expression(obj.expr) + if obj.sense == minimize: + sense = "minimize" + elif obj.sense == maximize: + sense = "maximize" + else: + raise ValueError(f"Objective sense is not recognized: {obj.sense}") - if obj.sense == minimize: - sense = "minimize" + if sense == "minimize": self._obj_con = self._solver_model.addCons(self._obj_var >= scip_expr) - elif obj.sense == maximize: - sense = "maximize" - self._obj_con = self._solver_model.addCons(self._obj_var <= scip_expr) else: - raise ValueError(f"Objective sense is not recognized: {obj.sense}") + self._obj_con = self._solver_model.addCons(self._obj_var <= scip_expr) self._solver_model.setObjective(self._obj_var, sense=sense) self._objective = obj diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index f4988ca5c8b..1b6f122219c 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -88,6 +88,13 @@ ] nl_solvers = [('ipopt', Ipopt)] nl_solvers_set = {i[0] for i in nl_solvers} +dual_solvers = [ + ('gurobi_persistent', GurobiPersistent), + ('gurobi_direct', GurobiDirect), + ('gurobi_direct_quadratic', GurobiDirectQuadratic), + ('ipopt', Ipopt), + ('highs', Highs), +] def _load_tests(solver_list): @@ -114,7 +121,7 @@ def test_all_solvers_list(): class TestDualSignConvention(unittest.TestCase): - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_equality(self, name: str, opt_class: Type[SolverBase], use_presolve: bool): opt: SolverBase = opt_class() if not opt.available(): @@ -166,7 +173,7 @@ def test_equality(self, name: str, opt_class: Type[SolverBase], use_presolve: bo self.assertAlmostEqual(duals[m.c1], 0) self.assertAlmostEqual(duals[m.c2], -1) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_inequality( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): @@ -228,7 +235,7 @@ def test_inequality( self.assertAlmostEqual(duals[m.c1], 0.5) self.assertAlmostEqual(duals[m.c2], 0.5) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_bounds(self, name: str, opt_class: Type[SolverBase], use_presolve: bool): opt: SolverBase = opt_class() if not opt.available(): @@ -283,7 +290,7 @@ def test_bounds(self, name: str, opt_class: Type[SolverBase], use_presolve: bool rc = res.solution_loader.get_reduced_costs() self.assertAlmostEqual(rc[m.x], -1) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_range(self, name: str, opt_class: Type[SolverBase], use_presolve: bool): opt: SolverBase = opt_class() if not opt.available(): @@ -335,7 +342,7 @@ def test_range(self, name: str, opt_class: Type[SolverBase], use_presolve: bool) self.assertAlmostEqual(duals[m.c1], -0.5) self.assertAlmostEqual(duals[m.c2], -0.5) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_equality_max( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): @@ -389,7 +396,7 @@ def test_equality_max( self.assertAlmostEqual(duals[m.c1], 0) self.assertAlmostEqual(duals[m.c2], 1) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_inequality_max( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): @@ -451,7 +458,7 @@ def test_inequality_max( self.assertAlmostEqual(duals[m.c1], -0.5) self.assertAlmostEqual(duals[m.c2], -0.5) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_bounds_max( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): @@ -508,7 +515,7 @@ def test_bounds_max( rc = res.solution_loader.get_reduced_costs() self.assertAlmostEqual(rc[m.x], 1) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_range_max( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): @@ -664,16 +671,18 @@ def test_range_constraint( res = opt.solve(m) self.assertEqual(res.solution_status, SolutionStatus.optimal) self.assertAlmostEqual(m.x.value, -1) - duals = res.solution_loader.get_duals() - self.assertAlmostEqual(duals[m.c], 1) + if (name, opt_class) in dual_solvers: + duals = res.solution_loader.get_duals() + self.assertAlmostEqual(duals[m.c], 1) m.obj.sense = pyo.maximize res = opt.solve(m) self.assertEqual(res.solution_status, SolutionStatus.optimal) self.assertAlmostEqual(m.x.value, 1) - duals = res.solution_loader.get_duals() - self.assertAlmostEqual(duals[m.c], 1) + if (name, opt_class) in dual_solvers: + duals = res.solution_loader.get_duals() + self.assertAlmostEqual(duals[m.c], 1) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_reduced_costs( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): @@ -702,7 +711,7 @@ def test_reduced_costs( self.assertAlmostEqual(rc[m.x], -3) self.assertAlmostEqual(rc[m.y], -4) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_reduced_costs2( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): @@ -768,9 +777,10 @@ def test_param_changes( else: bound = res.objective_bound self.assertTrue(bound <= m.y.value) - duals = res.solution_loader.get_duals() - self.assertAlmostEqual(duals[m.c1], (1 + a1 / (a2 - a1))) - self.assertAlmostEqual(duals[m.c2], a1 / (a2 - a1)) + if (name, opt_class) in dual_solvers: + duals = res.solution_loader.get_duals() + self.assertAlmostEqual(duals[m.c1], (1 + a1 / (a2 - a1))) + self.assertAlmostEqual(duals[m.c2], a1 / (a2 - a1)) @parameterized.expand(input=_load_tests(all_solvers)) def test_immutable_param( @@ -815,9 +825,10 @@ def test_immutable_param( else: bound = res.objective_bound self.assertTrue(bound <= m.y.value) - duals = res.solution_loader.get_duals() - self.assertAlmostEqual(duals[m.c1], (1 + a1 / (a2 - a1))) - self.assertAlmostEqual(duals[m.c2], a1 / (a2 - a1)) + if (name, opt_class) in dual_solvers: + duals = res.solution_loader.get_duals() + self.assertAlmostEqual(duals[m.c1], (1 + a1 / (a2 - a1))) + self.assertAlmostEqual(duals[m.c2], a1 / (a2 - a1)) @parameterized.expand(input=_load_tests(all_solvers)) def test_equality(self, name: str, opt_class: Type[SolverBase], use_presolve: bool): @@ -831,6 +842,8 @@ def test_equality(self, name: str, opt_class: Type[SolverBase], use_presolve: bo check_duals = False else: opt.config.writer_config.linear_presolve = False + if (name, opt_class) not in dual_solvers: + check_duals = False m = pyo.ConcreteModel() m.x = pyo.Var() m.y = pyo.Var() @@ -922,6 +935,8 @@ def test_no_objective( opt.config.writer_config.linear_presolve = True else: opt.config.writer_config.linear_presolve = False + if (name, opt_class) not in dual_solvers: + check_duals = False m = pyo.ConcreteModel() m.x = pyo.Var() m.y = pyo.Var() @@ -983,9 +998,10 @@ def test_add_remove_cons( else: bound = res.objective_bound self.assertTrue(bound <= m.y.value) - duals = res.solution_loader.get_duals() - self.assertAlmostEqual(duals[m.c1], -(1 + a1 / (a2 - a1))) - self.assertAlmostEqual(duals[m.c2], a1 / (a2 - a1)) + if (name, opt_class) in dual_solvers: + duals = res.solution_loader.get_duals() + self.assertAlmostEqual(duals[m.c1], -(1 + a1 / (a2 - a1))) + self.assertAlmostEqual(duals[m.c2], a1 / (a2 - a1)) m.c3 = pyo.Constraint(expr=m.y >= a3 * m.x + b3) res = opt.solve(m) @@ -994,10 +1010,11 @@ def test_add_remove_cons( self.assertAlmostEqual(m.y.value, a1 * (b3 - b1) / (a1 - a3) + b1) self.assertAlmostEqual(res.incumbent_objective, m.y.value) self.assertTrue(res.objective_bound is None or res.objective_bound <= m.y.value) - duals = res.solution_loader.get_duals() - self.assertAlmostEqual(duals[m.c1], -(1 + a1 / (a3 - a1))) - self.assertAlmostEqual(duals[m.c2], 0) - self.assertAlmostEqual(duals[m.c3], a1 / (a3 - a1)) + if (name, opt_class) in dual_solvers: + duals = res.solution_loader.get_duals() + self.assertAlmostEqual(duals[m.c1], -(1 + a1 / (a3 - a1))) + self.assertAlmostEqual(duals[m.c2], 0) + self.assertAlmostEqual(duals[m.c3], a1 / (a3 - a1)) del m.c3 res = opt.solve(m) @@ -1006,9 +1023,10 @@ def test_add_remove_cons( self.assertAlmostEqual(m.y.value, a1 * (b2 - b1) / (a1 - a2) + b1) self.assertAlmostEqual(res.incumbent_objective, m.y.value) self.assertTrue(res.objective_bound is None or res.objective_bound <= m.y.value) - duals = res.solution_loader.get_duals() - self.assertAlmostEqual(duals[m.c1], -(1 + a1 / (a2 - a1))) - self.assertAlmostEqual(duals[m.c2], a1 / (a2 - a1)) + if (name, opt_class) in dual_solvers: + duals = res.solution_loader.get_duals() + self.assertAlmostEqual(duals[m.c1], -(1 + a1 / (a2 - a1))) + self.assertAlmostEqual(duals[m.c2], a1 / (a2 - a1)) @parameterized.expand(input=_load_tests(all_solvers)) def test_results_infeasible( @@ -1057,14 +1075,15 @@ def test_results_infeasible( NoSolutionError, '.*does not currently have a valid solution.*' ): res.solution_loader.load_vars() - with self.assertRaisesRegex( - NoDualsError, '.*does not currently have valid duals.*' - ): - res.solution_loader.get_duals() - with self.assertRaisesRegex( - NoReducedCostsError, '.*does not currently have valid reduced costs.*' - ): - res.solution_loader.get_reduced_costs() + if (name, opt_class) in dual_solvers: + with self.assertRaisesRegex( + NoDualsError, '.*does not currently have valid duals.*' + ): + res.solution_loader.get_duals() + with self.assertRaisesRegex( + NoReducedCostsError, '.*does not currently have valid reduced costs.*' + ): + res.solution_loader.get_reduced_costs() @parameterized.expand(input=_load_tests(all_solvers)) def test_trivial_constraints( @@ -1118,7 +1137,7 @@ def test_trivial_constraints( self.assertIn(res.termination_condition, acceptable_termination_conditions) self.assertIsNone(res.incumbent_objective) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_duals(self, name: str, opt_class: Type[SolverBase], use_presolve: bool): opt: SolverBase = opt_class() if not opt.available(): @@ -1167,13 +1186,13 @@ def test_mutable_quadratic_coefficient( m.c = pyo.Constraint(expr=m.y >= (m.a * m.x + m.b) ** 2) res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0.41024548525899274, 4) - self.assertAlmostEqual(m.y.value, 0.34781038127030117, 4) + self.assertAlmostEqual(m.x.value, 0.41024548525899274, 3) + self.assertAlmostEqual(m.y.value, 0.34781038127030117, 3) m.a.value = 2 m.b.value = -0.5 res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0.10256137418973625, 4) - self.assertAlmostEqual(m.y.value, 0.0869525991355825, 4) + self.assertAlmostEqual(m.x.value, 0.10256137418973625, 3) + self.assertAlmostEqual(m.y.value, 0.0869525991355825, 3) @parameterized.expand(input=_load_tests(qcp_solvers)) def test_mutable_quadratic_objective_qcp( @@ -1198,14 +1217,14 @@ def test_mutable_quadratic_objective_qcp( m.ccon = pyo.Constraint(expr=m.y >= (m.a * m.x + m.b) ** 2) res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0.2719178742733325, 4) - self.assertAlmostEqual(m.y.value, 0.5301035741688002, 4) + self.assertAlmostEqual(m.x.value, 0.2719178742733325, 3) + self.assertAlmostEqual(m.y.value, 0.5301035741688002, 3) m.c.value = 3.5 m.d.value = -1 res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0.6962249634573562, 4) - self.assertAlmostEqual(m.y.value, 0.09227926676152151, 4) + self.assertAlmostEqual(m.x.value, 0.6962249634573562, 3) + self.assertAlmostEqual(m.y.value, 0.09227926676152151, 3) @parameterized.expand(input=_load_tests(qp_solvers)) def test_mutable_quadratic_objective_qp( @@ -1412,7 +1431,7 @@ def test_fixed_vars_4( else: opt.config.writer_config.linear_presolve = False m = pyo.ConcreteModel() - m.x = pyo.Var() + m.x = pyo.Var(bounds=(0, None)) m.y = pyo.Var() m.obj = pyo.Objective(expr=m.x**2 + m.y**2) m.c1 = pyo.Constraint(expr=m.x == 2 / m.y) @@ -1421,8 +1440,8 @@ def test_fixed_vars_4( self.assertAlmostEqual(m.x.value, 2) m.y.unfix() res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 2**0.5) - self.assertAlmostEqual(m.y.value, 2**0.5) + self.assertAlmostEqual(m.x.value, 2**0.5, 3) + self.assertAlmostEqual(m.y.value, 2**0.5, 3) @parameterized.expand(input=_load_tests(all_solvers)) def test_mutable_param_with_range( @@ -1506,9 +1525,10 @@ def test_mutable_param_with_range( res.objective_bound is None or res.objective_bound <= m.y.value + 1e-12 ) - duals = res.solution_loader.get_duals() - self.assertAlmostEqual(duals[m.con1], (1 + a1 / (a2 - a1)), 6) - self.assertAlmostEqual(duals[m.con2], -a1 / (a2 - a1), 6) + if (name, opt_class) in dual_solvers: + duals = res.solution_loader.get_duals() + self.assertAlmostEqual(duals[m.con1], (1 + a1 / (a2 - a1)), 6) + self.assertAlmostEqual(duals[m.con2], -a1 / (a2 - a1), 6) else: self.assertAlmostEqual(m.x.value, (c2 - c1) / (a1 - a2), 6) self.assertAlmostEqual(m.y.value, a1 * (c2 - c1) / (a1 - a2) + c1, 6) @@ -1517,9 +1537,10 @@ def test_mutable_param_with_range( res.objective_bound is None or res.objective_bound >= m.y.value - 1e-12 ) - duals = res.solution_loader.get_duals() - self.assertAlmostEqual(duals[m.con1], (1 + a1 / (a2 - a1)), 6) - self.assertAlmostEqual(duals[m.con2], -a1 / (a2 - a1), 6) + if (name, opt_class) in dual_solvers: + duals = res.solution_loader.get_duals() + self.assertAlmostEqual(duals[m.con1], (1 + a1 / (a2 - a1)), 6) + self.assertAlmostEqual(duals[m.con2], -a1 / (a2 - a1), 6) @parameterized.expand(input=_load_tests(all_solvers)) def test_add_and_remove_vars( @@ -1590,8 +1611,8 @@ def test_exp(self, name: str, opt_class: Type[SolverBase], use_presolve: bool): m.obj = pyo.Objective(expr=m.x**2 + m.y**2) m.c1 = pyo.Constraint(expr=m.y >= pyo.exp(m.x)) res = opt.solve(m) - self.assertAlmostEqual(m.x.value, -0.42630274815985264) - self.assertAlmostEqual(m.y.value, 0.6529186341994245) + self.assertAlmostEqual(m.x.value, -0.42630274815985264, 4) + self.assertAlmostEqual(m.y.value, 0.6529186341994245, 4) @parameterized.expand(input=_load_tests(nlp_solvers)) def test_log(self, name: str, opt_class: Type[SolverBase], use_presolve: bool): @@ -1609,8 +1630,8 @@ def test_log(self, name: str, opt_class: Type[SolverBase], use_presolve: bool): m.obj = pyo.Objective(expr=m.x**2 + m.y**2) m.c1 = pyo.Constraint(expr=m.y <= pyo.log(m.x)) res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0.6529186341994245) - self.assertAlmostEqual(m.y.value, -0.42630274815985264) + self.assertAlmostEqual(m.x.value, 0.6529186341994245, 3) + self.assertAlmostEqual(m.y.value, -0.42630274815985264, 3) @parameterized.expand(input=_load_tests(all_solvers)) def test_with_numpy( @@ -1720,24 +1741,25 @@ def test_solution_loader( self.assertNotIn(m.x, primals) self.assertIn(m.y, primals) self.assertAlmostEqual(primals[m.y], 1) - reduced_costs = res.solution_loader.get_reduced_costs() - self.assertIn(m.x, reduced_costs) - self.assertIn(m.y, reduced_costs) - self.assertAlmostEqual(reduced_costs[m.x], 1) - self.assertAlmostEqual(reduced_costs[m.y], 0) - reduced_costs = res.solution_loader.get_reduced_costs([m.y]) - self.assertNotIn(m.x, reduced_costs) - self.assertIn(m.y, reduced_costs) - self.assertAlmostEqual(reduced_costs[m.y], 0) - duals = res.solution_loader.get_duals() - self.assertIn(m.c1, duals) - self.assertIn(m.c2, duals) - self.assertAlmostEqual(duals[m.c1], 1) - self.assertAlmostEqual(duals[m.c2], 0) - duals = res.solution_loader.get_duals([m.c1]) - self.assertNotIn(m.c2, duals) - self.assertIn(m.c1, duals) - self.assertAlmostEqual(duals[m.c1], 1) + if (name, opt_class) in dual_solvers: + reduced_costs = res.solution_loader.get_reduced_costs() + self.assertIn(m.x, reduced_costs) + self.assertIn(m.y, reduced_costs) + self.assertAlmostEqual(reduced_costs[m.x], 1) + self.assertAlmostEqual(reduced_costs[m.y], 0) + reduced_costs = res.solution_loader.get_reduced_costs([m.y]) + self.assertNotIn(m.x, reduced_costs) + self.assertIn(m.y, reduced_costs) + self.assertAlmostEqual(reduced_costs[m.y], 0) + duals = res.solution_loader.get_duals() + self.assertIn(m.c1, duals) + self.assertIn(m.c2, duals) + self.assertAlmostEqual(duals[m.c1], 1) + self.assertAlmostEqual(duals[m.c2], 0) + duals = res.solution_loader.get_duals([m.c1]) + self.assertNotIn(m.c2, duals) + self.assertIn(m.c1, duals) + self.assertAlmostEqual(duals[m.c1], 1) @parameterized.expand(input=_load_tests(all_solvers)) def test_time_limit( @@ -2219,6 +2241,8 @@ def test_scaling(self, name: str, opt_class: Type[SolverBase], use_presolve: boo opt.config.writer_config.linear_presolve = True else: opt.config.writer_config.linear_presolve = False + if (name, opt_class) not in dual_solvers: + check_duals = False m = pyo.ConcreteModel() m.x = pyo.Var() From 37a31a7fc7b7fe8cb14179e4dc18df4dbdd8bc87 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Fri, 15 Aug 2025 01:50:18 -0600 Subject: [PATCH 036/107] scip direct --- pyomo/contrib/solver/plugins.py | 2 +- .../contrib/solver/solvers/scip/scip_direct.py | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index a4d6f5f9004..81e3677b19e 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -44,4 +44,4 @@ def load(): name='scip_direct', legacy_name='scip_direct_v2', doc='Direct interface pyscipopt', - ) + )(SCIPDirect) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 1ff470223bd..3926dd25a1c 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -16,6 +16,7 @@ from typing import Tuple, List, Optional, Sequence, Mapping, Dict from pyomo.common.collections import ComponentMap +from pyomo.core.expr.numvalue import is_constant from pyomo.common.numeric_types import native_numeric_types from pyomo.common.errors import InfeasibleConstraintException, ApplicationError from pyomo.common.timing import HierarchicalTimer @@ -67,6 +68,8 @@ ) from pyomo.common.config import ConfigValue from pyomo.common.tee import capture_output, TeeStream +from pyomo.core.base.units_container import _PyomoUnit +from pyomo.contrib.fbbt.fbbt import compute_bounds_on_expr logger = logging.getLogger(__name__) @@ -132,7 +135,15 @@ def _handle_negation(node, data, opt): def _handle_pow(node, data, opt): - return data[0] ** data[1] + x, y = data # x ** y = exp(log(x**y)) = exp(y*log(x)) + if is_constant(node.args[1]): + return x**y + else: + xlb, xub = compute_bounds_on_expr(node.args[0]) + if xlb > 0: + return scip.exp(y*scip.log(x)) + else: + return x**y # scip will probably raise an error here def _handle_product(node, data, opt): @@ -210,6 +221,10 @@ def _handle_named_expression(node, data, opt): return data[0] +def _handle_unit(node, data, opt): + return node.value + + _operator_map = { NegationExpression: _handle_negation, PowExpression: _handle_pow, @@ -237,6 +252,7 @@ def _handle_named_expression(node, data, opt): float: _handle_float, int: _handle_float, AutoLinkedBinaryVar: _handle_var, + _PyomoUnit: _handle_unit, } From 0b84dcc77e826022c705ae95d187dc29e0b24e1f Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 18 Aug 2025 04:57:27 -0600 Subject: [PATCH 037/107] more expression types for scip --- .../solver/solvers/scip/scip_direct.py | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 3926dd25a1c..d72ce47ef02 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -42,6 +42,7 @@ NPV_SumExpression, NPV_UnaryFunctionExpression, ) +from pyomo.core.expr.numvalue import NumericConstant from pyomo.gdp.disjunct import AutoLinkedBinaryVar from pyomo.core.base.expression import ExpressionData, ScalarExpression from pyomo.core.expr.relational_expr import EqualityExpression, InequalityExpression, RangedExpression @@ -126,6 +127,10 @@ def _handle_param(node, data, opt): return scip_param +def _handle_constant(node, data, opt): + return node.value + + def _handle_float(node, data, opt): return float(node) @@ -167,6 +172,10 @@ def _handle_log(node, data, opt): return scip.log(data[0]) +def _handle_log10(node, data, opt): + return scip.log(data[0]) / math.log(10) + + def _handle_sin(node, data, opt): return scip.sin(data[0]) @@ -187,6 +196,12 @@ def _handle_tan(node, data, opt): return scip.sin(data[0]) / scip.cos(data[0]) +def _handle_tanh(node, data, opt): + x = data[0] + _exp = scip.exp + return (_exp(x) - _exp(-x)) / (_exp(x) + _exp(-x)) + + _unary_map = { 'exp': _handle_exp, 'log': _handle_log, @@ -195,6 +210,8 @@ def _handle_tan(node, data, opt): 'sqrt': _handle_sqrt, 'abs': _handle_abs, 'tan': _handle_tan, + 'log10': _handle_log10, + 'tanh': _handle_tanh, } @@ -253,6 +270,7 @@ def _handle_unit(node, data, opt): int: _handle_float, AutoLinkedBinaryVar: _handle_var, _PyomoUnit: _handle_unit, + NumericConstant: _handle_constant, } @@ -427,7 +445,7 @@ def solve(self, model: BlockData, **kwds) -> Results: scip_model.setParam(key, option) timer.start('optimize') - with capture_output(TeeStream(*ostreams), capture_fd=True): + with capture_output(TeeStream(*ostreams), capture_fd=False): scip_model.optimize() timer.stop('optimize') @@ -690,7 +708,7 @@ def _postsolve( if has_obj: try: - if scip_model.getObjVal() < scip_model.infinity(): + if scip_model.getNSols() > 0 and scip_model.getObjVal() < scip_model.infinity(): results.incumbent_objective = scip_model.getObjVal() else: results.incumbent_objective = None From 3180462477261e7c1eaa63e349b35abf080c6f05 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 20 Aug 2025 09:08:52 -0600 Subject: [PATCH 038/107] capture_fd for scip --- pyomo/contrib/solver/solvers/scip/scip_direct.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index d72ce47ef02..8deca600b40 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -445,7 +445,7 @@ def solve(self, model: BlockData, **kwds) -> Results: scip_model.setParam(key, option) timer.start('optimize') - with capture_output(TeeStream(*ostreams), capture_fd=False): + with capture_output(TeeStream(*ostreams), capture_fd=True): scip_model.optimize() timer.stop('optimize') From 72912e0d9e5397a3403061aff3daf32a4e9b281c Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Fri, 22 Aug 2025 08:49:40 -0600 Subject: [PATCH 039/107] working on persistent interface to scip --- .../solver/solvers/scip/scip_direct.py | 161 +++++++++++++++++- 1 file changed, 153 insertions(+), 8 deletions(-) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 8deca600b40..344b1741552 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -9,6 +9,7 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ +from __future__ import annotations import datetime import io import logging @@ -49,7 +50,7 @@ from pyomo.core.staleflag import StaleFlagManager from pyomo.core.expr.visitor import StreamBasedExpressionVisitor from pyomo.common.dependencies import attempt_import -from pyomo.contrib.solver.common.base import SolverBase, Availability +from pyomo.contrib.solver.common.base import SolverBase, Availability, PersistentSolverBase from pyomo.contrib.solver.common.config import BranchAndBoundConfig from pyomo.contrib.solver.common.util import ( NoFeasibleSolutionError, @@ -71,6 +72,7 @@ from pyomo.common.tee import capture_output, TeeStream from pyomo.core.base.units_container import _PyomoUnit from pyomo.contrib.fbbt.fbbt import compute_bounds_on_expr +from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector logger = logging.getLogger(__name__) @@ -354,7 +356,72 @@ def load_import_suffixes(self, solution_id=None): load_import_suffixes(self._pyomo_model, self, solution_id=solution_id) -class SCIPDirect(SolverBase): +class ScipPersistentSolutionLoader(ScipDirectSolutionLoader): + def __init__( + self, + solver_model, + var_id_map, + var_map, + con_map, + pyomo_model, + opt, + ) -> None: + super().__init__( + solver_model, + var_id_map, + var_map, + con_map, + pyomo_model, + opt, + ) + self._valid = False + + def invalidate(self): + self._valid = False + + def _assert_solution_still_valid(self): + if not self._valid: + raise RuntimeError('The results in the solver are no longer valid.') + + def load_vars( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None + ) -> None: + self._assert_solution_still_valid() + return super().load_vars(vars_to_load, solution_id) + + def get_vars( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None + ) -> Mapping[VarData, float]: + self._assert_solution_still_valid() + return super().get_vars(vars_to_load, solution_id) + + def get_duals( + self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=None + ) -> Dict[ConstraintData, float]: + self._assert_solution_still_valid() + return super().get_duals(cons_to_load) + + def get_reduced_costs( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None + ) -> Mapping[VarData, float]: + self._assert_solution_still_valid() + return super().get_reduced_costs(vars_to_load) + + def get_number_of_solutions(self) -> int: + self._assert_solution_still_valid() + return super().get_number_of_solutions() + + def get_solution_ids(self) -> List: + self._assert_solution_still_valid() + return super().get_solution_ids() + + def load_import_suffixes(self, solution_id=None): + self._assert_solution_still_valid() + super().load_import_suffixes(solution_id) + + + +class ScipDirect(SolverBase): _available = None _tc_map = None @@ -393,11 +460,11 @@ def available(self) -> Availability: return self._available if not scip_available: - SCIPDirect._available = Availability.NotFound + ScipDirect._available = Availability.NotFound elif self.version() < self._minimum_version: - SCIPDirect._available = Availability.BadVersion + ScipDirect._available = Availability.BadVersion else: - SCIPDirect._available = Availability.FullLicense + ScipDirect._available = Availability.FullLicense return self._available @@ -465,9 +532,9 @@ def solve(self, model: BlockData, **kwds) -> Results: return results def _get_tc_map(self): - if SCIPDirect._tc_map is None: + if ScipDirect._tc_map is None: tc = TerminationCondition - SCIPDirect._tc_map = { + ScipDirect._tc_map = { "unknown": tc.unknown, "userinterrupt": tc.interrupted, "nodelimit": tc.iterationLimit, @@ -487,7 +554,7 @@ def _get_tc_map(self): "inforunbd": tc.infeasibleOrUnbounded, "terminate": tc.unknown, } - return SCIPDirect._tc_map + return ScipDirect._tc_map def _get_infeasible_results(self): res = Results() @@ -753,3 +820,81 @@ def _mipstart(self): if pyomo_var.is_integer(): sol[scip_var] = pyomo_var.value self._solver_model.addSol(sol) + + +class _SCIPObserver(Observer): + def __init__(self, opt: ScipPersistent) -> None: + self.opt = opt + + def add_variables(self, variables: List[VarData]): + self.opt._add_variables(variables) + + def add_parameters(self, params: List[ParamData]): + pass + + def add_constraints(self, cons: List[ConstraintData]): + self.opt._add_constraints(cons) + + def add_sos_constraints(self, cons: List[SOSConstraintData]): + self.opt._add_sos_constraints(cons) + + def set_objective(self, obj: ObjectiveData | None): + self.opt._set_objective(obj) + + def remove_constraints(self, cons: List[ConstraintData]): + self.opt._remove_constraints(cons) + + def remove_sos_constraints(self, cons: List[SOSConstraintData]): + self.opt._remove_sos_constraints(cons) + + def remove_variables(self, variables: List[VarData]): + self.opt._remove_variables(variables) + + def remove_parameters(self, params: List[ParamData]): + pass + + def update_variables(self, variables: List[VarData]): + self.opt._update_variables(variables) + + def update_parameters(self, params: List[ParamData]): + self.opt._update_parameters(params) + + +class ScipPersistent(ScipDirect, PersistentSolverBase): + _minimum_version = (5, 5, 0) # this is probably conservative + + CONFIG = ScipConfig() + + def __init__(self, **kwds): + super().__init__(**kwds) + self._pyomo_model = None + self._objective = None + self._observer = _SCIPObserver(self) + self._change_detector = ModelChangeDetector(observers=[self._observer]) + + @property + def auto_updates(self): + return self._change_detector.config + + def _clear(self): + super()._clear() + self._pyomo_model = None + self._objective = None + + def _create_solver_model(self, model): + if model is self._pyomo_model: + self.update() + else: + self.set_instance(model=model) + + solution_loader = ScipPersistentSolutionLoader( + solver_model=self._solver_model, + var_id_map=self._vars, + var_map=self._pyomo_var_to_solver_var_map, + con_map=self._pyomo_con_to_solver_con_map, + pyomo_model=model, + opt=self, + ) + + has_obj = self._objective is not None: + return self._solver_model, solution_loader, has_obj \ No newline at end of file From cfa1e9108f6edd05d238056e6f071122a3164d39 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 1 Sep 2025 17:10:19 -0600 Subject: [PATCH 040/107] minor fixes --- pyomo/contrib/solver/plugins.py | 4 ++-- pyomo/contrib/solver/solvers/scip/scip_direct.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index 81e3677b19e..895b6387725 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -15,7 +15,7 @@ from .solvers.gurobi.gurobi_direct import GurobiDirect from .solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic, GurobiPersistent from .solvers.highs import Highs -from .solvers.scip.scip_direct import SCIPDirect +from .solvers.scip.scip_direct import ScipDirect def load(): @@ -44,4 +44,4 @@ def load(): name='scip_direct', legacy_name='scip_direct_v2', doc='Direct interface pyscipopt', - )(SCIPDirect) + )(ScipDirect) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 344b1741552..1032affd597 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -896,5 +896,5 @@ def _create_solver_model(self, model): opt=self, ) - has_obj = self._objective is not None: + has_obj = self._objective is not None return self._solver_model, solution_loader, has_obj \ No newline at end of file From e6331dfa67f60224a075f0fb1a8c4bc57baf43df Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 5 Oct 2025 21:28:15 -0600 Subject: [PATCH 041/107] persistent interface to scip --- .../solver/solvers/scip/scip_direct.py | 223 ++++++++++++++++-- 1 file changed, 204 insertions(+), 19 deletions(-) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 1032affd597..cf0c71606f8 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -25,6 +25,7 @@ from pyomo.core.base.var import VarData, ScalarVar from pyomo.core.base.param import ParamData, ScalarParam from pyomo.core.base.constraint import Constraint, ConstraintData +from pyomo.core.base.objective import ObjectiveData from pyomo.core.base.sos import SOSConstraint, SOSConstraintData from pyomo.core.kernel.objective import minimize, maximize from pyomo.core.expr.numeric_expr import ( @@ -72,7 +73,7 @@ from pyomo.common.tee import capture_output, TeeStream from pyomo.core.base.units_container import _PyomoUnit from pyomo.contrib.fbbt.fbbt import compute_bounds_on_expr -from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector +from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector, AutoUpdateConfig logger = logging.getLogger(__name__) @@ -420,7 +421,6 @@ def load_import_suffixes(self, solution_id=None): super().load_import_suffixes(solution_id) - class ScipDirect(SolverBase): _available = None @@ -822,7 +822,27 @@ def _mipstart(self): self._solver_model.addSol(sol) -class _SCIPObserver(Observer): +class ScipPersistentConfig(ScipConfig): + def __init__( + self, + description=None, + doc=None, + implicit=False, + implicit_domain=None, + visibility=0, + ): + ScipConfig.__init__( + self, + description=description, + doc=doc, + implicit=implicit, + implicit_domain=implicit_domain, + visibility=visibility, + ) + self.auto_updates: bool = self.declare('auto_updates', AutoUpdateConfig()) + + +class _ScipObserver(Observer): def __init__(self, opt: ScipPersistent) -> None: self.opt = opt @@ -838,8 +858,11 @@ def add_constraints(self, cons: List[ConstraintData]): def add_sos_constraints(self, cons: List[SOSConstraintData]): self.opt._add_sos_constraints(cons) - def set_objective(self, obj: ObjectiveData | None): - self.opt._set_objective(obj) + def add_objectives(self, objs: List[ObjectiveData]): + self.opt._add_objectives(objs) + + def remove_objectives(self, objs: List[ObjectiveData]): + self.opt._remove_objectives(objs) def remove_constraints(self, cons: List[ConstraintData]): self.opt._remove_constraints(cons) @@ -862,39 +885,201 @@ def update_parameters(self, params: List[ParamData]): class ScipPersistent(ScipDirect, PersistentSolverBase): _minimum_version = (5, 5, 0) # this is probably conservative - - CONFIG = ScipConfig() + CONFIG = ScipPersistentConfig() def __init__(self, **kwds): super().__init__(**kwds) self._pyomo_model = None - self._objective = None - self._observer = _SCIPObserver(self) - self._change_detector = ModelChangeDetector(observers=[self._observer]) - - @property - def auto_updates(self): - return self._change_detector.config + self._observer = None + self._change_detector = None + self._last_results_object: Optional[Results] = None def _clear(self): super()._clear() self._pyomo_model = None self._objective = None + self._observer = None + self._change_detector = None - def _create_solver_model(self, model): - if model is self._pyomo_model: + def _create_solver_model(self, pyomo_model): + if pyomo_model is self._pyomo_model: self.update() else: - self.set_instance(model=model) + self.set_instance(pyomo_model=pyomo_model) solution_loader = ScipPersistentSolutionLoader( solver_model=self._solver_model, var_id_map=self._vars, var_map=self._pyomo_var_to_solver_var_map, con_map=self._pyomo_con_to_solver_con_map, - pyomo_model=model, + pyomo_model=pyomo_model, opt=self, ) has_obj = self._objective is not None - return self._solver_model, solution_loader, has_obj \ No newline at end of file + return self._solver_model, solution_loader, has_obj + + def solve(self, model, **kwds) -> Results: + res = super().solve(model, **kwds) + return res + + def update(self): + if self.config.timer is None: + timer = HierarchicalTimer() + else: + timer = self.config.timer + if self._pyomo_model is None: + raise RuntimeError('must call set_instance or solve before update') + timer.start('update') + self._change_detector.update(timer=timer) + timer.stop('update') + + def set_instance(self, pyomo_model): + if self.config.timer is None: + timer = HierarchicalTimer() + else: + timer = self.config.timer + self._clear() + self._pyomo_model = pyomo_model + self._solver_model = scip.Model() + self._observer = _ScipObserver(self) + timer.start('set_instance') + self._change_detector = ModelChangeDetector( + model=self._pyomo_model, + observers=[self._observer], + **dict(self.config.auto_updates), + ) + self._change_detector.config = self.config.auto_updates + timer.stop('set_instance') + + def _invalidate_last_results(self): + if self._last_results_object is not None: + self._last_results_object.solution_loader.invalidate() + + def _add_variables(self, variables: List[VarData]): + self._invalidate_last_results() + for v in variables: + self._add_var(v) + + def _add_constraints(self, cons: List[ConstraintData]): + self._invalidate_last_results() + super()._add_constraints(cons) + + def _add_sos_constraints(self, cons: List[SOSConstraintData]): + self._invalidate_last_results() + return super()._add_sos_constraints(cons) + + def _add_objectives(self, objs: List[ObjectiveData]): + if len(objs) > 1: + raise NotImplementedError( + 'the persistent interface to gurobi currently ' + f'only supports single-objective problems; got {len(objs)}: ' + f'{[str(i) for i in objs]}' + ) + + if len(objs) == 0: + return + + obj = objs[0] + + if self._objective is not None: + raise NotImplementedError( + 'the persistent interface to gurobi currently ' + 'only supports single-objective problems; tried to add ' + f'an objective ({str(obj)}), but there is already an ' + f'active objective ({str(self._objective)})' + ) + + self._invalidate_last_results() + self._set_objective(obj) + + def _remove_objectives(self, objs: List[ObjectiveData]): + for obj in objs: + if obj is not self._objective: + raise RuntimeError( + 'tried to remove an objective that has not been added: ' + f'{str(obj)}' + ) + else: + self._invalidate_last_results() + self._set_objective(None) + + def _remove_constraints(self, cons: List[ConstraintData]): + for con in cons: + scip_con = self._pyomo_con_to_solver_con_map.pop(con) + self._solver_model.delCons(scip_con) + + def _remove_sos_constraints(self, cons: List[SOSConstraintData]): + for con in cons: + scip_con = self._pyomo_con_to_solver_con_map.pop(con) + self._solver_model.delCons(scip_con) + + def _remove_variables(self, variables: List[VarData]): + for v in variables: + vid = id(v) + scip_var = self._pyomo_var_to_solver_var_map.pop(vid) + self._solver_model.delVar(scip_var) + self._vars.pop(vid) + + def _update_variables(self, variables: List[VarData]): + for v in variables: + vid = id(v) + scip_var = self._pyomo_var_to_solver_var_map[vid] + vtype = self._scip_vtype_from_var(v) + lb, ub = self._scip_lb_ub_from_var(v) + self._solver_model.chgVarLb(scip_var, lb) + self._solver_model.chgVarUb(scip_var, ub) + self._solver_model.chgVarType(scip_var, vtype) + + def _update_parameters(self, params: List[ParamData]): + for p in params: + pid = id(p) + scip_var = self._pyomo_param_to_solver_param_map[pid] + lb = ub = p.value + self._solver_model.chgVarLb(scip_var, lb) + self._solver_model.chgVarUb(scip_var, ub) + + def add_variables(self, variables): + if self._change_detector is None: + raise RuntimeError('call set_instance first') + self._change_detector.add_variables(variables) + + def add_constraints(self, cons): + if self._change_detector is None: + raise RuntimeError('call set_instance first') + self._change_detector.add_constraints(cons) + + def add_sos_constraints(self, cons): + if self._change_detector is None: + raise RuntimeError('call set_instance first') + self._change_detector.add_sos_constraints(cons) + + def set_objective(self, obj: ObjectiveData): + if self._change_detector is None: + raise RuntimeError('call set_instance first') + self._change_detector.add_objectives([obj]) + + def remove_constraints(self, cons): + if self._change_detector is None: + raise RuntimeError('call set_instance first') + self._change_detector.remove_constraints(cons) + + def remove_sos_constraints(self, cons): + if self._change_detector is None: + raise RuntimeError('call set_instance first') + self._change_detector.remove_sos_constraints(cons) + + def remove_variables(self, variables): + if self._change_detector is None: + raise RuntimeError('call set_instance first') + self._change_detector.remove_variables(variables) + + def update_variables(self, variables): + if self._change_detector is None: + raise RuntimeError('call set_instance first') + self._change_detector.update_variables(variables) + + def update_parameters(self, params): + if self._change_detector is None: + raise RuntimeError('call set_instance first') + self._change_detector.update_parameters(params) From 98e2c9a9904b688eb4cb735474bbfd0ea092370f Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 5 Oct 2025 21:40:47 -0600 Subject: [PATCH 042/107] update docs --- .../reference/topical/solvers/scip_persistent.rst | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 doc/OnlineDocs/reference/topical/solvers/scip_persistent.rst diff --git a/doc/OnlineDocs/reference/topical/solvers/scip_persistent.rst b/doc/OnlineDocs/reference/topical/solvers/scip_persistent.rst deleted file mode 100644 index 63ed55b74e3..00000000000 --- a/doc/OnlineDocs/reference/topical/solvers/scip_persistent.rst +++ /dev/null @@ -1,7 +0,0 @@ -SCIPPersistent -================ - -.. autoclass:: pyomo.solvers.plugins.solvers.scip_persistent.SCIPPersistent - :members: - :inherited-members: - :show-inheritance: \ No newline at end of file From f0be4ffd5a9cfed6b51d8aa22453358426d873cd Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 5 Oct 2025 21:41:07 -0600 Subject: [PATCH 043/107] update docs --- doc/OnlineDocs/reference/topical/solvers/index.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/OnlineDocs/reference/topical/solvers/index.rst b/doc/OnlineDocs/reference/topical/solvers/index.rst index 628f9cfdab0..400032df076 100644 --- a/doc/OnlineDocs/reference/topical/solvers/index.rst +++ b/doc/OnlineDocs/reference/topical/solvers/index.rst @@ -9,4 +9,3 @@ Solver Interfaces gurobi_direct.rst gurobi_persistent.rst xpress_persistent.rst - scip_persistent.rst From 7ec95a8f4f2c4a9e678cc61aa97c71be20411171 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 5 Oct 2025 21:46:55 -0600 Subject: [PATCH 044/107] persistent interface to scip --- pyomo/contrib/solver/plugins.py | 7 ++++++- .../solver/tests/solvers/test_solvers.py | 17 +++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index 2c7bab3bf03..4ac74ecf560 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -15,7 +15,7 @@ from .solvers.gurobi.gurobi_direct import GurobiDirect from .solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic, GurobiPersistent from .solvers.highs import Highs -from .solvers.scip.scip_direct import ScipDirect +from .solvers.scip.scip_direct import ScipDirect, ScipPersistent def load(): @@ -45,3 +45,8 @@ def load(): legacy_name='scip_direct_v2', doc='Direct interface pyscipopt', )(ScipDirect) + SolverFactory.register( + name='scip_persistent', + legacy_name='scip_persistent_v2', + doc='Persistent interface pyscipopt', + )(ScipPersistent) diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index b49d80baa37..6bd7d01e679 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -34,7 +34,7 @@ SolutionStatus, Results, ) -from pyomo.contrib.solver.solvers.scip.scip_direct import SCIPDirect +from pyomo.contrib.solver.solvers.scip.scip_direct import ScipDirect, ScipPersistent from pyomo.contrib.solver.common.util import ( NoDualsError, NoOptimalSolutionError, @@ -60,30 +60,35 @@ ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), ('highs', Highs), - ('scip_direct', SCIPDirect), + ('scip_direct', ScipDirect), + ('scip_persistent', ScipPersistent), ] mip_solvers = [ ('gurobi_persistent', GurobiPersistent), ('gurobi_direct', GurobiDirect), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('highs', Highs), - ('scip_direct', SCIPDirect), + ('scip_direct', ScipDirect), + ('scip_persistent', ScipPersistent), ] nlp_solvers = [ ('ipopt', Ipopt), - ('scip_direct', SCIPDirect), + ('scip_direct', ScipDirect), + ('scip_persistent', ScipPersistent), ] qcp_solvers = [ ('gurobi_persistent', GurobiPersistent), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), - ('scip_direct', SCIPDirect), + ('scip_direct', ScipDirect), + ('scip_persistent', ScipPersistent), ] qp_solvers = qcp_solvers + [("highs", Highs)] miqcqp_solvers = [ ('gurobi_persistent', GurobiPersistent), ('gurobi_direct_quadratic', GurobiDirectQuadratic), - ('scip_direct', SCIPDirect), + ('scip_direct', ScipDirect), + ('scip_persistent', ScipPersistent), ] nl_solvers = [('ipopt', Ipopt)] nl_solvers_set = {i[0] for i in nl_solvers} From 75903d6cbbfbcd19f96bb709535e6b2a12b0086c Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 6 Oct 2025 08:44:16 -0600 Subject: [PATCH 045/107] persistent interface to scip --- pyomo/contrib/observer/model_observer.py | 9 ++ .../solver/solvers/scip/scip_direct.py | 135 +++++++++++++----- 2 files changed, 112 insertions(+), 32 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 2da340aab4f..77356ac1b57 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -1202,3 +1202,12 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): finally: if is_gc_enabled: gc.enable() + + def get_variables_impacted_by_param(self, p: ParamData): + return [self._vars[vid][0] for vid in self._referenced_params[id(p)][3]] + + def get_constraints_impacted_by_param(self, p: ParamData): + return list(self._referenced_params[id(p)][0]) + + def get_constraints_impacted_by_var(self, v: VarData): + return list(self._referenced_variables[id(v)][0]) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index cf0c71606f8..99de1d80125 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -110,7 +110,7 @@ def __init__( ) -def _handle_var(node, data, opt): +def _handle_var(node, data, opt, visitor): if id(node) not in opt._pyomo_var_to_solver_var_map: scip_var = opt._add_var(node) else: @@ -118,7 +118,13 @@ def _handle_var(node, data, opt): return scip_var -def _handle_param(node, data, opt): +def _handle_param(node, data, opt, visitor): + # for the persistent interface, we create scip variables in place + # of parameters. However, this makes things complicated for range + # constraints because scip does not allow variables in the + # lower and upper parts of range constraints + if visitor.in_range: + return node.value if not opt.is_persistent(): return node.value if not node.mutable: @@ -130,19 +136,19 @@ def _handle_param(node, data, opt): return scip_param -def _handle_constant(node, data, opt): +def _handle_constant(node, data, opt, visitor): return node.value -def _handle_float(node, data, opt): +def _handle_float(node, data, opt, visitor): return float(node) -def _handle_negation(node, data, opt): +def _handle_negation(node, data, opt, visitor): return -data[0] -def _handle_pow(node, data, opt): +def _handle_pow(node, data, opt, visitor): x, y = data # x ** y = exp(log(x**y)) = exp(y*log(x)) if is_constant(node.args[1]): return x**y @@ -154,52 +160,52 @@ def _handle_pow(node, data, opt): return x**y # scip will probably raise an error here -def _handle_product(node, data, opt): +def _handle_product(node, data, opt, visitor): assert len(data) == 2 return data[0] * data[1] -def _handle_division(node, data, opt): +def _handle_division(node, data, opt, visitor): return data[0] / data[1] -def _handle_sum(node, data, opt): +def _handle_sum(node, data, opt, visitor): return sum(data) -def _handle_exp(node, data, opt): +def _handle_exp(node, data, opt, visitor): return scip.exp(data[0]) -def _handle_log(node, data, opt): +def _handle_log(node, data, opt, visitor): return scip.log(data[0]) -def _handle_log10(node, data, opt): +def _handle_log10(node, data, opt, visitor): return scip.log(data[0]) / math.log(10) -def _handle_sin(node, data, opt): +def _handle_sin(node, data, opt, visitor): return scip.sin(data[0]) -def _handle_cos(node, data, opt): +def _handle_cos(node, data, opt, visitor): return scip.cos(data[0]) -def _handle_sqrt(node, data, opt): +def _handle_sqrt(node, data, opt, visitor): return scip.sqrt(data[0]) -def _handle_abs(node, data, opt): +def _handle_abs(node, data, opt, visitor): return abs(data[0]) -def _handle_tan(node, data, opt): +def _handle_tan(node, data, opt, visitor): return scip.sin(data[0]) / scip.cos(data[0]) -def _handle_tanh(node, data, opt): +def _handle_tanh(node, data, opt, visitor): x = data[0] _exp = scip.exp return (_exp(x) - _exp(-x)) / (_exp(x) + _exp(-x)) @@ -218,30 +224,32 @@ def _handle_tanh(node, data, opt): } -def _handle_unary(node, data, opt): +def _handle_unary(node, data, opt, visitor): if node.getname() in _unary_map: - return _unary_map[node.getname()](node, data, opt) + return _unary_map[node.getname()](node, data, opt, visitor) else: raise NotImplementedError(f'unable to handle unary expression: {str(node)}') -def _handle_equality(node, data, opt): +def _handle_equality(node, data, opt, visitor): return data[0] == data[1] -def _handle_ranged(node, data, opt): +def _handle_ranged(node, data, opt, visitor): + # note that the lower and upper parts of the + # range constraint cannot have variables return data[0] <= (data[1] <= data[2]) -def _handle_inequality(node, data, opt): +def _handle_inequality(node, data, opt, visitor): return data[0] <= data[1] -def _handle_named_expression(node, data, opt): +def _handle_named_expression(node, data, opt, visitor): return data[0] -def _handle_unit(node, data, opt): +def _handle_unit(node, data, opt, visitor): return node.value @@ -281,16 +289,26 @@ class _PyomoToScipVisitor(StreamBasedExpressionVisitor): def __init__(self, solver, **kwds): super().__init__(**kwds) self.solver = solver + self.in_range = False + + def initializeWalker(self, expr): + self.in_range = False + return True, None def exitNode(self, node, data): nt = type(node) if nt in _operator_map: - return _operator_map[nt](node, data, self.solver) + return _operator_map[nt](node, data, self.solver, self) elif nt in native_numeric_types: _operator_map[nt] = _handle_float - return _handle_float(node, data, self.solver) + return _handle_float(node, data, self.solver, self) else: raise NotImplementedError(f'unrecognized expression type: {nt}') + + def enterNode(self, node): + if type(node) is RangedExpression: + self.in_range = True + return None, [] logger = logging.getLogger("pyomo.solvers") @@ -375,7 +393,7 @@ def __init__( pyomo_model, opt, ) - self._valid = False + self._valid = True def invalidate(self): self._valid = False @@ -513,6 +531,7 @@ def solve(self, model: BlockData, **kwds) -> Results: timer.start('optimize') with capture_output(TeeStream(*ostreams), capture_fd=True): + # scip_model.writeProblem(filename='foo.lp') scip_model.optimize() timer.stop('optimize') @@ -723,7 +742,7 @@ def _set_objective(self, obj): vtype="C" ) - if self._objective is not None: + if self._obj_con is not None: self._solver_model.delCons(self._obj_con) if obj is None: @@ -850,7 +869,7 @@ def add_variables(self, variables: List[VarData]): self.opt._add_variables(variables) def add_parameters(self, params: List[ParamData]): - pass + self.opt._add_parameters(params) def add_constraints(self, cons: List[ConstraintData]): self.opt._add_constraints(cons) @@ -874,7 +893,7 @@ def remove_variables(self, variables: List[VarData]): self.opt._remove_variables(variables) def remove_parameters(self, params: List[ParamData]): - pass + self.opt._remove_parameters(params) def update_variables(self, variables: List[VarData]): self.opt._update_variables(variables) @@ -893,13 +912,22 @@ def __init__(self, **kwds): self._observer = None self._change_detector = None self._last_results_object: Optional[Results] = None - + self._needs_reopt = False + self._range_constraints = set() + def _clear(self): super()._clear() self._pyomo_model = None self._objective = None self._observer = None self._change_detector = None + self._needs_reopt = False + + def _check_reopt(self): + if self._needs_reopt: + # self._solver_model.freeReoptSolve() # when is it safe to use this one??? + self._solver_model.freeTransform() + self._needs_reopt = False def _create_solver_model(self, pyomo_model): if pyomo_model is self._pyomo_model: @@ -921,6 +949,7 @@ def _create_solver_model(self, pyomo_model): def solve(self, model, **kwds) -> Results: res = super().solve(model, **kwds) + self._needs_reopt = True return res def update(self): @@ -957,19 +986,32 @@ def _invalidate_last_results(self): self._last_results_object.solution_loader.invalidate() def _add_variables(self, variables: List[VarData]): + self._check_reopt() self._invalidate_last_results() for v in variables: self._add_var(v) + def _add_parameters(self, params: List[ParamData]): + self._check_reopt() + self._invalidate_last_results() + for p in params: + self._add_param(p) + def _add_constraints(self, cons: List[ConstraintData]): + self._check_reopt() self._invalidate_last_results() + for con in cons: + if type(con.expr) is RangedExpression: + self._range_constraints.add(con) super()._add_constraints(cons) def _add_sos_constraints(self, cons: List[SOSConstraintData]): + self._check_reopt() self._invalidate_last_results() return super()._add_sos_constraints(cons) def _add_objectives(self, objs: List[ObjectiveData]): + self._check_reopt() if len(objs) > 1: raise NotImplementedError( 'the persistent interface to gurobi currently ' @@ -994,6 +1036,7 @@ def _add_objectives(self, objs: List[ObjectiveData]): self._set_objective(obj) def _remove_objectives(self, objs: List[ObjectiveData]): + self._check_reopt() for obj in objs: if obj is not self._objective: raise RuntimeError( @@ -1005,23 +1048,41 @@ def _remove_objectives(self, objs: List[ObjectiveData]): self._set_objective(None) def _remove_constraints(self, cons: List[ConstraintData]): + self._check_reopt() + self._invalidate_last_results() for con in cons: scip_con = self._pyomo_con_to_solver_con_map.pop(con) self._solver_model.delCons(scip_con) + self._range_constraints.discard(con) def _remove_sos_constraints(self, cons: List[SOSConstraintData]): + self._check_reopt() + self._invalidate_last_results() for con in cons: scip_con = self._pyomo_con_to_solver_con_map.pop(con) self._solver_model.delCons(scip_con) def _remove_variables(self, variables: List[VarData]): + self._check_reopt() + self._invalidate_last_results() for v in variables: vid = id(v) scip_var = self._pyomo_var_to_solver_var_map.pop(vid) self._solver_model.delVar(scip_var) self._vars.pop(vid) + def _remove_parameters(self, params: List[ParamData]): + self._check_reopt() + self._invalidate_last_results() + for p in params: + pid = id(p) + scip_var = self._pyomo_param_to_solver_param_map.pop(pid) + self._solver_model.delVar(scip_var) + self._params.pop(pid) + def _update_variables(self, variables: List[VarData]): + self._check_reopt() + self._invalidate_last_results() for v in variables: vid = id(v) scip_var = self._pyomo_var_to_solver_var_map[vid] @@ -1032,12 +1093,22 @@ def _update_variables(self, variables: List[VarData]): self._solver_model.chgVarType(scip_var, vtype) def _update_parameters(self, params: List[ParamData]): + self._check_reopt() + self._invalidate_last_results() for p in params: pid = id(p) scip_var = self._pyomo_param_to_solver_param_map[pid] lb = ub = p.value self._solver_model.chgVarLb(scip_var, lb) self._solver_model.chgVarUb(scip_var, ub) + impacted_vars = self._change_detector.get_variables_impacted_by_param(p) + if impacted_vars: + self._update_variables(impacted_vars) + impacted_cons = self._change_detector.get_constraints_impacted_by_param(p) + for con in impacted_cons: + if con in self._range_constraints: + self._remove_constraints([con]) + self._add_constraints([con]) def add_variables(self, variables): if self._change_detector is None: From 0051024e7a44ab7d4c3df43c96890c49669e3b69 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 6 Oct 2025 10:46:14 -0600 Subject: [PATCH 046/107] updating tests --- .../solver/tests/solvers/test_solvers.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 6bd7d01e679..3665de4521a 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -2381,7 +2381,8 @@ def test_param_updates(self, name: str, opt_class: Type[SolverBase]): m.obj = pyo.Objective(expr=m.y) m.c1 = pyo.Constraint(expr=(0, m.y - m.a1 * m.x - m.b1, None)) m.c2 = pyo.Constraint(expr=(None, -m.y + m.a2 * m.x + m.b2, 0)) - m.dual = pyo.Suffix(direction=pyo.Suffix.IMPORT) + if (name, opt_class) in dual_solvers: + m.dual = pyo.Suffix(direction=pyo.Suffix.IMPORT) params_to_test = [(1, -1, 2, 1), (1, -2, 2, 1), (1, -1, 3, 1)] for a1, a2, b1, b2 in params_to_test: @@ -2393,8 +2394,9 @@ def test_param_updates(self, name: str, opt_class: Type[SolverBase]): pyo.assert_optimal_termination(res) self.assertAlmostEqual(m.x.value, (b2 - b1) / (a1 - a2)) self.assertAlmostEqual(m.y.value, a1 * (b2 - b1) / (a1 - a2) + b1) - self.assertAlmostEqual(m.dual[m.c1], (1 + a1 / (a2 - a1))) - self.assertAlmostEqual(m.dual[m.c2], a1 / (a2 - a1)) + if (name, opt_class) in dual_solvers: + self.assertAlmostEqual(m.dual[m.c1], (1 + a1 / (a2 - a1))) + self.assertAlmostEqual(m.dual[m.c2], a1 / (a2 - a1)) @parameterized.expand(input=all_solvers) def test_load_solutions(self, name: str, opt_class: Type[SolverBase]): @@ -2405,11 +2407,14 @@ def test_load_solutions(self, name: str, opt_class: Type[SolverBase]): m.x = pyo.Var() m.obj = pyo.Objective(expr=m.x) m.c = pyo.Constraint(expr=(-1, m.x, 1)) - m.dual = pyo.Suffix(direction=pyo.Suffix.IMPORT) + if (name, opt_class) in dual_solvers: + m.dual = pyo.Suffix(direction=pyo.Suffix.IMPORT) res = opt.solve(m, load_solutions=False) pyo.assert_optimal_termination(res) self.assertIsNone(m.x.value) - self.assertNotIn(m.c, m.dual) + if (name, opt_class) in dual_solvers: + self.assertNotIn(m.c, m.dual) m.solutions.load_from(res) self.assertAlmostEqual(m.x.value, -1) - self.assertAlmostEqual(m.dual[m.c], 1) + if (name, opt_class) in dual_solvers: + self.assertAlmostEqual(m.dual[m.c], 1) From b037b9c356cd9caac3bf04924bece2c247962f91 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 6 Oct 2025 10:57:00 -0600 Subject: [PATCH 047/107] forgot to delete/revert some files --- .../solver/solvers/scip/scip_persistent.py | 192 ----------- pyomo/solvers/tests/checks/test_SCIPDirect.py | 310 ----------------- .../tests/checks/test_SCIPPersistent.py | 318 ------------------ pyomo/solvers/tests/solvers.py | 15 - pyomo/solvers/tests/testcases.py | 9 - 5 files changed, 844 deletions(-) delete mode 100644 pyomo/contrib/solver/solvers/scip/scip_persistent.py delete mode 100644 pyomo/solvers/tests/checks/test_SCIPDirect.py delete mode 100644 pyomo/solvers/tests/checks/test_SCIPPersistent.py diff --git a/pyomo/contrib/solver/solvers/scip/scip_persistent.py b/pyomo/contrib/solver/solvers/scip/scip_persistent.py deleted file mode 100644 index bc64edc28a8..00000000000 --- a/pyomo/contrib/solver/solvers/scip/scip_persistent.py +++ /dev/null @@ -1,192 +0,0 @@ -# ___________________________________________________________________________ -# -# Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2025 -# National Technology and Engineering Solutions of Sandia, LLC -# Under the terms of Contract DE-NA0003525 with National Technology and -# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain -# rights in this software. -# This software is distributed under the 3-clause BSD License. -# ___________________________________________________________________________ -from pyomo.solvers.plugins.solvers.scip_direct import SCIPDirect -from pyomo.solvers.plugins.solvers.persistent_solver import PersistentSolver -from pyomo.opt.base import SolverFactory - - -@SolverFactory.register("scip_persistent", doc="Persistent python interface to SCIP") -class SCIPPersistent(PersistentSolver, SCIPDirect): - """ - A class that provides a persistent interface to SCIP. Direct solver interfaces do not use any file io. - Rather, they interface directly with the python bindings for the specific solver. Persistent solver interfaces - are similar except that they "remember" their model. Thus, persistent solver interfaces allow incremental changes - to the solver model (e.g., the gurobi python model or the cplex python model). Note that users are responsible - for notifying the persistent solver interfaces when changes are made to the corresponding pyomo model. - - Keyword Arguments - ----------------- - model: ConcreteModel - Passing a model to the constructor is equivalent to calling the set_instance method. - type: str - String indicating the class type of the solver instance. - name: str - String representing either the class type of the solver instance or an assigned name. - doc: str - Documentation for the solver - options: dict - Dictionary of solver options - """ - - def __init__(self, **kwds): - kwds["type"] = "scip_persistent" - PersistentSolver.__init__(self, **kwds) - SCIPDirect._init(self) - - self._pyomo_model = kwds.pop("model", None) - if self._pyomo_model is not None: - self.set_instance(self._pyomo_model, **kwds) - - def _remove_constraint(self, solver_conname): - con = self._solver_con_to_pyomo_con_map[solver_conname] - scip_con = self._pyomo_con_to_solver_con_expr_map[con] - self._solver_model.delCons(scip_con) - del self._pyomo_con_to_solver_con_expr_map[con] - - def _remove_sos_constraint(self, solver_sos_conname): - con = self._solver_con_to_pyomo_con_map[solver_sos_conname] - scip_con = self._pyomo_con_to_solver_con_expr_map[con] - self._solver_model.delCons(scip_con) - del self._pyomo_con_to_solver_con_expr_map[con] - - def _remove_var(self, solver_varname): - var = self._solver_var_to_pyomo_var_map[solver_varname] - scip_var = self._pyomo_var_to_solver_var_expr_map[var] - self._solver_model.delVar(scip_var) - del self._pyomo_var_to_solver_var_expr_map[var] - - def _warm_start(self): - SCIPDirect._warm_start(self) - - def update_var(self, var): - """Update a single variable in the solver's model. - - This will update bounds, fix/unfix the variable as needed, and - update the variable type. - - Parameters - ---------- - var: Var (scalar Var or single _VarData) - - """ - # see PR #366 for discussion about handling indexed - # objects and keeping compatibility with the - # pyomo.kernel objects - # if var.is_indexed(): - # for child_var in var.values(): - # self.compile_var(child_var) - # return - if var not in self._pyomo_var_to_solver_var_map: - raise ValueError( - f"The Var provided to compile_var needs to be added first: {var}" - ) - scip_var = self._pyomo_var_to_solver_var_map[var] - vtype = self._scip_vtype_from_var(var) - lb, ub = self._scip_lb_ub_from_var(var) - - self._solver_model.chgVarLb(scip_var, lb) - self._solver_model.chgVarUb(scip_var, ub) - self._solver_model.chgVarType(scip_var, vtype) - - def write(self, filename, filetype=""): - """ - Write the model to a file (e.g., an lp file). - - Parameters - ---------- - filename: str - Name of the file to which the model should be written. - filetype: str - The file type (e.g., lp). - """ - self._solver_model.writeProblem(filename + filetype) - - def set_scip_param(self, param, val): - """ - Set a SCIP parameter. - - Parameters - ---------- - param: str - The SCIP parameter to set. Options include any SCIP parameter. - Please see the SCIP documentation for options. - Link at: https://www.scipopt.org/doc/html/PARAMETERS.php - val: any - The value to set the parameter to. See SCIP documentation for possible values. - """ - self._solver_model.setParam(param, val) - - def get_scip_param(self, param): - """ - Get the value of the SCIP parameter. - - Parameters - ---------- - param: str or int or float - The SCIP parameter to get the value of. See SCIP documentation for possible options. - Link at: https://www.scipopt.org/doc/html/PARAMETERS.php - """ - return self._solver_model.getParam(param) - - def _add_column(self, var, obj_coef, constraints, coefficients): - """Add a column to the solver's model - - This will add the Pyomo variable var to the solver's - model, and put the coefficients on the associated - constraints in the solver model. If the obj_coef is - not zero, it will add obj_coef*var to the objective - of the solver's model. - - Parameters - ---------- - var: Var (scalar Var or single _VarData) - obj_coef: float - constraints: list of solver constraints - coefficients: list of coefficients to put on var in the associated constraint - """ - - # Set-up add var - varname = self._symbol_map.getSymbol(var, self._labeler) - vtype = self._scip_vtype_from_var(var) - lb, ub = self._scip_lb_ub_from_var(var) - - # Add the variable to the model and then to all the constraints - scip_var = self._solver_model.addVar(lb=lb, ub=ub, vtype=vtype, name=varname) - self._pyomo_var_to_solver_var_expr_map[var] = scip_var - self._solver_var_to_pyomo_var_map[varname] = var - self._referenced_variables[var] = len(coefficients) - - # Get the SCIP cons by passing through two dictionaries - pyomo_cons = [self._solver_con_to_pyomo_con_map[con] for con in constraints] - scip_cons = [ - self._pyomo_con_to_solver_con_expr_map[pyomo_con] - for pyomo_con in pyomo_cons - ] - - for i, scip_con in enumerate(scip_cons): - if not scip_con.isLinear(): - raise ValueError( - "_add_column functionality not supported for non-linear constraints" - ) - self._solver_model.addConsCoeff(scip_con, scip_var, coefficients[i]) - con = self._solver_con_to_pyomo_con_map[scip_con.name] - self._vars_referenced_by_con[con].add(var) - - sense = self._solver_model.getObjectiveSense() - self._solver_model.setObjective(obj_coef * scip_var, sense=sense, clear=False) - - def reset(self): - """This function is necessary to call before making any changes to the - SCIP model after optimizing. It frees solution run specific information - that is not automatically done when changes to an already solved model - are made. Making changes to an already optimized model, e.g. adding additional - constraints will raise an error unless this function is called.""" - self._solver_model.freeTransform() diff --git a/pyomo/solvers/tests/checks/test_SCIPDirect.py b/pyomo/solvers/tests/checks/test_SCIPDirect.py deleted file mode 100644 index 186de0eaf58..00000000000 --- a/pyomo/solvers/tests/checks/test_SCIPDirect.py +++ /dev/null @@ -1,310 +0,0 @@ -# ___________________________________________________________________________ -# -# Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2025 -# National Technology and Engineering Solutions of Sandia, LLC -# Under the terms of Contract DE-NA0003525 with National Technology and -# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain -# rights in this software. -# This software is distributed under the 3-clause BSD License. -# ___________________________________________________________________________ - -import sys - -import pyomo.common.unittest as unittest - -from pyomo.environ import ( - ConcreteModel, - AbstractModel, - Var, - Objective, - Block, - Constraint, - Suffix, - NonNegativeIntegers, - NonNegativeReals, - Integers, - Binary, - value, -) -from pyomo.opt import SolverFactory, TerminationCondition, SolutionStatus - -try: - import pyscipopt - - scip_available = True -except ImportError: - scip_available = False - - -@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") -class SCIPDirectTests(unittest.TestCase): - def setUp(self): - self.stderr = sys.stderr - sys.stderr = None - - def tearDown(self): - sys.stderr = self.stderr - - def test_infeasible_lp(self): - with SolverFactory("scip_direct", solver_io="python") as opt: - model = ConcreteModel() - model.X = Var(within=NonNegativeReals) - model.C1 = Constraint(expr=model.X == 1) - model.C2 = Constraint(expr=model.X == 2) - model.O = Objective(expr=model.X) - - results = opt.solve(model) - - self.assertEqual( - results.solver.termination_condition, TerminationCondition.infeasible - ) - - def test_unbounded_lp(self): - with SolverFactory("scip_direct", solver_io="python") as opt: - model = ConcreteModel() - model.X = Var() - model.O = Objective(expr=model.X) - - results = opt.solve(model) - - self.assertIn( - results.solver.termination_condition, - ( - TerminationCondition.unbounded, - TerminationCondition.infeasibleOrUnbounded, - ), - ) - - def test_optimal_lp(self): - with SolverFactory("scip_direct", solver_io="python") as opt: - model = ConcreteModel() - model.X = Var(within=NonNegativeReals) - model.O = Objective(expr=model.X) - - results = opt.solve(model, load_solutions=False) - - self.assertEqual(results.solution.status, SolutionStatus.optimal) - - def test_infeasible_mip(self): - with SolverFactory("scip_direct", solver_io="python") as opt: - model = ConcreteModel() - model.X = Var(within=NonNegativeIntegers) - model.C1 = Constraint(expr=model.X == 1) - model.C2 = Constraint(expr=model.X == 2) - model.O = Objective(expr=model.X) - - results = opt.solve(model) - - self.assertEqual( - results.solver.termination_condition, TerminationCondition.infeasible - ) - - def test_unbounded_mip(self): - with SolverFactory("scip_direct", solver_io="python") as opt: - model = AbstractModel() - model.X = Var(within=Integers) - model.O = Objective(expr=model.X) - - instance = model.create_instance() - results = opt.solve(instance) - - self.assertIn( - results.solver.termination_condition, - ( - TerminationCondition.unbounded, - TerminationCondition.infeasibleOrUnbounded, - ), - ) - - def test_optimal_mip(self): - with SolverFactory("scip_direct", solver_io="python") as opt: - model = ConcreteModel() - model.X = Var(within=NonNegativeIntegers) - model.O = Objective(expr=model.X) - - results = opt.solve(model, load_solutions=False) - - self.assertEqual(results.solution.status, SolutionStatus.optimal) - - -@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") -class TestAddVar(unittest.TestCase): - def test_add_single_variable(self): - """Test that the variable is added correctly to `solver_model`.""" - model = ConcreteModel() - - opt = SolverFactory("scip_direct", solver_io="python") - opt._set_instance(model) - - self.assertEqual(opt._solver_model.getNVars(), 0) - - model.X = Var(within=Binary) - - opt._add_var(model.X) - - self.assertEqual(opt._solver_model.getNVars(), 1) - self.assertEqual(opt._solver_model.getVars()[0].vtype(), "BINARY") - - def test_add_block_containing_single_variable(self): - """Test that the variable is added correctly to `solver_model`.""" - model = ConcreteModel() - - opt = SolverFactory("scip_direct", solver_io="python") - opt._set_instance(model) - - self.assertEqual(opt._solver_model.getNVars(), 0) - - model.X = Var(within=Binary) - - opt._add_block(model) - - self.assertEqual(opt._solver_model.getNVars(), 1) - self.assertEqual(opt._solver_model.getVars()[0].vtype(), "BINARY") - - def test_add_block_containing_multiple_variables(self): - """Test that: - - The variable is added correctly to `solver_model` - - Fixed variable bounds are set correctly - """ - model = ConcreteModel() - - opt = SolverFactory("scip_direct", solver_io="python") - opt._set_instance(model) - - self.assertEqual(opt._solver_model.getNVars(), 0) - - model.X1 = Var(within=Binary) - model.X2 = Var(within=NonNegativeReals) - model.X3 = Var(within=NonNegativeIntegers) - - model.X3.fix(5) - - opt._add_block(model) - - self.assertEqual(opt._solver_model.getNVars(), 3) - scip_vars = opt._solver_model.getVars() - vtypes = [scip_var.vtype() for scip_var in scip_vars] - assert "BINARY" in vtypes and "CONTINUOUS" in vtypes and "INTEGER" in vtypes - lbs = [scip_var.getLbGlobal() for scip_var in scip_vars] - ubs = [scip_var.getUbGlobal() for scip_var in scip_vars] - assert 0 in lbs and 5 in lbs - assert ( - 1 in ubs - and 5 in ubs - and any([opt._solver_model.isInfinity(ub) for ub in ubs]) - ) - - -@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") -class TestAddCon(unittest.TestCase): - def test_add_single_constraint(self): - model = ConcreteModel() - model.X = Var(within=Binary) - - opt = SolverFactory("scip_direct", solver_io="python") - opt._set_instance(model) - - self.assertEqual(opt._solver_model.getNConss(), 0) - - model.C = Constraint(expr=model.X == 1) - - opt._add_constraint(model.C) - - self.assertEqual(opt._solver_model.getNConss(), 1) - con = opt._solver_model.getConss()[0] - self.assertEqual(con.isLinear(), 1) - self.assertEqual(opt._solver_model.getRhs(con), 1) - - def test_add_block_containing_single_constraint(self): - model = ConcreteModel() - model.X = Var(within=Binary) - - opt = SolverFactory("scip_direct", solver_io="python") - opt._set_instance(model) - - self.assertEqual(opt._solver_model.getNConss(), 0) - - model.B = Block() - model.B.C = Constraint(expr=model.X == 1) - - opt._add_block(model.B) - - self.assertEqual(opt._solver_model.getNConss(), 1) - con = opt._solver_model.getConss()[0] - self.assertEqual(con.isLinear(), 1) - self.assertEqual(opt._solver_model.getRhs(con), 1) - - def test_add_block_containing_multiple_constraints(self): - model = ConcreteModel() - model.X = Var(within=Binary) - - opt = SolverFactory("scip_direct", solver_io="python") - opt._set_instance(model) - - self.assertEqual(opt._solver_model.getNConss(), 0) - - model.B = Block() - model.B.C1 = Constraint(expr=model.X == 1) - model.B.C2 = Constraint(expr=model.X <= 1) - model.B.C3 = Constraint(expr=model.X >= 1) - - opt._add_block(model.B) - - self.assertEqual(opt._solver_model.getNConss(), 3) - - -@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") -class TestLoadVars(unittest.TestCase): - def setUp(self): - opt = SolverFactory("scip_direct", solver_io="python") - model = ConcreteModel() - model.X = Var(within=NonNegativeReals, initialize=0) - model.Y = Var(within=NonNegativeReals, initialize=0) - - model.C1 = Constraint(expr=2 * model.X + model.Y >= 8) - model.C2 = Constraint(expr=model.X + 3 * model.Y >= 6) - - model.O = Objective(expr=model.X + model.Y) - - opt.solve(model, load_solutions=False, save_results=False) - - self._model = model - self._opt = opt - - def test_all_vars_are_loaded(self): - self.assertTrue(self._model.X.stale) - self.assertTrue(self._model.Y.stale) - self.assertEqual(value(self._model.X), 0) - self.assertEqual(value(self._model.Y), 0) - - self._opt.load_vars() - - self.assertFalse(self._model.X.stale) - self.assertFalse(self._model.Y.stale) - self.assertAlmostEqual(value(self._model.X), 3.6) - self.assertAlmostEqual(value(self._model.Y), 0.8) - - def test_only_specified_vars_are_loaded(self): - self.assertTrue(self._model.X.stale) - self.assertTrue(self._model.Y.stale) - self.assertEqual(value(self._model.X), 0) - self.assertEqual(value(self._model.Y), 0) - - self._opt.load_vars([self._model.X]) - - self.assertFalse(self._model.X.stale) - self.assertTrue(self._model.Y.stale) - self.assertAlmostEqual(value(self._model.X), 3.6) - self.assertEqual(value(self._model.Y), 0) - - self._opt.load_vars([self._model.Y]) - - self.assertFalse(self._model.X.stale) - self.assertFalse(self._model.Y.stale) - self.assertAlmostEqual(value(self._model.X), 3.6) - self.assertAlmostEqual(value(self._model.Y), 0.8) - - -if __name__ == "__main__": - unittest.main() diff --git a/pyomo/solvers/tests/checks/test_SCIPPersistent.py b/pyomo/solvers/tests/checks/test_SCIPPersistent.py deleted file mode 100644 index 61cf7385352..00000000000 --- a/pyomo/solvers/tests/checks/test_SCIPPersistent.py +++ /dev/null @@ -1,318 +0,0 @@ -# ___________________________________________________________________________ -# -# Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2025 -# National Technology and Engineering Solutions of Sandia, LLC -# Under the terms of Contract DE-NA0003525 with National Technology and -# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain -# rights in this software. -# This software is distributed under the 3-clause BSD License. -# ___________________________________________________________________________ - -import pyomo.environ -import pyomo.common.unittest as unittest - -from pyomo.core import ( - ConcreteModel, - Var, - Objective, - Constraint, - NonNegativeReals, - NonNegativeIntegers, - Reals, - Binary, - SOSConstraint, - Set, - sin, - cos, - exp, - log, -) -from pyomo.opt import SolverFactory - -try: - import pyscipopt - - scip_available = True -except ImportError: - scip_available = False - - -@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") -class TestQuadraticObjective(unittest.TestCase): - def test_quadratic_objective_linear_surrogate_is_set(self): - m = ConcreteModel() - m.X = Var(bounds=(-2, 2)) - m.Y = Var(bounds=(-2, 2)) - m.Z = Var(within=Reals) - m.O = Objective(expr=m.Z) - m.C1 = Constraint(expr=m.Y >= 2 * m.X - 1) - m.C2 = Constraint(expr=m.Y >= -m.X + 2) - m.C3 = Constraint(expr=m.Z >= m.X**2 + m.Y**2) - opt = SolverFactory("scip_persistent") - opt.set_instance(m) - opt.solve() - - self.assertAlmostEqual(m.X.value, 1, places=3) - self.assertAlmostEqual(m.Y.value, 1, places=3) - - opt.reset() - - opt.remove_constraint(m.C3) - del m.C3 - m.C3 = Constraint(expr=m.Z >= m.X**2) - opt.add_constraint(m.C3) - opt.solve() - self.assertAlmostEqual(m.X.value, 0, places=3) - self.assertAlmostEqual(m.Y.value, 2, places=3) - - def test_add_and_remove_sos(self): - m = ConcreteModel() - m.I = Set(initialize=[1, 2, 3]) - m.X = Var(m.I, bounds=(-2, 2)) - - m.C = SOSConstraint(var=m.X, sos=1) - - m.O = Objective(expr=m.X[1] + m.X[2]) - - opt = SolverFactory("scip_persistent") - - opt.set_instance(m) - opt.solve() - - zero_val_var = 0 - for i in range(1, 4): - if -0.001 < m.X[i].value < 0.001: - zero_val_var += 1 - assert zero_val_var == 2 - - opt.reset() - - opt.remove_sos_constraint(m.C) - del m.C - - m.C = SOSConstraint(var=m.X, sos=2) - opt.add_sos_constraint(m.C) - - opt.solve() - - zero_val_var = 0 - for i in range(1, 4): - if -0.001 < m.X[i].value < 0.001: - zero_val_var += 1 - assert zero_val_var == 1 - - def test_get_and_set_param(self): - m = ConcreteModel() - m.X = Var(bounds=(-2, 2)) - m.O = Objective(expr=m.X) - m.C3 = Constraint(expr=m.X <= 2) - opt = SolverFactory("scip_persistent") - opt.set_instance(m) - - opt.set_scip_param("limits/time", 60) - - assert opt.get_scip_param("limits/time") == 60 - - def test_non_linear(self): - - PI = 3.141592653589793238462643 - NWIRES = 11 - DIAMETERS = [ - 0.207, - 0.225, - 0.244, - 0.263, - 0.283, - 0.307, - 0.331, - 0.362, - 0.394, - 0.4375, - 0.500, - ] - PRELOAD = 300.0 - MAXWORKLOAD = 1000.0 - MAXDEFLECT = 6.0 - DEFLECTPRELOAD = 1.25 - MAXFREELEN = 14.0 - MAXCOILDIAM = 3.0 - MAXSHEARSTRESS = 189000.0 - SHEARMOD = 11500000.0 - - m = ConcreteModel() - m.coil = Var(within=NonNegativeReals) - m.wire = Var(within=NonNegativeReals) - m.defl = Var( - bounds=(DEFLECTPRELOAD / (MAXWORKLOAD - PRELOAD), MAXDEFLECT / PRELOAD) - ) - m.ncoils = Var(within=NonNegativeIntegers) - m.const1 = Var(within=NonNegativeReals) - m.const2 = Var(within=NonNegativeReals) - m.volume = Var(within=NonNegativeReals) - m.I = Set(initialize=[i for i in range(NWIRES)]) - m.y = Var(m.I, within=Binary) - - m.O = Objective(expr=m.volume) - - m.c1 = Constraint( - expr=PI / 2 * (m.ncoils + 2) * m.coil * m.wire**2 - m.volume == 0 - ) - - m.c2 = Constraint(expr=m.coil / m.wire - m.const1 == 0) - - m.c3 = Constraint( - expr=(4 * m.const1 - 1) / (4 * m.const1 - 4) + 0.615 / m.const1 - m.const2 - == 0 - ) - - m.c4 = Constraint( - expr=8.0 * MAXWORKLOAD / PI * m.const1 * m.const2 - - MAXSHEARSTRESS * m.wire**2 - <= 0 - ) - - m.c5 = Constraint( - expr=8 / SHEARMOD * m.ncoils * m.const1**3 / m.wire - m.defl == 0 - ) - - m.c6 = Constraint( - expr=MAXWORKLOAD * m.defl + 1.05 * m.ncoils * m.wire + 2.1 * m.wire - <= MAXFREELEN - ) - - m.c7 = Constraint(expr=m.coil + m.wire <= MAXCOILDIAM) - - m.c8 = Constraint( - expr=sum(m.y[i] * DIAMETERS[i] for i in range(NWIRES)) - m.wire == 0 - ) - - m.c9 = Constraint(expr=sum(m.y[i] for i in range(NWIRES)) == 1) - - opt = SolverFactory("scip_persistent") - opt.set_instance(m) - - opt.solve() - - self.assertAlmostEqual(m.volume.value, 1.6924910128, places=2) - - def test_non_linear_unary_expressions(self): - - m = ConcreteModel() - m.X = Var(bounds=(1, 2)) - m.Y = Var(within=Reals) - - m.O = Objective(expr=m.Y) - - m.C = Constraint(expr=exp(m.X) == m.Y) - - opt = SolverFactory("scip_persistent") - opt.set_instance(m) - - opt.solve() - self.assertAlmostEqual(m.X.value, 1, places=3) - self.assertAlmostEqual(m.Y.value, exp(1), places=3) - - opt.reset() - opt.remove_constraint(m.C) - del m.C - - m.C = Constraint(expr=log(m.X) == m.Y) - opt.add_constraint(m.C) - opt.solve() - self.assertAlmostEqual(m.X.value, 1, places=3) - self.assertAlmostEqual(m.Y.value, 0, places=3) - - opt.reset() - opt.remove_constraint(m.C) - del m.C - - m.C = Constraint(expr=sin(m.X) == m.Y) - opt.add_constraint(m.C) - opt.solve() - self.assertAlmostEqual(m.X.value, 1, places=3) - self.assertAlmostEqual(m.Y.value, sin(1), places=3) - - opt.reset() - opt.remove_constraint(m.C) - del m.C - - m.C = Constraint(expr=cos(m.X) == m.Y) - opt.add_constraint(m.C) - opt.solve() - self.assertAlmostEqual(m.X.value, 2, places=3) - self.assertAlmostEqual(m.Y.value, cos(2), places=3) - - def test_add_column(self): - m = ConcreteModel() - m.x = Var(within=NonNegativeReals) - m.c = Constraint(expr=(0, m.x, 1)) - m.obj = Objective(expr=-m.x) - - opt = SolverFactory("scip_persistent") - opt.set_instance(m) - opt.solve() - self.assertAlmostEqual(m.x.value, 1) - - m.y = Var(within=NonNegativeReals) - - opt.reset() - - opt.add_column(m, m.y, -3, [m.c], [2]) - opt.solve() - - self.assertAlmostEqual(m.x.value, 0) - self.assertAlmostEqual(m.y.value, 0.5) - - def test_add_column_exceptions(self): - m = ConcreteModel() - m.x = Var() - m.c = Constraint(expr=(0, m.x, 1)) - m.ci = Constraint([1, 2], rule=lambda m, i: (0, m.x, i + 1)) - m.cd = Constraint(expr=(0, -m.x, 1)) - m.cd.deactivate() - m.obj = Objective(expr=-m.x) - - opt = SolverFactory("scip_persistent") - - # set_instance not called - self.assertRaises(RuntimeError, opt.add_column, m, m.x, 0, [m.c], [1]) - - opt.set_instance(m) - - m2 = ConcreteModel() - m2.y = Var() - m2.c = Constraint(expr=(0, m.x, 1)) - - # different model than attached to opt - self.assertRaises(RuntimeError, opt.add_column, m2, m2.y, 0, [], []) - # pyomo var attached to different model - self.assertRaises(RuntimeError, opt.add_column, m, m2.y, 0, [], []) - - z = Var() - # pyomo var floating - self.assertRaises(RuntimeError, opt.add_column, m, z, -2, [m.c, z], [1]) - - m.y = Var() - # len(coefficients) == len(constraints) - self.assertRaises(RuntimeError, opt.add_column, m, m.y, -2, [m.c], [1, 2]) - self.assertRaises(RuntimeError, opt.add_column, m, m.y, -2, [m.c, z], [1]) - - # add indexed constraint - self.assertRaises(AttributeError, opt.add_column, m, m.y, -2, [m.ci], [1]) - # add something not a _ConstraintData - self.assertRaises(AttributeError, opt.add_column, m, m.y, -2, [m.x], [1]) - - # constraint not on solver model - self.assertRaises(KeyError, opt.add_column, m, m.y, -2, [m2.c], [1]) - - # inactive constraint - self.assertRaises(KeyError, opt.add_column, m, m.y, -2, [m.cd], [1]) - - opt.add_var(m.y) - # var already in solver model - self.assertRaises(RuntimeError, opt.add_column, m, m.y, -2, [m.c], [1]) - - -if __name__ == "__main__": - unittest.main() diff --git a/pyomo/solvers/tests/solvers.py b/pyomo/solvers/tests/solvers.py index e9967cd1ce2..e5058e8894b 100644 --- a/pyomo/solvers/tests/solvers.py +++ b/pyomo/solvers/tests/solvers.py @@ -369,21 +369,6 @@ def test_solver_cases(*args): name='scip', io='nl', capabilities=_scip_capabilities, import_suffixes=[] ) - # - # SCIP PERSISTENT - # - - _scip_persistent_capabilities = set( - ["linear", "integer", "quadratic_constraint", "sos1", "sos2"] - ) - - _test_solver_cases["scip_persistent", "python"] = initialize( - name="scip_persistent", - io="python", - capabilities=_scip_persistent_capabilities, - import_suffixes=[], - ) - # # CONOPT # diff --git a/pyomo/solvers/tests/testcases.py b/pyomo/solvers/tests/testcases.py index c1725bedee7..696936ddf05 100644 --- a/pyomo/solvers/tests/testcases.py +++ b/pyomo/solvers/tests/testcases.py @@ -248,15 +248,6 @@ "inside NL files. A ticket has been filed.", ) -# -# SCIP Persistent -# - -ExpectedFailures["scip_persistent", "python", "LP_trivial_constraints"] = ( - lambda v: v <= _trunk_version, - "SCIP does not allow empty constraints with no variables to be added to the Model.", -) - # # BARON # From c200e2e996920c0eccb2e12d3b74376847e3b448 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 6 Oct 2025 15:09:16 -0600 Subject: [PATCH 048/107] run black --- pyomo/contrib/observer/model_observer.py | 2 +- pyomo/contrib/solver/plugins.py | 8 +- .../solver/solvers/scip/scip_direct.py | 138 +++++++++--------- .../solver/tests/solvers/test_solvers.py | 3 +- 4 files changed, 75 insertions(+), 76 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 77356ac1b57..9bb9c917fc5 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -1208,6 +1208,6 @@ def get_variables_impacted_by_param(self, p: ParamData): def get_constraints_impacted_by_param(self, p: ParamData): return list(self._referenced_params[id(p)][0]) - + def get_constraints_impacted_by_var(self, v: VarData): return list(self._referenced_variables[id(v)][0]) diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index 4ac74ecf560..ff24148dd73 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -41,12 +41,12 @@ def load(): name='highs', legacy_name='highs_v2', doc='Persistent interface to HiGHS' )(Highs) SolverFactory.register( - name='scip_direct', - legacy_name='scip_direct_v2', + name='scip_direct', + legacy_name='scip_direct_v2', doc='Direct interface pyscipopt', )(ScipDirect) SolverFactory.register( - name='scip_persistent', - legacy_name='scip_persistent_v2', + name='scip_persistent', + legacy_name='scip_persistent_v2', doc='Persistent interface pyscipopt', )(ScipPersistent) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 99de1d80125..7e39d6e8595 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -47,11 +47,19 @@ from pyomo.core.expr.numvalue import NumericConstant from pyomo.gdp.disjunct import AutoLinkedBinaryVar from pyomo.core.base.expression import ExpressionData, ScalarExpression -from pyomo.core.expr.relational_expr import EqualityExpression, InequalityExpression, RangedExpression +from pyomo.core.expr.relational_expr import ( + EqualityExpression, + InequalityExpression, + RangedExpression, +) from pyomo.core.staleflag import StaleFlagManager from pyomo.core.expr.visitor import StreamBasedExpressionVisitor from pyomo.common.dependencies import attempt_import -from pyomo.contrib.solver.common.base import SolverBase, Availability, PersistentSolverBase +from pyomo.contrib.solver.common.base import ( + SolverBase, + Availability, + PersistentSolverBase, +) from pyomo.contrib.solver.common.config import BranchAndBoundConfig from pyomo.contrib.solver.common.util import ( NoFeasibleSolutionError, @@ -73,7 +81,11 @@ from pyomo.common.tee import capture_output, TeeStream from pyomo.core.base.units_container import _PyomoUnit from pyomo.contrib.fbbt.fbbt import compute_bounds_on_expr -from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector, AutoUpdateConfig +from pyomo.contrib.observer.model_observer import ( + Observer, + ModelChangeDetector, + AutoUpdateConfig, +) logger = logging.getLogger(__name__) @@ -120,8 +132,8 @@ def _handle_var(node, data, opt, visitor): def _handle_param(node, data, opt, visitor): # for the persistent interface, we create scip variables in place - # of parameters. However, this makes things complicated for range - # constraints because scip does not allow variables in the + # of parameters. However, this makes things complicated for range + # constraints because scip does not allow variables in the # lower and upper parts of range constraints if visitor.in_range: return node.value @@ -155,7 +167,7 @@ def _handle_pow(node, data, opt, visitor): else: xlb, xub = compute_bounds_on_expr(node.args[0]) if xlb > 0: - return scip.exp(y*scip.log(x)) + return scip.exp(y * scip.log(x)) else: return x**y # scip will probably raise an error here @@ -236,7 +248,7 @@ def _handle_equality(node, data, opt, visitor): def _handle_ranged(node, data, opt, visitor): - # note that the lower and upper parts of the + # note that the lower and upper parts of the # range constraint cannot have variables return data[0] <= (data[1] <= data[2]) @@ -304,7 +316,7 @@ def exitNode(self, node, data): return _handle_float(node, data, self.solver, self) else: raise NotImplementedError(f'unrecognized expression type: {nt}') - + def enterNode(self, node): if type(node) is RangedExpression: self.in_range = True @@ -316,13 +328,7 @@ def enterNode(self, node): class ScipDirectSolutionLoader(SolutionLoaderBase): def __init__( - self, - solver_model, - var_id_map, - var_map, - con_map, - pyomo_model, - opt, + self, solver_model, var_id_map, var_map, con_map, pyomo_model, opt ) -> None: super().__init__() self._solver_model = solver_model @@ -342,7 +348,9 @@ def get_solution_ids(self) -> List: def load_vars( self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> None: - for v, val in self.get_vars(vars_to_load=vars_to_load, solution_id=solution_id).items(): + for v, val in self.get_vars( + vars_to_load=vars_to_load, solution_id=solution_id + ).items(): v.value = val def get_vars( @@ -377,22 +385,9 @@ def load_import_suffixes(self, solution_id=None): class ScipPersistentSolutionLoader(ScipDirectSolutionLoader): def __init__( - self, - solver_model, - var_id_map, - var_map, - con_map, - pyomo_model, - opt, + self, solver_model, var_id_map, var_map, con_map, pyomo_model, opt ) -> None: - super().__init__( - solver_model, - var_id_map, - var_map, - con_map, - pyomo_model, - opt, - ) + super().__init__(solver_model, var_id_map, var_map, con_map, pyomo_model, opt) self._valid = True def invalidate(self): @@ -401,7 +396,7 @@ def invalidate(self): def _assert_solution_still_valid(self): if not self._valid: raise RuntimeError('The results in the solver are no longer valid.') - + def load_vars( self, vars_to_load: Sequence[VarData] | None = None, solution_id=None ) -> None: @@ -454,11 +449,15 @@ def __init__(self, **kwds): self._params = {} # param id to param self._pyomo_var_to_solver_var_map = {} # var id to scip var self._pyomo_con_to_solver_con_map = {} - self._pyomo_param_to_solver_param_map = {} # param id to scip var with equal bounds + self._pyomo_param_to_solver_param_map = ( + {} + ) # param id to scip var with equal bounds self._pyomo_sos_to_solver_sos_map = {} self._expr_visitor = _PyomoToScipVisitor(self) self._objective = None # pyomo objective - self._obj_var = None # a scip variable because the objective cannot be nonlinear + self._obj_var = ( + None # a scip variable because the objective cannot be nonlinear + ) self._obj_con = None # a scip constraint (obj_var >= obj_expr) def _clear(self): @@ -476,7 +475,7 @@ def _clear(self): def available(self) -> Availability: if self._available is not None: return self._available - + if not scip_available: ScipDirect._available = Availability.NotFound elif self.version() < self._minimum_version: @@ -485,7 +484,7 @@ def available(self) -> Availability: ScipDirect._available = Availability.FullLicense return self._available - + def version(self) -> Tuple: return tuple(int(i) for i in scip.__version__.split('.')) @@ -493,9 +492,7 @@ def solve(self, model: BlockData, **kwds) -> Results: start_timestamp = datetime.datetime.now(datetime.timezone.utc) orig_config = self.config if not self.available(): - raise ApplicationError( - f'{self.name} is not available: {self.available()}' - ) + raise ApplicationError(f'{self.name} is not available: {self.available()}') try: config = self.config(value=kwds, preserve_implicit=True) @@ -546,7 +543,9 @@ def solve(self, model: BlockData, **kwds) -> Results: results.solver_log = ostreams[0].getvalue() end_timestamp = datetime.datetime.now(datetime.timezone.utc) results.timing_info.start_timestamp = start_timestamp - results.timing_info.wall_time = (end_timestamp - start_timestamp).total_seconds() + results.timing_info.wall_time = ( + end_timestamp - start_timestamp + ).total_seconds() results.timing_info.timer = timer return results @@ -616,7 +615,7 @@ def _add_var(self, var): self._vars[id(var)] = var self._pyomo_var_to_solver_var_map[id(var)] = scip_var return scip_var - + def _add_param(self, p): vtype = "C" lb = ub = p.value @@ -646,9 +645,7 @@ def _create_solver_model(self, model): self._solver_model = scip.Model() timer.start('collect constraints') cons = list( - model.component_data_objects( - Constraint, descend_into=True, active=True - ) + model.component_data_objects(Constraint, descend_into=True, active=True) ) timer.stop('collect constraints') timer.start('translate constraints') @@ -656,9 +653,7 @@ def _create_solver_model(self, model): timer.stop('translate constraints') timer.start('sos') sos = list( - model.component_data_objects( - SOSConstraint, descend_into=True, active=True - ) + model.component_data_objects(SOSConstraint, descend_into=True, active=True) ) self._add_sos_constraints(sos) timer.stop('sos') @@ -688,7 +683,9 @@ def _add_constraint(self, con): def _add_sos_constraint(self, con): level = con.level if level not in [1, 2]: - raise ValueError(f"{self.name} does not support SOS level {level} constraints") + raise ValueError( + f"{self.name} does not support SOS level {level} constraints" + ) scip_vars = [] weights = [] @@ -701,13 +698,9 @@ def _add_sos_constraint(self, con): weights.append(w) if level == 1: - scip_cons = self._solver_model.addConsSOS1( - scip_vars, weights=weights - ) + scip_cons = self._solver_model.addConsSOS1(scip_vars, weights=weights) else: - scip_cons = self._solver_model.addConsSOS2( - scip_vars, weights=weights - ) + scip_cons = self._solver_model.addConsSOS2(scip_vars, weights=weights) self._pyomo_con_to_solver_con_map[con] = scip_cons def _scip_vtype_from_var(self, var): @@ -737,9 +730,9 @@ def _scip_vtype_from_var(self, var): def _set_objective(self, obj): if self._obj_var is None: self._obj_var = self._solver_model.addVar( - lb=-self._solver_model.infinity(), - ub=self._solver_model.infinity(), - vtype="C" + lb=-self._solver_model.infinity(), + ub=self._solver_model.infinity(), + vtype="C", ) if self._obj_con is not None: @@ -766,19 +759,21 @@ def _set_objective(self, obj): self._objective = obj def _postsolve( - self, - scip_model, - solution_loader: ScipDirectSolutionLoader, - has_obj + self, scip_model, solution_loader: ScipDirectSolutionLoader, has_obj ): results = Results() results.solution_loader = solution_loader - results.timing_info.scip_time = scip_model.getSolvingTime() - results.termination_condition = self._get_tc_map().get(scip_model.getStatus(), TerminationCondition.unknown) - + results.timing_info.scip_time = scip_model.getSolvingTime() + results.termination_condition = self._get_tc_map().get( + scip_model.getStatus(), TerminationCondition.unknown + ) + if solution_loader.get_number_of_solutions() > 0: - if results.termination_condition == TerminationCondition.convergenceCriteriaSatisfied: + if ( + results.termination_condition + == TerminationCondition.convergenceCriteriaSatisfied + ): results.solution_status = SolutionStatus.optimal else: results.solution_status = SolutionStatus.feasible @@ -786,15 +781,18 @@ def _postsolve( results.solution_status = SolutionStatus.noSolution if ( - results.termination_condition + results.termination_condition != TerminationCondition.convergenceCriteriaSatisfied and self.config.raise_exception_on_nonoptimal_result ): raise NoOptimalSolutionError() - + if has_obj: try: - if scip_model.getNSols() > 0 and scip_model.getObjVal() < scip_model.infinity(): + if ( + scip_model.getNSols() > 0 + and scip_model.getObjVal() < scip_model.infinity() + ): results.incumbent_objective = scip_model.getObjVal() else: results.incumbent_objective = None @@ -831,7 +829,7 @@ def _postsolve( return results def _mipstart(self): - # TODO: it is also possible to specify continuous variables, but + # TODO: it is also possible to specify continuous variables, but # I think we should have a differnt option for that sol = self._solver_model.createPartialSol() for vid, scip_var in self._pyomo_var_to_solver_var_map.items(): @@ -1009,7 +1007,7 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): self._check_reopt() self._invalidate_last_results() return super()._add_sos_constraints(cons) - + def _add_objectives(self, objs: List[ObjectiveData]): self._check_reopt() if len(objs) > 1: diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 3665de4521a..e6686266028 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -1152,7 +1152,8 @@ def test_results_infeasible( ): res.solution_loader.get_duals() with self.assertRaisesRegex( - NoReducedCostsError, '.*does not currently have valid reduced costs.*' + NoReducedCostsError, + '.*does not currently have valid reduced costs.*', ): res.solution_loader.get_reduced_costs() From 960c531bba4c9bc9ea65cb652c699c5a131a8382 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 6 Oct 2025 17:06:23 -0600 Subject: [PATCH 049/107] typos --- pyomo/contrib/solver/common/solution_loader.py | 2 +- pyomo/contrib/solver/solvers/scip/scip_direct.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index f8723b6e0f4..6be23b63c77 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -63,7 +63,7 @@ def get_solution_ids(self) -> List[Any]: """ If there are multiple solutions available, this will return a list of the solution ids which can then be used with other - methods like `load_soltuion`. If only one solution is + methods like `load_solution`. If only one solution is available, this will return [None]. If no solutions are available, this will return None diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 7e39d6e8595..05f39b0cb16 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -830,7 +830,7 @@ def _postsolve( def _mipstart(self): # TODO: it is also possible to specify continuous variables, but - # I think we should have a differnt option for that + # I think we should have a different option for that sol = self._solver_model.createPartialSol() for vid, scip_var in self._pyomo_var_to_solver_var_map.items(): pyomo_var = self._vars[vid] From 879ed3aea700887204873d1dd42b2063d1327f85 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 18 Dec 2025 11:21:10 -0700 Subject: [PATCH 050/107] update scip interface to use observer --- pyomo/contrib/observer/model_observer.py | 3 + .../solver/solvers/scip/scip_direct.py | 273 +++++++++--------- 2 files changed, 145 insertions(+), 131 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index b1eda200a9a..9a4e8f27563 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -741,6 +741,9 @@ def remove_objectives(self, objs: Collection[ObjectiveData]): def _check_for_unknown_active_components(self): for ctype in self._model.collect_ctypes(active=True, descend_into=True): + if not issubclass(ctype, ActiveComponent): + # strangly, this is needed to skip things like Param + continue if ctype in self._known_active_ctypes: continue if ctype is Suffix: diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 05f39b0cb16..2d820e92028 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -85,6 +85,7 @@ Observer, ModelChangeDetector, AutoUpdateConfig, + Reason, ) @@ -111,8 +112,8 @@ def __init__( implicit_domain=implicit_domain, visibility=visibility, ) - self.use_mipstart: bool = self.declare( - 'use_mipstart', + self.warmstart_discrete_vars: bool = self.declare( + 'warmstart_discrete_vars', ConfigValue( default=False, domain=bool, @@ -328,11 +329,10 @@ def enterNode(self, node): class ScipDirectSolutionLoader(SolutionLoaderBase): def __init__( - self, solver_model, var_id_map, var_map, con_map, pyomo_model, opt + self, solver_model, var_map, con_map, pyomo_model, opt ) -> None: super().__init__() self._solver_model = solver_model - self._vars = var_id_map self._var_map = var_map self._con_map = con_map self._pyomo_model = pyomo_model @@ -359,13 +359,13 @@ def get_vars( if self.get_number_of_solutions() == 0: raise NoSolutionError() if vars_to_load is None: - vars_to_load = list(self._vars.values()) + vars_to_load = list(self._var_map.keys()) if solution_id is None: solution_id = 0 sol = self._solver_model.getSols()[solution_id] res = ComponentMap() for v in vars_to_load: - sv = self._var_map[id(v)] + sv = self._var_map[v] res[v] = sol[sv] return res @@ -385,9 +385,9 @@ def load_import_suffixes(self, solution_id=None): class ScipPersistentSolutionLoader(ScipDirectSolutionLoader): def __init__( - self, solver_model, var_id_map, var_map, con_map, pyomo_model, opt + self, solver_model, var_map, con_map, pyomo_model, opt ) -> None: - super().__init__(solver_model, var_id_map, var_map, con_map, pyomo_model, opt) + super().__init__(solver_model, var_map, con_map, pyomo_model, opt) self._valid = True def invalidate(self): @@ -445,13 +445,11 @@ class ScipDirect(SolverBase): def __init__(self, **kwds): super().__init__(**kwds) self._solver_model = None - self._vars = {} # var id to var - self._params = {} # param id to param - self._pyomo_var_to_solver_var_map = {} # var id to scip var + self._pyomo_var_to_solver_var_map = ComponentMap() self._pyomo_con_to_solver_con_map = {} self._pyomo_param_to_solver_param_map = ( - {} - ) # param id to scip var with equal bounds + ComponentMap() + ) # param to scip var with equal bounds self._pyomo_sos_to_solver_sos_map = {} self._expr_visitor = _PyomoToScipVisitor(self) self._objective = None # pyomo objective @@ -462,11 +460,9 @@ def __init__(self, **kwds): def _clear(self): self._solver_model = None - self._vars = {} - self._params = {} - self._pyomo_var_to_solver_var_map = {} + self._pyomo_var_to_solver_var_map = ComponentMap() self._pyomo_con_to_solver_con_map = {} - self._pyomo_param_to_solver_param_map = {} + self._pyomo_param_to_solver_param_map = ComponentMap() self._pyomo_sos_to_solver_sos_map = {} self._objective = None self._obj_var = None @@ -490,16 +486,9 @@ def version(self) -> Tuple: def solve(self, model: BlockData, **kwds) -> Results: start_timestamp = datetime.datetime.now(datetime.timezone.utc) - orig_config = self.config - if not self.available(): - raise ApplicationError(f'{self.name} is not available: {self.available()}') try: config = self.config(value=kwds, preserve_implicit=True) - # hack to work around legacy solver wrapper __setattr__ - # otherwise, this would just be self.config = config - object.__setattr__(self, 'config', config) - StaleFlagManager.mark_all_as_stale() if config.timer is None: @@ -508,7 +497,7 @@ def solve(self, model: BlockData, **kwds) -> Results: ostreams = [io.StringIO()] + config.tee - scip_model, solution_loader, has_obj = self._create_solver_model(model) + scip_model, solution_loader, has_obj = self._create_solver_model(model, config) scip_model.hideOutput(quiet=False) if config.threads is not None: @@ -520,7 +509,7 @@ def solve(self, model: BlockData, **kwds) -> Results: if config.abs_gap is not None: scip_model.setParam('limits/absgap', config.abs_gap) - if config.use_mipstart: + if config.warmstart_discrete_vars: self._mipstart() for key, option in config.solver_options.items(): @@ -532,13 +521,10 @@ def solve(self, model: BlockData, **kwds) -> Results: scip_model.optimize() timer.stop('optimize') - results = self._postsolve(scip_model, solution_loader, has_obj) + results = self._populate_results(scip_model, solution_loader, has_obj, config) except InfeasibleConstraintException: + # is it possible to hit this? results = self._get_infeasible_results() - finally: - # hack to work around legacy solver wrapper __setattr__ - # otherwise, this would just be self.config = orig_config - object.__setattr__(self, 'config', orig_config) results.solver_log = ostreams[0].getvalue() end_timestamp = datetime.datetime.now(datetime.timezone.utc) @@ -612,16 +598,14 @@ def _add_var(self, var): scip_var = self._solver_model.addVar(lb=lb, ub=ub, vtype=vtype) - self._vars[id(var)] = var - self._pyomo_var_to_solver_var_map[id(var)] = scip_var + self._pyomo_var_to_solver_var_map[var] = scip_var return scip_var def _add_param(self, p): vtype = "C" lb = ub = p.value scip_var = self._solver_model.addVar(lb=lb, ub=ub, vtype=vtype) - self._params[id(p)] = p - self._pyomo_param_to_solver_param_map[id(p)] = scip_var + self._pyomo_param_to_solver_param_map[p] = scip_var return scip_var def __del__(self): @@ -638,8 +622,8 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): for on in cons: self._add_sos_constraint(con) - def _create_solver_model(self, model): - timer = self.config.timer + def _create_solver_model(self, model, config): + timer = config.timer timer.start('create scip model') self._clear() self._solver_model = scip.Model() @@ -666,7 +650,6 @@ def _create_solver_model(self, model): has_obj = obj is not None solution_loader = ScipDirectSolutionLoader( solver_model=self._solver_model, - var_id_map=self._vars, var_map=self._pyomo_var_to_solver_var_map, con_map=self._pyomo_con_to_solver_con_map, pyomo_model=model, @@ -758,8 +741,8 @@ def _set_objective(self, obj): self._solver_model.setObjective(self._obj_var, sense=sense) self._objective = obj - def _postsolve( - self, scip_model, solution_loader: ScipDirectSolutionLoader, has_obj + def _populate_results( + self, scip_model, solution_loader: ScipDirectSolutionLoader, has_obj, config ): results = Results() @@ -783,7 +766,7 @@ def _postsolve( if ( results.termination_condition != TerminationCondition.convergenceCriteriaSatisfied - and self.config.raise_exception_on_nonoptimal_result + and config.raise_exception_on_nonoptimal_result ): raise NoOptimalSolutionError() @@ -813,16 +796,16 @@ def _postsolve( results.incumbent_objective = None results.objective_bound = None - self.config.timer.start('load solution') - if self.config.load_solutions: + config.timer.start('load solution') + if config.load_solutions: if solution_loader.get_number_of_solutions() > 0: solution_loader.load_solution() else: raise NoFeasibleSolutionError() - self.config.timer.stop('load solution') + config.timer.stop('load solution') - results.iteration_count = scip_model.getNNodes() - results.solver_config = self.config + results.extra_info['NNodes'] = scip_model.getNNodes() + results.solver_config = config results.solver_name = self.name results.solver_version = self.version() @@ -832,8 +815,7 @@ def _mipstart(self): # TODO: it is also possible to specify continuous variables, but # I think we should have a different option for that sol = self._solver_model.createPartialSol() - for vid, scip_var in self._pyomo_var_to_solver_var_map.items(): - pyomo_var = self._vars[vid] + for pyomo_var, scip_var in self._pyomo_var_to_solver_var_map.items(): if pyomo_var.is_integer(): sol[scip_var] = pyomo_var.value self._solver_model.addSol(sol) @@ -859,55 +841,13 @@ def __init__( self.auto_updates: bool = self.declare('auto_updates', AutoUpdateConfig()) -class _ScipObserver(Observer): - def __init__(self, opt: ScipPersistent) -> None: - self.opt = opt - - def add_variables(self, variables: List[VarData]): - self.opt._add_variables(variables) - - def add_parameters(self, params: List[ParamData]): - self.opt._add_parameters(params) - - def add_constraints(self, cons: List[ConstraintData]): - self.opt._add_constraints(cons) - - def add_sos_constraints(self, cons: List[SOSConstraintData]): - self.opt._add_sos_constraints(cons) - - def add_objectives(self, objs: List[ObjectiveData]): - self.opt._add_objectives(objs) - - def remove_objectives(self, objs: List[ObjectiveData]): - self.opt._remove_objectives(objs) - - def remove_constraints(self, cons: List[ConstraintData]): - self.opt._remove_constraints(cons) - - def remove_sos_constraints(self, cons: List[SOSConstraintData]): - self.opt._remove_sos_constraints(cons) - - def remove_variables(self, variables: List[VarData]): - self.opt._remove_variables(variables) - - def remove_parameters(self, params: List[ParamData]): - self.opt._remove_parameters(params) - - def update_variables(self, variables: List[VarData]): - self.opt._update_variables(variables) - - def update_parameters(self, params: List[ParamData]): - self.opt._update_parameters(params) - - -class ScipPersistent(ScipDirect, PersistentSolverBase): +class ScipPersistent(ScipDirect, PersistentSolverBase, Observer): _minimum_version = (5, 5, 0) # this is probably conservative CONFIG = ScipPersistentConfig() def __init__(self, **kwds): super().__init__(**kwds) self._pyomo_model = None - self._observer = None self._change_detector = None self._last_results_object: Optional[Results] = None self._needs_reopt = False @@ -916,10 +856,9 @@ def __init__(self, **kwds): def _clear(self): super()._clear() self._pyomo_model = None - self._objective = None - self._observer = None self._change_detector = None self._needs_reopt = False + self._range_constraints = set() def _check_reopt(self): if self._needs_reopt: @@ -927,15 +866,14 @@ def _check_reopt(self): self._solver_model.freeTransform() self._needs_reopt = False - def _create_solver_model(self, pyomo_model): + def _create_solver_model(self, pyomo_model, config): if pyomo_model is self._pyomo_model: - self.update() + self.update(**config) else: - self.set_instance(pyomo_model=pyomo_model) + self.set_instance(pyomo_model, **config) solution_loader = ScipPersistentSolutionLoader( solver_model=self._solver_model, - var_id_map=self._vars, var_map=self._pyomo_var_to_solver_var_map, con_map=self._pyomo_con_to_solver_con_map, pyomo_model=pyomo_model, @@ -950,39 +888,128 @@ def solve(self, model, **kwds) -> Results: self._needs_reopt = True return res - def update(self): - if self.config.timer is None: + def update(self, **kwds): + config = self.config(value=kwds, preserve_implicit=True) + if config.timer is None: timer = HierarchicalTimer() else: - timer = self.config.timer + timer = config.timer if self._pyomo_model is None: raise RuntimeError('must call set_instance or solve before update') timer.start('update') - self._change_detector.update(timer=timer) + self._change_detector.update(timer=timer, **config.auto_updates) timer.stop('update') - def set_instance(self, pyomo_model): - if self.config.timer is None: + def set_instance(self, pyomo_model, **kwds): + config = self.config(value=kwds, preserve_implicit=True) + if config.timer is None: timer = HierarchicalTimer() else: - timer = self.config.timer + timer = config.timer self._clear() self._pyomo_model = pyomo_model self._solver_model = scip.Model() - self._observer = _ScipObserver(self) timer.start('set_instance') self._change_detector = ModelChangeDetector( model=self._pyomo_model, - observers=[self._observer], - **dict(self.config.auto_updates), + observers=[self], + **config.auto_updates, ) - self._change_detector.config = self.config.auto_updates timer.stop('set_instance') def _invalidate_last_results(self): if self._last_results_object is not None: self._last_results_object.solution_loader.invalidate() + def _update_variables(self, variables: Mapping[VarData, Reason]): + new_vars = [] + old_vars = [] + mod_vars = [] + for v, reason in variables.items(): + if reason & Reason.added: + new_vars.append(v) + elif reason & Reason.removed: + old_vars.append(v) + else: + mod_vars.append(v) + + if new_vars: + self._add_variables(new_vars) + if old_vars: + self._remove_variables(old_vars) + if mod_vars: + self._update_vars_for_real(mod_vars) + + def _update_parameters(self, params: Mapping[ParamData, Reason]): + new_params = [] + old_params = [] + mod_params = [] + for p, reason in params.items(): + if reason & Reason.added: + new_params.append(p) + elif reason & Reason.removed: + old_params.append(p) + else: + mod_params.append(p) + + if new_params: + self._add_parameters(new_params) + if old_params: + self._remove_parameters(old_params) + if mod_params: + self._update_params_for_real(mod_params) + + def _update_constraints(self, cons: Mapping[ConstraintData, Reason]): + new_cons = [] + old_cons = [] + for c, reason in cons.items(): + if reason & Reason.added: + new_cons.append(c) + elif reason & Reason.removed: + old_cons.append(c) + elif reason & Reason.expr: + old_cons.append(c) + new_cons.append(c) + + if old_cons: + self._remove_constraints(old_cons) + if new_cons: + self._add_constraints(new_cons) + + def _update_sos_constraints(self, cons: Mapping[SOSConstraintData, Reason]): + new_cons = [] + old_cons = [] + for c, reason in cons.items(): + if reason & Reason.added: + new_cons.append(c) + elif reason & Reason.removed: + old_cons.append(c) + elif reason & Reason.sos_items: + old_cons.append(c) + new_cons.append(c) + + if old_cons: + self._remove_sos_constraints(old_cons) + if new_cons: + self._add_sos_constraints(new_cons) + + def _update_objectives(self, objs: Mapping[ObjectiveData, Reason]): + new_objs = [] + old_objs = [] + for obj, reason in objs.items(): + if reason & Reason.added: + new_objs.append(obj) + elif reason & Reason.removed: + old_objs.append(obj) + elif reason & (Reason.expr | Reason.sense): + old_objs.append(obj) + new_objs.append(obj) + + if old_objs: + self._remove_objectives(old_objs) + if new_objs: + self._add_objectives(new_objs) + def _add_variables(self, variables: List[VarData]): self._check_reopt() self._invalidate_last_results() @@ -1024,7 +1051,7 @@ def _add_objectives(self, objs: List[ObjectiveData]): if self._objective is not None: raise NotImplementedError( - 'the persistent interface to gurobi currently ' + 'the persistent interface to scip currently ' 'only supports single-objective problems; tried to add ' f'an objective ({str(obj)}), but there is already an ' f'active objective ({str(self._objective)})' @@ -1064,38 +1091,32 @@ def _remove_variables(self, variables: List[VarData]): self._check_reopt() self._invalidate_last_results() for v in variables: - vid = id(v) - scip_var = self._pyomo_var_to_solver_var_map.pop(vid) + scip_var = self._pyomo_var_to_solver_var_map.pop(v) self._solver_model.delVar(scip_var) - self._vars.pop(vid) def _remove_parameters(self, params: List[ParamData]): self._check_reopt() self._invalidate_last_results() for p in params: - pid = id(p) - scip_var = self._pyomo_param_to_solver_param_map.pop(pid) + scip_var = self._pyomo_param_to_solver_param_map.pop(p) self._solver_model.delVar(scip_var) - self._params.pop(pid) - def _update_variables(self, variables: List[VarData]): + def _update_vars_for_real(self, variables: List[VarData]): self._check_reopt() self._invalidate_last_results() for v in variables: - vid = id(v) - scip_var = self._pyomo_var_to_solver_var_map[vid] + scip_var = self._pyomo_var_to_solver_var_map[v] vtype = self._scip_vtype_from_var(v) lb, ub = self._scip_lb_ub_from_var(v) self._solver_model.chgVarLb(scip_var, lb) self._solver_model.chgVarUb(scip_var, ub) self._solver_model.chgVarType(scip_var, vtype) - def _update_parameters(self, params: List[ParamData]): + def _update_params_for_real(self, params: List[ParamData]): self._check_reopt() self._invalidate_last_results() for p in params: - pid = id(p) - scip_var = self._pyomo_param_to_solver_param_map[pid] + scip_var = self._pyomo_param_to_solver_param_map[p] lb = ub = p.value self._solver_model.chgVarLb(scip_var, lb) self._solver_model.chgVarUb(scip_var, ub) @@ -1108,11 +1129,6 @@ def _update_parameters(self, params: List[ParamData]): self._remove_constraints([con]) self._add_constraints([con]) - def add_variables(self, variables): - if self._change_detector is None: - raise RuntimeError('call set_instance first') - self._change_detector.add_variables(variables) - def add_constraints(self, cons): if self._change_detector is None: raise RuntimeError('call set_instance first') @@ -1138,11 +1154,6 @@ def remove_sos_constraints(self, cons): raise RuntimeError('call set_instance first') self._change_detector.remove_sos_constraints(cons) - def remove_variables(self, variables): - if self._change_detector is None: - raise RuntimeError('call set_instance first') - self._change_detector.remove_variables(variables) - def update_variables(self, variables): if self._change_detector is None: raise RuntimeError('call set_instance first') From 3bfa5cbd9e5ba3856e8b4c3d73a26d79c0e623c2 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 8 Jan 2026 14:40:23 -0700 Subject: [PATCH 051/107] run black --- .../solver/solvers/scip/scip_direct.py | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 2d820e92028..b5cb1a6946a 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -328,9 +328,7 @@ def enterNode(self, node): class ScipDirectSolutionLoader(SolutionLoaderBase): - def __init__( - self, solver_model, var_map, con_map, pyomo_model, opt - ) -> None: + def __init__(self, solver_model, var_map, con_map, pyomo_model, opt) -> None: super().__init__() self._solver_model = solver_model self._var_map = var_map @@ -384,9 +382,7 @@ def load_import_suffixes(self, solution_id=None): class ScipPersistentSolutionLoader(ScipDirectSolutionLoader): - def __init__( - self, solver_model, var_map, con_map, pyomo_model, opt - ) -> None: + def __init__(self, solver_model, var_map, con_map, pyomo_model, opt) -> None: super().__init__(solver_model, var_map, con_map, pyomo_model, opt) self._valid = True @@ -497,7 +493,9 @@ def solve(self, model: BlockData, **kwds) -> Results: ostreams = [io.StringIO()] + config.tee - scip_model, solution_loader, has_obj = self._create_solver_model(model, config) + scip_model, solution_loader, has_obj = self._create_solver_model( + model, config + ) scip_model.hideOutput(quiet=False) if config.threads is not None: @@ -521,7 +519,9 @@ def solve(self, model: BlockData, **kwds) -> Results: scip_model.optimize() timer.stop('optimize') - results = self._populate_results(scip_model, solution_loader, has_obj, config) + results = self._populate_results( + scip_model, solution_loader, has_obj, config + ) except InfeasibleConstraintException: # is it possible to hit this? results = self._get_infeasible_results() @@ -911,9 +911,7 @@ def set_instance(self, pyomo_model, **kwds): self._solver_model = scip.Model() timer.start('set_instance') self._change_detector = ModelChangeDetector( - model=self._pyomo_model, - observers=[self], - **config.auto_updates, + model=self._pyomo_model, observers=[self], **config.auto_updates ) timer.stop('set_instance') @@ -951,7 +949,7 @@ def _update_parameters(self, params: Mapping[ParamData, Reason]): old_params.append(p) else: mod_params.append(p) - + if new_params: self._add_parameters(new_params) if old_params: From 4863d1fbf196d607c71ac1114ba75f28a7431536 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Fri, 16 Jan 2026 20:50:58 -0700 Subject: [PATCH 052/107] initial draft of factorable programming transformation for pwl approximations --- .../contrib/piecewise/transform/factorable.py | 448 ++++++++++++++++++ 1 file changed, 448 insertions(+) create mode 100644 pyomo/contrib/piecewise/transform/factorable.py diff --git a/pyomo/contrib/piecewise/transform/factorable.py b/pyomo/contrib/piecewise/transform/factorable.py new file mode 100644 index 00000000000..8527702baae --- /dev/null +++ b/pyomo/contrib/piecewise/transform/factorable.py @@ -0,0 +1,448 @@ +from pyomo.core.expr.visitor import StreamBasedExpressionVisitor +from pyomo.common.collections import ComponentMap, ComponentSet +from pyomo.common.numeric_types import native_numeric_types +from pyomo.core.expr.numeric_expr import ( + NegationExpression, + PowExpression, + ProductExpression, + MonomialTermExpression, + DivisionExpression, + SumExpression, + LinearExpression, + UnaryFunctionExpression, +) + +from pyomo.repn.util import ExitNodeDispatcher +from pyomo.core.base import ( + VarData, + ParamData, + ExpressionData, + VarList, + ConstraintList, + Block, + Constraint, + Objective, +) +from pyomo.core.base.var import ScalarVar +from pyomo.contrib.fbbt.fbbt import compute_bounds_on_expr, fbbt +from pyomo.core.base.expression import ScalarExpression +from pyomo.core.base.transformation import Transformation, TransformationFactory +from pyomo.common.modeling import unique_component_name +from pyomo.core.base.component import ActiveComponent +from pyomo.core.base.suffix import Suffix +from pyomo.gdp import Disjunct +from pyomo.repn.linear import LinearRepn, LinearRepnVisitor + + +""" +The purpose of this module/transformation is to convert any nonlinear model +to the following form: + +min/max f(x_i) +s.t. + g_j(x_i)*h_j(x_k) + a_j^T*x >/// 1: + arg1 = visitor.create_aux_var(arg1) + arg1_vars = (arg1,) + arg1_nvars = 1 + arg1_degree = 1 + if arg2_nvars > 1: + arg2 = visitor.create_aux_var(arg2) + arg2_vars = (arg2,) + arg2_nvars = 1 + arg2_degree = 1 + res = arg1 * arg2 + # at this point arg1 should have at most 1 variable + # and arg2 should have at most 1 variable + if arg1_nvars == 0: + visitor.node_to_var_map[res] = arg2_vars + elif arg2_nvars == 0: + visitor.node_to_var_map[res] = arg1_vars + else: + x = arg1_vars[0] + y = arg2_vars[0] + if x is y: + visitor.node_to_var_map[res] = (x,) + else: + visitor.node_to_var_map[res] = (x, y) + if arg1_degree == 0: + visitor.degree_map[res] = arg2_degree + elif arg2_degree == 0: + visitor.degree_map[res] = arg1_degree + else: + visitor.degree_map[res] = -1 + visitor.substitution_map[node] = res + return res + + +def _handle_sum(node, data, visitor): + # remember to separate the linear parts of the root expression + # from the nonlinear parts prior to using this visitor + # otherwise, we will get auxilliary variables that we don't + # strictly need. If this is not done, we don't do anything + # incorrect, we just get extra variables and constraints + # I'm not sure we should even worry about it, because pretty much + # any presolve should remove them + arg_list = [] + new_degree = 0 + vset = ComponentSet() + for arg in data: + arg_vars = visitor.node_to_var_map[arg] + arg_degree = visitor.degree_map[arg] + if arg_degree == -1: + arg = visitor.create_aux_var(arg) + arg_vars = (arg,) + arg_degree = 1 + arg_list.append(arg) + if arg_degree != 0: + new_degree = 1 + vset.update(arg_vars) + res = sum(arg_list) + visitor.node_to_var_map[res] = tuple(vset) + visitor.degree_map[res] = new_degree + visitor.substitution_map[node] = res + return res + + +def _handle_division(node, data, visitor): + """ + This one is a bit tricky. If we encounter both x/z and y/z + at different places in the model, we only want one auxilliary + variable for 1/z. + """ + arg1, arg2 = data + arg1_vars = visitor.node_to_var_map[arg1] + arg2_vars = visitor.node_to_var_map[arg2] + arg1_nvars = len(arg1_vars) + arg2_nvars = len(arg2_vars) + arg1_degree = visitor.degree_map[arg1] + arg2_degree = visitor.degree_map[arg2] + + if arg2_degree == 0: + res = arg1 / arg2 + visitor.node_to_var_map[res] = arg1_vars + visitor.degree_map[res] = arg1_degree + visitor.substitution_map[node] = res + return res + + if arg1_nvars > 1: + arg1 = visitor.create_aux_var(arg1) + arg1_vars = (arg1,) + arg1_nvars = 1 + arg1_degree = 1 + + if arg2_nvars > 1: + arg2 = visitor.create_aux_var(arg2) + arg2_vars = (arg2,) + arg2_nvars = 1 + arg2_degree = 1 + + if "div" not in visitor.substitution_map: + visitor.substitution_map["div"] = ComponentMap() + + # now we need to figure out if we have seen 1/arg2 before + if arg2 in visitor.substitution_map["div"]: + aux = visitor.substitution_map["div"][arg2] + else: + aux = visitor.block.x.add() + visitor.substitution_map["div"][arg2] = aux + """ + we can only create a piecewise linear function of 1/arg2 if arg2 is either + strictly greater than 0 or strictly less than 0 + + otherwise, we do + aux = 1 / arg2 + aux * arg2 = 1 + """ + arg2_lb, arg2_ub = compute_bounds_on_expr(arg2) + if (arg2_lb is not None and arg2_lb > 0) or (arg2_ub is not None and arg2_ub < 0): + c = visitor.block.c.add(aux == 1/arg2) # keep it univariate if we can + else: + c = visitor.block.c.add(aux * arg2 == 1) + fbbt(c) + + arg2 = aux + arg2_vars = (arg2,) + arg2_nvars = 1 + arg2_degree = 1 + res = arg1 * arg2 + # at this point arg1 should have at most 1 variable + # and arg2 should have exactly 1 variable + if arg1_nvars == 0: + visitor.node_to_var_map[res] = arg2_vars + else: + x = arg1_vars[0] + y = arg2_vars[0] + if x is y: + visitor.node_to_var_map[res] = (x,) + else: + visitor.node_to_var_map[res] = (x, y) + if arg1_degree == 0: + visitor.degree_map[res] = arg2_degree + else: + visitor.degree_map[res] = -1 + visitor.substitution_map[node] = res + return res + + +def _handle_pow(node, data, visitor): + # arg1 ** arg2 + # exp(arg2 * log(arg1)) + arg1, arg2 = data + arg1_vars = visitor.node_to_var_map[arg1] + arg2_vars = visitor.node_to_var_map[arg2] + arg1_nvars = len(arg1_vars) + arg2_nvars = len(arg2_vars) + arg1_degree = visitor.degree_map[arg1] + arg2_degree = visitor.degree_map[arg2] + + if arg1_nvars > 1: + arg1 = visitor.create_aux_var(arg1) + arg1_vars = (arg1,) + arg1_nvars = 1 + arg1_degree = 1 + + if arg2_nvars > 1: + arg2 = visitor.create_aux_var(arg2) + arg2_vars = (arg2,) + arg2_nvars = 1 + arg2_degree = 1 + + res = arg1**arg2 + # at this point arg1 should have at most 1 variable + # and arg2 should have at most 1 variable + if arg1_nvars == 0: + visitor.node_to_var_map[res] = arg2_vars + elif arg2_nvars == 0: + visitor.node_to_var_map[res] = arg1_vars + else: + x = arg1_vars[0] + y = arg2_vars[0] + if x is y: + visitor.node_to_var_map[res] = (x,) + else: + visitor.node_to_var_map[res] = (x, y) + if arg1_degree == 0 and arg2_degree == 0: + visitor.degree_map[res] = 0 + else: + visitor.degree_map[res] = -1 + visitor.substitution_map[node] = res + return res + + +def _handle_named_expression(node, data, visitor): + assert len(data) == 1 + res = data[0] + visitor.substitution_map[node] = res + return res + + +def _handle_negation(node, data, visitor): + arg = data[0] + res = -arg + visitor.node_to_var_map[res] = visitor.node_to_var_map[arg] + visitor.degree_map[res] = visitor.degree_map[arg] + visitor.substitution_map[node] = res + return res + + +def _handle_unary(node, data, visitor): + arg = data[0] + arg_vars = visitor.node_to_var_map[arg] + arg_nvars = len(arg_vars) + arg_degree = visitor.degree_map[arg] + + if arg_nvars > 1: + arg = visitor.create_aux_var(arg) + arg_vars = (arg,) + arg_nvars = 1 + arg_degree = 1 + res = node.create_node_with_local_data((arg,)) + visitor.node_to_var_map[res] = arg_vars + if arg_degree == 0: + visitor.degree_map[res] = 0 + else: + visitor.degree_map[res] = -1 + visitor.substitution_map[node] = res + return res + + +handlers = ExitNodeDispatcher() +handlers[VarData] = _handle_var +handlers[ScalarVar] = _handle_var +handlers[ProductExpression] = _handle_product +handlers[SumExpression] = _handle_sum +handlers[DivisionExpression] = _handle_division +handlers[PowExpression] = _handle_pow +handlers[MonomialTermExpression] = _handle_product +handlers[LinearExpression] = _handle_sum +handlers[ParamData] = _handle_param +handlers[ExpressionData] = _handle_named_expression +handlers[ScalarExpression] = _handle_named_expression +handlers[NegationExpression] = _handle_negation +handlers[UnaryFunctionExpression] = _handle_unary +handlers[int] = _handle_float +handlers[float] = _handle_float + + +class _UnivariateNonlinearDecompositionVisitor(StreamBasedExpressionVisitor): + def __init__(self, **kwds): + self.block = kwds.pop('aux_block') + super().__init__(**kwds) + self.node_to_var_map = ComponentMap() + self.degree_map = ComponentMap() # values will be 0 (constant), 1 (linear), or -1 (nonlinear) + + self.substitution_map = ComponentMap() + + self.block.x = VarList() + self.block.c = ConstraintList() + + def initializeWalker(self, expr): + if expr in self.substitution_map: + return False, self.substitution_map[expr] + return True, None + + def beforeChild(self, node, child, child_idx): + if child in self.substitution_map: + return False, self.substitution_map[child] + return True, None + + def exitNode(self, node, data): + nt = type(node) + if nt in handlers: + return handlers[type(node)](node, data, self) + elif nt in native_numeric_types: + handlers[nt] = _handle_float + return _handle_float(node, data, self) + else: + raise NotImplementedError(f'unrecognized expression type: {nt}') + + def create_aux_var(self, expr): + if expr in self.substitution_map: + x = self.substitution_map[expr] + else: + x = self.block.x.add() + self.substitution_map[expr] = x + c = self.block.c.add(x == expr) + # we need to compute bounds on x now because some of the + # handlers depend on variable bounds (e.g., division) + fbbt(c) + return x + + +class UnivariateNonlinearDecompositionTransformation(Transformation): + def __init__(self): + super().__init__() + + def _check_for_unknown_active_components(self, model): + known_ctypes = {Constraint, Objective, Block} + for ctype in model.collect_ctypes(active=True, descend_into=True): + if not issubclass(ctype, ActiveComponent): + continue + if ctype in known_ctypes: + continue + if ctype is Suffix: + continue + raise NotImplementedError( + f'UnivariateNonlinearDecompositionTransformation does not know how to ' + f'handle components with ctype {ctype}' + ) + + def _apply_to(self, model, **kwds): + if kwds: + raise ValueError('UnivariateNonlinearDecompositionTransformation does not take any keyword arguments') + + self._check_for_unknown_active_components(model) + + objectives = list(model.component_data_objects(Objective, active=True, descend_into=True)) + constraints = list(model.component_data_objects(Constraint, active=True, descend_into=True)) + + bname = unique_component_name(model, 'auxiliary') + setattr(model, bname, Block()) + block = getattr(model, bname) + visitor = _UnivariateNonlinearDecompositionVisitor(aux_block=block) + linear_repn_visitor = LinearRepnVisitor(subexpression_cache={}) + + for con in constraints: + lower, body, upper = con.to_bounded_expression() + repn = linear_repn_visitor.walk_expression(body) + if repn.nonlinear is None: + continue + nonlinear_part = repn.nonlinear + repn.nonlinear = None + linear_part = repn.to_expression(linear_repn_visitor) + nonlinear_part = visitor.walk_expression(nonlinear_part) + new_body = linear_part + nonlinear_part + con.set_value((lower, new_body, upper)) + + for obj in objectives: + repn = linear_repn_visitor.walk_expression(obj.expr) + if repn.nonlinear is None: + continue + nonlinear_part = repn.nonlinear + repn.nonlinear = None + linear_part = repn.to_expression(linear_repn_visitor) + nonlinear_part = visitor.walk_expression(nonlinear_part) + obj.expr = linear_part + nonlinear_part From c8223818033ed13cf87fc1f6fea8fd71c00e2e5e Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 20 Jan 2026 09:59:21 -0700 Subject: [PATCH 053/107] tests for piecewise univariate nonlinear decomposition --- pyomo/contrib/piecewise/__init__.py | 3 + ...test_univariate_nonlinear_decomposition.py | 57 +++++++++++++++++++ .../contrib/piecewise/transform/factorable.py | 42 ++++++-------- 3 files changed, 77 insertions(+), 25 deletions(-) create mode 100644 pyomo/contrib/piecewise/tests/test_univariate_nonlinear_decomposition.py diff --git a/pyomo/contrib/piecewise/__init__.py b/pyomo/contrib/piecewise/__init__.py index e402628b423..3362fb0122d 100644 --- a/pyomo/contrib/piecewise/__init__.py +++ b/pyomo/contrib/piecewise/__init__.py @@ -37,6 +37,9 @@ DomainPartitioningMethod, NonlinearToPWL, ) +from pyomo.contrib.piecewise.transform.factorable import ( + UnivariateNonlinearDecompositionTransformation, +) from pyomo.contrib.piecewise.transform.nested_inner_repn import ( NestedInnerRepresentationGDPTransformation, ) diff --git a/pyomo/contrib/piecewise/tests/test_univariate_nonlinear_decomposition.py b/pyomo/contrib/piecewise/tests/test_univariate_nonlinear_decomposition.py new file mode 100644 index 00000000000..da247d56335 --- /dev/null +++ b/pyomo/contrib/piecewise/tests/test_univariate_nonlinear_decomposition.py @@ -0,0 +1,57 @@ +from pyomo.common.unittest import TestCase +import pyomo.environ as pyo +from pyomo.contrib import piecewise +from pyomo.core.expr.compare import assertExpressionsEqual + + +pe = pyo + + +def _get_trans(): + return pyo.TransformationFactory('contrib.piecewise.univariate_nonlinear_decomposition') + + +class TestUnivariateNonlinearDecomposition(TestCase): + def test_multiterm(self): + m = pe.ConcreteModel() + m.x = pe.Var() + m.y = pe.Var() + m.z = pe.Var() + m.c = pe.Constraint(expr=m.x + pe.log(m.y + m.z) + 1/pe.exp(m.x**0.5) <= 0) + + trans = _get_trans() + trans.apply_to(m) + aux = m.auxiliary + + assertExpressionsEqual( + self, + m.c.body, + aux.x[3] + aux.x[2] + m.x, + ) + assertExpressionsEqual( + self, + aux.c[1].expr, + aux.x[1] == (m.y + m.z), + ) + assertExpressionsEqual( + self, + aux.c[2].expr, + aux.x[2] == 1/pyo.exp(m.x**0.5), + ) + assertExpressionsEqual( + self, + aux.c[3].expr, + aux.x[3] == pyo.log(aux.x[1]), + ) + self.assertEqual(m.x.lb, 0) + self.assertIsNone(m.x.ub) + self.assertIsNone(m.y.lb) + self.assertIsNone(m.y.ub) + self.assertIsNone(m.z.lb) + self.assertIsNone(m.z.ub) + self.assertEqual(aux.x[1].lb, 0) + self.assertIsNone(aux.x[1].ub) + self.assertEqual(aux.x[2].lb, 0) + self.assertEqual(aux.x[2].ub, 1) + self.assertIsNone(aux.x[3].lb) + self.assertIsNone(aux.x[3].ub) diff --git a/pyomo/contrib/piecewise/transform/factorable.py b/pyomo/contrib/piecewise/transform/factorable.py index 8527702baae..a9c3e337379 100644 --- a/pyomo/contrib/piecewise/transform/factorable.py +++ b/pyomo/contrib/piecewise/transform/factorable.py @@ -34,31 +34,6 @@ from pyomo.repn.linear import LinearRepn, LinearRepnVisitor -""" -The purpose of this module/transformation is to convert any nonlinear model -to the following form: - -min/max f(x_i) -s.t. - g_j(x_i)*h_j(x_k) + a_j^T*x >////// Date: Tue, 20 Jan 2026 12:39:43 -0700 Subject: [PATCH 054/107] testing for univariate nonlinear decomposition --- ...test_univariate_nonlinear_decomposition.py | 213 +++++++++++++++++- .../contrib/piecewise/transform/factorable.py | 51 ++--- 2 files changed, 227 insertions(+), 37 deletions(-) diff --git a/pyomo/contrib/piecewise/tests/test_univariate_nonlinear_decomposition.py b/pyomo/contrib/piecewise/tests/test_univariate_nonlinear_decomposition.py index da247d56335..aa027570313 100644 --- a/pyomo/contrib/piecewise/tests/test_univariate_nonlinear_decomposition.py +++ b/pyomo/contrib/piecewise/tests/test_univariate_nonlinear_decomposition.py @@ -1,7 +1,9 @@ -from pyomo.common.unittest import TestCase +from pyomo.common.unittest import TestCase, skipUnless import pyomo.environ as pyo from pyomo.contrib import piecewise from pyomo.core.expr.compare import assertExpressionsEqual +from pyomo.common.dependencies import numpy_available, numpy +from pyomo.core.expr.numeric_expr import ProductExpression pe = pyo @@ -26,7 +28,7 @@ def test_multiterm(self): assertExpressionsEqual( self, m.c.body, - aux.x[3] + aux.x[2] + m.x, + m.x + aux.x[3] + aux.x[2], ) assertExpressionsEqual( self, @@ -55,3 +57,210 @@ def test_multiterm(self): self.assertEqual(aux.x[2].ub, 1) self.assertIsNone(aux.x[3].lb) self.assertIsNone(aux.x[3].ub) + + def test_common_subexpressions(self): + m = pe.ConcreteModel() + m.x = pe.Var() + m.y = pe.Var() + m.z1 = pe.Var() + m.z2 = pe.Var() + e = -pe.log(m.x + m.y) + m.c1 = pe.Constraint(expr=m.z1 + e == 0) + m.c2 = pe.Constraint(expr=m.z2 + e == 0) + + trans = _get_trans() + trans.apply_to(m) + aux = m.auxiliary + + assertExpressionsEqual( + self, + m.c1.expr, + m.z1 + aux.x[2] == 0, + ) + assertExpressionsEqual( + self, + m.c2.expr, + m.z2 + aux.x[2] == 0, + ) + assertExpressionsEqual( + self, + aux.c[1].expr, + aux.x[1] == m.x + m.y, + ) + assertExpressionsEqual( + self, + aux.c[2].expr, + aux.x[2] == -pe.log(aux.x[1]), + ) + + def test_product_fixed_variable(self): + m = pe.ConcreteModel() + m.x = pe.Var() + m.y = pe.Var() + m.z = pe.Var() + m.c = pe.Constraint(expr=2*pe.log(m.x + m.y) <= 0) + + trans = _get_trans() + trans.apply_to(m) + aux = m.auxiliary + + assertExpressionsEqual( + self, + m.c.expr, + 2*pe.log(aux.x[1]) <= 0, + ) + assertExpressionsEqual( + self, + aux.c[1].expr, + aux.x[1] == m.x + m.y, + ) + + def test_product_variable_fixed(self): + m = pe.ConcreteModel() + m.x = pe.Var() + m.y = pe.Var() + m.z = pe.Var() + m.c = pe.Constraint(expr=pe.log(m.x + m.y)*2 <= 0) + + trans = _get_trans() + trans.apply_to(m) + aux = m.auxiliary + + assertExpressionsEqual( + self, + m.c.expr, + pe.log(aux.x[1])*2 <= 0, + ) + assertExpressionsEqual( + self, + aux.c[1].expr, + aux.x[1] == m.x + m.y, + ) + + def test_prod_sum_sum(self): + m = pe.ConcreteModel() + m.x1 = pe.Var() + m.x2 = pe.Var() + m.x3 = pe.Var() + m.x4 = pe.Var() + m.c = pe.Constraint(expr=(m.x1 + m.x2) * (m.x3 + m.x4) <= 1) + + trans = _get_trans() + trans.apply_to(m) + aux = m.auxiliary + + assertExpressionsEqual( + self, + m.c.expr, + aux.x[1] * aux.x[2] <= 1, + ) + assertExpressionsEqual( + self, + aux.c[1].expr, + aux.x[1] == m.x1 + m.x2, + ) + assertExpressionsEqual( + self, + aux.c[2].expr, + aux.x[2] == m.x3 + m.x4, + ) + + def test_pow_sum_sum(self): + m = pe.ConcreteModel() + m.x1 = pe.Var() + m.x2 = pe.Var() + m.x3 = pe.Var() + m.x4 = pe.Var() + m.c = pe.Constraint(expr=(m.x1 + m.x2) ** (m.x3 + m.x4) <= 1) + + trans = _get_trans() + trans.apply_to(m) + aux = m.auxiliary + + assertExpressionsEqual( + self, + m.c.expr, + aux.x[1] ** aux.x[2] <= 1, + ) + assertExpressionsEqual( + self, + aux.c[1].expr, + aux.x[1] == m.x1 + m.x2, + ) + assertExpressionsEqual( + self, + aux.c[2].expr, + aux.x[2] == m.x3 + m.x4, + ) + + def test_division_var_const(self): + m = pe.ConcreteModel() + m.x = pe.Var() + m.y = pe.Var() + m.c = pe.Constraint(expr=(m.x + m.y) / 2 <= 0) + + trans = _get_trans() + trans.apply_to(m) + aux = m.auxiliary + + assertExpressionsEqual( + self, + m.c.expr, + (m.x + m.y) / 2 <= 0, + ) + + def test_division_sum_sum(self): + m = pe.ConcreteModel() + m.x1 = pe.Var() + m.x2 = pe.Var() + m.x3 = pe.Var() + m.x4 = pe.Var() + m.c = pe.Constraint(expr=(m.x1 + m.x2) / (m.x3 + m.x4) <= 1) + + trans = _get_trans() + trans.apply_to(m) + aux = m.auxiliary + + assertExpressionsEqual( + self, + m.c.expr, + aux.x[1] * aux.x[3] <= 1, + ) + assertExpressionsEqual( + self, + aux.c[1].expr, + aux.x[1] == m.x1 + m.x2, + ) + assertExpressionsEqual( + self, + aux.c[2].expr, + aux.x[2] == m.x3 + m.x4, + ) + assertExpressionsEqual( + self, + aux.c[3].expr, + aux.x[3] * aux.x[2] == 1, + ) + + @skipUnless(numpy_available, "Numpy is not available") + def test_numpy_float(self): + m = pe.ConcreteModel() + m.x = pe.Var() + m.y = pe.Var() + m.z = pe.Var() + m.c = pe.Constraint(expr=ProductExpression((numpy.float64(2.5), pe.log(m.x + m.y))) <= 0) + + trans = _get_trans() + trans.apply_to(m) + aux = m.auxiliary + + assertExpressionsEqual( + self, + m.c.expr, + 2.5*pe.log(aux.x[1]) <= 0, + ) + assertExpressionsEqual( + self, + aux.c[1].expr, + aux.x[1] == m.x + m.y, + ) diff --git a/pyomo/contrib/piecewise/transform/factorable.py b/pyomo/contrib/piecewise/transform/factorable.py index a9c3e337379..3e6af881735 100644 --- a/pyomo/contrib/piecewise/transform/factorable.py +++ b/pyomo/contrib/piecewise/transform/factorable.py @@ -24,31 +24,29 @@ Objective, ) from pyomo.core.base.var import ScalarVar +from pyomo.core.base.param import ScalarParam from pyomo.contrib.fbbt.fbbt import compute_bounds_on_expr, fbbt from pyomo.core.base.expression import ScalarExpression from pyomo.core.base.transformation import Transformation, TransformationFactory from pyomo.common.modeling import unique_component_name from pyomo.core.base.component import ActiveComponent from pyomo.core.base.suffix import Suffix -from pyomo.gdp import Disjunct -from pyomo.repn.linear import LinearRepn, LinearRepnVisitor def _handle_var(node, data, visitor): + if node.fixed: + return _handle_float(node.value, data, visitor) visitor.node_to_var_map[node] = (node,) visitor.degree_map[node] = 1 visitor.substitution_map[node] = node return node -def _handle_float(node, data, visitor): - visitor.node_to_var_map[node] = tuple() - visitor.degree_map[node] = 0 - visitor.substitution_map[node] = node - return node +def _handle_param(node, data, visitor): + return _handle_float(node.value, data, visitor) -def _handle_param(node, data, visitor): +def _handle_float(node, data, visitor): visitor.node_to_var_map[node] = tuple() visitor.degree_map[node] = 0 visitor.substitution_map[node] = node @@ -113,13 +111,6 @@ def _handle_product(node, data, visitor): def _handle_sum(node, data, visitor): - # remember to separate the linear parts of the root expression - # from the nonlinear parts prior to using this visitor - # otherwise, we will get auxilliary variables that we don't - # strictly need. If this is not done, we don't do anything - # incorrect, we just get extra variables and constraints - # I'm not sure we should even worry about it, because pretty much - # any presolve should remove them arg_list = [] new_degree = 0 vset = ComponentSet() @@ -307,13 +298,14 @@ def _handle_unary(node, data, visitor): handlers = ExitNodeDispatcher() handlers[VarData] = _handle_var handlers[ScalarVar] = _handle_var +handlers[ParamData] = _handle_param +handlers[ScalarParam] = _handle_param handlers[ProductExpression] = _handle_product handlers[SumExpression] = _handle_sum handlers[DivisionExpression] = _handle_division handlers[PowExpression] = _handle_pow handlers[MonomialTermExpression] = _handle_product handlers[LinearExpression] = _handle_sum -handlers[ParamData] = _handle_param handlers[ExpressionData] = _handle_named_expression handlers[ScalarExpression] = _handle_named_expression handlers[NegationExpression] = _handle_negation @@ -415,26 +407,15 @@ def _apply_to(self, model, **kwds): setattr(model, bname, Block()) block = getattr(model, bname) visitor = _UnivariateNonlinearDecompositionVisitor(aux_block=block) - linear_repn_visitor = LinearRepnVisitor(subexpression_cache={}) for con in constraints: - lower, body, upper = con.to_bounded_expression() - repn = linear_repn_visitor.walk_expression(body) - if repn.nonlinear is None: - continue - nonlinear_part = repn.nonlinear - repn.nonlinear = None - linear_part = repn.to_expression(linear_repn_visitor) - nonlinear_part = visitor.walk_expression(nonlinear_part) - new_body = linear_part + nonlinear_part - con.set_value((lower, new_body, upper)) + lower, body, upper = con.to_bounded_expression(evaluate_bounds=True) + new_body = visitor.walk_expression(body) + if lower == upper: + con.set_value(new_body == lower) + else: + con.set_value((lower, new_body, upper)) for obj in objectives: - repn = linear_repn_visitor.walk_expression(obj.expr) - if repn.nonlinear is None: - continue - nonlinear_part = repn.nonlinear - repn.nonlinear = None - linear_part = repn.to_expression(linear_repn_visitor) - nonlinear_part = visitor.walk_expression(nonlinear_part) - obj.expr = linear_part + nonlinear_part + new_expr = visitor.walk_expression(obj.expr) + obj.expr = new_expr From 6ae744988773399c8944a1f1e8d1e1f204a40332 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 20 Jan 2026 14:09:29 -0700 Subject: [PATCH 055/107] add an option to univariate nonlinear decomposition to aggressively add auxiliary variables --- .../contrib/piecewise/transform/factorable.py | 60 +++++++++++++++---- 1 file changed, 50 insertions(+), 10 deletions(-) diff --git a/pyomo/contrib/piecewise/transform/factorable.py b/pyomo/contrib/piecewise/transform/factorable.py index 3e6af881735..ac044b6840f 100644 --- a/pyomo/contrib/piecewise/transform/factorable.py +++ b/pyomo/contrib/piecewise/transform/factorable.py @@ -31,6 +31,7 @@ from pyomo.common.modeling import unique_component_name from pyomo.core.base.component import ActiveComponent from pyomo.core.base.suffix import Suffix +from pyomo.common.config import ConfigDict, ConfigValue def _handle_var(node, data, visitor): @@ -76,12 +77,12 @@ def _handle_product(node, data, visitor): visitor.substitution_map[node] = res return res - if arg1_nvars > 1: + if arg1_nvars > 1 or visitor.aggressive_substitution: arg1 = visitor.create_aux_var(arg1) arg1_vars = (arg1,) arg1_nvars = 1 arg1_degree = 1 - if arg2_nvars > 1: + if arg2_nvars > 1 or visitor.aggressive_substitution: arg2 = visitor.create_aux_var(arg2) arg2_vars = (arg2,) arg2_nvars = 1 @@ -153,13 +154,13 @@ def _handle_division(node, data, visitor): visitor.substitution_map[node] = res return res - if arg1_nvars > 1: + if arg1_nvars > 1 or visitor.aggressive_substitution: arg1 = visitor.create_aux_var(arg1) arg1_vars = (arg1,) arg1_nvars = 1 arg1_degree = 1 - if arg2_nvars > 1: + if arg2_nvars > 1 or visitor.aggressive_substitution: arg2 = visitor.create_aux_var(arg2) arg2_vars = (arg2,) arg2_nvars = 1 @@ -224,13 +225,13 @@ def _handle_pow(node, data, visitor): arg1_degree = visitor.degree_map[arg1] arg2_degree = visitor.degree_map[arg2] - if arg1_nvars > 1: + if arg1_nvars > 1 or visitor.aggressive_substitution: arg1 = visitor.create_aux_var(arg1) arg1_vars = (arg1,) arg1_nvars = 1 arg1_degree = 1 - if arg2_nvars > 1: + if arg2_nvars > 1 or visitor.aggressive_substitution: arg2 = visitor.create_aux_var(arg2) arg2_vars = (arg2,) arg2_nvars = 1 @@ -280,7 +281,7 @@ def _handle_unary(node, data, visitor): arg_nvars = len(arg_vars) arg_degree = visitor.degree_map[arg] - if arg_nvars > 1: + if arg_nvars > 1 or visitor.aggressive_substitution: arg = visitor.create_aux_var(arg) arg_vars = (arg,) arg_nvars = 1 @@ -317,6 +318,7 @@ def _handle_unary(node, data, visitor): class _UnivariateNonlinearDecompositionVisitor(StreamBasedExpressionVisitor): def __init__(self, **kwds): self.block = kwds.pop('aux_block') + self.aggressive_substitution = kwds.pop('aggressive_substitution') super().__init__(**kwds) self.node_to_var_map = ComponentMap() self.degree_map = ComponentMap() # values will be 0 (constant), 1 (linear), or -1 (nonlinear) @@ -325,6 +327,7 @@ def __init__(self, **kwds): self.block.x = VarList() self.block.c = ConstraintList() + self._leaf_types = {VarData, ScalarVar, ParamData, ScalarVar, float, int} def initializeWalker(self, expr): if expr in self.substitution_map: @@ -345,10 +348,18 @@ def exitNode(self, node, data): return _handle_float(node, data, self) else: raise NotImplementedError(f'unrecognized expression type: {nt}') + + def _is_leaf(self, x): + if type(x) in self._leaf_types or type(x) in native_numeric_types: + return True + return False def create_aux_var(self, expr): if expr in self.substitution_map: x = self.substitution_map[expr] + elif self._is_leaf(expr): + self.substitution_map[expr] = expr + return expr else: x = self.block.x.add() self.substitution_map[expr] = x @@ -377,8 +388,35 @@ def create_aux_var(self, expr): """ ) class UnivariateNonlinearDecompositionTransformation(Transformation): + + CONFIG = ConfigDict('contrib.piecewise.univariate_nonlinear_decomposition') + CONFIG.declare( + 'aggressive_substitution', + ConfigValue( + default=False, + domain=bool, + description="introduce auxiliary variables more frequently", + doc=""" + If aggressive_substitution is True, then auxiliary variables will be introduced + more aggressively. For example, + + x**2 * y**3 + + will become + + z1 * z2 + z1 == x**2 + z2 == y**3 + + even though this does not reduce the dimensionality of the PWL approximation. + The reason to do this is to be able to bound z1 and z2. + """ + ) + ) + def __init__(self): super().__init__() + self._config = UnivariateNonlinearDecompositionTransformation.CONFIG() def _check_for_unknown_active_components(self, model): known_ctypes = {Constraint, Objective, Block} @@ -395,8 +433,7 @@ def _check_for_unknown_active_components(self, model): ) def _apply_to(self, model, **kwds): - if kwds: - raise ValueError('UnivariateNonlinearDecompositionTransformation does not take any keyword arguments') + self._config = UnivariateNonlinearDecompositionTransformation.CONFIG(value=kwds, preserve_implicit=True) self._check_for_unknown_active_components(model) @@ -406,7 +443,10 @@ def _apply_to(self, model, **kwds): bname = unique_component_name(model, 'auxiliary') setattr(model, bname, Block()) block = getattr(model, bname) - visitor = _UnivariateNonlinearDecompositionVisitor(aux_block=block) + visitor = _UnivariateNonlinearDecompositionVisitor( + aux_block=block, + aggressive_substitution=self._config.aggressive_substitution, + ) for con in constraints: lower, body, upper = con.to_bounded_expression(evaluate_bounds=True) From 2b0bea70de66b04387a106c4f7cb8b8149120dcc Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 29 Jan 2026 13:40:20 -0700 Subject: [PATCH 056/107] handle NPV nodes in univariate nonlinear decomposition transformation --- .../contrib/piecewise/transform/factorable.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pyomo/contrib/piecewise/transform/factorable.py b/pyomo/contrib/piecewise/transform/factorable.py index ac044b6840f..1be8022c6d9 100644 --- a/pyomo/contrib/piecewise/transform/factorable.py +++ b/pyomo/contrib/piecewise/transform/factorable.py @@ -10,7 +10,14 @@ SumExpression, LinearExpression, UnaryFunctionExpression, + NPV_NegationExpression, + NPV_PowExpression, + NPV_ProductExpression, + NPV_DivisionExpression, + NPV_SumExpression, + NPV_UnaryFunctionExpression, ) +from pyomo.core.base.units_container import _PyomoUnit from pyomo.repn.util import ExitNodeDispatcher from pyomo.core.base import ( @@ -43,6 +50,10 @@ def _handle_var(node, data, visitor): return node +def _handle_unit(node, data, visitor): + return _handle_float(node.value, data, visitor) + + def _handle_param(node, data, visitor): return _handle_float(node.value, data, visitor) @@ -313,6 +324,13 @@ def _handle_unary(node, data, visitor): handlers[UnaryFunctionExpression] = _handle_unary handlers[int] = _handle_float handlers[float] = _handle_float +handlers[NPV_NegationExpression] = _handle_negation +handlers[NPV_PowExpression] = _handle_pow +handlers[NPV_ProductExpression] = _handle_product +handlers[NPV_DivisionExpression] = _handle_division +handlers[NPV_SumExpression] = _handle_sum +handlers[NPV_UnaryFunctionExpression] = _handle_unary +handlers[_PyomoUnit] = _handle_unit class _UnivariateNonlinearDecompositionVisitor(StreamBasedExpressionVisitor): From d29a2cfe9b985857e25c76c8a8f40be7be1c42aa Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 29 Jan 2026 15:24:10 -0700 Subject: [PATCH 057/107] starting a module for initialization --- pyomo/contrib/initialization/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 pyomo/contrib/initialization/__init__.py diff --git a/pyomo/contrib/initialization/__init__.py b/pyomo/contrib/initialization/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From 8914a638203b048d28c06ef2d2dac7a6529c3396 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 29 Jan 2026 16:13:14 -0700 Subject: [PATCH 058/107] starting a module for initialization --- pyomo/contrib/initialization/README.md | 1 + .../contrib/initialization/bounds/__init__.py | 0 .../initialization/bounds/bound_variables.py | 10 +++ pyomo/contrib/initialization/initialize.py | 42 ++++++++++++ pyomo/contrib/initialization/pwl_init.py | 68 +++++++++++++++++++ pyomo/contrib/initialization/utils.py | 38 +++++++++++ 6 files changed, 159 insertions(+) create mode 100644 pyomo/contrib/initialization/README.md create mode 100644 pyomo/contrib/initialization/bounds/__init__.py create mode 100644 pyomo/contrib/initialization/bounds/bound_variables.py create mode 100644 pyomo/contrib/initialization/initialize.py create mode 100644 pyomo/contrib/initialization/pwl_init.py create mode 100644 pyomo/contrib/initialization/utils.py diff --git a/pyomo/contrib/initialization/README.md b/pyomo/contrib/initialization/README.md new file mode 100644 index 00000000000..24ff0a0de86 --- /dev/null +++ b/pyomo/contrib/initialization/README.md @@ -0,0 +1 @@ +The purpose of this module is to provide methods for initializing nonlinear programming models. \ No newline at end of file diff --git a/pyomo/contrib/initialization/bounds/__init__.py b/pyomo/contrib/initialization/bounds/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pyomo/contrib/initialization/bounds/bound_variables.py b/pyomo/contrib/initialization/bounds/bound_variables.py new file mode 100644 index 00000000000..5eb59ec3c35 --- /dev/null +++ b/pyomo/contrib/initialization/bounds/bound_variables.py @@ -0,0 +1,10 @@ +from pyomo.core.base.block import BlockData + + +def bound_all_nonlinear_variables(m: BlockData, default_bound: float = 1.0e8): + """ + Attempt to obtain valid bounds on all nonlinear variables based on the + constraints in the model, m. If variable bounds cannot be obtained, + we use default_bound. + """ + raise NotImplementedError('not done yet') diff --git a/pyomo/contrib/initialization/initialize.py b/pyomo/contrib/initialization/initialize.py new file mode 100644 index 00000000000..e20b18d51df --- /dev/null +++ b/pyomo/contrib/initialization/initialize.py @@ -0,0 +1,42 @@ +from pyomo.core.base.block import BlockData +from enum import Enum +from pyomo.contrib.initialization.utils import get_vars, shallow_clone +from pyomo.common.collections import ComponentMap +from pyomo.contrib.initialization.pwl_init import _initialize_with_piecewise_linear_approximation + + +class InitializationMethod(Enum): + pwl_approximation = "pwl_approximation" + + +def initialize_nlp( + nlp: BlockData, + method: InitializationMethod = InitializationMethod.pwl_approximation +): + # get all variable bounds, domains, etc. to restore them later + orig_vars = get_vars(nlp) + orig_var_data = ComponentMap( + (v, (v.lower, v.upper, v.domain, v.fixed, v.value)) for v in orig_vars + ) + + # create a shallow clone of the model so that the initialization method can + # can work with the original variables but not make any other + # modifications to the model + nlp = shallow_clone(nlp) + + # run the initialization + if method == InitializationMethod.pwl_approximation: + _initialize_with_piecewise_linear_approximation(nlp) + else: + raise ValueError(f'unexpected initialization method: {method}') + + # restore variable bounds, domain, etc. + for v, (lb, ub, domain, fixed, value) in orig_var_data.items(): + v.setlb(lb) + v.setub(ub) + v.domain = domain + if fixed: + assert v.value == value + assert v.fixed + else: + v.unfix() diff --git a/pyomo/contrib/initialization/pwl_init.py b/pyomo/contrib/initialization/pwl_init.py new file mode 100644 index 00000000000..a5b0ee8aa84 --- /dev/null +++ b/pyomo/contrib/initialization/pwl_init.py @@ -0,0 +1,68 @@ +from pyomo.core.base.block import BlockData +import pyomo.environ as pe +from pyomo.contrib.initialization.bounds.bound_variables import bound_all_nonlinear_variables +from pyomo.contrib.initialization.utils import fix_vars_with_equal_bounds + + +def _minimize_infeasibility(m): + m.slacks = pe.VarList() + m.extra_cons = pe.ConstraintList() + + obj_expr = 0 + + found_obj = False + for obj in m.component_data_objects(pe.Objective, active=True, descend_into=True): + assert not found_obj + if obj.sense == pe.minimize: + obj_expr += 0.1*obj.expr + else: + obj_expr -= 0.1*obj_expr + obj.deactivate() + found_obj = True + + for con in m.component_data_objects(pe.Constraint, active=True, descend_into=True): + lb, body, ub = con.to_bounded_expression(evaluate_bounds=True) + if lb == ub: + ps = m.slacks.add() + ns = m.slacks.add() + ps.setlb(0) + ns.setlb(0) + con.set_value(body - lb - ps + ns == 0) + elif lb is None: + ps = m.slacks.add() + ps.setlb(0) + con.set_value(body - ub - ps <= 0) + elif ub is None: + ns = m.slacks.add() + ns.setlb(0) + con.set_value(body - lb + ns >= 0) + else: + con.deactivate() + ps = m.slacks.add() + ns = m.slacks.add() + ps.setlb(0) + ns.setlb(0) + m.extra_cons.add(body - ub - ps <= 0) + m.extra_cons.add(body - lb + ns >= 0) + + m.slack_obj = pe.Objective(expr=10*sum(m.slacks.values()) + obj_expr) + + +def _initialize_with_piecewise_linear_approximation(nlp: BlockData): + # first introduce auxiliary variables so that we don't try to + # approximate any functions of more than two variables + trans = pe.TransformationFactory('contrib.piecewise.univariate_nonlinear_decomposition') + trans.apply_to(nlp, aggressive_substitution=True) + + # now we need to try to get bounds on all of the nonlinear variables + bound_all_nonlinear_variables(nlp) + + # Now, we need to fix variables with equal (or nearly equal) bounds. + # Otherwise, the PWL transformation complains + fix_vars_with_equal_bounds(nlp) + + # now we modify the model by introducing slacks to make sure the PWL + # approximatin is feasible + _minimize_infeasibility(nlp) + + raise NotImplementedError('not done yet') diff --git a/pyomo/contrib/initialization/utils.py b/pyomo/contrib/initialization/utils.py new file mode 100644 index 00000000000..792509583f0 --- /dev/null +++ b/pyomo/contrib/initialization/utils.py @@ -0,0 +1,38 @@ +import pyomo.environ as pe +from pyomo.common.collections import ComponentSet +from pyomo.core.base.block import BlockData +from pyomo.core.expr.visitor import identify_variables +import math + + +def get_vars(m: BlockData): + vset = ComponentSet() + for c in m.component_data_objects(pe.Constraint, active=True, descend_into=True): + vset.update(identify_variables(c.body, include_fixed=False)) + for o in m.component_data_objects(pe.Objective, active=True, descend_into=True): + vset.update(identify_variables(c.expr, include_fixed=False)) + return vset + + +def shallow_clone(m1): + m2 = pe.ConcreteModel() + m2.cons = pe.ConstraintList() + + for con in m1.component_data_objects(pe.Constraint, active=True, descend_into=True): + m2.cons.add(con.expr) + + objlist = list(m1.component_data_objects(pe.Objective, active=True, descend_into=True)) + assert len(objlist) <= 1 + if objlist: + obj = objlist[0] + m2.obj = pe.Objective(expr=obj.expr, sense=obj.sense) + + return m2 + + +def fix_vars_with_equal_bounds(m): + for v in get_vars(m): + if v.fixed: + continue + if v.lb is not None and v.ub is not None and math.isclose(v.lb, v.ub, abs_tol=1e-4, rel_tol=1e-4): + v.fix(0.5 * (v.lb + v.ub)) From 5ebfdd4dc3dda7407f89b2032cbfac939b865804 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 3 Feb 2026 08:26:01 -0700 Subject: [PATCH 059/107] initialization --- .../initialization/bounds/bound_variables.py | 16 +++++++++- pyomo/contrib/initialization/pwl_init.py | 32 +++++++++++++++---- 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/pyomo/contrib/initialization/bounds/bound_variables.py b/pyomo/contrib/initialization/bounds/bound_variables.py index 5eb59ec3c35..0c243b8eb33 100644 --- a/pyomo/contrib/initialization/bounds/bound_variables.py +++ b/pyomo/contrib/initialization/bounds/bound_variables.py @@ -1,4 +1,10 @@ from pyomo.core.base.block import BlockData +from pyomo.contrib.fbbt.fbbt import fbbt +from pyomo.contrib.initialization.utils import get_vars +import logging + + +logger = logging.getLogger(__name__) def bound_all_nonlinear_variables(m: BlockData, default_bound: float = 1.0e8): @@ -7,4 +13,12 @@ def bound_all_nonlinear_variables(m: BlockData, default_bound: float = 1.0e8): constraints in the model, m. If variable bounds cannot be obtained, we use default_bound. """ - raise NotImplementedError('not done yet') + fbbt(m) + for v in get_vars(m): + if v.lb is None or v.lb < -default_bound: + logger.warning(f'Could not obtain a lower bound for {str(v)} better than {-default_bound}; setting the lower bound to {-default_bound}') + v.setlb(-default_bound) + if v.ub is None or v.ub > default_bound: + logger.warning(f'Could not obtain an upper bound for {str(v)} better than {default_bound}; setting the upper bound to {default_bound}') + v.setub(default_bound) + fbbt(m) diff --git a/pyomo/contrib/initialization/pwl_init.py b/pyomo/contrib/initialization/pwl_init.py index a5b0ee8aa84..969adbecd99 100644 --- a/pyomo/contrib/initialization/pwl_init.py +++ b/pyomo/contrib/initialization/pwl_init.py @@ -1,7 +1,7 @@ from pyomo.core.base.block import BlockData import pyomo.environ as pe from pyomo.contrib.initialization.bounds.bound_variables import bound_all_nonlinear_variables -from pyomo.contrib.initialization.utils import fix_vars_with_equal_bounds +from pyomo.contrib.initialization.utils import fix_vars_with_equal_bounds, shallow_clone def _minimize_infeasibility(m): @@ -48,21 +48,41 @@ def _minimize_infeasibility(m): m.slack_obj = pe.Objective(expr=10*sum(m.slacks.values()) + obj_expr) -def _initialize_with_piecewise_linear_approximation(nlp: BlockData): +def _refine_pwl_approx(m): + + +def _initialize_with_piecewise_linear_approximation(nlp: BlockData, default_bound=1.0e8): + pwl = shallow_clone(nlp) + # first introduce auxiliary variables so that we don't try to # approximate any functions of more than two variables trans = pe.TransformationFactory('contrib.piecewise.univariate_nonlinear_decomposition') - trans.apply_to(nlp, aggressive_substitution=True) + trans.apply_to(pwl, aggressive_substitution=True) # now we need to try to get bounds on all of the nonlinear variables - bound_all_nonlinear_variables(nlp) + bound_all_nonlinear_variables(pwl, default_bound=default_bound) # Now, we need to fix variables with equal (or nearly equal) bounds. # Otherwise, the PWL transformation complains - fix_vars_with_equal_bounds(nlp) + fix_vars_with_equal_bounds(pwl) # now we modify the model by introducing slacks to make sure the PWL # approximatin is feasible - _minimize_infeasibility(nlp) + # all of the slacks appear linearly, so we don't need to worry about + # upper bounds for them + _minimize_infeasibility(pwl) + + # build the PWL approximation + trans = pe.TransformationFactory('contrib.piecewise.nonlinear_to_pwl') + trans.apply_to(pwl, num_points=num_points, additively_decompose=False) + + """ + Now we want to + 1. solve the PWL approximation + 2. Initialize the NLP to the solution + 3. Try solving the NLP + 4. If the NLP converges => done + 5. If the NLP does not converge, refine the PWL approximation and repeat + """ raise NotImplementedError('not done yet') From 488a539925fecda3e57f08e894650e7e6d80cf0c Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 3 Feb 2026 18:57:28 -0700 Subject: [PATCH 060/107] pwl initialization is almost working --- pyomo/contrib/initialization/initialize.py | 17 +- pyomo/contrib/initialization/pwl_init.py | 237 +++++++++++++++++- .../piecewise/transform/nonlinear_to_pwl.py | 16 +- 3 files changed, 259 insertions(+), 11 deletions(-) diff --git a/pyomo/contrib/initialization/initialize.py b/pyomo/contrib/initialization/initialize.py index e20b18d51df..7b7e8af7338 100644 --- a/pyomo/contrib/initialization/initialize.py +++ b/pyomo/contrib/initialization/initialize.py @@ -3,6 +3,7 @@ from pyomo.contrib.initialization.utils import get_vars, shallow_clone from pyomo.common.collections import ComponentMap from pyomo.contrib.initialization.pwl_init import _initialize_with_piecewise_linear_approximation +from pyomo.contrib.solver.common.base import SolverBase class InitializationMethod(Enum): @@ -11,7 +12,12 @@ class InitializationMethod(Enum): def initialize_nlp( nlp: BlockData, - method: InitializationMethod = InitializationMethod.pwl_approximation + mip_solver: SolverBase, + nlp_solver: SolverBase, + method: InitializationMethod = InitializationMethod.pwl_approximation, + default_bound=1.0e8, + max_pwl_refinement_iter=100, + num_pwl_cons_to_refine_per_iter=5, ): # get all variable bounds, domains, etc. to restore them later orig_vars = get_vars(nlp) @@ -26,7 +32,14 @@ def initialize_nlp( # run the initialization if method == InitializationMethod.pwl_approximation: - _initialize_with_piecewise_linear_approximation(nlp) + _initialize_with_piecewise_linear_approximation( + nlp=nlp, + mip_solver=mip_solver, + nlp_solver=nlp_solver, + default_bound=default_bound, + max_iter=max_pwl_refinement_iter, + num_cons_to_refine_per_iter=num_pwl_cons_to_refine_per_iter, + ) else: raise ValueError(f'unexpected initialization method: {method}') diff --git a/pyomo/contrib/initialization/pwl_init.py b/pyomo/contrib/initialization/pwl_init.py index 969adbecd99..d6fdae92c25 100644 --- a/pyomo/contrib/initialization/pwl_init.py +++ b/pyomo/contrib/initialization/pwl_init.py @@ -1,7 +1,44 @@ from pyomo.core.base.block import BlockData import pyomo.environ as pe from pyomo.contrib.initialization.bounds.bound_variables import bound_all_nonlinear_variables -from pyomo.contrib.initialization.utils import fix_vars_with_equal_bounds, shallow_clone +from pyomo.contrib.initialization.utils import fix_vars_with_equal_bounds, shallow_clone, get_vars +from pyomo.core.expr.visitor import identify_components +from pyomo.contrib.piecewise.piecewise_linear_expression import PiecewiseLinearExpression +from pyomo.contrib.piecewise.piecewise_linear_function import PiecewiseLinearFunction +from pyomo.common.collections import ComponentMap, ComponentSet +from typing import MutableMapping, Sequence, List +from pyomo.core.base.constraint import ConstraintData +from pyomo.core.expr.visitor import StreamBasedExpressionVisitor +from pyomo.common.numeric_types import native_numeric_types +from pyomo.core.expr.numvalue import NumericConstant +from pyomo.core.expr.numeric_expr import ( + NegationExpression, + PowExpression, + ProductExpression, + MonomialTermExpression, + DivisionExpression, + SumExpression, + LinearExpression, + UnaryFunctionExpression, + NPV_NegationExpression, + NPV_PowExpression, + NPV_ProductExpression, + NPV_DivisionExpression, + NPV_SumExpression, + NPV_UnaryFunctionExpression, +) +from pyomo.core.expr.relational_expr import EqualityExpression, InequalityExpression, RangedExpression +from pyomo.repn.util import ExitNodeDispatcher +from pyomo.core.base.var import ScalarVar, VarData +from pyomo.core.base.param import ScalarParam, ParamData +from pyomo.core.base.expression import ScalarExpression, ExpressionData +import math +from pyomo.contrib.solver.common.base import SolverBase +import logging +from pyomo.common.modeling import unique_component_name + + +logger = logging.getLogger(__name__) def _minimize_infeasibility(m): @@ -48,33 +85,184 @@ def _minimize_infeasibility(m): m.slack_obj = pe.Objective(expr=10*sum(m.slacks.values()) + obj_expr) -def _refine_pwl_approx(m): +def _get_pwl_constraints(m: BlockData) -> MutableMapping[ + PiecewiseLinearExpression, + List[ConstraintData] +]: + comp_types = set() + comp_types.add(PiecewiseLinearExpression) + pwl_expr_to_con_map = ComponentMap() + for con in m.component_data_objects(pe.Constraint, active=True, descend_into=True): + pwl_exprs = list(identify_components(con.expr, comp_types)) + if not pwl_exprs: + continue + assert len(pwl_exprs) == 1 + e = pwl_exprs[0] + if e not in pwl_expr_to_con_map: + pwl_expr_to_con_map[e] = [] + pwl_expr_to_con_map[e].append(con) + return pwl_expr_to_con_map + + +def _handle_leaf(node, data): + return node + + +def _handle_node(node, data): + return node.create_node_with_local_data(data) + + +_handlers = ExitNodeDispatcher() +for t in [float, int, VarData, ScalarVar, ParamData, ScalarParam, NumericConstant]: + _handlers[t] = _handle_leaf +for t in [ + ProductExpression, + SumExpression, + DivisionExpression, + PowExpression, + MonomialTermExpression, + LinearExpression, + ExpressionData, + ScalarExpression, + NegationExpression, + UnaryFunctionExpression, + NPV_NegationExpression, + NPV_PowExpression, + NPV_ProductExpression, + NPV_DivisionExpression, + NPV_SumExpression, + NPV_UnaryFunctionExpression, + EqualityExpression, + InequalityExpression, + RangedExpression, +]: + _handlers[t] = _handle_node + + +class _PWLRefinementVisitor(StreamBasedExpressionVisitor): + def __init__(self, m, pwl_exprs, **kwds): + self.m = m + self.pwl_exprs = ComponentSet(pwl_exprs) + self.substitution = ComponentMap() + self.named_expr_map = ComponentMap() + super().__init__(**kwds) + + def exitNode(self, node, data): + if node in self.named_expr_map: + return self.named_expr_map[node] + nt = type(node) + if nt in _handlers: + return _handlers[nt](node, data) + elif nt in native_numeric_types: + _handlers[nt] = _handle_leaf + return _handle_leaf(node, data) + else: + raise NotImplementedError(f'unrecognized expression type: {nt}') + + def beforeChild(self, node, child, child_idx): + if child in self.substitution: + return False, self.substitution[child] + + if child not in self.pwl_exprs: + return True, None + old_func = child.pw_linear_function + _func = old_func._func + points = list(old_func._points) + variables = child.args + var_values = tuple(i.value for i in variables) + points.append(var_values) + points.sort() + if len(points[0]) == 1: + points = [i[0] for i in points] + new_func = PiecewiseLinearFunction(points=points, function=_func) + fname = unique_component_name(self.m.auxiliary._pyomo_contrib_nonlinear_to_pwl, 'f') + setattr(self.m.auxiliary._pyomo_contrib_nonlinear_to_pwl, fname, new_func) + new_expr = new_func(*variables) + self.named_expr_map[node] = new_expr + self.substitution[child] = new_expr.expr + return False, new_expr.expr -def _initialize_with_piecewise_linear_approximation(nlp: BlockData, default_bound=1.0e8): + +def _refine_pwl_approx( + m, + pwl_expr_to_con_map: MutableMapping[ + PiecewiseLinearExpression, + Sequence[ConstraintData], + ], + num_to_refine: int = 5, +): + violations = [] + for expr in pwl_expr_to_con_map.keys(): + func = expr.pw_linear_function + var_vals = tuple(i.value for i in expr.args) + if any(i is None for i in var_vals): + continue + approx_value = func(*var_vals) + true_value = func._func(*var_vals) + err = abs(true_value - approx_value) + violations.append((err, expr)) + violations.sort(key=lambda i: i[0], reverse=True) + + if len(violations) == 0: + raise RuntimeError('Did not find any piecewise linear functions with variable values') + + if math.isclose(violations[0][0], 0): + raise RuntimeError('All of the original nonlinear functions are satisfied!') + + for err, expr in violations[:num_to_refine]: + logger.info(f'refining {expr.pw_linear_function._func.expr} with error {err}') + + funcs_to_refine = ComponentSet(i[1] for i in violations[:num_to_refine]) + visitor = _PWLRefinementVisitor(m, funcs_to_refine) + + for expr in funcs_to_refine: + for con in pwl_expr_to_con_map[expr]: + con.set_value(visitor.walk_expression(con.expr)) + + for e1, e2 in visitor.substitution.items(): + cons = pwl_expr_to_con_map.pop(e1) + pwl_expr_to_con_map[e2] = cons + + +def _initialize_with_piecewise_linear_approximation( + nlp: BlockData, + mip_solver: SolverBase, + nlp_solver: SolverBase, + default_bound=1.0e8, + max_iter=100, + num_cons_to_refine_per_iter=5, +): + logger.info('Starting initialization using a piecewise linear approximation') pwl = shallow_clone(nlp) + logger.info('created a shallow clone of the model') # first introduce auxiliary variables so that we don't try to # approximate any functions of more than two variables trans = pe.TransformationFactory('contrib.piecewise.univariate_nonlinear_decomposition') trans.apply_to(pwl, aggressive_substitution=True) + logger.info('applied the univariate_nonlinear_decomposition transformation') # now we need to try to get bounds on all of the nonlinear variables bound_all_nonlinear_variables(pwl, default_bound=default_bound) + logger.info('bounded nonlinear variables') # Now, we need to fix variables with equal (or nearly equal) bounds. # Otherwise, the PWL transformation complains fix_vars_with_equal_bounds(pwl) + logger.info('fixed variables with equal bounds') # now we modify the model by introducing slacks to make sure the PWL # approximatin is feasible # all of the slacks appear linearly, so we don't need to worry about # upper bounds for them _minimize_infeasibility(pwl) + logger.info('reformulated model to minimize infeasibility') # build the PWL approximation trans = pe.TransformationFactory('contrib.piecewise.nonlinear_to_pwl') - trans.apply_to(pwl, num_points=num_points, additively_decompose=False) + trans.apply_to(pwl, num_points=2, additively_decompose=False) + logger.info('replaced nonlinear expressions with piecewise linear expressions') """ Now we want to @@ -84,5 +272,44 @@ def _initialize_with_piecewise_linear_approximation(nlp: BlockData, default_boun 4. If the NLP converges => done 5. If the NLP does not converge, refine the PWL approximation and repeat """ + pwl_expr_to_con_map = _get_pwl_constraints(pwl) + solved = False + for _iter in range(max_iter): + logger.info(f'PWL initialization: iter {_iter}') + + # PWL transformation (and map the variables) + orig_vars = list(get_vars(pwl)) + pwl.orig_vars = orig_vars + trans = pe.TransformationFactory('contrib.piecewise.disaggregated_logarithmic') + _pwl = trans.create_using(pwl) + new_vars = _pwl.orig_vars + del pwl.orig_vars + del _pwl.orig_vars + logger.info('applied the disaggregated logarithmic transformation') + + # solve the MILP + res = mip_solver.solve(_pwl, load_solutions=True) + logger.info(f'solved MILP: {res.solution_status}, {res.termination_condition}') + + #load the variable values back into orig_vars + for ov, nv in zip(orig_vars, new_vars): + ov.value = nv.value + + # refine the PWL approximation + _refine_pwl_approx( + pwl, + pwl_expr_to_con_map=pwl_expr_to_con_map, + num_to_refine=num_cons_to_refine_per_iter, + ) + logger.info('refined PWL approximation') + + # try solving the NLP + res = nlp_solver.solve(nlp, load_solutions=False, raise_exception_on_nonoptimal_result=False) + logger.info(f'solved NLP: {res.solution_status}, {res.termination_condition}') + if res.incumbent_objective is not None: + solved = True + res.solution_loader.load_vars() + # break - raise NotImplementedError('not done yet') + if not solved: + raise RuntimeError('no feasible solution found') diff --git a/pyomo/contrib/piecewise/transform/nonlinear_to_pwl.py b/pyomo/contrib/piecewise/transform/nonlinear_to_pwl.py index 65936eda109..e6239a1dcda 100644 --- a/pyomo/contrib/piecewise/transform/nonlinear_to_pwl.py +++ b/pyomo/contrib/piecewise/transform/nonlinear_to_pwl.py @@ -347,6 +347,17 @@ def _find_leaves(splits, leaves, input_node): return leaves_list +class _Evaluator: + def __init__(self, expr, expr_vars): + self.expr = expr + self.expr_vars = expr_vars + + def __call__(self, *args): + for i, v in enumerate(self.expr_vars): + v.value = args[i] + return value(self.expr) + + @TransformationFactory.register( 'contrib.piecewise.nonlinear_to_pwl', doc="Convert nonlinear constraints and objectives to piecewise-linear " @@ -742,10 +753,7 @@ def _approximate_expression( continue # else we approximate subexpr - def eval_expr(*args): - for i, v in enumerate(expr_vars): - v.value = args[i] - return value(subexpr) + eval_expr = _Evaluator(subexpr, expr_vars) pwlf = _get_pwl_function_approximation( eval_expr, config, self._get_bounds_list(expr_vars, obj) From e99362341fc69dec82359d12071e549f00a54d13 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 3 Feb 2026 20:58:24 -0700 Subject: [PATCH 061/107] debugging --- pyomo/contrib/initialization/initialize.py | 2 +- pyomo/contrib/initialization/pwl_init.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/initialization/initialize.py b/pyomo/contrib/initialization/initialize.py index 7b7e8af7338..6bc0c9e9ff6 100644 --- a/pyomo/contrib/initialization/initialize.py +++ b/pyomo/contrib/initialization/initialize.py @@ -28,7 +28,7 @@ def initialize_nlp( # create a shallow clone of the model so that the initialization method can # can work with the original variables but not make any other # modifications to the model - nlp = shallow_clone(nlp) + # nlp = shallow_clone(nlp) # run the initialization if method == InitializationMethod.pwl_approximation: diff --git a/pyomo/contrib/initialization/pwl_init.py b/pyomo/contrib/initialization/pwl_init.py index d6fdae92c25..b70b82a2838 100644 --- a/pyomo/contrib/initialization/pwl_init.py +++ b/pyomo/contrib/initialization/pwl_init.py @@ -179,6 +179,8 @@ def beforeChild(self, node, child, child_idx): fname = unique_component_name(self.m.auxiliary._pyomo_contrib_nonlinear_to_pwl, 'f') setattr(self.m.auxiliary._pyomo_contrib_nonlinear_to_pwl, fname, new_func) new_expr = new_func(*variables) + for v, val in zip(variables, var_values): + v.value = val self.named_expr_map[node] = new_expr self.substitution[child] = new_expr.expr return False, new_expr.expr @@ -196,6 +198,8 @@ def _refine_pwl_approx( for expr in pwl_expr_to_con_map.keys(): func = expr.pw_linear_function var_vals = tuple(i.value for i in expr.args) + # for v, val in zip(expr.args, var_vals): + # print(f'{str(v):<20}{val:<20.5f}{v.lb:<20.5f}{v.ub:<20.5f}{id(v):<20}') if any(i is None for i in var_vals): continue approx_value = func(*var_vals) @@ -304,7 +308,7 @@ def _initialize_with_piecewise_linear_approximation( logger.info('refined PWL approximation') # try solving the NLP - res = nlp_solver.solve(nlp, load_solutions=False, raise_exception_on_nonoptimal_result=False) + res = nlp_solver.solve(nlp, tee=True, load_solutions=False, raise_exception_on_nonoptimal_result=False) logger.info(f'solved NLP: {res.solution_status}, {res.termination_condition}') if res.incumbent_objective is not None: solved = True From aa450e1cedb4f69fbc0b5a9ddd8f8f84416e5138 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 3 Feb 2026 21:07:56 -0700 Subject: [PATCH 062/107] initialization --- pyomo/contrib/initialization/pwl_init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/initialization/pwl_init.py b/pyomo/contrib/initialization/pwl_init.py index b70b82a2838..65885f0ac5a 100644 --- a/pyomo/contrib/initialization/pwl_init.py +++ b/pyomo/contrib/initialization/pwl_init.py @@ -313,7 +313,7 @@ def _initialize_with_piecewise_linear_approximation( if res.incumbent_objective is not None: solved = True res.solution_loader.load_vars() - # break + break if not solved: raise RuntimeError('no feasible solution found') From ee96e0754662e90f66072285f0d216e25d39c59a Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 5 Mar 2026 21:25:03 -0700 Subject: [PATCH 063/107] NFC: apply black --- ...test_univariate_nonlinear_decomposition.py | 165 ++++-------------- .../contrib/piecewise/transform/factorable.py | 56 +++--- 2 files changed, 67 insertions(+), 154 deletions(-) diff --git a/pyomo/contrib/piecewise/tests/test_univariate_nonlinear_decomposition.py b/pyomo/contrib/piecewise/tests/test_univariate_nonlinear_decomposition.py index aa027570313..fdeb5ae6f05 100644 --- a/pyomo/contrib/piecewise/tests/test_univariate_nonlinear_decomposition.py +++ b/pyomo/contrib/piecewise/tests/test_univariate_nonlinear_decomposition.py @@ -5,12 +5,13 @@ from pyomo.common.dependencies import numpy_available, numpy from pyomo.core.expr.numeric_expr import ProductExpression - pe = pyo def _get_trans(): - return pyo.TransformationFactory('contrib.piecewise.univariate_nonlinear_decomposition') + return pyo.TransformationFactory( + 'contrib.piecewise.univariate_nonlinear_decomposition' + ) class TestUnivariateNonlinearDecomposition(TestCase): @@ -19,32 +20,16 @@ def test_multiterm(self): m.x = pe.Var() m.y = pe.Var() m.z = pe.Var() - m.c = pe.Constraint(expr=m.x + pe.log(m.y + m.z) + 1/pe.exp(m.x**0.5) <= 0) + m.c = pe.Constraint(expr=m.x + pe.log(m.y + m.z) + 1 / pe.exp(m.x**0.5) <= 0) trans = _get_trans() trans.apply_to(m) aux = m.auxiliary - assertExpressionsEqual( - self, - m.c.body, - m.x + aux.x[3] + aux.x[2], - ) - assertExpressionsEqual( - self, - aux.c[1].expr, - aux.x[1] == (m.y + m.z), - ) - assertExpressionsEqual( - self, - aux.c[2].expr, - aux.x[2] == 1/pyo.exp(m.x**0.5), - ) - assertExpressionsEqual( - self, - aux.c[3].expr, - aux.x[3] == pyo.log(aux.x[1]), - ) + assertExpressionsEqual(self, m.c.body, m.x + aux.x[3] + aux.x[2]) + assertExpressionsEqual(self, aux.c[1].expr, aux.x[1] == (m.y + m.z)) + assertExpressionsEqual(self, aux.c[2].expr, aux.x[2] == 1 / pyo.exp(m.x**0.5)) + assertExpressionsEqual(self, aux.c[3].expr, aux.x[3] == pyo.log(aux.x[1])) self.assertEqual(m.x.lb, 0) self.assertIsNone(m.x.ub) self.assertIsNone(m.y.lb) @@ -72,70 +57,38 @@ def test_common_subexpressions(self): trans.apply_to(m) aux = m.auxiliary - assertExpressionsEqual( - self, - m.c1.expr, - m.z1 + aux.x[2] == 0, - ) - assertExpressionsEqual( - self, - m.c2.expr, - m.z2 + aux.x[2] == 0, - ) - assertExpressionsEqual( - self, - aux.c[1].expr, - aux.x[1] == m.x + m.y, - ) - assertExpressionsEqual( - self, - aux.c[2].expr, - aux.x[2] == -pe.log(aux.x[1]), - ) + assertExpressionsEqual(self, m.c1.expr, m.z1 + aux.x[2] == 0) + assertExpressionsEqual(self, m.c2.expr, m.z2 + aux.x[2] == 0) + assertExpressionsEqual(self, aux.c[1].expr, aux.x[1] == m.x + m.y) + assertExpressionsEqual(self, aux.c[2].expr, aux.x[2] == -pe.log(aux.x[1])) def test_product_fixed_variable(self): m = pe.ConcreteModel() m.x = pe.Var() m.y = pe.Var() m.z = pe.Var() - m.c = pe.Constraint(expr=2*pe.log(m.x + m.y) <= 0) + m.c = pe.Constraint(expr=2 * pe.log(m.x + m.y) <= 0) trans = _get_trans() trans.apply_to(m) aux = m.auxiliary - assertExpressionsEqual( - self, - m.c.expr, - 2*pe.log(aux.x[1]) <= 0, - ) - assertExpressionsEqual( - self, - aux.c[1].expr, - aux.x[1] == m.x + m.y, - ) + assertExpressionsEqual(self, m.c.expr, 2 * pe.log(aux.x[1]) <= 0) + assertExpressionsEqual(self, aux.c[1].expr, aux.x[1] == m.x + m.y) def test_product_variable_fixed(self): m = pe.ConcreteModel() m.x = pe.Var() m.y = pe.Var() m.z = pe.Var() - m.c = pe.Constraint(expr=pe.log(m.x + m.y)*2 <= 0) + m.c = pe.Constraint(expr=pe.log(m.x + m.y) * 2 <= 0) trans = _get_trans() trans.apply_to(m) aux = m.auxiliary - assertExpressionsEqual( - self, - m.c.expr, - pe.log(aux.x[1])*2 <= 0, - ) - assertExpressionsEqual( - self, - aux.c[1].expr, - aux.x[1] == m.x + m.y, - ) + assertExpressionsEqual(self, m.c.expr, pe.log(aux.x[1]) * 2 <= 0) + assertExpressionsEqual(self, aux.c[1].expr, aux.x[1] == m.x + m.y) def test_prod_sum_sum(self): m = pe.ConcreteModel() @@ -149,21 +102,9 @@ def test_prod_sum_sum(self): trans.apply_to(m) aux = m.auxiliary - assertExpressionsEqual( - self, - m.c.expr, - aux.x[1] * aux.x[2] <= 1, - ) - assertExpressionsEqual( - self, - aux.c[1].expr, - aux.x[1] == m.x1 + m.x2, - ) - assertExpressionsEqual( - self, - aux.c[2].expr, - aux.x[2] == m.x3 + m.x4, - ) + assertExpressionsEqual(self, m.c.expr, aux.x[1] * aux.x[2] <= 1) + assertExpressionsEqual(self, aux.c[1].expr, aux.x[1] == m.x1 + m.x2) + assertExpressionsEqual(self, aux.c[2].expr, aux.x[2] == m.x3 + m.x4) def test_pow_sum_sum(self): m = pe.ConcreteModel() @@ -177,21 +118,9 @@ def test_pow_sum_sum(self): trans.apply_to(m) aux = m.auxiliary - assertExpressionsEqual( - self, - m.c.expr, - aux.x[1] ** aux.x[2] <= 1, - ) - assertExpressionsEqual( - self, - aux.c[1].expr, - aux.x[1] == m.x1 + m.x2, - ) - assertExpressionsEqual( - self, - aux.c[2].expr, - aux.x[2] == m.x3 + m.x4, - ) + assertExpressionsEqual(self, m.c.expr, aux.x[1] ** aux.x[2] <= 1) + assertExpressionsEqual(self, aux.c[1].expr, aux.x[1] == m.x1 + m.x2) + assertExpressionsEqual(self, aux.c[2].expr, aux.x[2] == m.x3 + m.x4) def test_division_var_const(self): m = pe.ConcreteModel() @@ -203,11 +132,7 @@ def test_division_var_const(self): trans.apply_to(m) aux = m.auxiliary - assertExpressionsEqual( - self, - m.c.expr, - (m.x + m.y) / 2 <= 0, - ) + assertExpressionsEqual(self, m.c.expr, (m.x + m.y) / 2 <= 0) def test_division_sum_sum(self): m = pe.ConcreteModel() @@ -221,26 +146,10 @@ def test_division_sum_sum(self): trans.apply_to(m) aux = m.auxiliary - assertExpressionsEqual( - self, - m.c.expr, - aux.x[1] * aux.x[3] <= 1, - ) - assertExpressionsEqual( - self, - aux.c[1].expr, - aux.x[1] == m.x1 + m.x2, - ) - assertExpressionsEqual( - self, - aux.c[2].expr, - aux.x[2] == m.x3 + m.x4, - ) - assertExpressionsEqual( - self, - aux.c[3].expr, - aux.x[3] * aux.x[2] == 1, - ) + assertExpressionsEqual(self, m.c.expr, aux.x[1] * aux.x[3] <= 1) + assertExpressionsEqual(self, aux.c[1].expr, aux.x[1] == m.x1 + m.x2) + assertExpressionsEqual(self, aux.c[2].expr, aux.x[2] == m.x3 + m.x4) + assertExpressionsEqual(self, aux.c[3].expr, aux.x[3] * aux.x[2] == 1) @skipUnless(numpy_available, "Numpy is not available") def test_numpy_float(self): @@ -248,19 +157,13 @@ def test_numpy_float(self): m.x = pe.Var() m.y = pe.Var() m.z = pe.Var() - m.c = pe.Constraint(expr=ProductExpression((numpy.float64(2.5), pe.log(m.x + m.y))) <= 0) + m.c = pe.Constraint( + expr=ProductExpression((numpy.float64(2.5), pe.log(m.x + m.y))) <= 0 + ) trans = _get_trans() trans.apply_to(m) aux = m.auxiliary - assertExpressionsEqual( - self, - m.c.expr, - 2.5*pe.log(aux.x[1]) <= 0, - ) - assertExpressionsEqual( - self, - aux.c[1].expr, - aux.x[1] == m.x + m.y, - ) + assertExpressionsEqual(self, m.c.expr, 2.5 * pe.log(aux.x[1]) <= 0) + assertExpressionsEqual(self, aux.c[1].expr, aux.x[1] == m.x + m.y) diff --git a/pyomo/contrib/piecewise/transform/factorable.py b/pyomo/contrib/piecewise/transform/factorable.py index 3e6af881735..5704b062a32 100644 --- a/pyomo/contrib/piecewise/transform/factorable.py +++ b/pyomo/contrib/piecewise/transform/factorable.py @@ -14,11 +14,11 @@ from pyomo.repn.util import ExitNodeDispatcher from pyomo.core.base import ( - VarData, - ParamData, - ExpressionData, - VarList, - ConstraintList, + VarData, + ParamData, + ExpressionData, + VarList, + ConstraintList, Block, Constraint, Objective, @@ -68,7 +68,7 @@ def _handle_product(node, data, visitor): visitor.degree_map[res] = arg2_degree visitor.substitution_map[node] = res return res - + if arg2_degree == 0: res = arg1 * arg2 visitor.node_to_var_map[res] = arg1_vars @@ -87,7 +87,7 @@ def _handle_product(node, data, visitor): arg2_nvars = 1 arg2_degree = 1 res = arg1 * arg2 - # at this point arg1 should have at most 1 variable + # at this point arg1 should have at most 1 variable # and arg2 should have at most 1 variable if arg1_nvars == 0: visitor.node_to_var_map[res] = arg2_vars @@ -134,8 +134,8 @@ def _handle_sum(node, data, visitor): def _handle_division(node, data, visitor): """ - This one is a bit tricky. If we encounter both x/z and y/z - at different places in the model, we only want one auxilliary + This one is a bit tricky. If we encounter both x/z and y/z + at different places in the model, we only want one auxilliary variable for 1/z. """ arg1, arg2 = data @@ -183,8 +183,10 @@ def _handle_division(node, data, visitor): aux * arg2 = 1 """ arg2_lb, arg2_ub = compute_bounds_on_expr(arg2) - if (arg2_lb is not None and arg2_lb > 0) or (arg2_ub is not None and arg2_ub < 0): - c = visitor.block.c.add(aux == 1/arg2) # keep it univariate if we can + if (arg2_lb is not None and arg2_lb > 0) or ( + arg2_ub is not None and arg2_ub < 0 + ): + c = visitor.block.c.add(aux == 1 / arg2) # keep it univariate if we can else: c = visitor.block.c.add(aux * arg2 == 1) fbbt(c) @@ -194,7 +196,7 @@ def _handle_division(node, data, visitor): arg2_nvars = 1 arg2_degree = 1 res = arg1 * arg2 - # at this point arg1 should have at most 1 variable + # at this point arg1 should have at most 1 variable # and arg2 should have exactly 1 variable if arg1_nvars == 0: visitor.node_to_var_map[res] = arg2_vars @@ -237,7 +239,7 @@ def _handle_pow(node, data, visitor): arg2_degree = 1 res = arg1**arg2 - # at this point arg1 should have at most 1 variable + # at this point arg1 should have at most 1 variable # and arg2 should have at most 1 variable if arg1_nvars == 0: visitor.node_to_var_map[res] = arg2_vars @@ -319,7 +321,9 @@ def __init__(self, **kwds): self.block = kwds.pop('aux_block') super().__init__(**kwds) self.node_to_var_map = ComponentMap() - self.degree_map = ComponentMap() # values will be 0 (constant), 1 (linear), or -1 (nonlinear) + self.degree_map = ( + ComponentMap() + ) # values will be 0 (constant), 1 (linear), or -1 (nonlinear) self.substitution_map = ComponentMap() @@ -330,12 +334,12 @@ def initializeWalker(self, expr): if expr in self.substitution_map: return False, self.substitution_map[expr] return True, None - + def beforeChild(self, node, child, child_idx): if child in self.substitution_map: return False, self.substitution_map[child] return True, None - + def exitNode(self, node, data): nt = type(node) if nt in handlers: @@ -353,7 +357,7 @@ def create_aux_var(self, expr): x = self.block.x.add() self.substitution_map[expr] = x c = self.block.c.add(x == expr) - # we need to compute bounds on x now because some of the + # we need to compute bounds on x now because some of the # handlers depend on variable bounds (e.g., division) fbbt(c) return x @@ -374,7 +378,7 @@ def create_aux_var(self, expr): By doing so, each nonlinear function is only a function of one or two variables. If this transformation is used prior to the nonlinear_to_pwl transformation, it can significantly reduce the complexity of the PWL approximation. - """ + """, ) class UnivariateNonlinearDecompositionTransformation(Transformation): def __init__(self): @@ -396,13 +400,19 @@ def _check_for_unknown_active_components(self, model): def _apply_to(self, model, **kwds): if kwds: - raise ValueError('UnivariateNonlinearDecompositionTransformation does not take any keyword arguments') - + raise ValueError( + 'UnivariateNonlinearDecompositionTransformation does not take any keyword arguments' + ) + self._check_for_unknown_active_components(model) - objectives = list(model.component_data_objects(Objective, active=True, descend_into=True)) - constraints = list(model.component_data_objects(Constraint, active=True, descend_into=True)) - + objectives = list( + model.component_data_objects(Objective, active=True, descend_into=True) + ) + constraints = list( + model.component_data_objects(Constraint, active=True, descend_into=True) + ) + bname = unique_component_name(model, 'auxiliary') setattr(model, bname, Block()) block = getattr(model, bname) From ce55a10c23979074db67bae060015b9bdc97b708 Mon Sep 17 00:00:00 2001 From: John Siirola Date: Thu, 5 Mar 2026 22:48:49 -0700 Subject: [PATCH 064/107] NFC: fix typo --- pyomo/contrib/piecewise/transform/factorable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/piecewise/transform/factorable.py b/pyomo/contrib/piecewise/transform/factorable.py index 5704b062a32..8e2661f42a0 100644 --- a/pyomo/contrib/piecewise/transform/factorable.py +++ b/pyomo/contrib/piecewise/transform/factorable.py @@ -135,7 +135,7 @@ def _handle_sum(node, data, visitor): def _handle_division(node, data, visitor): """ This one is a bit tricky. If we encounter both x/z and y/z - at different places in the model, we only want one auxilliary + at different places in the model, we only want one auxiliary variable for 1/z. """ arg1, arg2 = data From aec0e65de1414ca33e4b8c71372d6c30198248c1 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 23 Mar 2026 09:12:25 -0600 Subject: [PATCH 065/107] merge trivial_constraints into scip_port --- pyomo/contrib/solver/common/solution_loader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index 776aa8f258d..d0f14d403fa 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -35,12 +35,14 @@ def load_import_suffixes( elif suffix.local_name == 'rc': rc_suffix = suffix if dual_suffix is not None: + dual_suffix.clear() duals = solution_loader.get_duals(solution_id=solution_id) if duals is NotImplemented: logger.warning(f'Cannot load duals into suffix') else: dual_suffix.update(duals) if rc_suffix is not None: + rc_suffix.clear() rc = solution_loader.get_reduced_costs(solution_id=solution_id) if rc is NotImplemented: logger.warning(f'cannot load duals into suffix') From 043f8c8a6da16541116fb97e2ca21b586edece4b Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 23 Mar 2026 11:25:11 -0600 Subject: [PATCH 066/107] some updates to initialization module --- .../initialization/bounds/bound_variables.py | 4 +- pyomo/contrib/initialization/initialize.py | 12 +- .../contrib/initialization/lp_approx_init.py | 182 ++++++++++++++++++ pyomo/contrib/initialization/pwl_init.py | 27 ++- 4 files changed, 217 insertions(+), 8 deletions(-) create mode 100644 pyomo/contrib/initialization/lp_approx_init.py diff --git a/pyomo/contrib/initialization/bounds/bound_variables.py b/pyomo/contrib/initialization/bounds/bound_variables.py index 0c243b8eb33..0d0232f53b8 100644 --- a/pyomo/contrib/initialization/bounds/bound_variables.py +++ b/pyomo/contrib/initialization/bounds/bound_variables.py @@ -16,9 +16,9 @@ def bound_all_nonlinear_variables(m: BlockData, default_bound: float = 1.0e8): fbbt(m) for v in get_vars(m): if v.lb is None or v.lb < -default_bound: - logger.warning(f'Could not obtain a lower bound for {str(v)} better than {-default_bound}; setting the lower bound to {-default_bound}') + logger.debug(f'Could not obtain a lower bound for {str(v)} better than {-default_bound}; setting the lower bound to {-default_bound}') v.setlb(-default_bound) if v.ub is None or v.ub > default_bound: - logger.warning(f'Could not obtain an upper bound for {str(v)} better than {default_bound}; setting the upper bound to {default_bound}') + logger.debug(f'Could not obtain an upper bound for {str(v)} better than {default_bound}; setting the upper bound to {default_bound}') v.setub(default_bound) fbbt(m) diff --git a/pyomo/contrib/initialization/initialize.py b/pyomo/contrib/initialization/initialize.py index 6bc0c9e9ff6..baf5accaec6 100644 --- a/pyomo/contrib/initialization/initialize.py +++ b/pyomo/contrib/initialization/initialize.py @@ -3,11 +3,13 @@ from pyomo.contrib.initialization.utils import get_vars, shallow_clone from pyomo.common.collections import ComponentMap from pyomo.contrib.initialization.pwl_init import _initialize_with_piecewise_linear_approximation +from pyomo.contrib.initialization.lp_approx_init import _initialize_with_LP_approximation from pyomo.contrib.solver.common.base import SolverBase class InitializationMethod(Enum): pwl_approximation = "pwl_approximation" + lp_approximation = "lp_approximation" def initialize_nlp( @@ -32,7 +34,7 @@ def initialize_nlp( # run the initialization if method == InitializationMethod.pwl_approximation: - _initialize_with_piecewise_linear_approximation( + res = _initialize_with_piecewise_linear_approximation( nlp=nlp, mip_solver=mip_solver, nlp_solver=nlp_solver, @@ -40,6 +42,12 @@ def initialize_nlp( max_iter=max_pwl_refinement_iter, num_cons_to_refine_per_iter=num_pwl_cons_to_refine_per_iter, ) + elif method == InitializationMethod.lp_approximation: + res = _initialize_with_LP_approximation( + nlp=nlp, + lp_solver=mip_solver, + nlp_solver=nlp_solver, + ) else: raise ValueError(f'unexpected initialization method: {method}') @@ -53,3 +61,5 @@ def initialize_nlp( assert v.fixed else: v.unfix() + + return res diff --git a/pyomo/contrib/initialization/lp_approx_init.py b/pyomo/contrib/initialization/lp_approx_init.py new file mode 100644 index 00000000000..63d96011987 --- /dev/null +++ b/pyomo/contrib/initialization/lp_approx_init.py @@ -0,0 +1,182 @@ +from pyomo.core.base.block import BlockData +import pyomo.environ as pe +from pyomo.contrib.initialization.bounds.bound_variables import bound_all_nonlinear_variables +from pyomo.contrib.initialization.utils import fix_vars_with_equal_bounds, shallow_clone, get_vars +from pyomo.core.expr.visitor import identify_components +from pyomo.contrib.piecewise.piecewise_linear_expression import PiecewiseLinearExpression +from pyomo.contrib.piecewise.piecewise_linear_function import PiecewiseLinearFunction +from pyomo.common.collections import ComponentMap, ComponentSet +from typing import MutableMapping, Sequence, List +from pyomo.core.base.constraint import ConstraintData +from pyomo.core.expr.visitor import StreamBasedExpressionVisitor +from pyomo.common.numeric_types import native_numeric_types +from pyomo.core.expr.numvalue import NumericConstant +from pyomo.core.expr.numeric_expr import ( + NegationExpression, + PowExpression, + ProductExpression, + MonomialTermExpression, + DivisionExpression, + SumExpression, + LinearExpression, + UnaryFunctionExpression, + NPV_NegationExpression, + NPV_PowExpression, + NPV_ProductExpression, + NPV_DivisionExpression, + NPV_SumExpression, + NPV_UnaryFunctionExpression, +) +from pyomo.core.expr.relational_expr import EqualityExpression, InequalityExpression, RangedExpression +from pyomo.repn.util import ExitNodeDispatcher +from pyomo.core.base.var import ScalarVar, VarData +from pyomo.core.base.param import ScalarParam, ParamData +from pyomo.core.base.expression import ScalarExpression, ExpressionData +import math +from pyomo.contrib.solver.common.base import SolverBase +import logging +from pyomo.common.modeling import unique_component_name +from pyomo.contrib.initialization.pwl_init import _minimize_infeasibility +from pyomo.contrib.fbbt.fbbt import fbbt +from pyomo.repn.linear import LinearRepnVisitor, LinearRepn +from pyomo.core.expr.visitor import identify_variables +import numpy as np +from scipy.stats import qmc + + +logger = logging.getLogger(__name__) + + +def _replace_expression_with_linear_approx(expr, num_samples=100): + vset = ComponentSet(identify_variables(expr, include_fixed=False)) + vlist = list(vset) + n_vars = len(vlist) + bnds_list = [] + for v in vlist: + if v.lb is None: + lb = -1e6 + else: + lb = v.lb + if v.ub is None: + ub = 1e6 + else: + ub = v.ub + bnds_list.append((lb, ub)) + sampler = qmc.LatinHypercube(d=n_vars) + sample = sampler.random(n=num_samples) + l_bounds = [i[0] for i in bnds_list] + u_bounds = [i[1] for i in bnds_list] + sample = qmc.scale(sample, l_bounds, u_bounds) + + # we have our samples + # now we want to build the matrix and the right hand side + # we have a linear coefficient for each variable plus a constant + n_coefs = n_vars + 1 + A = np.zeros((num_samples, n_coefs), dtype=float) + b = np.zeros(num_samples, dtype=float) + A[:, :n_vars] = sample + A[:, n_vars] = 1 + for sample_ndx in range(num_samples): + for v, val in zip(vlist, sample[sample_ndx, :]): + v.value = float(val) + b[sample_ndx] = pe.value(expr) + coefs = np.linalg.solve(A.transpose().dot(A), A.transpose().dot(b)) + coefs = [float(i) for i in coefs] + + new_expr = 0 + for c, v in zip(coefs[:n_vars], vlist): + new_expr += c * v + new_expr += coefs[-1] + return new_expr + + +def _build_lp_approx(nlp: BlockData) -> BlockData: + lp = pe.Block(concrete=True) + lp.cons = pe.ConstraintList() + visitor = LinearRepnVisitor(subexpression_cache={}) + + objs = list(nlp.component_data_objects(pe.Objective, active=True, descend_into=True)) + if objs: + if len(objs) > 1: + raise NotImplementedError('lp approximation does not support multiple objectives') + obj = objs[0] + repn = visitor.walk_expression(obj) + assert repn.multiplier == 1 + linear_part = LinearRepn() + linear_part.multiplier = 1 + linear_part.constant = repn.constant + linear_part.linear = repn.linear + linear_part.nonlinear = None + new_obj_expr = linear_part.to_expression(visitor=visitor) + if repn.nonlinear is not None: + replacement = _replace_expression_with_linear_approx(repn.nonlinear) + new_obj_expr += replacement + lp.obj = pe.Objective(expr=new_obj_expr, sense=obj.sense) + + for con in nlp.component_data_objects(pe.Constraint, active=True, descend_into=True): + lb, body, ub = con.to_bounded_expression() + repn = visitor.walk_expression(body) + assert repn.multiplier == 1 + linear_part = LinearRepn() + linear_part.multiplier = 1 + linear_part.constant = repn.constant + linear_part.linear = repn.linear + linear_part.nonlinear = None + new_body = linear_part.to_expression(visitor=visitor) + if repn.nonlinear is not None: + replacement = _replace_expression_with_linear_approx(repn.nonlinear) + new_body += replacement + if lb == ub: + lp.cons.add(new_body == lb) + else: + lp.cons.add((lb, new_body, ub)) + return lp + + +def _initialize_with_LP_approximation( + nlp: BlockData, + lp_solver: SolverBase, + nlp_solver: SolverBase, +): + orig_nlp = nlp + logger.info('Starting initialization using a linear programming approximation') + nlp = shallow_clone(nlp) + logger.info('created a shallow clone of the model') + + # first introduce auxiliary variables so that we don't try to + # approximate any functions of more than two variables + trans = pe.TransformationFactory('contrib.piecewise.univariate_nonlinear_decomposition') + trans.apply_to(nlp, aggressive_substitution=False) + logger.info('applied the univariate_nonlinear_decomposition transformation') + + # let's see if we can get bounds on the nonlinear variables + fbbt(nlp) + logger.info('ran FBBT') + + # Now, we need to fix variables with equal (or nearly equal) bounds. + # Otherwise, the PWL transformation complains + fix_vars_with_equal_bounds(nlp) + logger.info('fixed variables with equal bounds') + + # now we modify the model by introducing slacks to make sure the LP + # approximatin is feasible + _minimize_infeasibility(nlp) + logger.info('reformulated model to minimize infeasibility') + + # build the LP approximation + lp = _build_lp_approx(nlp) + logger.info('replaced nonlinear expressions with linear approximations') + + # solve the LP + lp_res = lp_solver.solve(lp, load_solutions=True, raise_exception_on_nonoptimal_result=False) + logger.info(f'solved LP: {lp_res.solution_status}, {lp_res.termination_condition}') + + # try solving the NLP + nlp_res = nlp_solver.solve(orig_nlp, load_solutions=False, raise_exception_on_nonoptimal_result=False) + logger.info(f'solved NLP: {nlp_res.solution_status}, {nlp_res.termination_condition}') + if nlp_res.incumbent_objective is not None: + nlp_res.solution_loader.load_vars() + else: + raise RuntimeError('no feasible solution found') + + return nlp_res diff --git a/pyomo/contrib/initialization/pwl_init.py b/pyomo/contrib/initialization/pwl_init.py index 65885f0ac5a..30baa1d7cef 100644 --- a/pyomo/contrib/initialization/pwl_init.py +++ b/pyomo/contrib/initialization/pwl_init.py @@ -180,7 +180,7 @@ def beforeChild(self, node, child, child_idx): setattr(self.m.auxiliary._pyomo_contrib_nonlinear_to_pwl, fname, new_func) new_expr = new_func(*variables) for v, val in zip(variables, var_values): - v.value = val + v.set_value(val, skip_validation=True) self.named_expr_map[node] = new_expr self.substitution[child] = new_expr.expr return False, new_expr.expr @@ -197,7 +197,20 @@ def _refine_pwl_approx( violations = [] for expr in pwl_expr_to_con_map.keys(): func = expr.pw_linear_function - var_vals = tuple(i.value for i in expr.args) + var_vals = [] + for v in expr.args: + if math.isclose(v.lb, v.ub, rel_tol=1e-6, abs_tol=1e-6): + val = 0.5 * (v.lb + v.ub) + elif v.value is None: + val = None + else: + val = v.value + if val <= v.lb + 1e-6 + 1e-6 * abs(v.lb): + val += 1e-5 + if val >= v.ub - 1e-6 - 1e-6 * abs(v.ub): + val -= 1e-5 + var_vals.append(val) + # var_vals = tuple(i.value for i in expr.args) # for v, val in zip(expr.args, var_vals): # print(f'{str(v):<20}{val:<20.5f}{v.lb:<20.5f}{v.ub:<20.5f}{id(v):<20}') if any(i is None for i in var_vals): @@ -278,6 +291,7 @@ def _initialize_with_piecewise_linear_approximation( """ pwl_expr_to_con_map = _get_pwl_constraints(pwl) solved = False + last_nlp_res = None for _iter in range(max_iter): logger.info(f'PWL initialization: iter {_iter}') @@ -292,12 +306,12 @@ def _initialize_with_piecewise_linear_approximation( logger.info('applied the disaggregated logarithmic transformation') # solve the MILP - res = mip_solver.solve(_pwl, load_solutions=True) + res = mip_solver.solve(_pwl, load_solutions=True, raise_exception_on_nonoptimal_result=False) logger.info(f'solved MILP: {res.solution_status}, {res.termination_condition}') #load the variable values back into orig_vars for ov, nv in zip(orig_vars, new_vars): - ov.value = nv.value + ov.set_value(nv.value, skip_validation=True) # refine the PWL approximation _refine_pwl_approx( @@ -308,7 +322,8 @@ def _initialize_with_piecewise_linear_approximation( logger.info('refined PWL approximation') # try solving the NLP - res = nlp_solver.solve(nlp, tee=True, load_solutions=False, raise_exception_on_nonoptimal_result=False) + res = nlp_solver.solve(nlp, load_solutions=False, raise_exception_on_nonoptimal_result=False) + last_nlp_res = res logger.info(f'solved NLP: {res.solution_status}, {res.termination_condition}') if res.incumbent_objective is not None: solved = True @@ -317,3 +332,5 @@ def _initialize_with_piecewise_linear_approximation( if not solved: raise RuntimeError('no feasible solution found') + + return last_nlp_res From 1ae089076ec33a49a6c04aef3c6e59cd092d98ad Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 23 Mar 2026 11:39:41 -0600 Subject: [PATCH 067/107] bug --- pyomo/contrib/solver/solvers/scip/scip_direct.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index b5cb1a6946a..022878f918d 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -140,7 +140,7 @@ def _handle_param(node, data, opt, visitor): return node.value if not opt.is_persistent(): return node.value - if not node.mutable: + if node.is_constant(): return node.value if id(node) not in opt._pyomo_param_to_solver_param_map: scip_param = opt._add_param(node) From 8eabf28c9a3a873e3e49ba953bcb790aadaf00fb Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 23 Mar 2026 11:51:49 -0600 Subject: [PATCH 068/107] run black --- pyomo/contrib/piecewise/transform/factorable.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pyomo/contrib/piecewise/transform/factorable.py b/pyomo/contrib/piecewise/transform/factorable.py index f3061a7146e..f9829732e6b 100644 --- a/pyomo/contrib/piecewise/transform/factorable.py +++ b/pyomo/contrib/piecewise/transform/factorable.py @@ -370,7 +370,7 @@ def exitNode(self, node, data): return _handle_float(node, data, self) else: raise NotImplementedError(f'unrecognized expression type: {nt}') - + def _is_leaf(self, x): if type(x) in self._leaf_types or type(x) in native_numeric_types: return True @@ -432,8 +432,8 @@ class UnivariateNonlinearDecompositionTransformation(Transformation): even though this does not reduce the dimensionality of the PWL approximation. The reason to do this is to be able to bound z1 and z2. - """ - ) + """, + ), ) def __init__(self): @@ -455,8 +455,10 @@ def _check_for_unknown_active_components(self, model): ) def _apply_to(self, model, **kwds): - self._config = UnivariateNonlinearDecompositionTransformation.CONFIG(value=kwds, preserve_implicit=True) - + self._config = UnivariateNonlinearDecompositionTransformation.CONFIG( + value=kwds, preserve_implicit=True + ) + self._check_for_unknown_active_components(model) objectives = list( @@ -470,7 +472,7 @@ def _apply_to(self, model, **kwds): setattr(model, bname, Block()) block = getattr(model, bname) visitor = _UnivariateNonlinearDecompositionVisitor( - aux_block=block, + aux_block=block, aggressive_substitution=self._config.aggressive_substitution, ) From c8a07305a16cba2abeec0333d89f43b9c9269ade Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 23 Mar 2026 15:45:32 -0600 Subject: [PATCH 069/107] initialize with global solvers --- pyomo/contrib/initialization/global_init.py | 28 +++++++++++++++++++++ pyomo/contrib/initialization/initialize.py | 15 +++++++++-- 2 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 pyomo/contrib/initialization/global_init.py diff --git a/pyomo/contrib/initialization/global_init.py b/pyomo/contrib/initialization/global_init.py new file mode 100644 index 00000000000..e982570a2c9 --- /dev/null +++ b/pyomo/contrib/initialization/global_init.py @@ -0,0 +1,28 @@ +from pyomo.core.base.block import BlockData +from pyomo.contrib.solver.common.base import SolverBase +from pyomo.contrib.solver.solvers.scip.scip_direct import ScipDirect, ScipPersistent +from pyomo.contrib.solver.solvers.gurobi.gurobi_direct_minlp import GurobiDirectMINLP +import logging + + +logger = logging.getLogger(__name__) + + +def _initialize_with_global_solver( + nlp: BlockData, + global_solver: SolverBase, +): + if isinstance(global_solver, (ScipDirect, ScipPersistent)): + opts = {'limits/solutions': 1} + elif isinstance(global_solver, (GurobiDirectMINLP,)): + opts = {'SolutionLimit': 1} + else: + raise NotImplementedError('Currently, the initialization module only works with new solver interface, so the global solvers are limited to ScipDirect, ScipPersistent, and GurobiDirectMINLP.') + res = global_solver.solve(nlp, load_solutions=False, raise_exception_on_nonoptimal_result=False, solver_options=opts) + logger.info(f'solved NLP: {res.solution_status}, {res.termination_condition}') + if res.incumbent_objective is not None: + res.solution_loader.load_vars() + else: + raise RuntimeError('no feasible solution found') + + return res diff --git a/pyomo/contrib/initialization/initialize.py b/pyomo/contrib/initialization/initialize.py index baf5accaec6..b7e04d784d1 100644 --- a/pyomo/contrib/initialization/initialize.py +++ b/pyomo/contrib/initialization/initialize.py @@ -1,3 +1,4 @@ +from typing import Optional from pyomo.core.base.block import BlockData from enum import Enum from pyomo.contrib.initialization.utils import get_vars, shallow_clone @@ -5,17 +6,20 @@ from pyomo.contrib.initialization.pwl_init import _initialize_with_piecewise_linear_approximation from pyomo.contrib.initialization.lp_approx_init import _initialize_with_LP_approximation from pyomo.contrib.solver.common.base import SolverBase +from pyomo.contrib.initialization.global_init import _initialize_with_global_solver class InitializationMethod(Enum): pwl_approximation = "pwl_approximation" lp_approximation = "lp_approximation" + global_opt = "global_opt" def initialize_nlp( nlp: BlockData, - mip_solver: SolverBase, - nlp_solver: SolverBase, + mip_solver: Optional[SolverBase] = None, + nlp_solver: Optional[SolverBase] = None, + global_solver: Optional[SolverBase] = None, method: InitializationMethod = InitializationMethod.pwl_approximation, default_bound=1.0e8, max_pwl_refinement_iter=100, @@ -34,6 +38,8 @@ def initialize_nlp( # run the initialization if method == InitializationMethod.pwl_approximation: + assert mip_solver is not None, f"mip_solver must be specified for {method}" + assert nlp_solver is not None, f"nlp_solver must be specified for {method}" res = _initialize_with_piecewise_linear_approximation( nlp=nlp, mip_solver=mip_solver, @@ -43,11 +49,16 @@ def initialize_nlp( num_cons_to_refine_per_iter=num_pwl_cons_to_refine_per_iter, ) elif method == InitializationMethod.lp_approximation: + assert mip_solver is not None, f"mip_solver must be specified for {method}" + assert nlp_solver is not None, f"nlp_solver must be specified for {method}" res = _initialize_with_LP_approximation( nlp=nlp, lp_solver=mip_solver, nlp_solver=nlp_solver, ) + elif method == InitializationMethod.global_opt: + assert global_solver is not None, f"global_solver must be specified for {method}" + res = _initialize_with_global_solver(nlp=nlp, global_solver=global_solver) else: raise ValueError(f'unexpected initialization method: {method}') From e019bffc8f40fc2583623d3ff3cbd6a02b0b9233 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 24 Mar 2026 07:15:46 -0600 Subject: [PATCH 070/107] initialization work --- pyomo/contrib/initialization/global_init.py | 7 ++-- pyomo/contrib/initialization/initialize.py | 40 ++++++++++++++------- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/pyomo/contrib/initialization/global_init.py b/pyomo/contrib/initialization/global_init.py index e982570a2c9..4f858a4b7c3 100644 --- a/pyomo/contrib/initialization/global_init.py +++ b/pyomo/contrib/initialization/global_init.py @@ -11,6 +11,7 @@ def _initialize_with_global_solver( nlp: BlockData, global_solver: SolverBase, + nlp_solver: SolverBase, ): if isinstance(global_solver, (ScipDirect, ScipPersistent)): opts = {'limits/solutions': 1} @@ -18,8 +19,10 @@ def _initialize_with_global_solver( opts = {'SolutionLimit': 1} else: raise NotImplementedError('Currently, the initialization module only works with new solver interface, so the global solvers are limited to ScipDirect, ScipPersistent, and GurobiDirectMINLP.') - res = global_solver.solve(nlp, load_solutions=False, raise_exception_on_nonoptimal_result=False, solver_options=opts) - logger.info(f'solved NLP: {res.solution_status}, {res.termination_condition}') + res = global_solver.solve(nlp, load_solutions=True, raise_exception_on_nonoptimal_result=False, solver_options=opts) + logger.info(f'solved NLP with {global_solver.name}: {res.solution_status}, {res.termination_condition}') + res = nlp_solver.solve(nlp, load_solutions=False, raise_exception_on_nonoptimal_result=False) + logger.info(f'solved NLP with {nlp_solver.name}: {res.solution_status}, {res.termination_condition}') if res.incumbent_objective is not None: res.solution_loader.load_vars() else: diff --git a/pyomo/contrib/initialization/initialize.py b/pyomo/contrib/initialization/initialize.py index b7e04d784d1..f8918f95528 100644 --- a/pyomo/contrib/initialization/initialize.py +++ b/pyomo/contrib/initialization/initialize.py @@ -7,6 +7,11 @@ from pyomo.contrib.initialization.lp_approx_init import _initialize_with_LP_approximation from pyomo.contrib.solver.common.base import SolverBase from pyomo.contrib.initialization.global_init import _initialize_with_global_solver +from pyomo.contrib.solver.common.factory import SolverFactory +import logging + + +logger = logging.getLogger(__name__) class InitializationMethod(Enum): @@ -15,12 +20,21 @@ class InitializationMethod(Enum): global_opt = "global_opt" +def _get_solver(sname, reason): + opt = SolverFactory(sname) + if opt.available(): + logger.info(f'Using {sname} for {reason} because a solver was not specified') + else: + raise RuntimeError(f'No solver was specified for {reason} and the default ({sname}) is not available') + return opt + + def initialize_nlp( nlp: BlockData, mip_solver: Optional[SolverBase] = None, nlp_solver: Optional[SolverBase] = None, global_solver: Optional[SolverBase] = None, - method: InitializationMethod = InitializationMethod.pwl_approximation, + method: InitializationMethod = InitializationMethod.global_opt, default_bound=1.0e8, max_pwl_refinement_iter=100, num_pwl_cons_to_refine_per_iter=5, @@ -31,15 +45,12 @@ def initialize_nlp( (v, (v.lower, v.upper, v.domain, v.fixed, v.value)) for v in orig_vars ) - # create a shallow clone of the model so that the initialization method can - # can work with the original variables but not make any other - # modifications to the model - # nlp = shallow_clone(nlp) - # run the initialization if method == InitializationMethod.pwl_approximation: - assert mip_solver is not None, f"mip_solver must be specified for {method}" - assert nlp_solver is not None, f"nlp_solver must be specified for {method}" + if mip_solver is None: + mip_solver = _get_solver('gurobi_persistent', 'MILP solver') + if nlp_solver is None: + nlp_solver = _get_solver('ipopt', 'local NLP solver') res = _initialize_with_piecewise_linear_approximation( nlp=nlp, mip_solver=mip_solver, @@ -49,16 +60,21 @@ def initialize_nlp( num_cons_to_refine_per_iter=num_pwl_cons_to_refine_per_iter, ) elif method == InitializationMethod.lp_approximation: - assert mip_solver is not None, f"mip_solver must be specified for {method}" - assert nlp_solver is not None, f"nlp_solver must be specified for {method}" + if mip_solver is None: + mip_solver = _get_solver('gurobi_persistent', 'MILP solver') + if nlp_solver is None: + nlp_solver = _get_solver('ipopt', 'local NLP solver') res = _initialize_with_LP_approximation( nlp=nlp, lp_solver=mip_solver, nlp_solver=nlp_solver, ) elif method == InitializationMethod.global_opt: - assert global_solver is not None, f"global_solver must be specified for {method}" - res = _initialize_with_global_solver(nlp=nlp, global_solver=global_solver) + if global_solver is None: + global_solver = _get_solver('gurobi_direct_minlp', 'global NLP solver') + if nlp_solver is None: + nlp_solver = _get_solver('ipopt', 'local NLP solver') + res = _initialize_with_global_solver(nlp=nlp, global_solver=global_solver, nlp_solver=nlp_solver) else: raise ValueError(f'unexpected initialization method: {method}') From 9568bb0d6f9812bf3bbf0baeb35daa42bab25993 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 24 Mar 2026 07:37:49 -0600 Subject: [PATCH 071/107] lp init should bound all variables --- pyomo/contrib/initialization/lp_approx_init.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyomo/contrib/initialization/lp_approx_init.py b/pyomo/contrib/initialization/lp_approx_init.py index 63d96011987..0c72e82fbd7 100644 --- a/pyomo/contrib/initialization/lp_approx_init.py +++ b/pyomo/contrib/initialization/lp_approx_init.py @@ -137,6 +137,7 @@ def _initialize_with_LP_approximation( nlp: BlockData, lp_solver: SolverBase, nlp_solver: SolverBase, + default_bound=1.0e8, ): orig_nlp = nlp logger.info('Starting initialization using a linear programming approximation') @@ -149,9 +150,9 @@ def _initialize_with_LP_approximation( trans.apply_to(nlp, aggressive_substitution=False) logger.info('applied the univariate_nonlinear_decomposition transformation') - # let's see if we can get bounds on the nonlinear variables - fbbt(nlp) - logger.info('ran FBBT') + # bounds on the nonlinear variables + bound_all_nonlinear_variables(nlp, default_bound=default_bound) + logger.info('bounded nonlinear variables') # Now, we need to fix variables with equal (or nearly equal) bounds. # Otherwise, the PWL transformation complains From 387a646a36aa607b3d4c388c6c19de41b1654251 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 24 Mar 2026 07:58:48 -0600 Subject: [PATCH 072/107] move initialization module from contrib to devel --- pyomo/{contrib => devel}/initialization/README.md | 0 pyomo/{contrib => devel}/initialization/__init__.py | 0 pyomo/{contrib => devel}/initialization/bounds/__init__.py | 0 pyomo/{contrib => devel}/initialization/bounds/bound_variables.py | 0 pyomo/{contrib => devel}/initialization/global_init.py | 0 pyomo/{contrib => devel}/initialization/initialize.py | 0 pyomo/{contrib => devel}/initialization/lp_approx_init.py | 0 pyomo/{contrib => devel}/initialization/pwl_init.py | 0 pyomo/{contrib => devel}/initialization/utils.py | 0 9 files changed, 0 insertions(+), 0 deletions(-) rename pyomo/{contrib => devel}/initialization/README.md (100%) rename pyomo/{contrib => devel}/initialization/__init__.py (100%) rename pyomo/{contrib => devel}/initialization/bounds/__init__.py (100%) rename pyomo/{contrib => devel}/initialization/bounds/bound_variables.py (100%) rename pyomo/{contrib => devel}/initialization/global_init.py (100%) rename pyomo/{contrib => devel}/initialization/initialize.py (100%) rename pyomo/{contrib => devel}/initialization/lp_approx_init.py (100%) rename pyomo/{contrib => devel}/initialization/pwl_init.py (100%) rename pyomo/{contrib => devel}/initialization/utils.py (100%) diff --git a/pyomo/contrib/initialization/README.md b/pyomo/devel/initialization/README.md similarity index 100% rename from pyomo/contrib/initialization/README.md rename to pyomo/devel/initialization/README.md diff --git a/pyomo/contrib/initialization/__init__.py b/pyomo/devel/initialization/__init__.py similarity index 100% rename from pyomo/contrib/initialization/__init__.py rename to pyomo/devel/initialization/__init__.py diff --git a/pyomo/contrib/initialization/bounds/__init__.py b/pyomo/devel/initialization/bounds/__init__.py similarity index 100% rename from pyomo/contrib/initialization/bounds/__init__.py rename to pyomo/devel/initialization/bounds/__init__.py diff --git a/pyomo/contrib/initialization/bounds/bound_variables.py b/pyomo/devel/initialization/bounds/bound_variables.py similarity index 100% rename from pyomo/contrib/initialization/bounds/bound_variables.py rename to pyomo/devel/initialization/bounds/bound_variables.py diff --git a/pyomo/contrib/initialization/global_init.py b/pyomo/devel/initialization/global_init.py similarity index 100% rename from pyomo/contrib/initialization/global_init.py rename to pyomo/devel/initialization/global_init.py diff --git a/pyomo/contrib/initialization/initialize.py b/pyomo/devel/initialization/initialize.py similarity index 100% rename from pyomo/contrib/initialization/initialize.py rename to pyomo/devel/initialization/initialize.py diff --git a/pyomo/contrib/initialization/lp_approx_init.py b/pyomo/devel/initialization/lp_approx_init.py similarity index 100% rename from pyomo/contrib/initialization/lp_approx_init.py rename to pyomo/devel/initialization/lp_approx_init.py diff --git a/pyomo/contrib/initialization/pwl_init.py b/pyomo/devel/initialization/pwl_init.py similarity index 100% rename from pyomo/contrib/initialization/pwl_init.py rename to pyomo/devel/initialization/pwl_init.py diff --git a/pyomo/contrib/initialization/utils.py b/pyomo/devel/initialization/utils.py similarity index 100% rename from pyomo/contrib/initialization/utils.py rename to pyomo/devel/initialization/utils.py From 24dbd64bc36b26664ab3e4641a1bfc46ac288adf Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 24 Mar 2026 08:41:07 -0600 Subject: [PATCH 073/107] update imports --- pyomo/devel/initialization/__init__.py | 1 + pyomo/devel/initialization/bounds/bound_variables.py | 2 +- pyomo/devel/initialization/global_init.py | 3 ++- pyomo/devel/initialization/initialize.py | 8 ++++---- pyomo/devel/initialization/lp_approx_init.py | 10 ++++++---- pyomo/devel/initialization/pwl_init.py | 7 ++++--- 6 files changed, 18 insertions(+), 13 deletions(-) diff --git a/pyomo/devel/initialization/__init__.py b/pyomo/devel/initialization/__init__.py index e69de29bb2d..099f15af824 100644 --- a/pyomo/devel/initialization/__init__.py +++ b/pyomo/devel/initialization/__init__.py @@ -0,0 +1 @@ +from pyomo.devel.initialization.initialize import initialize_nlp, InitializationMethod \ No newline at end of file diff --git a/pyomo/devel/initialization/bounds/bound_variables.py b/pyomo/devel/initialization/bounds/bound_variables.py index 0d0232f53b8..8741b6ca979 100644 --- a/pyomo/devel/initialization/bounds/bound_variables.py +++ b/pyomo/devel/initialization/bounds/bound_variables.py @@ -1,6 +1,6 @@ from pyomo.core.base.block import BlockData from pyomo.contrib.fbbt.fbbt import fbbt -from pyomo.contrib.initialization.utils import get_vars +from pyomo.devel.initialization.utils import get_vars import logging diff --git a/pyomo/devel/initialization/global_init.py b/pyomo/devel/initialization/global_init.py index 4f858a4b7c3..142d58ec802 100644 --- a/pyomo/devel/initialization/global_init.py +++ b/pyomo/devel/initialization/global_init.py @@ -1,5 +1,6 @@ from pyomo.core.base.block import BlockData from pyomo.contrib.solver.common.base import SolverBase +from pyomo.contrib.solver.common.results import SolutionStatus from pyomo.contrib.solver.solvers.scip.scip_direct import ScipDirect, ScipPersistent from pyomo.contrib.solver.solvers.gurobi.gurobi_direct_minlp import GurobiDirectMINLP import logging @@ -23,7 +24,7 @@ def _initialize_with_global_solver( logger.info(f'solved NLP with {global_solver.name}: {res.solution_status}, {res.termination_condition}') res = nlp_solver.solve(nlp, load_solutions=False, raise_exception_on_nonoptimal_result=False) logger.info(f'solved NLP with {nlp_solver.name}: {res.solution_status}, {res.termination_condition}') - if res.incumbent_objective is not None: + if res.solution_status in {SolutionStatus.feasible, SolutionStatus.optimal}: res.solution_loader.load_vars() else: raise RuntimeError('no feasible solution found') diff --git a/pyomo/devel/initialization/initialize.py b/pyomo/devel/initialization/initialize.py index f8918f95528..f62bc9610b1 100644 --- a/pyomo/devel/initialization/initialize.py +++ b/pyomo/devel/initialization/initialize.py @@ -1,12 +1,12 @@ from typing import Optional from pyomo.core.base.block import BlockData from enum import Enum -from pyomo.contrib.initialization.utils import get_vars, shallow_clone +from pyomo.devel.initialization.utils import get_vars, shallow_clone from pyomo.common.collections import ComponentMap -from pyomo.contrib.initialization.pwl_init import _initialize_with_piecewise_linear_approximation -from pyomo.contrib.initialization.lp_approx_init import _initialize_with_LP_approximation +from pyomo.devel.initialization.pwl_init import _initialize_with_piecewise_linear_approximation +from pyomo.devel.initialization.lp_approx_init import _initialize_with_LP_approximation from pyomo.contrib.solver.common.base import SolverBase -from pyomo.contrib.initialization.global_init import _initialize_with_global_solver +from pyomo.devel.initialization.global_init import _initialize_with_global_solver from pyomo.contrib.solver.common.factory import SolverFactory import logging diff --git a/pyomo/devel/initialization/lp_approx_init.py b/pyomo/devel/initialization/lp_approx_init.py index 0c72e82fbd7..18752271c47 100644 --- a/pyomo/devel/initialization/lp_approx_init.py +++ b/pyomo/devel/initialization/lp_approx_init.py @@ -1,10 +1,11 @@ from pyomo.core.base.block import BlockData import pyomo.environ as pe -from pyomo.contrib.initialization.bounds.bound_variables import bound_all_nonlinear_variables -from pyomo.contrib.initialization.utils import fix_vars_with_equal_bounds, shallow_clone, get_vars +from pyomo.devel.initialization.bounds.bound_variables import bound_all_nonlinear_variables +from pyomo.devel.initialization.utils import fix_vars_with_equal_bounds, shallow_clone, get_vars from pyomo.core.expr.visitor import identify_components from pyomo.contrib.piecewise.piecewise_linear_expression import PiecewiseLinearExpression from pyomo.contrib.piecewise.piecewise_linear_function import PiecewiseLinearFunction +from pyomo.contrib.solver.common.results import SolutionStatus from pyomo.common.collections import ComponentMap, ComponentSet from typing import MutableMapping, Sequence, List from pyomo.core.base.constraint import ConstraintData @@ -36,7 +37,7 @@ from pyomo.contrib.solver.common.base import SolverBase import logging from pyomo.common.modeling import unique_component_name -from pyomo.contrib.initialization.pwl_init import _minimize_infeasibility +from pyomo.devel.initialization.pwl_init import _minimize_infeasibility from pyomo.contrib.fbbt.fbbt import fbbt from pyomo.repn.linear import LinearRepnVisitor, LinearRepn from pyomo.core.expr.visitor import identify_variables @@ -175,7 +176,8 @@ def _initialize_with_LP_approximation( # try solving the NLP nlp_res = nlp_solver.solve(orig_nlp, load_solutions=False, raise_exception_on_nonoptimal_result=False) logger.info(f'solved NLP: {nlp_res.solution_status}, {nlp_res.termination_condition}') - if nlp_res.incumbent_objective is not None: + + if nlp_res.solution_status in {SolutionStatus.feasible, SolutionStatus.optimal}: nlp_res.solution_loader.load_vars() else: raise RuntimeError('no feasible solution found') diff --git a/pyomo/devel/initialization/pwl_init.py b/pyomo/devel/initialization/pwl_init.py index 30baa1d7cef..d11d518608a 100644 --- a/pyomo/devel/initialization/pwl_init.py +++ b/pyomo/devel/initialization/pwl_init.py @@ -1,7 +1,7 @@ from pyomo.core.base.block import BlockData import pyomo.environ as pe -from pyomo.contrib.initialization.bounds.bound_variables import bound_all_nonlinear_variables -from pyomo.contrib.initialization.utils import fix_vars_with_equal_bounds, shallow_clone, get_vars +from pyomo.devel.initialization.bounds.bound_variables import bound_all_nonlinear_variables +from pyomo.devel.initialization.utils import fix_vars_with_equal_bounds, shallow_clone, get_vars from pyomo.core.expr.visitor import identify_components from pyomo.contrib.piecewise.piecewise_linear_expression import PiecewiseLinearExpression from pyomo.contrib.piecewise.piecewise_linear_function import PiecewiseLinearFunction @@ -36,6 +36,7 @@ from pyomo.contrib.solver.common.base import SolverBase import logging from pyomo.common.modeling import unique_component_name +from pyomo.contrib.solver.common.results import SolutionStatus logger = logging.getLogger(__name__) @@ -325,7 +326,7 @@ def _initialize_with_piecewise_linear_approximation( res = nlp_solver.solve(nlp, load_solutions=False, raise_exception_on_nonoptimal_result=False) last_nlp_res = res logger.info(f'solved NLP: {res.solution_status}, {res.termination_condition}') - if res.incumbent_objective is not None: + if res.solution_status in {SolutionStatus.feasible, SolutionStatus.optimal}: solved = True res.solution_loader.load_vars() break From 261650220c608f6581ed500eff0a7d9f8f1bcbcd Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 24 Mar 2026 08:41:58 -0600 Subject: [PATCH 074/107] initialization example --- .../solver/solvers/scip/scip_direct.py | 2 +- .../devel/initialization/examples/__init__.py | 0 .../examples/init_polynomial_ex.py | 33 +++++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 pyomo/devel/initialization/examples/__init__.py create mode 100644 pyomo/devel/initialization/examples/init_polynomial_ex.py diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 022878f918d..1371ef61530 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -349,7 +349,7 @@ def load_vars( for v, val in self.get_vars( vars_to_load=vars_to_load, solution_id=solution_id ).items(): - v.value = val + v.set_value(val, skip_validation=True) def get_vars( self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None diff --git a/pyomo/devel/initialization/examples/__init__.py b/pyomo/devel/initialization/examples/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pyomo/devel/initialization/examples/init_polynomial_ex.py b/pyomo/devel/initialization/examples/init_polynomial_ex.py new file mode 100644 index 00000000000..fe99923a23c --- /dev/null +++ b/pyomo/devel/initialization/examples/init_polynomial_ex.py @@ -0,0 +1,33 @@ +import pyomo.environ as pyo +import pyomo.devel.initialization as ini +from pyomo.contrib.solver.common.factory import SolverFactory +import logging + + +def build_model(): + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.c = pyo.Constraint(expr=(m.x+7)*(m.x+5)*(m.x-4) + 200 == 0) + return m + + +def main(method: ini.InitializationMethod): + m = build_model() + nlp_solver = SolverFactory('ipopt') + global_solver = SolverFactory('scip_direct') + mip_solver = SolverFactory('scip_direct') + results = ini.initialize_nlp( + nlp=m, + nlp_solver=nlp_solver, + mip_solver=mip_solver, + global_solver=global_solver, + method=method, + ) + + return results.solution_status, m.x.value + + +if __name__ == '__main__': + stat, x = main(ini.InitializationMethod.global_opt) + print(stat) + print(round(x, 4)) From 54d354e60383c66f4d385649160c5903afaa1087 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 24 Mar 2026 12:30:46 -0600 Subject: [PATCH 075/107] initialization: starting tests --- .../examples/init_polynomial_ex.py | 3 +- pyomo/devel/initialization/tests/__init__.py | 0 .../tests/test_initialization.py | 29 +++++++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 pyomo/devel/initialization/tests/__init__.py create mode 100644 pyomo/devel/initialization/tests/test_initialization.py diff --git a/pyomo/devel/initialization/examples/init_polynomial_ex.py b/pyomo/devel/initialization/examples/init_polynomial_ex.py index fe99923a23c..dc8dc88b31a 100644 --- a/pyomo/devel/initialization/examples/init_polynomial_ex.py +++ b/pyomo/devel/initialization/examples/init_polynomial_ex.py @@ -1,12 +1,11 @@ import pyomo.environ as pyo import pyomo.devel.initialization as ini from pyomo.contrib.solver.common.factory import SolverFactory -import logging def build_model(): m = pyo.ConcreteModel() - m.x = pyo.Var() + m.x = pyo.Var(bounds=(-20, 20)) m.c = pyo.Constraint(expr=(m.x+7)*(m.x+5)*(m.x-4) + 200 == 0) return m diff --git a/pyomo/devel/initialization/tests/__init__.py b/pyomo/devel/initialization/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pyomo/devel/initialization/tests/test_initialization.py b/pyomo/devel/initialization/tests/test_initialization.py new file mode 100644 index 00000000000..c1c6dd9bbd2 --- /dev/null +++ b/pyomo/devel/initialization/tests/test_initialization.py @@ -0,0 +1,29 @@ +import pyomo.environ as pyo +import pyomo.devel.initialization as ini +from pyomo.devel.initialization.examples.init_polynomial_ex import main +from pyomo.common import unittest +from pyomo.contrib.solver.common.factory import SolverFactory +from pyomo.contrib.solver.common.results import SolutionStatus + + +scip = SolverFactory('scip_direct') +ipopt = SolverFactory('ipopt') + + +@unittest.skipUnless(scip.available(), 'scip is not available') +@unittest.skipUnless(ipopt.available(), 'ipopt is not available') +class TestExamples(unittest.TestCase): + def test_poly_global(self): + stat, x = main(method=ini.InitializationMethod.global_opt) + self.assertEqual(stat, SolutionStatus.optimal) + self.assertAlmostEqual(x, -9.920159607881597) + + def test_poly_pwl(self): + stat, x = main(method=ini.InitializationMethod.pwl_approximation) + self.assertEqual(stat, SolutionStatus.optimal) + self.assertAlmostEqual(x, -9.920159607881597) + + def test_poly_lp(self): + stat, x = main(method=ini.InitializationMethod.lp_approximation) + self.assertEqual(stat, SolutionStatus.optimal) + self.assertAlmostEqual(x, -9.920159607881597) From 056cc993a4664f9db22ef644c250708451ad3dfd Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 24 Mar 2026 14:22:09 -0600 Subject: [PATCH 076/107] initialization: add copyright statements --- pyomo/devel/initialization/__init__.py | 9 +++++++++ pyomo/devel/initialization/bounds/__init__.py | 8 ++++++++ pyomo/devel/initialization/bounds/bound_variables.py | 9 +++++++++ .../initialization/examples/init_polynomial_ex.py | 10 ++++++++++ pyomo/devel/initialization/global_init.py | 9 +++++++++ pyomo/devel/initialization/initialize.py | 9 +++++++++ pyomo/devel/initialization/lp_approx_init.py | 9 +++++++++ pyomo/devel/initialization/pwl_init.py | 9 +++++++++ pyomo/devel/initialization/tests/__init__.py | 8 ++++++++ .../devel/initialization/tests/test_initialization.py | 9 +++++++++ pyomo/devel/initialization/utils.py | 9 +++++++++ 11 files changed, 98 insertions(+) diff --git a/pyomo/devel/initialization/__init__.py b/pyomo/devel/initialization/__init__.py index 099f15af824..42e52b2d8ff 100644 --- a/pyomo/devel/initialization/__init__.py +++ b/pyomo/devel/initialization/__init__.py @@ -1 +1,10 @@ +# ____________________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2026 National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and Engineering +# Solutions of Sandia, LLC, the U.S. Government retains certain rights in this +# software. This software is distributed under the 3-clause BSD License. +# ____________________________________________________________________________________ + from pyomo.devel.initialization.initialize import initialize_nlp, InitializationMethod \ No newline at end of file diff --git a/pyomo/devel/initialization/bounds/__init__.py b/pyomo/devel/initialization/bounds/__init__.py index e69de29bb2d..231b44987f6 100644 --- a/pyomo/devel/initialization/bounds/__init__.py +++ b/pyomo/devel/initialization/bounds/__init__.py @@ -0,0 +1,8 @@ +# ____________________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2026 National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and Engineering +# Solutions of Sandia, LLC, the U.S. Government retains certain rights in this +# software. This software is distributed under the 3-clause BSD License. +# ____________________________________________________________________________________ diff --git a/pyomo/devel/initialization/bounds/bound_variables.py b/pyomo/devel/initialization/bounds/bound_variables.py index 8741b6ca979..4402ea76510 100644 --- a/pyomo/devel/initialization/bounds/bound_variables.py +++ b/pyomo/devel/initialization/bounds/bound_variables.py @@ -1,3 +1,12 @@ +# ____________________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2026 National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and Engineering +# Solutions of Sandia, LLC, the U.S. Government retains certain rights in this +# software. This software is distributed under the 3-clause BSD License. +# ____________________________________________________________________________________ + from pyomo.core.base.block import BlockData from pyomo.contrib.fbbt.fbbt import fbbt from pyomo.devel.initialization.utils import get_vars diff --git a/pyomo/devel/initialization/examples/init_polynomial_ex.py b/pyomo/devel/initialization/examples/init_polynomial_ex.py index dc8dc88b31a..88b03aed0e0 100644 --- a/pyomo/devel/initialization/examples/init_polynomial_ex.py +++ b/pyomo/devel/initialization/examples/init_polynomial_ex.py @@ -1,3 +1,13 @@ +# ____________________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2026 National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and Engineering +# Solutions of Sandia, LLC, the U.S. Government retains certain rights in this +# software. This software is distributed under the 3-clause BSD License. +# ____________________________________________________________________________________ + +# === Required imports === import pyomo.environ as pyo import pyomo.devel.initialization as ini from pyomo.contrib.solver.common.factory import SolverFactory diff --git a/pyomo/devel/initialization/global_init.py b/pyomo/devel/initialization/global_init.py index 142d58ec802..11533d9f516 100644 --- a/pyomo/devel/initialization/global_init.py +++ b/pyomo/devel/initialization/global_init.py @@ -1,3 +1,12 @@ +# ____________________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2026 National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and Engineering +# Solutions of Sandia, LLC, the U.S. Government retains certain rights in this +# software. This software is distributed under the 3-clause BSD License. +# ____________________________________________________________________________________ + from pyomo.core.base.block import BlockData from pyomo.contrib.solver.common.base import SolverBase from pyomo.contrib.solver.common.results import SolutionStatus diff --git a/pyomo/devel/initialization/initialize.py b/pyomo/devel/initialization/initialize.py index f62bc9610b1..90eab0f3dad 100644 --- a/pyomo/devel/initialization/initialize.py +++ b/pyomo/devel/initialization/initialize.py @@ -2,6 +2,15 @@ from pyomo.core.base.block import BlockData from enum import Enum from pyomo.devel.initialization.utils import get_vars, shallow_clone +# ____________________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2026 National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and Engineering +# Solutions of Sandia, LLC, the U.S. Government retains certain rights in this +# software. This software is distributed under the 3-clause BSD License. +# ____________________________________________________________________________________ + from pyomo.common.collections import ComponentMap from pyomo.devel.initialization.pwl_init import _initialize_with_piecewise_linear_approximation from pyomo.devel.initialization.lp_approx_init import _initialize_with_LP_approximation diff --git a/pyomo/devel/initialization/lp_approx_init.py b/pyomo/devel/initialization/lp_approx_init.py index 18752271c47..4401a69e55d 100644 --- a/pyomo/devel/initialization/lp_approx_init.py +++ b/pyomo/devel/initialization/lp_approx_init.py @@ -1,3 +1,12 @@ +# ____________________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2026 National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and Engineering +# Solutions of Sandia, LLC, the U.S. Government retains certain rights in this +# software. This software is distributed under the 3-clause BSD License. +# ____________________________________________________________________________________ + from pyomo.core.base.block import BlockData import pyomo.environ as pe from pyomo.devel.initialization.bounds.bound_variables import bound_all_nonlinear_variables diff --git a/pyomo/devel/initialization/pwl_init.py b/pyomo/devel/initialization/pwl_init.py index d11d518608a..bec34ddb612 100644 --- a/pyomo/devel/initialization/pwl_init.py +++ b/pyomo/devel/initialization/pwl_init.py @@ -1,3 +1,12 @@ +# ____________________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2026 National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and Engineering +# Solutions of Sandia, LLC, the U.S. Government retains certain rights in this +# software. This software is distributed under the 3-clause BSD License. +# ____________________________________________________________________________________ + from pyomo.core.base.block import BlockData import pyomo.environ as pe from pyomo.devel.initialization.bounds.bound_variables import bound_all_nonlinear_variables diff --git a/pyomo/devel/initialization/tests/__init__.py b/pyomo/devel/initialization/tests/__init__.py index e69de29bb2d..231b44987f6 100644 --- a/pyomo/devel/initialization/tests/__init__.py +++ b/pyomo/devel/initialization/tests/__init__.py @@ -0,0 +1,8 @@ +# ____________________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2026 National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and Engineering +# Solutions of Sandia, LLC, the U.S. Government retains certain rights in this +# software. This software is distributed under the 3-clause BSD License. +# ____________________________________________________________________________________ diff --git a/pyomo/devel/initialization/tests/test_initialization.py b/pyomo/devel/initialization/tests/test_initialization.py index c1c6dd9bbd2..9e7aa50753e 100644 --- a/pyomo/devel/initialization/tests/test_initialization.py +++ b/pyomo/devel/initialization/tests/test_initialization.py @@ -1,3 +1,12 @@ +# ____________________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2026 National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and Engineering +# Solutions of Sandia, LLC, the U.S. Government retains certain rights in this +# software. This software is distributed under the 3-clause BSD License. +# ____________________________________________________________________________________ + import pyomo.environ as pyo import pyomo.devel.initialization as ini from pyomo.devel.initialization.examples.init_polynomial_ex import main diff --git a/pyomo/devel/initialization/utils.py b/pyomo/devel/initialization/utils.py index 792509583f0..3c69ba1a55f 100644 --- a/pyomo/devel/initialization/utils.py +++ b/pyomo/devel/initialization/utils.py @@ -1,3 +1,12 @@ +# ____________________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2026 National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and Engineering +# Solutions of Sandia, LLC, the U.S. Government retains certain rights in this +# software. This software is distributed under the 3-clause BSD License. +# ____________________________________________________________________________________ + import pyomo.environ as pe from pyomo.common.collections import ComponentSet from pyomo.core.base.block import BlockData From d70e677a751af53a9a19025025aa46e44f2c0ab7 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 24 Mar 2026 14:43:53 -0600 Subject: [PATCH 077/107] docstring --- pyomo/devel/initialization/initialize.py | 54 ++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/pyomo/devel/initialization/initialize.py b/pyomo/devel/initialization/initialize.py index 90eab0f3dad..c0cbcf35e68 100644 --- a/pyomo/devel/initialization/initialize.py +++ b/pyomo/devel/initialization/initialize.py @@ -17,6 +17,7 @@ from pyomo.contrib.solver.common.base import SolverBase from pyomo.devel.initialization.global_init import _initialize_with_global_solver from pyomo.contrib.solver.common.factory import SolverFactory +from pyomo.contrib.solver.common.results import Results import logging @@ -44,10 +45,55 @@ def initialize_nlp( nlp_solver: Optional[SolverBase] = None, global_solver: Optional[SolverBase] = None, method: InitializationMethod = InitializationMethod.global_opt, - default_bound=1.0e8, - max_pwl_refinement_iter=100, - num_pwl_cons_to_refine_per_iter=5, -): + default_bound: float = 1.0e8, + max_pwl_refinement_iter: int = 100, + num_pwl_cons_to_refine_per_iter: int = 5, +) -> Results: + """ + Attempt to initialize and subsequently solve the model given by ``nlp``. + The basic idea is to apply some method to find good initial values for + the variables and then try to solve the problem with ``nlp_solver``. + + Parameters + ---------- + nlp: BlockData + The pyomo model to be initialized. + mip_solver: Optional[SolverBase] + A solver interface appropriate for LPs and MILPs. + Needed for the following methods: + - pwl_approximation + - lp_approximation + Default: gurobi_persistent + nlp_solver: Optional[SolverBase] + A solver interface appropriate for NLPs. + Default: ipopt + global_solver: Optional[SolverBase] + A solver interface appropriate for global solution of NLPs + Default: gurobi_direct_minlp + method: InitializationMethod + The method used to initialize the model. + default_bound: float + Some initialize methods require all nonlinear variables to be bounded. + For these methods, all unbounded variables will be given lower and + upper bounds equal to default_bound. + Needed for the following methods: + - pwl_approximation + - lp_approximation + max_pwl_refinement_iter: int + Only used when method = InitializationMethod.pwl_approximation. This is + the maximum number of iterations used to refine the piecewise linear + approximation. + num_pwl_cons_to_refine_per_iter: int + Only used when method = InitializationMethod.pwl_approximation. This is + the maximum number of constraints to be refined with additional + segments in the piecewise linear approximation each iteration. + + Returns + ------- + res: Results + The results object obtained the last time the nlp_solver was used to + try and solve the model. + """ # get all variable bounds, domains, etc. to restore them later orig_vars = get_vars(nlp) orig_var_data = ComponentMap( From 823641d0dcf3207cb0a9c254d3b63f8acb07b336 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 24 Mar 2026 20:22:49 -0600 Subject: [PATCH 078/107] initialization: docs --- doc/OnlineDocs/explanation/analysis/index.rst | 1 + .../analysis/nlp_initialization/nlp_initialization.rst | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100644 doc/OnlineDocs/explanation/analysis/nlp_initialization/nlp_initialization.rst diff --git a/doc/OnlineDocs/explanation/analysis/index.rst b/doc/OnlineDocs/explanation/analysis/index.rst index 0a8e3c3b416..5537636b1bb 100644 --- a/doc/OnlineDocs/explanation/analysis/index.rst +++ b/doc/OnlineDocs/explanation/analysis/index.rst @@ -12,6 +12,7 @@ Analysis in Pyomo mpc/index parmest/index sensitivity_toolbox + nlp_initialization/nlp_initialization .. Reorganization notes: diff --git a/doc/OnlineDocs/explanation/analysis/nlp_initialization/nlp_initialization.rst b/doc/OnlineDocs/explanation/analysis/nlp_initialization/nlp_initialization.rst new file mode 100644 index 00000000000..0e7c07664ac --- /dev/null +++ b/doc/OnlineDocs/explanation/analysis/nlp_initialization/nlp_initialization.rst @@ -0,0 +1,10 @@ +.. _pyomo.devel.initialization + +NLP Initialization +================== +The initialization module within ``pyomo.devel.initialization`` is intended to provide methods to help initialize nonconvex nonlinear programs. Example usage is shown below. + +.. literalinclude:: /../../pyomo/devel/initialization/examples/ini_polynomial_ex.py + :start-after: # === Required imports === + +The `:meth:~pyomo.devel.initialization.initialize.initialize_nlp` function From 0b02132ec10aec2739e11f6209092a8c116a4f6b Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 7 Apr 2026 07:26:55 -0600 Subject: [PATCH 079/107] working on docs for initialization --- .../nlp_initialization/nlp_initialization.rst | 131 +++++++++++++++++- doc/OnlineDocs/explanation/index.rst | 1 + 2 files changed, 127 insertions(+), 5 deletions(-) diff --git a/doc/OnlineDocs/explanation/analysis/nlp_initialization/nlp_initialization.rst b/doc/OnlineDocs/explanation/analysis/nlp_initialization/nlp_initialization.rst index 0e7c07664ac..c18f0f4ac0a 100644 --- a/doc/OnlineDocs/explanation/analysis/nlp_initialization/nlp_initialization.rst +++ b/doc/OnlineDocs/explanation/analysis/nlp_initialization/nlp_initialization.rst @@ -1,10 +1,131 @@ -.. _pyomo.devel.initialization +.. _analysis_nlp_initialization: NLP Initialization -================== -The initialization module within ``pyomo.devel.initialization`` is intended to provide methods to help initialize nonconvex nonlinear programs. Example usage is shown below. +****************** -.. literalinclude:: /../../pyomo/devel/initialization/examples/ini_polynomial_ex.py +.. warning:: + + This package lives in :mod:`pyomo.devel`. APIs, options, and behavior may + change without notice. + +The initialization module within ``pyomo.devel.initialization`` is intended to +provide methods to help initialize nonconvex nonlinear programs (NLPs). The +goal is to increase the chance of finding a local minimizer (i.e., decrease the +chance of getting stuck a point that locally minimizes infeasibility). If +you are already able to solve your problem with a local NLP solver, these +tools will not help you. Example usage is shown below. + +.. literalinclude:: /../../pyomo/devel/initialization/examples/init_polynomial_ex.py :start-after: # === Required imports === -The `:meth:~pyomo.devel.initialization.initialize.initialize_nlp` function +The :func:`initialize_nlp ` +function uses the specified method to try to find a good starting point for the +NLP solver and then attempts to solve the problem with the given NLP solver. + +.. note:: + + Currently, this module only works with solvers from :mod:`pyomo.contrib.solver`. + + +Initialization Methods +====================== + +The initialization method is selected using the +:class:`InitializationMethod ` enum. + +.. note:: + + Not all of the methods described below require all nonlinear variables to be + bounded. However, all of the methods will perform better if all nonlinear + variables are bounded (the tighter the bounds, the better). + + +Method ``global_opt`` +--------------------- + +This method uses an MINLP solver to try to find a feasible solution. We +adjust the solver parameters so that the solver will stop as soon as any +feasible solution is found. We then initialize the NLP solver at that +feasible solution. Many MINLP solvers will default to a very large +time limit, so it can be useful to specify a time limit before +calling :func:`initialize_nlp `: + +.. testcode:: + :skipif: not scip_available + + import pyomo.environ as pyo + from pyomo.contrib.solver.common.factory import SolverFactory + + global_solver = SolverFactory('scip_direct') + global_solver.config.time_limit = 600 # 10 minutes + # now call initialize_nlp + +This method currently works with the following solver interfaces for MINLP solvers: + +* SCIP (:class:`direct ` and + :class:`persistent `) +* :class:`Gurobi MINLP ` + +Advantages +^^^^^^^^^^ + +* Currently, this is the method that is most likely to succeed in finding a + feasible solution. +* Does not strictly require variable bounds + +Disadvantages +^^^^^^^^^^^^^ + +* This method will only work if the model is completely algebraic. It will not + work with external functions. + + +Method ``pwl_approximation`` +---------------------------- + +This method builds a piecewise linear (PWL) approximation of the model, solves +it, and initializes the NLP solver at the solutin. If the NLP solver does not +converge, then the PWL approximation will be refined by adding additional +"segments". This is repeated until either a feasible solution is found or +the iteration limit is reached. + +This method does not currently work as well as ``global_opt``, but it does +have a great deal of potential. We expect future versions of this method +to perform significantly better. + +Advantages +^^^^^^^^^^ + +* Does not require an MINLP solver +* Future versions will work with external functions + +Disadvantages +^^^^^^^^^^^^^ + +* Current implemenation can be slow +* Requires all nonlinear variables to be bounded + + +Method ``lp_approximation`` +--------------------------- + +This method is similar to the PWL approximation method, but it builds +an LP approximation instead and does not do any refinement. Another +distinction is that the LP approximation uses a linear least-squares +fit, so the approximation may not equal the original function at the +variable bounds. This also means that variable bounds are not strictly +necessary, though they do help improve the approximation. + +Advantages +^^^^^^^^^^ + +* Fast +* Future versions will work with external functions +* Does not strictly require variable bounds +* Does not require an MINLP or even an MILP solver + +Disadvantages +^^^^^^^^^^^^^ + +* This method only attempts to initialize the problem once. If it does + not succeed, it is done. diff --git a/doc/OnlineDocs/explanation/index.rst b/doc/OnlineDocs/explanation/index.rst index 0121debcd32..27a3227de9d 100644 --- a/doc/OnlineDocs/explanation/index.rst +++ b/doc/OnlineDocs/explanation/index.rst @@ -43,6 +43,7 @@ Explanations `Design of Experiments` `MPC` `AOS` + `NLP Initialization` `Modeling Utilities` `Latex Printer` `FME` From 2869b9dff20ec889ab4ce1d19532b6a03e9dbc49 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 9 Apr 2026 06:16:48 -0600 Subject: [PATCH 080/107] speed up factorable programming transformation --- pyomo/contrib/piecewise/transform/factorable.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/piecewise/transform/factorable.py b/pyomo/contrib/piecewise/transform/factorable.py index f9829732e6b..2e7e0eba897 100644 --- a/pyomo/contrib/piecewise/transform/factorable.py +++ b/pyomo/contrib/piecewise/transform/factorable.py @@ -32,7 +32,8 @@ ) from pyomo.core.base.var import ScalarVar from pyomo.core.base.param import ScalarParam -from pyomo.contrib.fbbt.fbbt import compute_bounds_on_expr, fbbt +from pyomo.contrib.fbbt import interval +from pyomo.contrib.fbbt.fbbt import compute_bounds_on_expr, fbbt, ExpressionBoundsVisitor from pyomo.core.base.expression import ScalarExpression from pyomo.core.base.transformation import Transformation, TransformationFactory from pyomo.common.modeling import unique_component_name @@ -351,6 +352,8 @@ def __init__(self, **kwds): self.block.c = ConstraintList() self._leaf_types = {VarData, ScalarVar, ParamData, ScalarVar, float, int} + self._interval_visitor = ExpressionBoundsVisitor(use_fixed_var_values_as_bounds=True) + def initializeWalker(self, expr): if expr in self.substitution_map: return False, self.substitution_map[expr] @@ -388,7 +391,13 @@ def create_aux_var(self, expr): c = self.block.c.add(x == expr) # we need to compute bounds on x now because some of the # handlers depend on variable bounds (e.g., division) - fbbt(c) + xl, xu = self._interval_visitor.walk_expression(expr) + if xl == -interval.inf: + xl = None + if xu == interval.inf: + xu = None + x.setlb(xl) + x.setub(xu) return x From 235587037dd6328d551a2afdbfbf51302cc3ced8 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 9 Apr 2026 09:47:34 -0600 Subject: [PATCH 081/107] bug --- pyomo/devel/initialization/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/devel/initialization/utils.py b/pyomo/devel/initialization/utils.py index 3c69ba1a55f..b73da8625b6 100644 --- a/pyomo/devel/initialization/utils.py +++ b/pyomo/devel/initialization/utils.py @@ -19,7 +19,7 @@ def get_vars(m: BlockData): for c in m.component_data_objects(pe.Constraint, active=True, descend_into=True): vset.update(identify_variables(c.body, include_fixed=False)) for o in m.component_data_objects(pe.Objective, active=True, descend_into=True): - vset.update(identify_variables(c.expr, include_fixed=False)) + vset.update(identify_variables(o.expr, include_fixed=False)) return vset From e40f487286c1e70fb36d0d7e8b6e1de80b411148 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 9 Apr 2026 10:26:38 -0600 Subject: [PATCH 082/107] try nlp before initialization --- pyomo/devel/initialization/initialize.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pyomo/devel/initialization/initialize.py b/pyomo/devel/initialization/initialize.py index c0cbcf35e68..2793cda79ca 100644 --- a/pyomo/devel/initialization/initialize.py +++ b/pyomo/devel/initialization/initialize.py @@ -19,6 +19,7 @@ from pyomo.contrib.solver.common.factory import SolverFactory from pyomo.contrib.solver.common.results import Results import logging +from pyomo.contrib.solver.common.results import SolutionStatus logger = logging.getLogger(__name__) @@ -41,8 +42,8 @@ def _get_solver(sname, reason): def initialize_nlp( nlp: BlockData, + nlp_solver: SolverBase, mip_solver: Optional[SolverBase] = None, - nlp_solver: Optional[SolverBase] = None, global_solver: Optional[SolverBase] = None, method: InitializationMethod = InitializationMethod.global_opt, default_bound: float = 1.0e8, @@ -94,6 +95,15 @@ def initialize_nlp( The results object obtained the last time the nlp_solver was used to try and solve the model. """ + # in all cases, try to solve the nlp before doing extra work + res = nlp_solver.solve(nlp, load_solutions=False, raise_exception_on_nonoptimal_result=False) + logger.info(f'solved NLP: {res.solution_status}, {res.termination_condition}') + + if res.solution_status == SolutionStatus.optimal: + res.solution_loader.load_vars() + logger.info('NLP solved without any initialization') + return res + # get all variable bounds, domains, etc. to restore them later orig_vars = get_vars(nlp) orig_var_data = ComponentMap( From 42d7fff2866e8e0effc8d6184f5922a5f76cea75 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 9 Apr 2026 16:11:09 -0600 Subject: [PATCH 083/107] fix some docstrings --- .../analysis/nlp_initialization/nlp_initialization.rst | 2 +- pyomo/devel/initialization/initialize.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/OnlineDocs/explanation/analysis/nlp_initialization/nlp_initialization.rst b/doc/OnlineDocs/explanation/analysis/nlp_initialization/nlp_initialization.rst index c18f0f4ac0a..770f14369bf 100644 --- a/doc/OnlineDocs/explanation/analysis/nlp_initialization/nlp_initialization.rst +++ b/doc/OnlineDocs/explanation/analysis/nlp_initialization/nlp_initialization.rst @@ -84,7 +84,7 @@ Method ``pwl_approximation`` ---------------------------- This method builds a piecewise linear (PWL) approximation of the model, solves -it, and initializes the NLP solver at the solutin. If the NLP solver does not +it, and initializes the NLP solver at the solution. If the NLP solver does not converge, then the PWL approximation will be refined by adding additional "segments". This is repeated until either a feasible solution is found or the iteration limit is reached. diff --git a/pyomo/devel/initialization/initialize.py b/pyomo/devel/initialization/initialize.py index 2793cda79ca..e240e46a12c 100644 --- a/pyomo/devel/initialization/initialize.py +++ b/pyomo/devel/initialization/initialize.py @@ -1,7 +1,3 @@ -from typing import Optional -from pyomo.core.base.block import BlockData -from enum import Enum -from pyomo.devel.initialization.utils import get_vars, shallow_clone # ____________________________________________________________________________________ # # Pyomo: Python Optimization Modeling Objects @@ -11,6 +7,10 @@ # software. This software is distributed under the 3-clause BSD License. # ____________________________________________________________________________________ +from typing import Optional +from pyomo.core.base.block import BlockData +from enum import Enum +from pyomo.devel.initialization.utils import get_vars, shallow_clone from pyomo.common.collections import ComponentMap from pyomo.devel.initialization.pwl_init import _initialize_with_piecewise_linear_approximation from pyomo.devel.initialization.lp_approx_init import _initialize_with_LP_approximation @@ -91,7 +91,7 @@ def initialize_nlp( Returns ------- - res: Results + res: pyomo.contrib.solver.common.results.Results The results object obtained the last time the nlp_solver was used to try and solve the model. """ From fe98b7c72e86f32fcc2d4fc28c6eb55f08b7669e Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 11 Apr 2026 16:29:21 -0600 Subject: [PATCH 084/107] initialization: working on tests --- .../initialization/examples/init_polynomial_ex.py | 2 +- pyomo/devel/initialization/global_init.py | 2 +- pyomo/devel/initialization/lp_approx_init.py | 10 ++++++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pyomo/devel/initialization/examples/init_polynomial_ex.py b/pyomo/devel/initialization/examples/init_polynomial_ex.py index 88b03aed0e0..111ad418363 100644 --- a/pyomo/devel/initialization/examples/init_polynomial_ex.py +++ b/pyomo/devel/initialization/examples/init_polynomial_ex.py @@ -15,7 +15,7 @@ def build_model(): m = pyo.ConcreteModel() - m.x = pyo.Var(bounds=(-20, 20)) + m.x = pyo.Var(bounds=(-20, 20), initialize=-3.6) m.c = pyo.Constraint(expr=(m.x+7)*(m.x+5)*(m.x-4) + 200 == 0) return m diff --git a/pyomo/devel/initialization/global_init.py b/pyomo/devel/initialization/global_init.py index 11533d9f516..b924fb61273 100644 --- a/pyomo/devel/initialization/global_init.py +++ b/pyomo/devel/initialization/global_init.py @@ -36,6 +36,6 @@ def _initialize_with_global_solver( if res.solution_status in {SolutionStatus.feasible, SolutionStatus.optimal}: res.solution_loader.load_vars() else: - raise RuntimeError('no feasible solution found') + logger.warning('initialization was not successful via global optimization') return res diff --git a/pyomo/devel/initialization/lp_approx_init.py b/pyomo/devel/initialization/lp_approx_init.py index 4401a69e55d..91537436e10 100644 --- a/pyomo/devel/initialization/lp_approx_init.py +++ b/pyomo/devel/initialization/lp_approx_init.py @@ -156,9 +156,11 @@ def _initialize_with_LP_approximation( # first introduce auxiliary variables so that we don't try to # approximate any functions of more than two variables - trans = pe.TransformationFactory('contrib.piecewise.univariate_nonlinear_decomposition') - trans.apply_to(nlp, aggressive_substitution=False) - logger.info('applied the univariate_nonlinear_decomposition transformation') + # actually, this is not necessary for this method + # we will just comment this out for now + # trans = pe.TransformationFactory('contrib.piecewise.univariate_nonlinear_decomposition') + # trans.apply_to(nlp, aggressive_substitution=False) + # logger.info('applied the univariate_nonlinear_decomposition transformation') # bounds on the nonlinear variables bound_all_nonlinear_variables(nlp, default_bound=default_bound) @@ -189,6 +191,6 @@ def _initialize_with_LP_approximation( if nlp_res.solution_status in {SolutionStatus.feasible, SolutionStatus.optimal}: nlp_res.solution_loader.load_vars() else: - raise RuntimeError('no feasible solution found') + logger.warning('initialization was not successful via LP approximation') return nlp_res From 5d9660b08f09d59c9f41a80951f4b5bee5929651 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 11 Apr 2026 16:31:19 -0600 Subject: [PATCH 085/107] initialization: fixing bugs --- pyomo/devel/initialization/pwl_init.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pyomo/devel/initialization/pwl_init.py b/pyomo/devel/initialization/pwl_init.py index bec34ddb612..12f493f293b 100644 --- a/pyomo/devel/initialization/pwl_init.py +++ b/pyomo/devel/initialization/pwl_init.py @@ -63,7 +63,7 @@ def _minimize_infeasibility(m): if obj.sense == pe.minimize: obj_expr += 0.1*obj.expr else: - obj_expr -= 0.1*obj_expr + obj_expr -= 0.1*obj.expr obj.deactivate() found_obj = True @@ -102,15 +102,17 @@ def _get_pwl_constraints(m: BlockData) -> MutableMapping[ comp_types = set() comp_types.add(PiecewiseLinearExpression) pwl_expr_to_con_map = ComponentMap() - for con in m.component_data_objects(pe.Constraint, active=True, descend_into=True): - pwl_exprs = list(identify_components(con.expr, comp_types)) + con_list = list(m.component_data_objects(pe.Constraint, active=True, descend_into=True)) + obj_list = list(m.component_data_objects(pe.Objective, active=True, descend_into=True)) + for comp in con_list + obj_list: + pwl_exprs = list(identify_components(comp.expr, comp_types)) if not pwl_exprs: continue assert len(pwl_exprs) == 1 e = pwl_exprs[0] if e not in pwl_expr_to_con_map: pwl_expr_to_con_map[e] = [] - pwl_expr_to_con_map[e].append(con) + pwl_expr_to_con_map[e].append(comp) return pwl_expr_to_con_map @@ -224,6 +226,7 @@ def _refine_pwl_approx( # for v, val in zip(expr.args, var_vals): # print(f'{str(v):<20}{val:<20.5f}{v.lb:<20.5f}{v.ub:<20.5f}{id(v):<20}') if any(i is None for i in var_vals): + logger.info(f'missing variable values for {expr}') continue approx_value = func(*var_vals) true_value = func._func(*var_vals) @@ -341,6 +344,6 @@ def _initialize_with_piecewise_linear_approximation( break if not solved: - raise RuntimeError('no feasible solution found') + logger.warning('initialization was not successful via PWL approximation') return last_nlp_res From c08a92c4cfe40b9723b92120936f2849e6b24383 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 11 Apr 2026 17:01:30 -0600 Subject: [PATCH 086/107] fix nudge --- pyomo/contrib/piecewise/transform/nonlinear_to_pwl.py | 6 +++--- pyomo/devel/initialization/pwl_init.py | 7 +++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pyomo/contrib/piecewise/transform/nonlinear_to_pwl.py b/pyomo/contrib/piecewise/transform/nonlinear_to_pwl.py index db3f4b21621..6912811eb00 100644 --- a/pyomo/contrib/piecewise/transform/nonlinear_to_pwl.py +++ b/pyomo/contrib/piecewise/transform/nonlinear_to_pwl.py @@ -118,13 +118,13 @@ def _get_random_point_grid(bounds, n, func, config, seed=42): return list(itertools.product(*linspaces)) -def _get_uniform_point_grid(bounds, n, func, config): +def _get_uniform_point_grid(bounds, n, func, config, nudge_factor=0): # Generate non-randomized grid of points linspaces = [] for (lb, ub), is_integer in bounds: if not is_integer: # Issues happen when exactly using the boundary - nudge = (ub - lb) * 1e-4 + nudge = (ub - lb) * nudge_factor linspaces.append(np.linspace(lb + nudge, ub - nudge, n)) else: size = min(n, ub - lb + 1) @@ -139,7 +139,7 @@ def _get_points_lmt_random_sample(bounds, n, func, config, seed=42): def _get_points_lmt_uniform_sample(bounds, n, func, config, seed=42): - points = _get_uniform_point_grid(bounds, n, func, config) + points = _get_uniform_point_grid(bounds, n, func, config, nudge_factor=1e-4) return _get_points_lmt(points, bounds, func, config, seed) diff --git a/pyomo/devel/initialization/pwl_init.py b/pyomo/devel/initialization/pwl_init.py index 12f493f293b..786bcc2c699 100644 --- a/pyomo/devel/initialization/pwl_init.py +++ b/pyomo/devel/initialization/pwl_init.py @@ -237,8 +237,11 @@ def _refine_pwl_approx( if len(violations) == 0: raise RuntimeError('Did not find any piecewise linear functions with variable values') - if math.isclose(violations[0][0], 0): - raise RuntimeError('All of the original nonlinear functions are satisfied!') + tol = 1e-5 + if math.isclose(violations[0][0], 0, abs_tol=tol): + logger.info('All of the original nonlinear functions are satisfied!') + + violations = [i for i in violations if i[0] > tol] for err, expr in violations[:num_to_refine]: logger.info(f'refining {expr.pw_linear_function._func.expr} with error {err}') From ec9f9b9496009fa013da3767da2d612655b0883d Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 12 Apr 2026 19:41:04 -0600 Subject: [PATCH 087/107] initialization: working on tests --- .../tests/test_initialization.py | 154 +++++++++++++++++- 1 file changed, 153 insertions(+), 1 deletion(-) diff --git a/pyomo/devel/initialization/tests/test_initialization.py b/pyomo/devel/initialization/tests/test_initialization.py index 9e7aa50753e..177f32cec76 100644 --- a/pyomo/devel/initialization/tests/test_initialization.py +++ b/pyomo/devel/initialization/tests/test_initialization.py @@ -7,18 +7,54 @@ # software. This software is distributed under the 3-clause BSD License. # ____________________________________________________________________________________ +from typing import Tuple + import pyomo.environ as pyo import pyomo.devel.initialization as ini from pyomo.devel.initialization.examples.init_polynomial_ex import main from pyomo.common import unittest from pyomo.contrib.solver.common.factory import SolverFactory -from pyomo.contrib.solver.common.results import SolutionStatus +from pyomo.contrib.solver.common.results import SolutionStatus, Results, TerminationCondition +from pyomo.contrib.solver.common.base import Availability, SolverBase +import pytest scip = SolverFactory('scip_direct') ipopt = SolverFactory('ipopt') +class MockNLPSolver(SolverBase): + def __init__(self, varlist, sol_map, **kwds) -> None: + super().__init__(**kwds) + self.varlist = varlist + self.sol_map = sol_map + self.iter = 0 + + def available(self) -> Availability: + return Availability.FullLicense + + def version(self) -> Tuple: + return (1, 0, 0) + + def check_solution(self): + expected, rel_tol, abs_tol = self.sol_map[self.iter] + self.iter += 1 + for v, val in zip(self.varlist, expected): + # print(v, v.value, val) + assert v.value == pytest.approx(val, rel=rel_tol, abs=abs_tol) + + def solve(self, model, **kwds) -> Results: + self.check_solution() + res = Results() + res.termination_condition = TerminationCondition.error + res.solution_status = SolutionStatus.noSolution + res.incumbent_objective = None + res.objective_bound = None + res.solver_name = 'MockNLPSolver' + res.solver_version = self.version() + return res + + @unittest.skipUnless(scip.available(), 'scip is not available') @unittest.skipUnless(ipopt.available(), 'ipopt is not available') class TestExamples(unittest.TestCase): @@ -36,3 +72,119 @@ def test_poly_lp(self): stat, x = main(method=ini.InitializationMethod.lp_approximation) self.assertEqual(stat, SolutionStatus.optimal) self.assertAlmostEqual(x, -9.920159607881597) + + +class TestInit(unittest.TestCase): + def test_lp_init(self): + """ + For this test, we will create a small linear program, + but we will make it look nonlinear. Then, the linear + approximation should be exact. The LP is + + max 3*x1 + 2*x2 + s.t. + x1 + x2 <= 4 + 2*x1 + x2 <= 5 + x1 >= 0 + x2 >= 0 + + The solution is + + x1 = 1 + x2 = 3 + """ + m = pyo.ConcreteModel() + m.x1 = pyo.Var(bounds=(0, 100)) + m.x2 = pyo.Var(bounds=(0, 100)) + m.obj = pyo.Objective(expr=(3*m.x1*m.x1 + 2*m.x2*m.x1)/m.x1, sense=pyo.maximize) + m.c1 = pyo.Constraint(expr=pyo.exp(pyo.log(m.x1 + m.x2)) <= 4) + m.c2 = pyo.Constraint(expr=((2*m.x1 + m.x2)**2)**0.5 <= 5) + + # all the actual testing happens in the MockNLPSolver + nlp_solver = MockNLPSolver( + varlist=[m.x1, m.x2], + sol_map={ + 0: ([None, None], 0, 0), + 1: ([1, 3], 1e-6, 1e-6), + }, + ) + mip_solver = SolverFactory('highs') + results = ini.initialize_nlp( + nlp=m, + nlp_solver=nlp_solver, + mip_solver=mip_solver, + method=ini.InitializationMethod.lp_approximation, + ) + + def test_global_init(self): + """ + Same as test_lp_init + """ + m = pyo.ConcreteModel() + m.x1 = pyo.Var(bounds=(0, 100)) + m.x2 = pyo.Var(bounds=(0, 100)) + m.obj = pyo.Objective(expr=(3*m.x1*m.x1 + 2*m.x2*m.x1)/m.x1, sense=pyo.maximize) + m.c1 = pyo.Constraint(expr=pyo.exp(pyo.log(m.x1 + m.x2)) <= 4) + m.c2 = pyo.Constraint(expr=((2*m.x1 + m.x2)**2)**0.5 <= 5) + + # all the actual testing happens in the MockNLPSolver + nlp_solver = MockNLPSolver( + varlist=[m.x1, m.x2], + sol_map={ + 0: ([None, None], 0, 0), + 1: ([1, 3], 1e-6, 1e-6), + }, + ) + global_solver = SolverFactory('scip_direct') + results = ini.initialize_nlp( + nlp=m, + nlp_solver=nlp_solver, + global_solver=global_solver, + method=ini.InitializationMethod.global_opt, + ) + + def test_pwl_init(self): + """ + Here, we really just want to make sure that the + approximation improves as refinement is done. + """ + m = pyo.ConcreteModel() + m.x1 = pyo.Var(bounds=(0.5, 1.5)) + m.x2 = pyo.Var(bounds=(2.5, 3.5)) + m.obj = pyo.Objective(expr=(3*m.x1*m.x1 + 2*m.x2*m.x1)/m.x1, sense=pyo.maximize) + m.c1 = pyo.Constraint(expr=pyo.exp(pyo.log(m.x1 + m.x2)) <= 4) + m.c2 = pyo.Constraint(expr=((2*m.x1 + m.x2)**2)**0.5 <= 5) + + # all the actual testing happens in the MockNLPSolver + nlp_solver = MockNLPSolver( + varlist=[m.x1, m.x2], + sol_map={ + 0: ([None, None], 0, 0), + 1: ([1, 3], 1e-0, 1e-0), + 2: ([1, 3], 1e-1, 1e-1), + 3: ([1, 3], 1e-2, 1e-2), + 4: ([1, 3], 1e-3, 1e-3), + 5: ([1, 3], 1e-5, 1e-5), + 6: ([1, 3], 1e-6, 1e-6), + 7: ([1, 3], 1e-7, 1e-7), + 8: ([1, 3], 1e-7, 1e-7), + 9: ([1, 3], 1e-7, 1e-7), + 10: ([1, 3], 1e-7, 1e-7), + 11: ([1, 3], 1e-7, 1e-7), + }, + ) + mip_solver = SolverFactory('highs') + results = ini.initialize_nlp( + nlp=m, + nlp_solver=nlp_solver, + mip_solver=mip_solver, + method=ini.InitializationMethod.pwl_approximation, + max_pwl_refinement_iter=10, + ) + + +if __name__ == '__main__': + import logging + logging.basicConfig(level=logging.INFO) + t = TestInit() + t.test_pwl_init() From e52e1236921c09f0fadd84f78a2447740678e3c5 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 12 Apr 2026 19:42:00 -0600 Subject: [PATCH 088/107] debugging pwl initialization --- pyomo/devel/initialization/pwl_init.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pyomo/devel/initialization/pwl_init.py b/pyomo/devel/initialization/pwl_init.py index 786bcc2c699..7e32b8b31e6 100644 --- a/pyomo/devel/initialization/pwl_init.py +++ b/pyomo/devel/initialization/pwl_init.py @@ -292,6 +292,10 @@ def _initialize_with_piecewise_linear_approximation( _minimize_infeasibility(pwl) logger.info('reformulated model to minimize infeasibility') + m_before_pwl = shallow_clone(pwl) + vset = list(ComponentSet(get_vars(m_before_pwl))) + vset.sort(key=lambda i: i.local_name) + # build the PWL approximation trans = pe.TransformationFactory('contrib.piecewise.nonlinear_to_pwl') trans.apply_to(pwl, num_points=2, additively_decompose=False) @@ -329,6 +333,16 @@ def _initialize_with_piecewise_linear_approximation( for ov, nv in zip(orig_vars, new_vars): ov.set_value(nv.value, skip_validation=True) + print('\n\n**********************') + print(m_before_pwl.obj.expr) + print('\nConstraints:') + for c in m_before_pwl.component_data_objects(pe.Constraint, active=True, descend_into=True): + print(c) + print(' ', c.expr) + for v in vset: + print(v, v.value) + m_before_pwl.display() + # refine the PWL approximation _refine_pwl_approx( pwl, From 5b4fe02706a685c537290a77e818b6ef0e3aaec4 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 14 Apr 2026 08:19:16 -0600 Subject: [PATCH 089/107] comment out print statements --- pyomo/devel/initialization/pwl_init.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pyomo/devel/initialization/pwl_init.py b/pyomo/devel/initialization/pwl_init.py index 7e32b8b31e6..3d2a37251ed 100644 --- a/pyomo/devel/initialization/pwl_init.py +++ b/pyomo/devel/initialization/pwl_init.py @@ -292,9 +292,9 @@ def _initialize_with_piecewise_linear_approximation( _minimize_infeasibility(pwl) logger.info('reformulated model to minimize infeasibility') - m_before_pwl = shallow_clone(pwl) - vset = list(ComponentSet(get_vars(m_before_pwl))) - vset.sort(key=lambda i: i.local_name) + # m_before_pwl = shallow_clone(pwl) + # vset = list(ComponentSet(get_vars(m_before_pwl))) + # vset.sort(key=lambda i: i.local_name) # build the PWL approximation trans = pe.TransformationFactory('contrib.piecewise.nonlinear_to_pwl') @@ -333,15 +333,15 @@ def _initialize_with_piecewise_linear_approximation( for ov, nv in zip(orig_vars, new_vars): ov.set_value(nv.value, skip_validation=True) - print('\n\n**********************') - print(m_before_pwl.obj.expr) - print('\nConstraints:') - for c in m_before_pwl.component_data_objects(pe.Constraint, active=True, descend_into=True): - print(c) - print(' ', c.expr) - for v in vset: - print(v, v.value) - m_before_pwl.display() + # print('\n\n**********************') + # print(m_before_pwl.obj.expr) + # print('\nConstraints:') + # for c in m_before_pwl.component_data_objects(pe.Constraint, active=True, descend_into=True): + # print(c) + # print(' ', c.expr) + # for v in vset: + # print(v, v.value) + # m_before_pwl.display() # refine the PWL approximation _refine_pwl_approx( From 135b1527d70762aee4ccd0f6c2422c0eea0520fd Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 15 Apr 2026 05:54:18 -0600 Subject: [PATCH 090/107] initialization: testing --- pyomo/devel/initialization/initialize.py | 2 + pyomo/devel/initialization/pwl_init.py | 3 +- .../tests/test_initialization.py | 54 ++++++++++++------- 3 files changed, 38 insertions(+), 21 deletions(-) diff --git a/pyomo/devel/initialization/initialize.py b/pyomo/devel/initialization/initialize.py index e240e46a12c..64439212b1e 100644 --- a/pyomo/devel/initialization/initialize.py +++ b/pyomo/devel/initialization/initialize.py @@ -49,6 +49,7 @@ def initialize_nlp( default_bound: float = 1.0e8, max_pwl_refinement_iter: int = 100, num_pwl_cons_to_refine_per_iter: int = 5, + aggressive_substitution: bool = True, ) -> Results: """ Attempt to initialize and subsequently solve the model given by ``nlp``. @@ -123,6 +124,7 @@ def initialize_nlp( default_bound=default_bound, max_iter=max_pwl_refinement_iter, num_cons_to_refine_per_iter=num_pwl_cons_to_refine_per_iter, + aggressive_substitution=aggressive_substitution, ) elif method == InitializationMethod.lp_approximation: if mip_solver is None: diff --git a/pyomo/devel/initialization/pwl_init.py b/pyomo/devel/initialization/pwl_init.py index 3d2a37251ed..d3b6c3c25c6 100644 --- a/pyomo/devel/initialization/pwl_init.py +++ b/pyomo/devel/initialization/pwl_init.py @@ -265,6 +265,7 @@ def _initialize_with_piecewise_linear_approximation( default_bound=1.0e8, max_iter=100, num_cons_to_refine_per_iter=5, + aggressive_substitution=True, ): logger.info('Starting initialization using a piecewise linear approximation') pwl = shallow_clone(nlp) @@ -273,7 +274,7 @@ def _initialize_with_piecewise_linear_approximation( # first introduce auxiliary variables so that we don't try to # approximate any functions of more than two variables trans = pe.TransformationFactory('contrib.piecewise.univariate_nonlinear_decomposition') - trans.apply_to(pwl, aggressive_substitution=True) + trans.apply_to(pwl, aggressive_substitution=aggressive_substitution) logger.info('applied the univariate_nonlinear_decomposition transformation') # now we need to try to get bounds on all of the nonlinear variables diff --git a/pyomo/devel/initialization/tests/test_initialization.py b/pyomo/devel/initialization/tests/test_initialization.py index 177f32cec76..0ba33881570 100644 --- a/pyomo/devel/initialization/tests/test_initialization.py +++ b/pyomo/devel/initialization/tests/test_initialization.py @@ -40,7 +40,6 @@ def check_solution(self): expected, rel_tol, abs_tol = self.sol_map[self.iter] self.iter += 1 for v, val in zip(self.varlist, expected): - # print(v, v.value, val) assert v.value == pytest.approx(val, rel=rel_tol, abs=abs_tol) def solve(self, model, **kwds) -> Results: @@ -149,28 +148,42 @@ def test_pwl_init(self): approximation improves as refinement is done. """ m = pyo.ConcreteModel() - m.x1 = pyo.Var(bounds=(0.5, 1.5)) - m.x2 = pyo.Var(bounds=(2.5, 3.5)) - m.obj = pyo.Objective(expr=(3*m.x1*m.x1 + 2*m.x2*m.x1)/m.x1, sense=pyo.maximize) - m.c1 = pyo.Constraint(expr=pyo.exp(pyo.log(m.x1 + m.x2)) <= 4) - m.c2 = pyo.Constraint(expr=((2*m.x1 + m.x2)**2)**0.5 <= 5) + m.x = pyo.Var(bounds=(-15, 5)) + m.c = pyo.Constraint(expr=(m.x + 7) * (m.x + 5) * (m.x - 4) + 200 == 0) + m.obj = pyo.Objective(expr=m.x) # all the actual testing happens in the MockNLPSolver nlp_solver = MockNLPSolver( - varlist=[m.x1, m.x2], + varlist=[m.x], sol_map={ - 0: ([None, None], 0, 0), - 1: ([1, 3], 1e-0, 1e-0), - 2: ([1, 3], 1e-1, 1e-1), - 3: ([1, 3], 1e-2, 1e-2), - 4: ([1, 3], 1e-3, 1e-3), - 5: ([1, 3], 1e-5, 1e-5), - 6: ([1, 3], 1e-6, 1e-6), - 7: ([1, 3], 1e-7, 1e-7), - 8: ([1, 3], 1e-7, 1e-7), - 9: ([1, 3], 1e-7, 1e-7), - 10: ([1, 3], 1e-7, 1e-7), - 11: ([1, 3], 1e-7, 1e-7), + 0: ([None], 0, 0), + 1: ([1.0975609756097562], 1e-6, 1e-6), + 2: ([0.4346767574185112], 1e-6, 1e-6), + 3: ([-0.19286405313201946], 1e-6, 1e-6), + 4: ([-0.8653073960726083], 1e-6, 1e-6), + 5: ([-1.6404750700409576], 1e-6, 1e-6), + 6: ([-2.5676344169949443], 1e-6, 1e-6), + 7: ([-3.6759614495828297], 1e-6, 1e-6), + 8: ([-4.942429761325623], 1e-6, 1e-6), + 9: ([-6.259703235160286], 1e-6, 1e-6), + 10: ([-7.457220752001633], 1e-6, 1e-6), + 11: ([-8.393746738936832], 1e-6, 1e-6), + 12: ([-9.032852172775847], 1e-6, 1e-6), + 13: ([-9.426202540402329], 1e-6, 1e-6), + 14: ([-9.652335186743512], 1e-6, 1e-6), + 15: ([-9.777115808257673], 1e-6, 1e-6), + 16: ([-9.844390507666596], 1e-6, 1e-6), + 17: ([-9.880203709976758], 1e-6, 1e-6), + 18: ([-9.899139197799068], 1e-6, 1e-6), + 19: ([-9.90911480313665], 1e-6, 1e-6), + 20: ([-9.914360132504347], 1e-6, 1e-6), + 21: ([-9.91711543776506], 1e-6, 1e-6), + 22: ([-9.918562000338856], 1e-6, 1e-6), + 23: ([-9.919321249329018], 1e-6, 1e-6), + 24: ([-9.919719693944147], 1e-6, 1e-6), + 25: ([-9.91992877683681], 1e-6, 1e-6), + 26: ([-9.920038488200985], 1e-6, 1e-6), + 27: ([-9.920096055464825], 1e-6, 1e-6), }, ) mip_solver = SolverFactory('highs') @@ -179,7 +192,8 @@ def test_pwl_init(self): nlp_solver=nlp_solver, mip_solver=mip_solver, method=ini.InitializationMethod.pwl_approximation, - max_pwl_refinement_iter=10, + max_pwl_refinement_iter=27, + aggressive_substitution=False, ) From fc60afeb75d7f1e61a61b6b64c740a60ff6c0d95 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 15 Apr 2026 05:57:13 -0600 Subject: [PATCH 091/107] initialization: update docstring; remove debugging code --- pyomo/devel/initialization/initialize.py | 4 ++++ pyomo/devel/initialization/pwl_init.py | 14 -------------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/pyomo/devel/initialization/initialize.py b/pyomo/devel/initialization/initialize.py index 64439212b1e..1369db1a1f1 100644 --- a/pyomo/devel/initialization/initialize.py +++ b/pyomo/devel/initialization/initialize.py @@ -89,6 +89,10 @@ def initialize_nlp( Only used when method = InitializationMethod.pwl_approximation. This is the maximum number of constraints to be refined with additional segments in the piecewise linear approximation each iteration. + aggressive_substitution: bool + Only used when method = InitializationMethod.pwl_approximation. This is + passed along to the contrib.piecewise.univariate_nonlinear_decomposition + transformation. Returns ------- diff --git a/pyomo/devel/initialization/pwl_init.py b/pyomo/devel/initialization/pwl_init.py index d3b6c3c25c6..551e33a2f9a 100644 --- a/pyomo/devel/initialization/pwl_init.py +++ b/pyomo/devel/initialization/pwl_init.py @@ -293,10 +293,6 @@ def _initialize_with_piecewise_linear_approximation( _minimize_infeasibility(pwl) logger.info('reformulated model to minimize infeasibility') - # m_before_pwl = shallow_clone(pwl) - # vset = list(ComponentSet(get_vars(m_before_pwl))) - # vset.sort(key=lambda i: i.local_name) - # build the PWL approximation trans = pe.TransformationFactory('contrib.piecewise.nonlinear_to_pwl') trans.apply_to(pwl, num_points=2, additively_decompose=False) @@ -334,16 +330,6 @@ def _initialize_with_piecewise_linear_approximation( for ov, nv in zip(orig_vars, new_vars): ov.set_value(nv.value, skip_validation=True) - # print('\n\n**********************') - # print(m_before_pwl.obj.expr) - # print('\nConstraints:') - # for c in m_before_pwl.component_data_objects(pe.Constraint, active=True, descend_into=True): - # print(c) - # print(' ', c.expr) - # for v in vset: - # print(v, v.value) - # m_before_pwl.display() - # refine the PWL approximation _refine_pwl_approx( pwl, From 48bae5223495427db0adfd840da74cefbca703a4 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 15 Apr 2026 05:58:27 -0600 Subject: [PATCH 092/107] run black --- pyomo/devel/initialization/__init__.py | 2 +- .../initialization/bounds/bound_variables.py | 13 ++- .../examples/init_polynomial_ex.py | 2 +- pyomo/devel/initialization/global_init.py | 30 ++++-- pyomo/devel/initialization/initialize.py | 39 ++++---- pyomo/devel/initialization/lp_approx_init.py | 52 +++++++---- pyomo/devel/initialization/pwl_init.py | 93 ++++++++++++------- .../tests/test_initialization.py | 64 ++++++------- pyomo/devel/initialization/utils.py | 10 +- 9 files changed, 187 insertions(+), 118 deletions(-) diff --git a/pyomo/devel/initialization/__init__.py b/pyomo/devel/initialization/__init__.py index 42e52b2d8ff..0c9fd0b1950 100644 --- a/pyomo/devel/initialization/__init__.py +++ b/pyomo/devel/initialization/__init__.py @@ -7,4 +7,4 @@ # software. This software is distributed under the 3-clause BSD License. # ____________________________________________________________________________________ -from pyomo.devel.initialization.initialize import initialize_nlp, InitializationMethod \ No newline at end of file +from pyomo.devel.initialization.initialize import initialize_nlp, InitializationMethod diff --git a/pyomo/devel/initialization/bounds/bound_variables.py b/pyomo/devel/initialization/bounds/bound_variables.py index 4402ea76510..ca6eabbc7f8 100644 --- a/pyomo/devel/initialization/bounds/bound_variables.py +++ b/pyomo/devel/initialization/bounds/bound_variables.py @@ -12,22 +12,25 @@ from pyomo.devel.initialization.utils import get_vars import logging - logger = logging.getLogger(__name__) def bound_all_nonlinear_variables(m: BlockData, default_bound: float = 1.0e8): """ - Attempt to obtain valid bounds on all nonlinear variables based on the - constraints in the model, m. If variable bounds cannot be obtained, + Attempt to obtain valid bounds on all nonlinear variables based on the + constraints in the model, m. If variable bounds cannot be obtained, we use default_bound. """ fbbt(m) for v in get_vars(m): if v.lb is None or v.lb < -default_bound: - logger.debug(f'Could not obtain a lower bound for {str(v)} better than {-default_bound}; setting the lower bound to {-default_bound}') + logger.debug( + f'Could not obtain a lower bound for {str(v)} better than {-default_bound}; setting the lower bound to {-default_bound}' + ) v.setlb(-default_bound) if v.ub is None or v.ub > default_bound: - logger.debug(f'Could not obtain an upper bound for {str(v)} better than {default_bound}; setting the upper bound to {default_bound}') + logger.debug( + f'Could not obtain an upper bound for {str(v)} better than {default_bound}; setting the upper bound to {default_bound}' + ) v.setub(default_bound) fbbt(m) diff --git a/pyomo/devel/initialization/examples/init_polynomial_ex.py b/pyomo/devel/initialization/examples/init_polynomial_ex.py index 111ad418363..d327c6c9e3c 100644 --- a/pyomo/devel/initialization/examples/init_polynomial_ex.py +++ b/pyomo/devel/initialization/examples/init_polynomial_ex.py @@ -16,7 +16,7 @@ def build_model(): m = pyo.ConcreteModel() m.x = pyo.Var(bounds=(-20, 20), initialize=-3.6) - m.c = pyo.Constraint(expr=(m.x+7)*(m.x+5)*(m.x-4) + 200 == 0) + m.c = pyo.Constraint(expr=(m.x + 7) * (m.x + 5) * (m.x - 4) + 200 == 0) return m diff --git a/pyomo/devel/initialization/global_init.py b/pyomo/devel/initialization/global_init.py index b924fb61273..c573e4e8162 100644 --- a/pyomo/devel/initialization/global_init.py +++ b/pyomo/devel/initialization/global_init.py @@ -14,28 +14,38 @@ from pyomo.contrib.solver.solvers.gurobi.gurobi_direct_minlp import GurobiDirectMINLP import logging - logger = logging.getLogger(__name__) def _initialize_with_global_solver( - nlp: BlockData, - global_solver: SolverBase, - nlp_solver: SolverBase, + nlp: BlockData, global_solver: SolverBase, nlp_solver: SolverBase ): if isinstance(global_solver, (ScipDirect, ScipPersistent)): opts = {'limits/solutions': 1} elif isinstance(global_solver, (GurobiDirectMINLP,)): opts = {'SolutionLimit': 1} else: - raise NotImplementedError('Currently, the initialization module only works with new solver interface, so the global solvers are limited to ScipDirect, ScipPersistent, and GurobiDirectMINLP.') - res = global_solver.solve(nlp, load_solutions=True, raise_exception_on_nonoptimal_result=False, solver_options=opts) - logger.info(f'solved NLP with {global_solver.name}: {res.solution_status}, {res.termination_condition}') - res = nlp_solver.solve(nlp, load_solutions=False, raise_exception_on_nonoptimal_result=False) - logger.info(f'solved NLP with {nlp_solver.name}: {res.solution_status}, {res.termination_condition}') + raise NotImplementedError( + 'Currently, the initialization module only works with new solver interface, so the global solvers are limited to ScipDirect, ScipPersistent, and GurobiDirectMINLP.' + ) + res = global_solver.solve( + nlp, + load_solutions=True, + raise_exception_on_nonoptimal_result=False, + solver_options=opts, + ) + logger.info( + f'solved NLP with {global_solver.name}: {res.solution_status}, {res.termination_condition}' + ) + res = nlp_solver.solve( + nlp, load_solutions=False, raise_exception_on_nonoptimal_result=False + ) + logger.info( + f'solved NLP with {nlp_solver.name}: {res.solution_status}, {res.termination_condition}' + ) if res.solution_status in {SolutionStatus.feasible, SolutionStatus.optimal}: res.solution_loader.load_vars() else: logger.warning('initialization was not successful via global optimization') - + return res diff --git a/pyomo/devel/initialization/initialize.py b/pyomo/devel/initialization/initialize.py index 1369db1a1f1..759a4887f88 100644 --- a/pyomo/devel/initialization/initialize.py +++ b/pyomo/devel/initialization/initialize.py @@ -12,7 +12,9 @@ from enum import Enum from pyomo.devel.initialization.utils import get_vars, shallow_clone from pyomo.common.collections import ComponentMap -from pyomo.devel.initialization.pwl_init import _initialize_with_piecewise_linear_approximation +from pyomo.devel.initialization.pwl_init import ( + _initialize_with_piecewise_linear_approximation, +) from pyomo.devel.initialization.lp_approx_init import _initialize_with_LP_approximation from pyomo.contrib.solver.common.base import SolverBase from pyomo.devel.initialization.global_init import _initialize_with_global_solver @@ -21,7 +23,6 @@ import logging from pyomo.contrib.solver.common.results import SolutionStatus - logger = logging.getLogger(__name__) @@ -36,12 +37,14 @@ def _get_solver(sname, reason): if opt.available(): logger.info(f'Using {sname} for {reason} because a solver was not specified') else: - raise RuntimeError(f'No solver was specified for {reason} and the default ({sname}) is not available') + raise RuntimeError( + f'No solver was specified for {reason} and the default ({sname}) is not available' + ) return opt def initialize_nlp( - nlp: BlockData, + nlp: BlockData, nlp_solver: SolverBase, mip_solver: Optional[SolverBase] = None, global_solver: Optional[SolverBase] = None, @@ -53,7 +56,7 @@ def initialize_nlp( ) -> Results: """ Attempt to initialize and subsequently solve the model given by ``nlp``. - The basic idea is to apply some method to find good initial values for + The basic idea is to apply some method to find good initial values for the variables and then try to solve the problem with ``nlp_solver``. Parameters @@ -76,32 +79,34 @@ def initialize_nlp( The method used to initialize the model. default_bound: float Some initialize methods require all nonlinear variables to be bounded. - For these methods, all unbounded variables will be given lower and + For these methods, all unbounded variables will be given lower and upper bounds equal to default_bound. Needed for the following methods: - pwl_approximation - lp_approximation max_pwl_refinement_iter: int - Only used when method = InitializationMethod.pwl_approximation. This is + Only used when method = InitializationMethod.pwl_approximation. This is the maximum number of iterations used to refine the piecewise linear approximation. num_pwl_cons_to_refine_per_iter: int Only used when method = InitializationMethod.pwl_approximation. This is - the maximum number of constraints to be refined with additional + the maximum number of constraints to be refined with additional segments in the piecewise linear approximation each iteration. aggressive_substitution: bool - Only used when method = InitializationMethod.pwl_approximation. This is + Only used when method = InitializationMethod.pwl_approximation. This is passed along to the contrib.piecewise.univariate_nonlinear_decomposition - transformation. + transformation. Returns ------- res: pyomo.contrib.solver.common.results.Results - The results object obtained the last time the nlp_solver was used to + The results object obtained the last time the nlp_solver was used to try and solve the model. """ # in all cases, try to solve the nlp before doing extra work - res = nlp_solver.solve(nlp, load_solutions=False, raise_exception_on_nonoptimal_result=False) + res = nlp_solver.solve( + nlp, load_solutions=False, raise_exception_on_nonoptimal_result=False + ) logger.info(f'solved NLP: {res.solution_status}, {res.termination_condition}') if res.solution_status == SolutionStatus.optimal: @@ -136,18 +141,18 @@ def initialize_nlp( if nlp_solver is None: nlp_solver = _get_solver('ipopt', 'local NLP solver') res = _initialize_with_LP_approximation( - nlp=nlp, - lp_solver=mip_solver, - nlp_solver=nlp_solver, + nlp=nlp, lp_solver=mip_solver, nlp_solver=nlp_solver ) elif method == InitializationMethod.global_opt: if global_solver is None: global_solver = _get_solver('gurobi_direct_minlp', 'global NLP solver') if nlp_solver is None: nlp_solver = _get_solver('ipopt', 'local NLP solver') - res = _initialize_with_global_solver(nlp=nlp, global_solver=global_solver, nlp_solver=nlp_solver) + res = _initialize_with_global_solver( + nlp=nlp, global_solver=global_solver, nlp_solver=nlp_solver + ) else: - raise ValueError(f'unexpected initialization method: {method}') + raise ValueError(f'unexpected initialization method: {method}') # restore variable bounds, domain, etc. for v, (lb, ub, domain, fixed, value) in orig_var_data.items(): diff --git a/pyomo/devel/initialization/lp_approx_init.py b/pyomo/devel/initialization/lp_approx_init.py index 91537436e10..583e467c912 100644 --- a/pyomo/devel/initialization/lp_approx_init.py +++ b/pyomo/devel/initialization/lp_approx_init.py @@ -9,10 +9,18 @@ from pyomo.core.base.block import BlockData import pyomo.environ as pe -from pyomo.devel.initialization.bounds.bound_variables import bound_all_nonlinear_variables -from pyomo.devel.initialization.utils import fix_vars_with_equal_bounds, shallow_clone, get_vars +from pyomo.devel.initialization.bounds.bound_variables import ( + bound_all_nonlinear_variables, +) +from pyomo.devel.initialization.utils import ( + fix_vars_with_equal_bounds, + shallow_clone, + get_vars, +) from pyomo.core.expr.visitor import identify_components -from pyomo.contrib.piecewise.piecewise_linear_expression import PiecewiseLinearExpression +from pyomo.contrib.piecewise.piecewise_linear_expression import ( + PiecewiseLinearExpression, +) from pyomo.contrib.piecewise.piecewise_linear_function import PiecewiseLinearFunction from pyomo.contrib.solver.common.results import SolutionStatus from pyomo.common.collections import ComponentMap, ComponentSet @@ -37,7 +45,11 @@ NPV_SumExpression, NPV_UnaryFunctionExpression, ) -from pyomo.core.expr.relational_expr import EqualityExpression, InequalityExpression, RangedExpression +from pyomo.core.expr.relational_expr import ( + EqualityExpression, + InequalityExpression, + RangedExpression, +) from pyomo.repn.util import ExitNodeDispatcher from pyomo.core.base.var import ScalarVar, VarData from pyomo.core.base.param import ScalarParam, ParamData @@ -53,7 +65,6 @@ import numpy as np from scipy.stats import qmc - logger = logging.getLogger(__name__) @@ -105,10 +116,14 @@ def _build_lp_approx(nlp: BlockData) -> BlockData: lp.cons = pe.ConstraintList() visitor = LinearRepnVisitor(subexpression_cache={}) - objs = list(nlp.component_data_objects(pe.Objective, active=True, descend_into=True)) + objs = list( + nlp.component_data_objects(pe.Objective, active=True, descend_into=True) + ) if objs: if len(objs) > 1: - raise NotImplementedError('lp approximation does not support multiple objectives') + raise NotImplementedError( + 'lp approximation does not support multiple objectives' + ) obj = objs[0] repn = visitor.walk_expression(obj) assert repn.multiplier == 1 @@ -123,7 +138,9 @@ def _build_lp_approx(nlp: BlockData) -> BlockData: new_obj_expr += replacement lp.obj = pe.Objective(expr=new_obj_expr, sense=obj.sense) - for con in nlp.component_data_objects(pe.Constraint, active=True, descend_into=True): + for con in nlp.component_data_objects( + pe.Constraint, active=True, descend_into=True + ): lb, body, ub = con.to_bounded_expression() repn = visitor.walk_expression(body) assert repn.multiplier == 1 @@ -144,17 +161,14 @@ def _build_lp_approx(nlp: BlockData) -> BlockData: def _initialize_with_LP_approximation( - nlp: BlockData, - lp_solver: SolverBase, - nlp_solver: SolverBase, - default_bound=1.0e8, + nlp: BlockData, lp_solver: SolverBase, nlp_solver: SolverBase, default_bound=1.0e8 ): orig_nlp = nlp logger.info('Starting initialization using a linear programming approximation') nlp = shallow_clone(nlp) logger.info('created a shallow clone of the model') - # first introduce auxiliary variables so that we don't try to + # first introduce auxiliary variables so that we don't try to # approximate any functions of more than two variables # actually, this is not necessary for this method # we will just comment this out for now @@ -181,12 +195,18 @@ def _initialize_with_LP_approximation( logger.info('replaced nonlinear expressions with linear approximations') # solve the LP - lp_res = lp_solver.solve(lp, load_solutions=True, raise_exception_on_nonoptimal_result=False) + lp_res = lp_solver.solve( + lp, load_solutions=True, raise_exception_on_nonoptimal_result=False + ) logger.info(f'solved LP: {lp_res.solution_status}, {lp_res.termination_condition}') # try solving the NLP - nlp_res = nlp_solver.solve(orig_nlp, load_solutions=False, raise_exception_on_nonoptimal_result=False) - logger.info(f'solved NLP: {nlp_res.solution_status}, {nlp_res.termination_condition}') + nlp_res = nlp_solver.solve( + orig_nlp, load_solutions=False, raise_exception_on_nonoptimal_result=False + ) + logger.info( + f'solved NLP: {nlp_res.solution_status}, {nlp_res.termination_condition}' + ) if nlp_res.solution_status in {SolutionStatus.feasible, SolutionStatus.optimal}: nlp_res.solution_loader.load_vars() diff --git a/pyomo/devel/initialization/pwl_init.py b/pyomo/devel/initialization/pwl_init.py index 551e33a2f9a..f5513e34bd3 100644 --- a/pyomo/devel/initialization/pwl_init.py +++ b/pyomo/devel/initialization/pwl_init.py @@ -9,10 +9,18 @@ from pyomo.core.base.block import BlockData import pyomo.environ as pe -from pyomo.devel.initialization.bounds.bound_variables import bound_all_nonlinear_variables -from pyomo.devel.initialization.utils import fix_vars_with_equal_bounds, shallow_clone, get_vars +from pyomo.devel.initialization.bounds.bound_variables import ( + bound_all_nonlinear_variables, +) +from pyomo.devel.initialization.utils import ( + fix_vars_with_equal_bounds, + shallow_clone, + get_vars, +) from pyomo.core.expr.visitor import identify_components -from pyomo.contrib.piecewise.piecewise_linear_expression import PiecewiseLinearExpression +from pyomo.contrib.piecewise.piecewise_linear_expression import ( + PiecewiseLinearExpression, +) from pyomo.contrib.piecewise.piecewise_linear_function import PiecewiseLinearFunction from pyomo.common.collections import ComponentMap, ComponentSet from typing import MutableMapping, Sequence, List @@ -36,7 +44,11 @@ NPV_SumExpression, NPV_UnaryFunctionExpression, ) -from pyomo.core.expr.relational_expr import EqualityExpression, InequalityExpression, RangedExpression +from pyomo.core.expr.relational_expr import ( + EqualityExpression, + InequalityExpression, + RangedExpression, +) from pyomo.repn.util import ExitNodeDispatcher from pyomo.core.base.var import ScalarVar, VarData from pyomo.core.base.param import ScalarParam, ParamData @@ -47,7 +59,6 @@ from pyomo.common.modeling import unique_component_name from pyomo.contrib.solver.common.results import SolutionStatus - logger = logging.getLogger(__name__) @@ -61,9 +72,9 @@ def _minimize_infeasibility(m): for obj in m.component_data_objects(pe.Objective, active=True, descend_into=True): assert not found_obj if obj.sense == pe.minimize: - obj_expr += 0.1*obj.expr + obj_expr += 0.1 * obj.expr else: - obj_expr -= 0.1*obj.expr + obj_expr -= 0.1 * obj.expr obj.deactivate() found_obj = True @@ -92,18 +103,21 @@ def _minimize_infeasibility(m): m.extra_cons.add(body - ub - ps <= 0) m.extra_cons.add(body - lb + ns >= 0) - m.slack_obj = pe.Objective(expr=10*sum(m.slacks.values()) + obj_expr) + m.slack_obj = pe.Objective(expr=10 * sum(m.slacks.values()) + obj_expr) -def _get_pwl_constraints(m: BlockData) -> MutableMapping[ - PiecewiseLinearExpression, - List[ConstraintData] -]: +def _get_pwl_constraints( + m: BlockData, +) -> MutableMapping[PiecewiseLinearExpression, List[ConstraintData]]: comp_types = set() comp_types.add(PiecewiseLinearExpression) pwl_expr_to_con_map = ComponentMap() - con_list = list(m.component_data_objects(pe.Constraint, active=True, descend_into=True)) - obj_list = list(m.component_data_objects(pe.Objective, active=True, descend_into=True)) + con_list = list( + m.component_data_objects(pe.Constraint, active=True, descend_into=True) + ) + obj_list = list( + m.component_data_objects(pe.Objective, active=True, descend_into=True) + ) for comp in con_list + obj_list: pwl_exprs = list(identify_components(comp.expr, comp_types)) if not pwl_exprs: @@ -128,11 +142,11 @@ def _handle_node(node, data): for t in [float, int, VarData, ScalarVar, ParamData, ScalarParam, NumericConstant]: _handlers[t] = _handle_leaf for t in [ - ProductExpression, - SumExpression, - DivisionExpression, - PowExpression, - MonomialTermExpression, + ProductExpression, + SumExpression, + DivisionExpression, + PowExpression, + MonomialTermExpression, LinearExpression, ExpressionData, ScalarExpression, @@ -170,7 +184,7 @@ def exitNode(self, node, data): return _handle_leaf(node, data) else: raise NotImplementedError(f'unrecognized expression type: {nt}') - + def beforeChild(self, node, child, child_idx): if child in self.substitution: return False, self.substitution[child] @@ -188,7 +202,9 @@ def beforeChild(self, node, child, child_idx): if len(points[0]) == 1: points = [i[0] for i in points] new_func = PiecewiseLinearFunction(points=points, function=_func) - fname = unique_component_name(self.m.auxiliary._pyomo_contrib_nonlinear_to_pwl, 'f') + fname = unique_component_name( + self.m.auxiliary._pyomo_contrib_nonlinear_to_pwl, 'f' + ) setattr(self.m.auxiliary._pyomo_contrib_nonlinear_to_pwl, fname, new_func) new_expr = new_func(*variables) for v, val in zip(variables, var_values): @@ -201,8 +217,7 @@ def beforeChild(self, node, child, child_idx): def _refine_pwl_approx( m, pwl_expr_to_con_map: MutableMapping[ - PiecewiseLinearExpression, - Sequence[ConstraintData], + PiecewiseLinearExpression, Sequence[ConstraintData] ], num_to_refine: int = 5, ): @@ -235,12 +250,14 @@ def _refine_pwl_approx( violations.sort(key=lambda i: i[0], reverse=True) if len(violations) == 0: - raise RuntimeError('Did not find any piecewise linear functions with variable values') - + raise RuntimeError( + 'Did not find any piecewise linear functions with variable values' + ) + tol = 1e-5 if math.isclose(violations[0][0], 0, abs_tol=tol): logger.info('All of the original nonlinear functions are satisfied!') - + violations = [i for i in violations if i[0] > tol] for err, expr in violations[:num_to_refine]: @@ -259,11 +276,11 @@ def _refine_pwl_approx( def _initialize_with_piecewise_linear_approximation( - nlp: BlockData, + nlp: BlockData, mip_solver: SolverBase, nlp_solver: SolverBase, - default_bound=1.0e8, - max_iter=100, + default_bound=1.0e8, + max_iter=100, num_cons_to_refine_per_iter=5, aggressive_substitution=True, ): @@ -271,9 +288,11 @@ def _initialize_with_piecewise_linear_approximation( pwl = shallow_clone(nlp) logger.info('created a shallow clone of the model') - # first introduce auxiliary variables so that we don't try to + # first introduce auxiliary variables so that we don't try to # approximate any functions of more than two variables - trans = pe.TransformationFactory('contrib.piecewise.univariate_nonlinear_decomposition') + trans = pe.TransformationFactory( + 'contrib.piecewise.univariate_nonlinear_decomposition' + ) trans.apply_to(pwl, aggressive_substitution=aggressive_substitution) logger.info('applied the univariate_nonlinear_decomposition transformation') @@ -288,7 +307,7 @@ def _initialize_with_piecewise_linear_approximation( # now we modify the model by introducing slacks to make sure the PWL # approximatin is feasible - # all of the slacks appear linearly, so we don't need to worry about + # all of the slacks appear linearly, so we don't need to worry about # upper bounds for them _minimize_infeasibility(pwl) logger.info('reformulated model to minimize infeasibility') @@ -323,10 +342,12 @@ def _initialize_with_piecewise_linear_approximation( logger.info('applied the disaggregated logarithmic transformation') # solve the MILP - res = mip_solver.solve(_pwl, load_solutions=True, raise_exception_on_nonoptimal_result=False) + res = mip_solver.solve( + _pwl, load_solutions=True, raise_exception_on_nonoptimal_result=False + ) logger.info(f'solved MILP: {res.solution_status}, {res.termination_condition}') - #load the variable values back into orig_vars + # load the variable values back into orig_vars for ov, nv in zip(orig_vars, new_vars): ov.set_value(nv.value, skip_validation=True) @@ -339,7 +360,9 @@ def _initialize_with_piecewise_linear_approximation( logger.info('refined PWL approximation') # try solving the NLP - res = nlp_solver.solve(nlp, load_solutions=False, raise_exception_on_nonoptimal_result=False) + res = nlp_solver.solve( + nlp, load_solutions=False, raise_exception_on_nonoptimal_result=False + ) last_nlp_res = res logger.info(f'solved NLP: {res.solution_status}, {res.termination_condition}') if res.solution_status in {SolutionStatus.feasible, SolutionStatus.optimal}: diff --git a/pyomo/devel/initialization/tests/test_initialization.py b/pyomo/devel/initialization/tests/test_initialization.py index 0ba33881570..bf78d3ee9f2 100644 --- a/pyomo/devel/initialization/tests/test_initialization.py +++ b/pyomo/devel/initialization/tests/test_initialization.py @@ -14,11 +14,14 @@ from pyomo.devel.initialization.examples.init_polynomial_ex import main from pyomo.common import unittest from pyomo.contrib.solver.common.factory import SolverFactory -from pyomo.contrib.solver.common.results import SolutionStatus, Results, TerminationCondition +from pyomo.contrib.solver.common.results import ( + SolutionStatus, + Results, + TerminationCondition, +) from pyomo.contrib.solver.common.base import Availability, SolverBase import pytest - scip = SolverFactory('scip_direct') ipopt = SolverFactory('ipopt') @@ -32,16 +35,16 @@ def __init__(self, varlist, sol_map, **kwds) -> None: def available(self) -> Availability: return Availability.FullLicense - + def version(self) -> Tuple: return (1, 0, 0) - + def check_solution(self): expected, rel_tol, abs_tol = self.sol_map[self.iter] self.iter += 1 for v, val in zip(self.varlist, expected): assert v.value == pytest.approx(val, rel=rel_tol, abs=abs_tol) - + def solve(self, model, **kwds) -> Results: self.check_solution() res = Results() @@ -95,17 +98,16 @@ def test_lp_init(self): m = pyo.ConcreteModel() m.x1 = pyo.Var(bounds=(0, 100)) m.x2 = pyo.Var(bounds=(0, 100)) - m.obj = pyo.Objective(expr=(3*m.x1*m.x1 + 2*m.x2*m.x1)/m.x1, sense=pyo.maximize) + m.obj = pyo.Objective( + expr=(3 * m.x1 * m.x1 + 2 * m.x2 * m.x1) / m.x1, sense=pyo.maximize + ) m.c1 = pyo.Constraint(expr=pyo.exp(pyo.log(m.x1 + m.x2)) <= 4) - m.c2 = pyo.Constraint(expr=((2*m.x1 + m.x2)**2)**0.5 <= 5) + m.c2 = pyo.Constraint(expr=((2 * m.x1 + m.x2) ** 2) ** 0.5 <= 5) # all the actual testing happens in the MockNLPSolver nlp_solver = MockNLPSolver( - varlist=[m.x1, m.x2], - sol_map={ - 0: ([None, None], 0, 0), - 1: ([1, 3], 1e-6, 1e-6), - }, + varlist=[m.x1, m.x2], + sol_map={0: ([None, None], 0, 0), 1: ([1, 3], 1e-6, 1e-6)}, ) mip_solver = SolverFactory('highs') results = ini.initialize_nlp( @@ -122,17 +124,16 @@ def test_global_init(self): m = pyo.ConcreteModel() m.x1 = pyo.Var(bounds=(0, 100)) m.x2 = pyo.Var(bounds=(0, 100)) - m.obj = pyo.Objective(expr=(3*m.x1*m.x1 + 2*m.x2*m.x1)/m.x1, sense=pyo.maximize) + m.obj = pyo.Objective( + expr=(3 * m.x1 * m.x1 + 2 * m.x2 * m.x1) / m.x1, sense=pyo.maximize + ) m.c1 = pyo.Constraint(expr=pyo.exp(pyo.log(m.x1 + m.x2)) <= 4) - m.c2 = pyo.Constraint(expr=((2*m.x1 + m.x2)**2)**0.5 <= 5) + m.c2 = pyo.Constraint(expr=((2 * m.x1 + m.x2) ** 2) ** 0.5 <= 5) # all the actual testing happens in the MockNLPSolver nlp_solver = MockNLPSolver( - varlist=[m.x1, m.x2], - sol_map={ - 0: ([None, None], 0, 0), - 1: ([1, 3], 1e-6, 1e-6), - }, + varlist=[m.x1, m.x2], + sol_map={0: ([None, None], 0, 0), 1: ([1, 3], 1e-6, 1e-6)}, ) global_solver = SolverFactory('scip_direct') results = ini.initialize_nlp( @@ -144,7 +145,7 @@ def test_global_init(self): def test_pwl_init(self): """ - Here, we really just want to make sure that the + Here, we really just want to make sure that the approximation improves as refinement is done. """ m = pyo.ConcreteModel() @@ -154,18 +155,18 @@ def test_pwl_init(self): # all the actual testing happens in the MockNLPSolver nlp_solver = MockNLPSolver( - varlist=[m.x], + varlist=[m.x], sol_map={ - 0: ([None], 0, 0), - 1: ([1.0975609756097562], 1e-6, 1e-6), - 2: ([0.4346767574185112], 1e-6, 1e-6), - 3: ([-0.19286405313201946], 1e-6, 1e-6), - 4: ([-0.8653073960726083], 1e-6, 1e-6), - 5: ([-1.6404750700409576], 1e-6, 1e-6), - 6: ([-2.5676344169949443], 1e-6, 1e-6), - 7: ([-3.6759614495828297], 1e-6, 1e-6), - 8: ([-4.942429761325623], 1e-6, 1e-6), - 9: ([-6.259703235160286], 1e-6, 1e-6), + 0: ([None], 0, 0), + 1: ([1.0975609756097562], 1e-6, 1e-6), + 2: ([0.4346767574185112], 1e-6, 1e-6), + 3: ([-0.19286405313201946], 1e-6, 1e-6), + 4: ([-0.8653073960726083], 1e-6, 1e-6), + 5: ([-1.6404750700409576], 1e-6, 1e-6), + 6: ([-2.5676344169949443], 1e-6, 1e-6), + 7: ([-3.6759614495828297], 1e-6, 1e-6), + 8: ([-4.942429761325623], 1e-6, 1e-6), + 9: ([-6.259703235160286], 1e-6, 1e-6), 10: ([-7.457220752001633], 1e-6, 1e-6), 11: ([-8.393746738936832], 1e-6, 1e-6), 12: ([-9.032852172775847], 1e-6, 1e-6), @@ -199,6 +200,7 @@ def test_pwl_init(self): if __name__ == '__main__': import logging + logging.basicConfig(level=logging.INFO) t = TestInit() t.test_pwl_init() diff --git a/pyomo/devel/initialization/utils.py b/pyomo/devel/initialization/utils.py index b73da8625b6..b8ce5095081 100644 --- a/pyomo/devel/initialization/utils.py +++ b/pyomo/devel/initialization/utils.py @@ -30,7 +30,9 @@ def shallow_clone(m1): for con in m1.component_data_objects(pe.Constraint, active=True, descend_into=True): m2.cons.add(con.expr) - objlist = list(m1.component_data_objects(pe.Objective, active=True, descend_into=True)) + objlist = list( + m1.component_data_objects(pe.Objective, active=True, descend_into=True) + ) assert len(objlist) <= 1 if objlist: obj = objlist[0] @@ -43,5 +45,9 @@ def fix_vars_with_equal_bounds(m): for v in get_vars(m): if v.fixed: continue - if v.lb is not None and v.ub is not None and math.isclose(v.lb, v.ub, abs_tol=1e-4, rel_tol=1e-4): + if ( + v.lb is not None + and v.ub is not None + and math.isclose(v.lb, v.ub, abs_tol=1e-4, rel_tol=1e-4) + ): v.fix(0.5 * (v.lb + v.ub)) From 8c6e488d3f4eeaba6f5ef656749b3c9719ae7815 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 15 Apr 2026 06:52:28 -0600 Subject: [PATCH 093/107] run black --- pyomo/contrib/solver/common/solution_loader.py | 1 - pyomo/contrib/solver/solvers/scip/scip_direct.py | 1 - 2 files changed, 2 deletions(-) diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index d0f14d403fa..d9c5469f7e2 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -18,7 +18,6 @@ from .util import NoSolutionError import logging - logger = logging.getLogger(__name__) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 022878f918d..c43b243b9fd 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -88,7 +88,6 @@ Reason, ) - logger = logging.getLogger(__name__) From 9722188770c5528aed58e6579ebef302b7ed4078 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 15 Apr 2026 06:53:19 -0600 Subject: [PATCH 094/107] run black --- pyomo/contrib/piecewise/transform/factorable.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/piecewise/transform/factorable.py b/pyomo/contrib/piecewise/transform/factorable.py index 2e7e0eba897..6a0581c8381 100644 --- a/pyomo/contrib/piecewise/transform/factorable.py +++ b/pyomo/contrib/piecewise/transform/factorable.py @@ -33,7 +33,11 @@ from pyomo.core.base.var import ScalarVar from pyomo.core.base.param import ScalarParam from pyomo.contrib.fbbt import interval -from pyomo.contrib.fbbt.fbbt import compute_bounds_on_expr, fbbt, ExpressionBoundsVisitor +from pyomo.contrib.fbbt.fbbt import ( + compute_bounds_on_expr, + fbbt, + ExpressionBoundsVisitor, +) from pyomo.core.base.expression import ScalarExpression from pyomo.core.base.transformation import Transformation, TransformationFactory from pyomo.common.modeling import unique_component_name @@ -352,7 +356,9 @@ def __init__(self, **kwds): self.block.c = ConstraintList() self._leaf_types = {VarData, ScalarVar, ParamData, ScalarVar, float, int} - self._interval_visitor = ExpressionBoundsVisitor(use_fixed_var_values_as_bounds=True) + self._interval_visitor = ExpressionBoundsVisitor( + use_fixed_var_values_as_bounds=True + ) def initializeWalker(self, expr): if expr in self.substitution_map: From 0d1175f0afa360d7d17d20800a5ac6a83d3b009f Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 15 Apr 2026 07:09:50 -0600 Subject: [PATCH 095/107] fix typos --- .../analysis/nlp_initialization/nlp_initialization.rst | 2 +- pyomo/devel/initialization/lp_approx_init.py | 2 +- pyomo/devel/initialization/pwl_init.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/OnlineDocs/explanation/analysis/nlp_initialization/nlp_initialization.rst b/doc/OnlineDocs/explanation/analysis/nlp_initialization/nlp_initialization.rst index 770f14369bf..1ce1ffaa0fc 100644 --- a/doc/OnlineDocs/explanation/analysis/nlp_initialization/nlp_initialization.rst +++ b/doc/OnlineDocs/explanation/analysis/nlp_initialization/nlp_initialization.rst @@ -102,7 +102,7 @@ Advantages Disadvantages ^^^^^^^^^^^^^ -* Current implemenation can be slow +* Current implementation can be slow * Requires all nonlinear variables to be bounded diff --git a/pyomo/devel/initialization/lp_approx_init.py b/pyomo/devel/initialization/lp_approx_init.py index 583e467c912..44bdcb52a5c 100644 --- a/pyomo/devel/initialization/lp_approx_init.py +++ b/pyomo/devel/initialization/lp_approx_init.py @@ -186,7 +186,7 @@ def _initialize_with_LP_approximation( logger.info('fixed variables with equal bounds') # now we modify the model by introducing slacks to make sure the LP - # approximatin is feasible + # approximation is feasible _minimize_infeasibility(nlp) logger.info('reformulated model to minimize infeasibility') diff --git a/pyomo/devel/initialization/pwl_init.py b/pyomo/devel/initialization/pwl_init.py index f5513e34bd3..08ca99b9a75 100644 --- a/pyomo/devel/initialization/pwl_init.py +++ b/pyomo/devel/initialization/pwl_init.py @@ -306,7 +306,7 @@ def _initialize_with_piecewise_linear_approximation( logger.info('fixed variables with equal bounds') # now we modify the model by introducing slacks to make sure the PWL - # approximatin is feasible + # approximation is feasible # all of the slacks appear linearly, so we don't need to worry about # upper bounds for them _minimize_infeasibility(pwl) From 3646b41fc1231285fb2a211b6c8d5f1c99b023ea Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 15 Apr 2026 07:10:33 -0600 Subject: [PATCH 096/107] fix typos --- pyomo/contrib/observer/model_observer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 51f236b8ad0..c9814704b42 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -739,7 +739,7 @@ def remove_objectives(self, objs: Collection[ObjectiveData]): def _check_for_unknown_active_components(self): for ctype in self._model.collect_ctypes(active=True, descend_into=True): if not issubclass(ctype, ActiveComponent): - # strangly, this is needed to skip things like Param + # strangely, this is needed to skip things like Param continue if ctype in self._known_active_ctypes: continue From ae7d2d9c29d1c41df3b5b7cbe934851acf6e82fe Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 27 Apr 2026 05:17:16 -0600 Subject: [PATCH 097/107] run black --- pyomo/contrib/solver/common/solution_loader.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index 608e96cb51c..ce4d3ae9516 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -345,9 +345,7 @@ def get_number_of_solutions(self) -> int: def load_solution(self): raise NoSolutionError() - def load_vars( - self, vars_to_load: Sequence[VarData] | None = None - ) -> None: + def load_vars(self, vars_to_load: Sequence[VarData] | None = None) -> None: raise NoSolutionError() def get_vars( From d0c051513cd82d3a7b50a3c135b766e089a46545 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Fri, 1 May 2026 11:08:44 -0600 Subject: [PATCH 098/107] remove redundant code and propagate error messages --- .../contrib/solver/common/solution_loader.py | 24 +++++-------------- .../solvers/gurobi/gurobi_direct_base.py | 9 +++---- .../solvers/gurobi/gurobi_persistent.py | 1 - .../solver/tests/solvers/test_solvers.py | 1 + 4 files changed, 12 insertions(+), 23 deletions(-) diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index ce4d3ae9516..a71f245f891 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -333,38 +333,26 @@ def load_import_suffixes(self): class NoSolutionSolutionLoader(SolutionLoader): - def __init__(self) -> None: - pass - - def get_solution_ids(self) -> List[Any]: - return [] + def __init__(self, err_msg: str) -> None: + self.err_msg = err_msg def get_number_of_solutions(self) -> int: return 0 - def load_solution(self): - raise NoSolutionError() - - def load_vars(self, vars_to_load: Sequence[VarData] | None = None) -> None: - raise NoSolutionError() - def get_vars( self, vars_to_load: Sequence[VarData] | None = None ) -> Mapping[VarData, float]: - raise NoSolutionError() + raise NoSolutionError(self.err_msg) def get_duals( self, cons_to_load: Sequence[ConstraintData] | None = None - ) -> Dict[ConstraintData, float]: - raise NoSolutionError() + ) -> dict[ConstraintData, float]: + raise NoSolutionError(self.err_msg) def get_reduced_costs( self, vars_to_load: Sequence[VarData] | None = None ) -> Mapping[VarData, float]: - raise NoSolutionError() - - def load_import_suffixes(self): - raise NoSolutionError() + raise NoSolutionError(self.err_msg) class PersistentSolutionLoader(SolutionLoader): diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index 8e8771ffad0..e89e972b77e 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -376,8 +376,9 @@ def solve(self, model, **kwds) -> Results: has_obj=has_obj, config=config, ) - except InfeasibleConstraintException: - res = self._get_infeasible_results(config=config) + except InfeasibleConstraintException as err: + err_msg = f'Solution loader does not currently have a valid solution because the problem was proven to be infeasible ({str(err)}). Please check results.termination_condition and/or results.solution_status.' + res = self._get_infeasible_results(config=config, err_msg=err_msg) finally: os.chdir(orig_cwd) @@ -410,9 +411,9 @@ def _get_tc_map(self): } return GurobiDirectBase._tc_map - def _get_infeasible_results(self, config): + def _get_infeasible_results(self, config, err_msg): res = Results() - res.solution_loader = NoSolutionSolutionLoader() + res.solution_loader = NoSolutionSolutionLoader(err_msg) res.solution_status = SolutionStatus.noSolution res.termination_condition = TerminationCondition.provenInfeasible res.incumbent_objective = None diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 091481b152c..7e9ddcdeaf9 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -16,7 +16,6 @@ from pyomo.common.errors import PyomoException from pyomo.common.shutdown import python_is_shutting_down from pyomo.common.timing import HierarchicalTimer -from pyomo.common.errors import InfeasibleConstraintException from pyomo.core.base.objective import ObjectiveData from pyomo.core.kernel.objective import minimize, maximize from pyomo.core.base.var import VarData diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index f80e1c23d3e..136a1484525 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -1171,6 +1171,7 @@ def test_trivial_constraints( self.assertAlmostEqual(m.x.value, 0) self.assertAlmostEqual(m.y.value, 0) + # trivially feasible constraint m.x.fix(1) opt.config.tee = True res = opt.solve(m) From bf6f6fdbbdad43a31d411f60d24ba52b56789390 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Fri, 1 May 2026 15:36:50 -0600 Subject: [PATCH 099/107] fix highs interface --- pyomo/contrib/solver/solvers/highs.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyomo/contrib/solver/solvers/highs.py b/pyomo/contrib/solver/solvers/highs.py index 1bbbdd4a326..acb150832f6 100644 --- a/pyomo/contrib/solver/solvers/highs.py +++ b/pyomo/contrib/solver/solvers/highs.py @@ -686,7 +686,9 @@ def _postsolve(self, stream: io.StringIO): results.timing_info.highs_time = highs.getRunTime() self._sol = highs.getSolution() - has_feasible_solution = self._sol.value_valid + info = highs.getInfo() + # 0: None, 1: Infeasible, 2: Feasible + has_feasible_solution = info.primal_solution_status == 2 if status == highspy.HighsModelStatus.kOptimal: results.solution_status = SolutionStatus.optimal elif has_feasible_solution: @@ -744,12 +746,11 @@ def _postsolve(self, stream: io.StringIO): results.incumbent_objective = None results.objective_bound = None - info = highs.getInfo() if self._objective is not None: if has_feasible_solution: results.incumbent_objective = info.objective_function_value if info.mip_node_count == -1: - if has_feasible_solution: + if has_feasible_solution and results.termination_condition == TerminationCondition.convergenceCriteriaSatisfied: results.objective_bound = info.objective_function_value else: results.objective_bound = None From a1ed8e0670e1fdbc4724e0aa3acb98cd4692c5b3 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 3 May 2026 12:21:04 -0600 Subject: [PATCH 100/107] better handling of trivially feasible and infeasible constraints --- .../solver/solvers/gurobi/gurobi_direct_base.py | 1 - .../contrib/solver/solvers/gurobi/gurobi_persistent.py | 10 ++++++++-- pyomo/contrib/solver/tests/solvers/test_solvers.py | 1 + 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index e89e972b77e..de3bc0d8d6e 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -418,7 +418,6 @@ def _get_infeasible_results(self, config, err_msg): res.termination_condition = TerminationCondition.provenInfeasible res.incumbent_objective = None res.objective_bound = None - res.iteration_count = None res.timing_info.gurobi_time = None res.solver_config = config res.solver_name = self.name diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 7e9ddcdeaf9..32686c7341e 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -504,7 +504,13 @@ def _get_expr_from_pyomo_repn(self, repn): vlist = [self._pyomo_var_to_solver_var_map[v] for v in repn.linear_vars] new_expr = gurobipy.LinExpr(coef_list, vlist) else: - new_expr = 0.0 + # this still needs to be an expression object so that + # we don't generate a bool if both the body and + # the bounds are floats (i.e., a trivially feasible or + # trivially infeasible constraint). We don't want a bool + # because the constraint could swap between feasible and + # infeasible if the bounds are mutable. + new_expr = gurobipy.LinExpr() if len(repn.quadratic_vars) > 0: for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): @@ -535,7 +541,7 @@ def _add_constraints(self, cons: list[ConstraintData]): ) elif ub is None: rhs_expr = lb - repn.constant - gurobi_expr_list.append(float(value(rhs_expr)) <= gurobi_expr) + gurobi_expr_list.append(gurobi_expr >= float(value(rhs_expr))) if not is_constant(rhs_expr): mutable_constant = _MutableConstant( rhs_expr, con, self._pyomo_con_to_solver_con_map diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 136a1484525..7ae7f9827e8 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -1178,6 +1178,7 @@ def test_trivial_constraints( self.assertAlmostEqual(m.x.value, 1) self.assertAlmostEqual(m.y.value, 1) + # trivially infeasible constraint m.x.fix(-1) with self.assertRaises(NoOptimalSolutionError): res = opt.solve(m) From 29b9bf3a5d9cd0255aa957649a06c7dd9dc88d5e Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 3 May 2026 12:38:10 -0600 Subject: [PATCH 101/107] run black --- pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py | 4 ++-- pyomo/contrib/solver/solvers/highs.py | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 32686c7341e..7092b0f6558 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -505,10 +505,10 @@ def _get_expr_from_pyomo_repn(self, repn): new_expr = gurobipy.LinExpr(coef_list, vlist) else: # this still needs to be an expression object so that - # we don't generate a bool if both the body and + # we don't generate a bool if both the body and # the bounds are floats (i.e., a trivially feasible or # trivially infeasible constraint). We don't want a bool - # because the constraint could swap between feasible and + # because the constraint could swap between feasible and # infeasible if the bounds are mutable. new_expr = gurobipy.LinExpr() diff --git a/pyomo/contrib/solver/solvers/highs.py b/pyomo/contrib/solver/solvers/highs.py index acb150832f6..40596b2917e 100644 --- a/pyomo/contrib/solver/solvers/highs.py +++ b/pyomo/contrib/solver/solvers/highs.py @@ -750,7 +750,11 @@ def _postsolve(self, stream: io.StringIO): if has_feasible_solution: results.incumbent_objective = info.objective_function_value if info.mip_node_count == -1: - if has_feasible_solution and results.termination_condition == TerminationCondition.convergenceCriteriaSatisfied: + if ( + has_feasible_solution + and results.termination_condition + == TerminationCondition.convergenceCriteriaSatisfied + ): results.objective_bound = info.objective_function_value else: results.objective_bound = None From eb7abb65a54dad5267eb821fa88c04900aaef459 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 3 May 2026 15:20:04 -0600 Subject: [PATCH 102/107] better handling of trivially infeasible constraints --- pyomo/contrib/solver/common/results.py | 21 ++ pyomo/contrib/solver/solvers/gams.py | 191 +++++++++--------- .../solvers/gurobi/gurobi_direct_base.py | 20 +- pyomo/repn/plugins/gams_writer_v2.py | 3 + 4 files changed, 124 insertions(+), 111 deletions(-) diff --git a/pyomo/contrib/solver/common/results.py b/pyomo/contrib/solver/common/results.py index 4129386aa5f..b54fa4680c2 100644 --- a/pyomo/contrib/solver/common/results.py +++ b/pyomo/contrib/solver/common/results.py @@ -21,6 +21,8 @@ ADVANCED_OPTION, DEVELOPER_OPTION, ) +from pyomo.contrib.solver.common.util import NoOptimalSolutionError, NoSolutionError +from pyomo.contrib.solver.common.solution_loader import NoSolutionSolutionLoader from pyomo.opt.results.solution import SolutionStatus as LegacySolutionStatus from pyomo.opt.results.solver import ( TerminationCondition as LegacyTerminationCondition, @@ -242,6 +244,25 @@ def display( return super().display(content_filter, indent_spacing, ostream, visibility) +def get_infeasible_results(config, err_msg, solver_name, solver_version): + res = Results() + res.solution_loader = NoSolutionSolutionLoader(err_msg) + res.solution_status = SolutionStatus.noSolution + res.termination_condition = TerminationCondition.provenInfeasible + res.incumbent_objective = None + res.objective_bound = None + res.timing_info.gurobi_time = None + res.solver_config = config + res.solver_name = solver_name + res.solver_version = solver_version + if config.raise_exception_on_nonoptimal_result: + raise NoOptimalSolutionError(err_msg) + if config.load_solutions: + raise NoSolutionError(err_msg) + return res + + + # Everything below here preserves backwards compatibility legacy_termination_condition_map = { diff --git a/pyomo/contrib/solver/solvers/gams.py b/pyomo/contrib/solver/solvers/gams.py index 0aa24cd9a8f..c55c73897f6 100644 --- a/pyomo/contrib/solver/solvers/gams.py +++ b/pyomo/contrib/solver/solvers/gams.py @@ -19,6 +19,7 @@ import pathlib from pyomo.common.dependencies import attempt_import +from pyomo.common.errors import InfeasibleConstraintException from pyomo.common.fileutils import Executable, ExecutableData from pyomo.common.config import ( ConfigValue, @@ -37,6 +38,7 @@ Results, SolutionStatus, TerminationCondition, + get_infeasible_results, ) from pyomo.contrib.solver.solvers.gms_sol_reader import GMSSolutionLoader @@ -292,110 +294,113 @@ def solve(self, model, **kwds): output_filename = None with TempfileManager.new_context() as tempfile: - # IMPORTANT - only delete the whole tmpdir if the solver was the one - # that made the directory. Otherwise, just delete the files the solver - # made, if not keepfiles. That way the user can select a directory - # they already have, like the current directory, without having to - # worry about the rest of the contents of that directory being deleted. - if not config.working_dir: - dname = tempfile.mkdtemp() - else: - dname = config.working_dir - if not os.path.exists(dname): - os.mkdir(dname) - basename = os.path.join(dname, model_name) - output_filename = basename + '.gms' - lst_filename = os.path.join(dname, lst) - - timer.start(f'write_gms_file') - with open(output_filename, 'w', newline='\n', encoding='utf-8') as gms_file: - gms_info = GAMSWriter().write( - model, gms_file, config=config.writer_config - ) - # NOTE: omit InfeasibleConstraintException for now - timer.stop(f'write_gms_file') + try: + # IMPORTANT - only delete the whole tmpdir if the solver was the one + # that made the directory. Otherwise, just delete the files the solver + # made, if not keepfiles. That way the user can select a directory + # they already have, like the current directory, without having to + # worry about the rest of the contents of that directory being deleted. + if not config.working_dir: + dname = tempfile.mkdtemp() + else: + dname = config.working_dir + if not os.path.exists(dname): + os.mkdir(dname) + basename = os.path.join(dname, model_name) + output_filename = basename + '.gms' + lst_filename = os.path.join(dname, lst) + + timer.start(f'write_gms_file') + with open(output_filename, 'w', newline='\n', encoding='utf-8') as gms_file: + gms_info = GAMSWriter().write( + model, gms_file, config=config.writer_config + ) + timer.stop(f'write_gms_file') - if config.writer_config.put_results_format == 'gdx': - results_filename = os.path.join(dname, "GAMS_MODEL_p.gdx") - statresults_filename = os.path.join( - dname, "%s_s.gdx" % (config.writer_config.put_results,) - ) - else: - results_filename = os.path.join( - dname, "%s.dat" % (config.writer_config.put_results,) - ) - statresults_filename = os.path.join( - dname, "%sstat.dat" % (config.writer_config.put_results,) - ) + if config.writer_config.put_results_format == 'gdx': + results_filename = os.path.join(dname, "GAMS_MODEL_p.gdx") + statresults_filename = os.path.join( + dname, "%s_s.gdx" % (config.writer_config.put_results,) + ) + else: + results_filename = os.path.join( + dname, "%s.dat" % (config.writer_config.put_results,) + ) + statresults_filename = os.path.join( + dname, "%sstat.dat" % (config.writer_config.put_results,) + ) - #################################################################### - # Apply solver - #################################################################### - exe_path = config.executable.path() - command = [exe_path, output_filename, "o=" + lst, "curdir=" + dname] + #################################################################### + # Apply solver + #################################################################### + exe_path = config.executable.path() + command = [exe_path, output_filename, "o=" + lst, "curdir=" + dname] - # handled tee and logfile based on the length of list and - # string respectively - command.append(self._log_levels[(bool(config.tee), bool(config.logfile))]) + # handled tee and logfile based on the length of list and + # string respectively + command.append(self._log_levels[(bool(config.tee), bool(config.logfile))]) - ostreams = [StringIO()] - if config.tee: - ostreams.append(sys.stdout) + ostreams = [StringIO()] + if config.tee: + ostreams.append(sys.stdout) - with TeeStream(*ostreams) as t: - timer.start('subprocess') - subprocess_result = subprocess.run( - command, stdout=t.STDOUT, stderr=t.STDERR, cwd=dname - ) - timer.stop('subprocess') - rc = subprocess_result.returncode - txt = ostreams[0].getvalue() - if config.working_dir: - logger.info("\nGAMS WORKING DIRECTORY: %s\n" % config.working_dir) - - if rc: - # If nothing was raised, or for all other cases, raise this - error_message = f"GAMS process encountered an error (returncode={rc})." - if rc == 3: - # Execution Error - # Run check_expr_evaluation, which errors if necessary - error_message += ( - "\nError rc=3 (GAMS execution error), to be determined later." + with TeeStream(*ostreams) as t: + timer.start('subprocess') + subprocess_result = subprocess.run( + command, stdout=t.STDOUT, stderr=t.STDERR, cwd=dname ) - error_message += "\nCheck listing file for details.\n" - logger.error(error_message) - logger.error(txt.strip()) - if os.path.exists(lst_filename): - with open(lst_filename, 'r') as FILE: - logger.error( - "\nGAMS Listing file:\n\n%s" % (FILE.read().strip(),) + timer.stop('subprocess') + rc = subprocess_result.returncode + txt = ostreams[0].getvalue() + if config.working_dir: + logger.info("\nGAMS WORKING DIRECTORY: %s\n" % config.working_dir) + + if rc: + # If nothing was raised, or for all other cases, raise this + error_message = f"GAMS process encountered an error (returncode={rc})." + if rc == 3: + # Execution Error + # Run check_expr_evaluation, which errors if necessary + error_message += ( + "\nError rc=3 (GAMS execution error), to be determined later." ) - raise RuntimeError(error_message) + error_message += "\nCheck listing file for details.\n" + logger.error(error_message) + logger.error(txt.strip()) + if os.path.exists(lst_filename): + with open(lst_filename, 'r') as FILE: + logger.error( + "\nGAMS Listing file:\n\n%s" % (FILE.read().strip(),) + ) + raise RuntimeError(error_message) + + timer.start('parse_results') + if config.writer_config.put_results_format == 'gdx': + model_soln, stat_vars = self._parse_gdx_results( + config, results_filename, statresults_filename + ) + else: + model_soln, stat_vars = self._parse_dat_results( + config, results_filename, statresults_filename + ) + timer.stop('parse_results') - timer.start('parse_results') - if config.writer_config.put_results_format == 'gdx': - model_soln, stat_vars = self._parse_gdx_results( - config, results_filename, statresults_filename - ) - else: - model_soln, stat_vars = self._parse_dat_results( - config, results_filename, statresults_filename + #################################################################### + # Postsolve (WIP) + results = self._postsolve( + model, timer, config, model_soln, stat_vars, gms_info ) - timer.stop('parse_results') - - #################################################################### - # Postsolve (WIP) - results = self._postsolve( - model, timer, config, model_soln, stat_vars, gms_info - ) - results.solver_config = config - results.solver_log = ostreams[0].getvalue() + results.solver_config = config + results.solver_log = ostreams[0].getvalue() - tock = time.perf_counter() - results.timing_info.start_timestamp = start_timestamp - results.timing_info.wall_time = tock - tick - results.timing_info.timer = timer + tock = time.perf_counter() + results.timing_info.start_timestamp = start_timestamp + results.timing_info.wall_time = tock - tick + results.timing_info.timer = timer + except InfeasibleConstraintException as err: + err_msg = f'Solution loader does not currently have a valid solution because the problem was proven to be infeasible ({str(err)}). Please check results.termination_condition and/or results.solution_status.' + results = get_infeasible_results(config=config, err_msg=err_msg, solver_name=self.name, solver_version=self.version()) return results def _postsolve(self, model, timer, config, model_soln, stat_vars, gms_info): diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index de3bc0d8d6e..6991685423a 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -39,6 +39,7 @@ Results, SolutionStatus, TerminationCondition, + get_infeasible_results, ) from pyomo.contrib.solver.common.solution_loader import SolutionLoader import time @@ -378,7 +379,7 @@ def solve(self, model, **kwds) -> Results: ) except InfeasibleConstraintException as err: err_msg = f'Solution loader does not currently have a valid solution because the problem was proven to be infeasible ({str(err)}). Please check results.termination_condition and/or results.solution_status.' - res = self._get_infeasible_results(config=config, err_msg=err_msg) + res = get_infeasible_results(config=config, err_msg=err_msg, solver_name=self.name, solver_version=self.version()) finally: os.chdir(orig_cwd) @@ -411,23 +412,6 @@ def _get_tc_map(self): } return GurobiDirectBase._tc_map - def _get_infeasible_results(self, config, err_msg): - res = Results() - res.solution_loader = NoSolutionSolutionLoader(err_msg) - res.solution_status = SolutionStatus.noSolution - res.termination_condition = TerminationCondition.provenInfeasible - res.incumbent_objective = None - res.objective_bound = None - res.timing_info.gurobi_time = None - res.solver_config = config - res.solver_name = self.name - res.solver_version = self.version() - if config.raise_exception_on_nonoptimal_result: - raise NoOptimalSolutionError() - if config.load_solutions: - raise NoFeasibleSolutionError() - return res - def _populate_results(self, grb_model, solution_loader, has_obj, config): status = grb_model.Status diff --git a/pyomo/repn/plugins/gams_writer_v2.py b/pyomo/repn/plugins/gams_writer_v2.py index caf4455b51d..e5e60c3c898 100644 --- a/pyomo/repn/plugins/gams_writer_v2.py +++ b/pyomo/repn/plugins/gams_writer_v2.py @@ -18,6 +18,7 @@ In, ListOf, ) +from pyomo.common.errors import InfeasibleConstraintException from pyomo.common.gc_manager import PauseGC from pyomo.common.timing import TicTocTimer from pyomo.core.base import ( @@ -386,6 +387,8 @@ def write(self, model): if repn.linear or getattr(repn, 'quadratic', None): pass else: + if (lb is not None and lb > offset) or (ub is not None and ub < offset): + raise InfeasibleConstraintException(f'detected a trivially infeasible constraint: {con}') if ( skip_trivial_constraints and (lb is None or lb <= offset) From c987a9f93c03e748cc899f29933306591ba8e103 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 3 May 2026 15:21:17 -0600 Subject: [PATCH 103/107] run black --- pyomo/contrib/solver/common/results.py | 1 - pyomo/contrib/solver/solvers/gams.py | 23 +++++++++++++------ .../solvers/gurobi/gurobi_direct_base.py | 7 +++++- pyomo/repn/plugins/gams_writer_v2.py | 4 +++- 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/pyomo/contrib/solver/common/results.py b/pyomo/contrib/solver/common/results.py index b54fa4680c2..b3dcbe78dce 100644 --- a/pyomo/contrib/solver/common/results.py +++ b/pyomo/contrib/solver/common/results.py @@ -262,7 +262,6 @@ def get_infeasible_results(config, err_msg, solver_name, solver_version): return res - # Everything below here preserves backwards compatibility legacy_termination_condition_map = { diff --git a/pyomo/contrib/solver/solvers/gams.py b/pyomo/contrib/solver/solvers/gams.py index c55c73897f6..514af8ec2a8 100644 --- a/pyomo/contrib/solver/solvers/gams.py +++ b/pyomo/contrib/solver/solvers/gams.py @@ -311,7 +311,9 @@ def solve(self, model, **kwds): lst_filename = os.path.join(dname, lst) timer.start(f'write_gms_file') - with open(output_filename, 'w', newline='\n', encoding='utf-8') as gms_file: + with open( + output_filename, 'w', newline='\n', encoding='utf-8' + ) as gms_file: gms_info = GAMSWriter().write( model, gms_file, config=config.writer_config ) @@ -338,7 +340,9 @@ def solve(self, model, **kwds): # handled tee and logfile based on the length of list and # string respectively - command.append(self._log_levels[(bool(config.tee), bool(config.logfile))]) + command.append( + self._log_levels[(bool(config.tee), bool(config.logfile))] + ) ostreams = [StringIO()] if config.tee: @@ -357,13 +361,13 @@ def solve(self, model, **kwds): if rc: # If nothing was raised, or for all other cases, raise this - error_message = f"GAMS process encountered an error (returncode={rc})." + error_message = ( + f"GAMS process encountered an error (returncode={rc})." + ) if rc == 3: # Execution Error # Run check_expr_evaluation, which errors if necessary - error_message += ( - "\nError rc=3 (GAMS execution error), to be determined later." - ) + error_message += "\nError rc=3 (GAMS execution error), to be determined later." error_message += "\nCheck listing file for details.\n" logger.error(error_message) logger.error(txt.strip()) @@ -400,7 +404,12 @@ def solve(self, model, **kwds): results.timing_info.timer = timer except InfeasibleConstraintException as err: err_msg = f'Solution loader does not currently have a valid solution because the problem was proven to be infeasible ({str(err)}). Please check results.termination_condition and/or results.solution_status.' - results = get_infeasible_results(config=config, err_msg=err_msg, solver_name=self.name, solver_version=self.version()) + results = get_infeasible_results( + config=config, + err_msg=err_msg, + solver_name=self.name, + solver_version=self.version(), + ) return results def _postsolve(self, model, timer, config, model_soln, stat_vars, gms_info): diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index 6991685423a..28c52c1a1f7 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -379,7 +379,12 @@ def solve(self, model, **kwds) -> Results: ) except InfeasibleConstraintException as err: err_msg = f'Solution loader does not currently have a valid solution because the problem was proven to be infeasible ({str(err)}). Please check results.termination_condition and/or results.solution_status.' - res = get_infeasible_results(config=config, err_msg=err_msg, solver_name=self.name, solver_version=self.version()) + res = get_infeasible_results( + config=config, + err_msg=err_msg, + solver_name=self.name, + solver_version=self.version(), + ) finally: os.chdir(orig_cwd) diff --git a/pyomo/repn/plugins/gams_writer_v2.py b/pyomo/repn/plugins/gams_writer_v2.py index e5e60c3c898..1d9b39db974 100644 --- a/pyomo/repn/plugins/gams_writer_v2.py +++ b/pyomo/repn/plugins/gams_writer_v2.py @@ -388,7 +388,9 @@ def write(self, model): pass else: if (lb is not None and lb > offset) or (ub is not None and ub < offset): - raise InfeasibleConstraintException(f'detected a trivially infeasible constraint: {con}') + raise InfeasibleConstraintException( + f'detected a trivially infeasible constraint: {con}' + ) if ( skip_trivial_constraints and (lb is None or lb <= offset) From fe00fdd570fc9bf5ef62f6ec17662cbc9ab9be45 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 3 May 2026 16:55:15 -0600 Subject: [PATCH 104/107] better handling of trivially infeasible constraints --- pyomo/repn/plugins/gams_writer_v2.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyomo/repn/plugins/gams_writer_v2.py b/pyomo/repn/plugins/gams_writer_v2.py index 1d9b39db974..2cf3e56dddf 100644 --- a/pyomo/repn/plugins/gams_writer_v2.py +++ b/pyomo/repn/plugins/gams_writer_v2.py @@ -392,8 +392,9 @@ def write(self, model): f'detected a trivially infeasible constraint: {con}' ) if ( - skip_trivial_constraints - and (lb is None or lb <= offset) + # skip_trivial_constraints + # and (lb is None or lb <= offset) + (lb is None or lb <= offset) and (ub is None or ub >= offset) ): continue From 390df7c69f6d6b87719913771373e343dc920e3e Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 4 May 2026 07:11:55 -0600 Subject: [PATCH 105/107] factorable programming updates --- ...test_univariate_nonlinear_decomposition.py | 121 +++++++++--------- .../contrib/piecewise/transform/factorable.py | 33 +++-- pyomo/repn/util.py | 7 +- 3 files changed, 92 insertions(+), 69 deletions(-) diff --git a/pyomo/contrib/piecewise/tests/test_univariate_nonlinear_decomposition.py b/pyomo/contrib/piecewise/tests/test_univariate_nonlinear_decomposition.py index fdeb5ae6f05..67ecdbe411d 100644 --- a/pyomo/contrib/piecewise/tests/test_univariate_nonlinear_decomposition.py +++ b/pyomo/contrib/piecewise/tests/test_univariate_nonlinear_decomposition.py @@ -1,3 +1,12 @@ +# ____________________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2026 National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and Engineering +# Solutions of Sandia, LLC, the U.S. Government retains certain rights in this +# software. This software is distributed under the 3-clause BSD License. +# ____________________________________________________________________________________ + from pyomo.common.unittest import TestCase, skipUnless import pyomo.environ as pyo from pyomo.contrib import piecewise @@ -5,8 +14,6 @@ from pyomo.common.dependencies import numpy_available, numpy from pyomo.core.expr.numeric_expr import ProductExpression -pe = pyo - def _get_trans(): return pyo.TransformationFactory( @@ -16,11 +23,11 @@ def _get_trans(): class TestUnivariateNonlinearDecomposition(TestCase): def test_multiterm(self): - m = pe.ConcreteModel() - m.x = pe.Var() - m.y = pe.Var() - m.z = pe.Var() - m.c = pe.Constraint(expr=m.x + pe.log(m.y + m.z) + 1 / pe.exp(m.x**0.5) <= 0) + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.z = pyo.Var() + m.c = pyo.Constraint(expr=m.x + pyo.log(m.y + m.z) + 1 / pyo.exp(m.x**0.5) <= 0) trans = _get_trans() trans.apply_to(m) @@ -44,14 +51,14 @@ def test_multiterm(self): self.assertIsNone(aux.x[3].ub) def test_common_subexpressions(self): - m = pe.ConcreteModel() - m.x = pe.Var() - m.y = pe.Var() - m.z1 = pe.Var() - m.z2 = pe.Var() - e = -pe.log(m.x + m.y) - m.c1 = pe.Constraint(expr=m.z1 + e == 0) - m.c2 = pe.Constraint(expr=m.z2 + e == 0) + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.z1 = pyo.Var() + m.z2 = pyo.Var() + e = -pyo.log(m.x + m.y) + m.c1 = pyo.Constraint(expr=m.z1 + e == 0) + m.c2 = pyo.Constraint(expr=m.z2 + e == 0) trans = _get_trans() trans.apply_to(m) @@ -60,43 +67,43 @@ def test_common_subexpressions(self): assertExpressionsEqual(self, m.c1.expr, m.z1 + aux.x[2] == 0) assertExpressionsEqual(self, m.c2.expr, m.z2 + aux.x[2] == 0) assertExpressionsEqual(self, aux.c[1].expr, aux.x[1] == m.x + m.y) - assertExpressionsEqual(self, aux.c[2].expr, aux.x[2] == -pe.log(aux.x[1])) + assertExpressionsEqual(self, aux.c[2].expr, aux.x[2] == -pyo.log(aux.x[1])) def test_product_fixed_variable(self): - m = pe.ConcreteModel() - m.x = pe.Var() - m.y = pe.Var() - m.z = pe.Var() - m.c = pe.Constraint(expr=2 * pe.log(m.x + m.y) <= 0) + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.z = pyo.Var() + m.c = pyo.Constraint(expr=2 * pyo.log(m.x + m.y) <= 0) trans = _get_trans() trans.apply_to(m) aux = m.auxiliary - assertExpressionsEqual(self, m.c.expr, 2 * pe.log(aux.x[1]) <= 0) + assertExpressionsEqual(self, m.c.expr, 2 * pyo.log(aux.x[1]) <= 0) assertExpressionsEqual(self, aux.c[1].expr, aux.x[1] == m.x + m.y) def test_product_variable_fixed(self): - m = pe.ConcreteModel() - m.x = pe.Var() - m.y = pe.Var() - m.z = pe.Var() - m.c = pe.Constraint(expr=pe.log(m.x + m.y) * 2 <= 0) + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.z = pyo.Var() + m.c = pyo.Constraint(expr=pyo.log(m.x + m.y) * 2 <= 0) trans = _get_trans() trans.apply_to(m) aux = m.auxiliary - assertExpressionsEqual(self, m.c.expr, pe.log(aux.x[1]) * 2 <= 0) + assertExpressionsEqual(self, m.c.expr, pyo.log(aux.x[1]) * 2 <= 0) assertExpressionsEqual(self, aux.c[1].expr, aux.x[1] == m.x + m.y) def test_prod_sum_sum(self): - m = pe.ConcreteModel() - m.x1 = pe.Var() - m.x2 = pe.Var() - m.x3 = pe.Var() - m.x4 = pe.Var() - m.c = pe.Constraint(expr=(m.x1 + m.x2) * (m.x3 + m.x4) <= 1) + m = pyo.ConcreteModel() + m.x1 = pyo.Var() + m.x2 = pyo.Var() + m.x3 = pyo.Var() + m.x4 = pyo.Var() + m.c = pyo.Constraint(expr=(m.x1 + m.x2) * (m.x3 + m.x4) <= 1) trans = _get_trans() trans.apply_to(m) @@ -107,12 +114,12 @@ def test_prod_sum_sum(self): assertExpressionsEqual(self, aux.c[2].expr, aux.x[2] == m.x3 + m.x4) def test_pow_sum_sum(self): - m = pe.ConcreteModel() - m.x1 = pe.Var() - m.x2 = pe.Var() - m.x3 = pe.Var() - m.x4 = pe.Var() - m.c = pe.Constraint(expr=(m.x1 + m.x2) ** (m.x3 + m.x4) <= 1) + m = pyo.ConcreteModel() + m.x1 = pyo.Var() + m.x2 = pyo.Var() + m.x3 = pyo.Var() + m.x4 = pyo.Var() + m.c = pyo.Constraint(expr=(m.x1 + m.x2) ** (m.x3 + m.x4) <= 1) trans = _get_trans() trans.apply_to(m) @@ -123,10 +130,10 @@ def test_pow_sum_sum(self): assertExpressionsEqual(self, aux.c[2].expr, aux.x[2] == m.x3 + m.x4) def test_division_var_const(self): - m = pe.ConcreteModel() - m.x = pe.Var() - m.y = pe.Var() - m.c = pe.Constraint(expr=(m.x + m.y) / 2 <= 0) + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.c = pyo.Constraint(expr=(m.x + m.y) / 2 <= 0) trans = _get_trans() trans.apply_to(m) @@ -135,12 +142,12 @@ def test_division_var_const(self): assertExpressionsEqual(self, m.c.expr, (m.x + m.y) / 2 <= 0) def test_division_sum_sum(self): - m = pe.ConcreteModel() - m.x1 = pe.Var() - m.x2 = pe.Var() - m.x3 = pe.Var() - m.x4 = pe.Var() - m.c = pe.Constraint(expr=(m.x1 + m.x2) / (m.x3 + m.x4) <= 1) + m = pyo.ConcreteModel() + m.x1 = pyo.Var() + m.x2 = pyo.Var() + m.x3 = pyo.Var() + m.x4 = pyo.Var() + m.c = pyo.Constraint(expr=(m.x1 + m.x2) / (m.x3 + m.x4) <= 1) trans = _get_trans() trans.apply_to(m) @@ -153,17 +160,17 @@ def test_division_sum_sum(self): @skipUnless(numpy_available, "Numpy is not available") def test_numpy_float(self): - m = pe.ConcreteModel() - m.x = pe.Var() - m.y = pe.Var() - m.z = pe.Var() - m.c = pe.Constraint( - expr=ProductExpression((numpy.float64(2.5), pe.log(m.x + m.y))) <= 0 + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.z = pyo.Var() + m.c = pyo.Constraint( + expr=ProductExpression((numpy.float64(2.5), pyo.log(m.x + m.y))) <= 0 ) trans = _get_trans() trans.apply_to(m) aux = m.auxiliary - assertExpressionsEqual(self, m.c.expr, 2.5 * pe.log(aux.x[1]) <= 0) + assertExpressionsEqual(self, m.c.expr, 2.5 * pyo.log(aux.x[1]) <= 0) assertExpressionsEqual(self, aux.c[1].expr, aux.x[1] == m.x + m.y) diff --git a/pyomo/contrib/piecewise/transform/factorable.py b/pyomo/contrib/piecewise/transform/factorable.py index 6a0581c8381..566089fe766 100644 --- a/pyomo/contrib/piecewise/transform/factorable.py +++ b/pyomo/contrib/piecewise/transform/factorable.py @@ -1,3 +1,12 @@ +# ____________________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2026 National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and Engineering +# Solutions of Sandia, LLC, the U.S. Government retains certain rights in this +# software. This software is distributed under the 3-clause BSD License. +# ____________________________________________________________________________________ + from pyomo.core.expr.visitor import StreamBasedExpressionVisitor from pyomo.common.collections import ComponentMap, ComponentSet from pyomo.common.numeric_types import native_numeric_types @@ -44,6 +53,7 @@ from pyomo.core.base.component import ActiveComponent from pyomo.core.base.suffix import Suffix from pyomo.common.config import ConfigDict, ConfigValue +from pyomo.repn.util import categorize_valid_components def _handle_var(node, data, visitor): @@ -279,9 +289,9 @@ def _handle_pow(node, data, visitor): def _handle_named_expression(node, data, visitor): assert len(data) == 1 - res = data[0] - visitor.substitution_map[node] = res - return res + node.expr = data[0] + visitor.substitution_map[node] = node + return node def _handle_negation(node, data, visitor): @@ -457,16 +467,17 @@ def __init__(self): def _check_for_unknown_active_components(self, model): known_ctypes = {Constraint, Objective, Block} - for ctype in model.collect_ctypes(active=True, descend_into=True): - if not issubclass(ctype, ActiveComponent): - continue - if ctype in known_ctypes: - continue - if ctype is Suffix: - continue + ignore = {Suffix} + valid, invalid = categorize_valid_components( + model=model, + active=True, + valid=ignore, + targets=known_ctypes, + ) + if invalid: raise NotImplementedError( f'UnivariateNonlinearDecompositionTransformation does not know how to ' - f'handle components with ctype {ctype}' + f'handle components with ctype {list(invalid.keys())}' ) def _apply_to(self, model, **kwds): diff --git a/pyomo/repn/util.py b/pyomo/repn/util.py index 1cdadd2cdd1..69c5ae50421 100644 --- a/pyomo/repn/util.py +++ b/pyomo/repn/util.py @@ -589,7 +589,7 @@ def complex_number_error(value, visitor, expr, node=""): def categorize_valid_components( - model, active=True, sort=None, valid=set(), targets=set() + model, active=True, sort=None, valid=None, targets=None ): """Walk model and check for valid component types @@ -633,6 +633,11 @@ def categorize_valid_components( list of component data objects found on the model. """ + if valid is None: + valid = set() + if targets is None: + targets = set() + assert active in (True, None) # Note: we assume every target component is valid but that we expect # there to be far mode valid components than target components. From 3b58e8c2d164183eaf86a950984cf4a5c9a0380e Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 4 May 2026 07:12:34 -0600 Subject: [PATCH 106/107] run black --- pyomo/contrib/piecewise/transform/factorable.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pyomo/contrib/piecewise/transform/factorable.py b/pyomo/contrib/piecewise/transform/factorable.py index 566089fe766..328777e6f1b 100644 --- a/pyomo/contrib/piecewise/transform/factorable.py +++ b/pyomo/contrib/piecewise/transform/factorable.py @@ -469,10 +469,7 @@ def _check_for_unknown_active_components(self, model): known_ctypes = {Constraint, Objective, Block} ignore = {Suffix} valid, invalid = categorize_valid_components( - model=model, - active=True, - valid=ignore, - targets=known_ctypes, + model=model, active=True, valid=ignore, targets=known_ctypes ) if invalid: raise NotImplementedError( From 15d87609f59a1c23211cab3e95bf1ddd6a7c5fc6 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 4 May 2026 07:25:44 -0600 Subject: [PATCH 107/107] update test --- .../piecewise/tests/test_univariate_nonlinear_decomposition.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/piecewise/tests/test_univariate_nonlinear_decomposition.py b/pyomo/contrib/piecewise/tests/test_univariate_nonlinear_decomposition.py index 67ecdbe411d..721349528be 100644 --- a/pyomo/contrib/piecewise/tests/test_univariate_nonlinear_decomposition.py +++ b/pyomo/contrib/piecewise/tests/test_univariate_nonlinear_decomposition.py @@ -43,7 +43,7 @@ def test_multiterm(self): self.assertIsNone(m.y.ub) self.assertIsNone(m.z.lb) self.assertIsNone(m.z.ub) - self.assertEqual(aux.x[1].lb, 0) + self.assertTrue(aux.x[1].lb is None or aux.x[1].lb <= 0) self.assertIsNone(aux.x[1].ub) self.assertEqual(aux.x[2].lb, 0) self.assertEqual(aux.x[2].ub, 1)