From 27a3129a8b642d1d71596fbedf41f67fc9b1a11b Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 19 May 2026 15:19:41 +0200 Subject: [PATCH 01/21] feat(constraints): add _coef_dirty flag + rhs setter short-circuit Tracks per-Constraint coefficient mutation via a single boolean slot, flipped in coeffs/vars/lhs setters. Pure-constant rhs writes now short-circuit and leave coeffs/vars buffers untouched (by identity), so rhs-only updates don't trigger expensive coefficient recompare on the persistent-solver fast path. --- linopy/constraints.py | 12 ++++- test/test_constraint_coef_dirty.py | 73 ++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 test/test_constraint_coef_dirty.py diff --git a/linopy/constraints.py b/linopy/constraints.py index b74dee5c..1b51f48d 100644 --- a/linopy/constraints.py +++ b/linopy/constraints.py @@ -1043,7 +1043,7 @@ class Constraint(ConstraintBase): Supports setters, xarray operations via conwrap, and from_rule construction. """ - __slots__ = ("_data", "_model", "_assigned") + __slots__ = ("_data", "_model", "_assigned", "_coef_dirty") def __init__( self, @@ -1072,6 +1072,7 @@ def __init__( self._assigned = "labels" in data self._data = data self._model = model + self._coef_dirty = False @property def data(self) -> Dataset: @@ -1121,6 +1122,7 @@ def coeffs(self) -> DataArray: def coeffs(self, value: ConstantLike) -> None: value = DataArray(value).broadcast_like(self.vars, exclude=[self.term_dim]) self._data = assign_multiindex_safe(self.data, coeffs=value) + self._coef_dirty = True @property def vars(self) -> DataArray: @@ -1134,6 +1136,7 @@ def vars(self, value: variables.Variable | DataArray) -> None: raise TypeError("Expected value to be of type DataArray or Variable") value = value.broadcast_like(self.coeffs, exclude=[self.term_dim]) self._data = assign_multiindex_safe(self.data, vars=value) + self._coef_dirty = True @property def sign(self) -> DataArray: @@ -1154,7 +1157,11 @@ def rhs(self, value: ExpressionLike) -> None: value = expressions.as_expression( value, self.model, coords=self.coords, dims=self.coord_dims ) - self.lhs = self.lhs - value.reset_const() + residual = value.reset_const() + if residual.nterm == 0: + self._data = assign_multiindex_safe(self.data, rhs=value.const) + return + self.lhs = self.lhs - residual self._data = assign_multiindex_safe(self.data, rhs=value.const) @property @@ -1170,6 +1177,7 @@ def lhs(self, value: ExpressionLike | VariableLike | ConstantLike) -> None: self._data = self.data.drop_vars(["coeffs", "vars"]).assign( coeffs=value.coeffs, vars=value.vars, rhs=self.rhs - value.const ) + self._coef_dirty = True @property @has_optimized_model diff --git a/test/test_constraint_coef_dirty.py b/test/test_constraint_coef_dirty.py new file mode 100644 index 00000000..682eb6d8 --- /dev/null +++ b/test/test_constraint_coef_dirty.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import pytest + +from linopy import Model + + +@pytest.fixture +def m_with_c() -> tuple[Model, str]: + m = Model() + x = m.add_variables(0, 10, coords=[range(3)], name="x") + y = m.add_variables(0, 10, coords=[range(3)], name="y") + m.add_constraints(2 * x + y >= 5, name="c") + return m, "c" + + +def test_initial_coef_dirty_false(m_with_c: tuple[Model, str]) -> None: + m, name = m_with_c + assert m.constraints[name]._coef_dirty is False + + +def test_coeffs_setter_sets_dirty(m_with_c: tuple[Model, str]) -> None: + m, name = m_with_c + c = m.constraints[name] + c.coeffs = c.coeffs * 2 + assert c._coef_dirty is True + + +def test_vars_setter_sets_dirty(m_with_c: tuple[Model, str]) -> None: + m, name = m_with_c + c = m.constraints[name] + c.vars = c.vars + assert c._coef_dirty is True + + +def test_lhs_setter_sets_dirty(m_with_c: tuple[Model, str]) -> None: + m, name = m_with_c + c = m.constraints[name] + x = m.variables["x"] + c.lhs = 3 * x + assert c._coef_dirty is True + + +def test_pure_constant_rhs_short_circuits(m_with_c: tuple[Model, str]) -> None: + m, name = m_with_c + c = m.constraints[name] + coeffs_buf = c.data["coeffs"].values + vars_buf = c.data["vars"].values + c.rhs = 9 + assert c._coef_dirty is False + assert c.data["coeffs"].values is coeffs_buf + assert c.data["vars"].values is vars_buf + + +def test_rhs_with_variable_sets_dirty(m_with_c: tuple[Model, str]) -> None: + m, name = m_with_c + c = m.constraints[name] + x = m.variables["x"] + c.rhs = x + 3 + assert c._coef_dirty is True + + +def test_sign_setter_does_not_set_dirty(m_with_c: tuple[Model, str]) -> None: + m, name = m_with_c + c = m.constraints[name] + c.sign = "<=" + assert c._coef_dirty is False + + +def test_flag_persists_across_container_access(m_with_c: tuple[Model, str]) -> None: + m, name = m_with_c + m.constraints[name].coeffs = m.constraints[name].coeffs * 2 + assert m.constraints[name]._coef_dirty is True From 663ab92fec310eb02e71736f535eb87bfe84ae51 Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 19 May 2026 15:25:45 +0200 Subject: [PATCH 02/21] feat(persistent): add ModelSnapshot, CoefPattern, StructuralKey Pure-Python snapshot primitives for the persistent-solver Phase 1. Deep-copies value-side fields (var_lb/ub, con_rhs/sign, obj_linear), holds vlabels/clabels by reference, stores canonical CSR (indptr, indices) per constraint container. No Solver import. --- linopy/persistent/__init__.py | 13 +++ linopy/persistent/errors.py | 5 ++ linopy/persistent/snapshot.py | 165 ++++++++++++++++++++++++++++++++++ 3 files changed, 183 insertions(+) create mode 100644 linopy/persistent/__init__.py create mode 100644 linopy/persistent/errors.py create mode 100644 linopy/persistent/snapshot.py diff --git a/linopy/persistent/__init__.py b/linopy/persistent/__init__.py new file mode 100644 index 00000000..64f557e7 --- /dev/null +++ b/linopy/persistent/__init__.py @@ -0,0 +1,13 @@ +"""Persistent-solver snapshot and diff primitives.""" + +from __future__ import annotations + +from linopy.persistent.errors import UnsupportedUpdate +from linopy.persistent.snapshot import CoefPattern, ModelSnapshot, StructuralKey + +__all__ = [ + "CoefPattern", + "ModelSnapshot", + "StructuralKey", + "UnsupportedUpdate", +] diff --git a/linopy/persistent/errors.py b/linopy/persistent/errors.py new file mode 100644 index 00000000..839adedb --- /dev/null +++ b/linopy/persistent/errors.py @@ -0,0 +1,5 @@ +from __future__ import annotations + + +class UnsupportedUpdate(Exception): + pass diff --git a/linopy/persistent/snapshot.py b/linopy/persistent/snapshot.py new file mode 100644 index 00000000..ea35c996 --- /dev/null +++ b/linopy/persistent/snapshot.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +import numpy as np +import xarray as xr + +from linopy import expressions + +if TYPE_CHECKING: + from linopy.constraints import ConstraintBase + from linopy.model import Model + from linopy.variables import Variable + + +def _variable_type(var: Variable) -> str: + attrs = var.attrs + if attrs.get("binary"): + return "binary" + if attrs.get("integer"): + return "integer" + if attrs.get("semi_continuous"): + return "semi_continuous" + return "continuous" + + +def _coord_snapshot(obj: Variable | ConstraintBase) -> dict[str, np.ndarray]: + return {str(name): np.asarray(idx) for name, idx in obj.indexes.items()} + + +def _canonical_csr( + constraint: ConstraintBase, label_index +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + csr, _ = constraint.to_matrix(label_index) + csr.sort_indices() + csr.eliminate_zeros() + indptr = csr.indptr.astype(np.int64) + indices = csr.indices.astype(np.int64) + return indptr, indices, csr.data + + +def _objective_linear_vector(model: Model) -> xr.DataArray: + vlabels = model.variables.label_index.vlabels + label_to_pos = model.variables.label_index.label_to_pos + result = np.zeros(len(vlabels)) + expr = model.objective.expression + if isinstance(expr, expressions.QuadraticExpression): + vars_2d = expr.data.vars.values + coeffs_all = expr.data.coeffs.values.ravel() + vars1, vars2 = vars_2d[0], vars_2d[1] + linear = (vars1 == -1) | (vars2 == -1) + var_labels = np.where(vars1[linear] != -1, vars1[linear], vars2[linear]) + coeffs = coeffs_all[linear] + else: + var_labels = expr.data.vars.values.ravel() + coeffs = expr.data.coeffs.values.ravel() + mask = var_labels != -1 + np.add.at(result, label_to_pos[var_labels[mask]], coeffs[mask]) + return xr.DataArray(result, dims="vlabel", coords={"vlabel": vlabels}) + + +@dataclass(frozen=True) +class CoefPattern: + indptr: np.ndarray + indices: np.ndarray + + def __eq__(self, other: object) -> bool: + return ( + isinstance(other, CoefPattern) + and np.array_equal(self.indptr, other.indptr) + and np.array_equal(self.indices, other.indices) + ) + + __hash__ = None # type: ignore[assignment] + + +@dataclass(frozen=True) +class StructuralKey: + var_container_names: tuple[str, ...] + con_container_names: tuple[str, ...] + vlabels: np.ndarray + clabels: np.ndarray + + def __eq__(self, other: object) -> bool: + return ( + isinstance(other, StructuralKey) + and self.var_container_names == other.var_container_names + and self.con_container_names == other.con_container_names + and np.array_equal(self.vlabels, other.vlabels) + and np.array_equal(self.clabels, other.clabels) + ) + + __hash__ = None # type: ignore[assignment] + + +@dataclass +class ModelSnapshot: + structural_key: StructuralKey + + var_lb: dict[str, xr.DataArray] = field(default_factory=dict) + var_ub: dict[str, xr.DataArray] = field(default_factory=dict) + var_type: dict[str, str] = field(default_factory=dict) + var_coords: dict[str, dict[str, np.ndarray]] = field(default_factory=dict) + + con_rhs: dict[str, xr.DataArray] = field(default_factory=dict) + con_sign: dict[str, xr.DataArray] = field(default_factory=dict) + con_coords: dict[str, dict[str, np.ndarray]] = field(default_factory=dict) + con_coef_pattern: dict[str, CoefPattern] = field(default_factory=dict) + + obj_linear: xr.DataArray = field(default_factory=lambda: xr.DataArray([])) + obj_quad_present: bool = False + obj_sense: str = "min" + + @classmethod + def capture(cls, model: Model) -> ModelSnapshot: + var_label_index = model.variables.label_index + con_label_index = model.constraints.label_index + + structural_key = StructuralKey( + var_container_names=tuple(model.variables), + con_container_names=tuple(model.constraints), + vlabels=var_label_index.vlabels, + clabels=con_label_index.clabels, + ) + + var_lb: dict[str, xr.DataArray] = {} + var_ub: dict[str, xr.DataArray] = {} + var_type: dict[str, str] = {} + var_coords: dict[str, dict[str, np.ndarray]] = {} + for name, var in model.variables.items(): + var_lb[name] = var.lower.copy(deep=True) + var_ub[name] = var.upper.copy(deep=True) + var_type[name] = _variable_type(var) + var_coords[name] = _coord_snapshot(var) + + con_rhs: dict[str, xr.DataArray] = {} + con_sign: dict[str, xr.DataArray] = {} + con_coords: dict[str, dict[str, np.ndarray]] = {} + con_coef_pattern: dict[str, CoefPattern] = {} + for name, con in model.constraints.items(): + con_rhs[name] = con.rhs.copy(deep=True) + con_sign[name] = con.sign.copy(deep=True) + con_coords[name] = _coord_snapshot(con) + indptr, indices, _ = _canonical_csr(con, var_label_index) + con_coef_pattern[name] = CoefPattern(indptr=indptr, indices=indices) + + obj_linear = _objective_linear_vector(model).copy(deep=True) + obj_quad_present = model.objective.is_quadratic + obj_sense = model.objective.sense + + return cls( + structural_key=structural_key, + var_lb=var_lb, + var_ub=var_ub, + var_type=var_type, + var_coords=var_coords, + con_rhs=con_rhs, + con_sign=con_sign, + con_coords=con_coords, + con_coef_pattern=con_coef_pattern, + obj_linear=obj_linear, + obj_quad_present=obj_quad_present, + obj_sense=obj_sense, + ) From c1da075a660c54fcb6854fccad415d4829033e53 Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 19 May 2026 15:25:52 +0200 Subject: [PATCH 03/21] feat(persistent): add ModelDiff and compute_diff Pure-function diff for the persistent-solver Phase 1. Detects structural, coord, sparsity, quadratic-objective, value-only var/con, and objective-linear/sense changes. Supports same_model fast path via _coef_dirty and cross-model full re-scan. Includes a focused test suite covering capture, mutation paths, deep-copy invariant, and the same_model toggle. --- linopy/persistent/__init__.py | 4 + linopy/persistent/diff.py | 154 ++++++++++++++++++++++++++ test/test_persistent_snapshot_diff.py | 148 +++++++++++++++++++++++++ 3 files changed, 306 insertions(+) create mode 100644 linopy/persistent/diff.py create mode 100644 test/test_persistent_snapshot_diff.py diff --git a/linopy/persistent/__init__.py b/linopy/persistent/__init__.py index 64f557e7..5823ee2f 100644 --- a/linopy/persistent/__init__.py +++ b/linopy/persistent/__init__.py @@ -2,12 +2,16 @@ from __future__ import annotations +from linopy.persistent.diff import ModelDiff, RebuildReason, compute_diff from linopy.persistent.errors import UnsupportedUpdate from linopy.persistent.snapshot import CoefPattern, ModelSnapshot, StructuralKey __all__ = [ "CoefPattern", + "ModelDiff", "ModelSnapshot", + "RebuildReason", "StructuralKey", "UnsupportedUpdate", + "compute_diff", ] diff --git a/linopy/persistent/diff.py b/linopy/persistent/diff.py new file mode 100644 index 00000000..28b7a4d8 --- /dev/null +++ b/linopy/persistent/diff.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import enum +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +import numpy as np +import xarray as xr + +from linopy.persistent.snapshot import ( + CoefPattern, + ModelSnapshot, + _canonical_csr, + _coord_snapshot, + _objective_linear_vector, + _variable_type, +) + +if TYPE_CHECKING: + from linopy.model import Model + + +class RebuildReason(enum.Enum): + NONE = "none" + STRUCTURAL_LABELS = "vlabels/clabels mismatch" + STRUCTURAL_CONTAINERS = "container set changed" + COORD_REINDEX = "coordinates changed" + SPARSITY = "coefficient sparsity changed" + QUAD_OBJ = "quadratic objective changed" + BACKEND_REJECTED = "backend raised UnsupportedUpdate" + + +@dataclass +class ModelDiff: + rebuild_reason: RebuildReason = RebuildReason.NONE + + var_lb: dict[str, xr.DataArray] = field(default_factory=dict) + var_ub: dict[str, xr.DataArray] = field(default_factory=dict) + var_type: dict[str, str] = field(default_factory=dict) + con_rhs: dict[str, xr.DataArray] = field(default_factory=dict) + con_sign: dict[str, xr.DataArray] = field(default_factory=dict) + con_coef_updates: dict[str, np.ndarray] = field(default_factory=dict) + + obj_linear: xr.DataArray | None = None + obj_sense: str | None = None + + @property + def is_empty(self) -> bool: + return ( + self.rebuild_reason is RebuildReason.NONE + and not self.var_lb + and not self.var_ub + and not self.var_type + and not self.con_rhs + and not self.con_sign + and not self.con_coef_updates + and self.obj_linear is None + and self.obj_sense is None + ) + + @property + def rebuild_required(self) -> bool: + return self.rebuild_reason is not RebuildReason.NONE + + +def _coords_equal(a: dict[str, np.ndarray], b: dict[str, np.ndarray]) -> bool: + if a.keys() != b.keys(): + return False + return all(np.array_equal(a[k], b[k]) for k in a) + + +def _any_diff(a: xr.DataArray, b: xr.DataArray) -> bool: + return bool((a != b).any().item()) + + +def compute_diff( + snapshot: ModelSnapshot, model: Model, same_model: bool = True +) -> ModelDiff: + diff = ModelDiff() + + var_names = tuple(model.variables) + con_names = tuple(model.constraints) + if ( + snapshot.structural_key.var_container_names != var_names + or snapshot.structural_key.con_container_names != con_names + ): + diff.rebuild_reason = RebuildReason.STRUCTURAL_CONTAINERS + return diff + + var_label_index = model.variables.label_index + con_label_index = model.constraints.label_index + if not np.array_equal(snapshot.structural_key.vlabels, var_label_index.vlabels): + diff.rebuild_reason = RebuildReason.STRUCTURAL_LABELS + return diff + if not np.array_equal(snapshot.structural_key.clabels, con_label_index.clabels): + diff.rebuild_reason = RebuildReason.STRUCTURAL_LABELS + return diff + + for name, var in model.variables.items(): + if not _coords_equal(snapshot.var_coords[name], _coord_snapshot(var)): + diff.rebuild_reason = RebuildReason.COORD_REINDEX + return diff + if _any_diff(snapshot.var_lb[name], var.lower): + diff.var_lb[name] = var.lower.copy(deep=True) + if _any_diff(snapshot.var_ub[name], var.upper): + diff.var_ub[name] = var.upper.copy(deep=True) + vtype = _variable_type(var) + if snapshot.var_type[name] != vtype: + diff.var_type[name] = vtype + + for name, con in model.constraints.items(): + if not _coords_equal(snapshot.con_coords[name], _coord_snapshot(con)): + diff.rebuild_reason = RebuildReason.COORD_REINDEX + return diff + if _any_diff(snapshot.con_rhs[name], con.rhs): + diff.con_rhs[name] = con.rhs.copy(deep=True) + if _any_diff(snapshot.con_sign[name], con.sign): + diff.con_sign[name] = con.sign.copy(deep=True) + + if same_model: + dirty_names = [n for n, c in model.constraints.items() if c._coef_dirty] + else: + dirty_names = list(con_names) + + for name in dirty_names: + con = model.constraints[name] + indptr, indices, data = _canonical_csr(con, var_label_index) + pattern = CoefPattern(indptr=indptr, indices=indices) + if pattern == snapshot.con_coef_pattern[name]: + diff.con_coef_updates[name] = data + else: + diff.rebuild_reason = RebuildReason.SPARSITY + return diff + + obj_quad_present = model.objective.is_quadratic + if obj_quad_present != snapshot.obj_quad_present: + diff.rebuild_reason = RebuildReason.QUAD_OBJ + return diff + if obj_quad_present: + diff.rebuild_reason = RebuildReason.QUAD_OBJ + return diff + + obj_linear = _objective_linear_vector(model) + if not np.array_equal( + obj_linear.values, snapshot.obj_linear.values + ) or not np.array_equal( + obj_linear["vlabel"].values, snapshot.obj_linear["vlabel"].values + ): + diff.obj_linear = obj_linear.copy(deep=True) + + if model.objective.sense != snapshot.obj_sense: + diff.obj_sense = model.objective.sense + + return diff diff --git a/test/test_persistent_snapshot_diff.py b/test/test_persistent_snapshot_diff.py new file mode 100644 index 00000000..53bff0ad --- /dev/null +++ b/test/test_persistent_snapshot_diff.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +import numpy as np +import pytest + +from linopy import Model +from linopy.persistent import ( + CoefPattern, + ModelDiff, + ModelSnapshot, + RebuildReason, + StructuralKey, + compute_diff, +) + + +@pytest.fixture +def baseline() -> Model: + m = Model() + x = m.add_variables(0, 10, coords=[range(3)], name="x") + y = m.add_variables(0, 5, coords=[range(2)], name="y") + m.add_constraints(2 * x + 1 >= 4, name="c1") + m.add_constraints(x.sum() + y.sum() <= 20, name="c2") + m.add_objective(x.sum() + 2 * y.sum()) + return m + + +def test_capture_structural_key(baseline: Model) -> None: + snap = ModelSnapshot.capture(baseline) + assert isinstance(snap, ModelSnapshot) + assert isinstance(snap.structural_key, StructuralKey) + assert snap.structural_key.var_container_names == ("x", "y") + assert snap.structural_key.con_container_names == ("c1", "c2") + np.testing.assert_array_equal( + snap.structural_key.vlabels, baseline.variables.label_index.vlabels + ) + np.testing.assert_array_equal( + snap.structural_key.clabels, baseline.constraints.label_index.clabels + ) + assert isinstance(snap.con_coef_pattern["c1"], CoefPattern) + + +def test_is_empty_on_unmutated(baseline: Model) -> None: + snap = ModelSnapshot.capture(baseline) + diff = compute_diff(snap, baseline) + assert diff.is_empty + assert diff.rebuild_reason is RebuildReason.NONE + assert not diff.rebuild_required + + +def test_bounds_only_mutation(baseline: Model) -> None: + snap = ModelSnapshot.capture(baseline) + baseline.variables["x"].lower = 1 + diff = compute_diff(snap, baseline) + assert diff.rebuild_reason is RebuildReason.NONE + assert "x" in diff.var_lb + assert "x" not in diff.var_ub + + +def test_rhs_only_mutation(baseline: Model) -> None: + snap = ModelSnapshot.capture(baseline) + baseline.constraints["c1"].rhs = 9 + diff = compute_diff(snap, baseline) + assert diff.rebuild_reason is RebuildReason.NONE + assert "c1" in diff.con_rhs + assert not diff.con_coef_updates + + +def test_objective_linear_change(baseline: Model) -> None: + snap = ModelSnapshot.capture(baseline) + x = baseline.variables["x"] + y = baseline.variables["y"] + baseline.add_objective(3 * x.sum() + 2 * y.sum(), overwrite=True) + diff = compute_diff(snap, baseline) + assert diff.rebuild_reason is RebuildReason.NONE + assert diff.obj_linear is not None + + +def test_objective_sense_flip(baseline: Model) -> None: + snap = ModelSnapshot.capture(baseline) + baseline.objective.sense = "max" + diff = compute_diff(snap, baseline) + assert diff.rebuild_reason is RebuildReason.NONE + assert diff.obj_sense == "max" + + +def test_add_constraints_is_structural(baseline: Model) -> None: + snap = ModelSnapshot.capture(baseline) + x = baseline.variables["x"] + baseline.add_constraints(x.sum() <= 99, name="c3") + diff = compute_diff(snap, baseline) + assert diff.rebuild_reason in ( + RebuildReason.STRUCTURAL_LABELS, + RebuildReason.STRUCTURAL_CONTAINERS, + ) + + +def test_remove_variables_is_structural(baseline: Model) -> None: + snap = ModelSnapshot.capture(baseline) + baseline.remove_variables("y") + diff = compute_diff(snap, baseline) + assert diff.rebuild_reason in ( + RebuildReason.STRUCTURAL_LABELS, + RebuildReason.STRUCTURAL_CONTAINERS, + ) + + +def test_coef_value_change_same_sparsity(baseline: Model) -> None: + snap = ModelSnapshot.capture(baseline) + c = baseline.constraints["c1"] + c.coeffs = c.coeffs * 3 + diff = compute_diff(snap, baseline) + assert diff.rebuild_reason is RebuildReason.NONE + assert "c1" in diff.con_coef_updates + values = diff.con_coef_updates["c1"] + np.testing.assert_array_equal(values, np.full_like(values, 6.0)) + + +def test_coef_sparsity_change(baseline: Model) -> None: + snap = ModelSnapshot.capture(baseline) + x = baseline.variables["x"] + baseline.constraints["c2"].lhs = 2 * x.sum() + diff = compute_diff(snap, baseline) + assert diff.rebuild_reason is RebuildReason.SPARSITY + + +def test_deep_copy_invariant(baseline: Model) -> None: + snap = ModelSnapshot.capture(baseline) + baseline.variables["x"].lower.values[...] = 99 + diff = compute_diff(snap, baseline) + assert "x" in diff.var_lb + + +def test_same_model_false_ignores_dirty_flag(baseline: Model) -> None: + snap = ModelSnapshot.capture(baseline) + c = baseline.constraints["c1"] + c.coeffs = c.coeffs * 5 + c._coef_dirty = False + diff_fast = compute_diff(snap, baseline, same_model=True) + assert "c1" not in diff_fast.con_coef_updates + diff_full = compute_diff(snap, baseline, same_model=False) + assert "c1" in diff_full.con_coef_updates + + +def test_modeldiff_default_is_empty() -> None: + d = ModelDiff() + assert d.is_empty + assert not d.rebuild_required From 572b42617e7150a0f597f929d1b86e4e7135a93c Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 19 May 2026 15:33:30 +0200 Subject: [PATCH 04/21] feat(solvers): add persistent-update orchestration to Solver - supports_persistent_update class flag (default False) - snapshot/_rebuilds/_in_place_updates/_last_rebuild_reason fields - snapshot capture at end of direct _build, _clear_coef_dirty helper - apply_update stub raising UnsupportedUpdate - solve(model, assign) dispatcher with diff-or-rebuild path - update(model, apply=True) primitive returning ModelDiff - threading.Lock around diff+apply+resnapshot - __getstate__/__setstate__ drop native handle and snapshot --- linopy/solvers.py | 140 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 118 insertions(+), 22 deletions(-) diff --git a/linopy/solvers.py b/linopy/solvers.py index 44db983f..8e69bf21 100644 --- a/linopy/solvers.py +++ b/linopy/solvers.py @@ -46,6 +46,13 @@ Status, TerminationCondition, ) +from linopy.persistent import ( + ModelDiff, + ModelSnapshot, + RebuildReason, + UnsupportedUpdate, + compute_diff, +) def _parse_int_label(name: str) -> int: @@ -106,6 +113,11 @@ def _solution_from_labels( return values_to_lookup_array(np.asarray(values, dtype=float), labels, size=size) +def _clear_coef_dirty(constraints: Any) -> None: + for c in constraints.data.values(): + c._coef_dirty = False + + class SolverFeature(Enum): """Enumeration of all solver capabilities tracked by linopy.""" @@ -403,9 +415,17 @@ class Solver(ABC, Generic[EnvType]): _n_cons: int = field(init=False, default=0, repr=False) _problem_fn: Path | None = field(init=False, default=None, repr=False) + snapshot: ModelSnapshot | None = field(init=False, default=None, repr=False) + _rebuilds: int = field(init=False, default=0, repr=False) + _in_place_updates: int = field(init=False, default=0, repr=False) + _last_rebuild_reason: RebuildReason = field( + init=False, default=RebuildReason.NONE, repr=False + ) + display_name: ClassVar[str] = "" features: ClassVar[frozenset[SolverFeature]] = frozenset() accepted_io_apis: ClassVar[frozenset[str]] = frozenset() + supports_persistent_update: ClassVar[bool] = False def __post_init__(self) -> None: if type(self) is Solver: @@ -418,6 +438,15 @@ def __post_init__(self) -> None: "Please install first to initialize solver instance." ) raise ImportError(msg) + self._lock: threading.Lock = threading.Lock() + + def apply_update( + self, + diff: ModelDiff, + var_label_index: Any, + con_label_index: Any, + ) -> None: + raise UnsupportedUpdate(type(self).__name__) @property def solver_options(self) -> dict[str, Any]: @@ -521,6 +550,8 @@ def _build(self, **build_kwargs: Any) -> None: self.model._check_sos_unmasked() if self.io_api == "direct": self._build_direct(**build_kwargs) + self.snapshot = ModelSnapshot.capture(self.model) + _clear_coef_dirty(self.model.constraints) else: self._build_file(**build_kwargs) @@ -590,38 +621,91 @@ def _build_file(self, **build_kwargs: Any) -> None: self.io_api = read_io_api_from_problem_file(problem_fn) self._cache_model_sizes(model) - def solve(self, **run_kwargs: Any) -> Result: + def solve( + self, + model: Model | None = None, + assign: bool = False, + **run_kwargs: Any, + ) -> Result: """ Run the prepared solver and return a :class:`Result`. - The canonical low-level pattern is:: - - solver = Solver.from_name("gurobi", model, io_api="direct") - result = solver.solve() - model.assign_result(result, solver=solver) - - Passing ``solver=`` to :meth:`Model.assign_result` wires - ``model.solver`` so post-solve helpers like - :meth:`Model.compute_infeasibilities` keep working. - - Raises - ------ - ValueError - If the attached model has no objective set. Submit-time check - shared by both ``Model.solve()`` and direct-Solver callers. + With ``model`` supplied, diff against the held snapshot and either + apply in place or rebuild before running. Requires ``io_api='direct'``. + With ``assign=True`` the Result is written back to the target Model + via :meth:`Model.assign_result`. """ + if model is not None: + if self.io_api != "direct": + raise ValueError("solve(model=...) requires io_api='direct'") + with self._lock: + if self.solver_model is None: + self.model = model + self._build() + else: + self._update_locked(model, apply=True) + target = model + else: + target = self.model # type: ignore[assignment] + if self.model is not None and self.model.objective.expression.empty: raise ValueError( "No objective has been set on the model. Use `m.add_objective(...)` " "first (e.g. `m.add_objective(0 * x)` for a pure feasibility problem)." ) if self.io_api == "direct" or self.solver_model is not None: - return self._run_direct(**run_kwargs) - if self._problem_fn is not None: - return self._run_file(**run_kwargs) - raise RuntimeError( - "Solver has not been built; call Solver.from_name(...) or _build() first." - ) + result = self._run_direct(**run_kwargs) + elif self._problem_fn is not None: + result = self._run_file(**run_kwargs) + else: + raise RuntimeError( + "Solver has not been built; call Solver.from_name(...) or _build() first." + ) + + if assign and target is not None: + target.assign_result(result, solver=self) + return result + + def update(self, model: Model, apply: bool = True) -> ModelDiff: + if self.io_api != "direct": + raise ValueError("update requires io_api='direct'") + if self.snapshot is None or self.solver_model is None: + raise RuntimeError("Solver has not been built") + with self._lock: + return self._update_locked(model, apply=apply) + + def _update_locked(self, model: Model, apply: bool) -> ModelDiff: + assert self.snapshot is not None + same_model = model is self.model + diff = compute_diff(self.snapshot, model, same_model=same_model) + if not apply: + return diff + if diff.rebuild_required: + self._rebuild(model, diff.rebuild_reason) + return diff + try: + self.apply_update( + diff, + model.variables.label_index, + model.constraints.label_index, + ) + except Exception: + self._last_rebuild_reason = RebuildReason.BACKEND_REJECTED + self._rebuild(model, RebuildReason.BACKEND_REJECTED) + return diff + self.model = model + self.snapshot = ModelSnapshot.capture(model) + _clear_coef_dirty(model.constraints) + self._in_place_updates += 1 + self._last_rebuild_reason = RebuildReason.NONE + return diff + + def _rebuild(self, model: Model, reason: RebuildReason) -> None: + self.close() + self.model = model + self._build() + self._rebuilds += 1 + self._last_rebuild_reason = reason def _run_direct(self, **run_kwargs: Any) -> Result: """Run the pre-built native solver model. Override per-solver.""" @@ -775,6 +859,18 @@ def __del__(self) -> None: with contextlib.suppress(Exception): self.close() + def __getstate__(self) -> dict[str, Any]: + drop = {"solver_model", "env", "_env_stack", "snapshot", "_lock"} + return {k: v for k, v in self.__dict__.items() if k not in drop} + + def __setstate__(self, state: dict[str, Any]) -> None: + self.__dict__.update(state) + self.solver_model = None + self.env = None + self._env_stack = None + self.snapshot = None + self._lock = threading.Lock() + def __repr__(self) -> str: status = self.status.status.value if self.status is not None else "unsolved" parts = [f"name={self.solver_name.value!r}", f"status={status!r}"] From 61b3647290775210005182df93456c56a6e9d5ce Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 19 May 2026 15:33:33 +0200 Subject: [PATCH 05/21] test(persistent): smoke test Solver orchestrator with fake backend --- test/test_persistent_solver_orchestrator.py | 116 ++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 test/test_persistent_solver_orchestrator.py diff --git a/test/test_persistent_solver_orchestrator.py b/test/test_persistent_solver_orchestrator.py new file mode 100644 index 00000000..dbe40a48 --- /dev/null +++ b/test/test_persistent_solver_orchestrator.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import pickle +from typing import Any + +import pytest + +from linopy import Model +from linopy.constants import ( + Result, + Solution, + SolverStatus, + Status, + TerminationCondition, +) +from linopy.persistent import ModelDiff, RebuildReason +from linopy.solvers import Solver, SolverFeature + + +class FakeSolver(Solver[None]): + display_name = "Fake" + features = frozenset({SolverFeature.DIRECT_API}) + accepted_io_apis = frozenset({"direct"}) + supports_persistent_update = False + + @classmethod + def is_available(cls) -> bool: + return True + + @property + def solver_name(self): # type: ignore[override] + class _N: + value = "fake" + + return _N() + + def _validate_model(self) -> None: + return None + + def _build_direct(self, **kwargs: Any) -> None: + self.solver_model = object() + + def _run_direct(self, **kwargs: Any) -> Result: + status = Status(SolverStatus.ok, TerminationCondition.optimal) + return Result( + status=status, solution=Solution(objective=0.0), solver_name="fake" + ) + + +@pytest.fixture +def model() -> Model: + m = Model() + x = m.add_variables(0, 10, coords=[range(3)], name="x") + m.add_constraints(2 * x >= 4, name="c1") + m.add_objective(x.sum()) + return m + + +@pytest.fixture +def other_model() -> Model: + m = Model() + x = m.add_variables(0, 10, coords=[range(3)], name="x") + m.add_constraints(2 * x >= 4, name="c1") + m.add_objective(x.sum()) + return m + + +def _built(model: Model) -> FakeSolver: + s = FakeSolver(model=model, io_api="direct") + s._build() + return s + + +def test_unsupported_falls_through_to_rebuild(model: Model, other_model: Model) -> None: + s = _built(model) + assert s._rebuilds == 0 + s.solve(other_model) + assert s._rebuilds == 1 + assert s._last_rebuild_reason is RebuildReason.BACKEND_REJECTED + assert s.model is other_model + + +def test_update_apply_false_returns_diff(model: Model) -> None: + s = _built(model) + diff = s.update(model, apply=False) + assert isinstance(diff, ModelDiff) + assert s._in_place_updates == 0 + assert s._rebuilds == 0 + + +def test_solve_no_model_still_works(model: Model) -> None: + s = _built(model) + result = s.solve() + assert result.status.status is SolverStatus.ok + + +def test_getstate_drops_native_fields(model: Model) -> None: + s = _built(model) + state = s.__getstate__() + for k in ("solver_model", "env", "_env_stack", "snapshot", "_lock"): + assert k not in state + restored = pickle.loads(pickle.dumps(s)) + assert restored.solver_model is None + assert restored.snapshot is None + + +def test_update_without_snapshot_raises(model: Model) -> None: + s = FakeSolver(model=model, io_api="direct") + with pytest.raises(RuntimeError, match="not been built"): + s.update(model) + + +def test_unmutated_resolve_diff_is_empty(model: Model) -> None: + s = _built(model) + diff = s.update(model, apply=False) + assert diff.is_empty From ff1ae15c2b20cbc67913e4910e9333bc343efa4a Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 19 May 2026 15:34:52 +0200 Subject: [PATCH 06/21] feat(solvers): short-circuit rebuild when backend lacks persistent-update support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Skip diff computation entirely when supports_persistent_update is False on apply, per plan: 'dispatcher checks flag before calling — if False, skips diffing entirely and goes to rebuild.' --- linopy/solvers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/linopy/solvers.py b/linopy/solvers.py index 8e69bf21..74fa911a 100644 --- a/linopy/solvers.py +++ b/linopy/solvers.py @@ -676,6 +676,10 @@ def update(self, model: Model, apply: bool = True) -> ModelDiff: def _update_locked(self, model: Model, apply: bool) -> ModelDiff: assert self.snapshot is not None + if apply and not type(self).supports_persistent_update: + diff = ModelDiff(rebuild_reason=RebuildReason.BACKEND_REJECTED) + self._rebuild(model, RebuildReason.BACKEND_REJECTED) + return diff same_model = model is self.model diff = compute_diff(self.snapshot, model, same_model=same_model) if not apply: From f67cec6d158587e8a2df8e01ef850508950d73c7 Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 19 May 2026 15:41:04 +0200 Subject: [PATCH 07/21] feat(solvers): Gurobi apply_update for persistent solves --- linopy/solvers.py | 256 +++++++++++++++++++++++++++++++++ test/test_persistent_gurobi.py | 149 +++++++++++++++++++ 2 files changed, 405 insertions(+) create mode 100644 test/test_persistent_gurobi.py diff --git a/linopy/solvers.py b/linopy/solvers.py index 74fa911a..586a9f4c 100644 --- a/linopy/solvers.py +++ b/linopy/solvers.py @@ -1283,12 +1283,121 @@ class Highs(Solver[None]): SolverFeature.MIP_DUAL_BOUND_REPORT, } ) + supports_persistent_update: ClassVar[bool] = True @classmethod @functools.cache def is_available(cls) -> bool: return _has_module("highspy") + def apply_update( + self, + diff: ModelDiff, + var_label_index: Any, + con_label_index: Any, + ) -> None: + if diff.con_sign: + raise UnsupportedUpdate( + "HiGHS does not support in-place constraint sign change" + ) + + variables = var_label_index._variables + constraints = con_label_index._constraints + var_pos = var_label_index.label_to_pos + con_pos = con_label_index.label_to_pos + + def _container_positions_and_mask(obj: Any) -> tuple[np.ndarray, np.ndarray]: + labels = obj.labels.values.ravel() + mask = labels != -1 + positions = var_pos[labels[mask]].astype(np.int32) + return positions, mask + + def _con_positions_and_mask(obj: Any) -> tuple[np.ndarray, np.ndarray]: + labels = obj.labels.values.ravel() + mask = labels != -1 + positions = con_pos[labels[mask]].astype(np.int32) + return positions, mask + + h = self.solver_model + + bounds_for_type_binary: dict[str, None] = {} + for name, vtype in diff.var_type.items(): + if vtype == "binary": + bounds_for_type_binary[name] = None + + updated_bounds: set[str] = set() + names_for_bounds = set(diff.var_lb) | set(diff.var_ub) | set(bounds_for_type_binary) + for name in names_for_bounds: + var = variables[name] + positions, mask = _container_positions_and_mask(var) + if name in bounds_for_type_binary: + lb = np.zeros(positions.size, dtype=np.float64) + ub = np.ones(positions.size, dtype=np.float64) + else: + lb_src = diff.var_lb.get(name, var.lower).values.ravel()[mask] + ub_src = diff.var_ub.get(name, var.upper).values.ravel()[mask] + lb = np.asarray(lb_src, dtype=np.float64) + ub = np.asarray(ub_src, dtype=np.float64) + h.changeColsBounds(positions.size, positions, lb, ub) + updated_bounds.add(name) + + type_map = { + "continuous": highspy.HighsVarType.kContinuous, + "binary": highspy.HighsVarType.kInteger, + "integer": highspy.HighsVarType.kInteger, + "semi_continuous": highspy.HighsVarType.kSemiContinuous, + } + for name, vtype in diff.var_type.items(): + var = variables[name] + positions, _ = _container_positions_and_mask(var) + integrality = np.full( + positions.size, int(type_map[vtype]), dtype=np.uint8 + ) + h.changeColsIntegrality(positions.size, positions, integrality) + + for name, rhs in diff.con_rhs.items(): + con = constraints[name] + positions, mask = _con_positions_and_mask(con) + rhs_values = np.asarray(rhs.values.ravel()[mask], dtype=np.float64) + sign_values = con.sign.values.ravel()[mask] + inf = np.inf + lower = np.where(sign_values == "<=", -inf, rhs_values) + upper = np.where(sign_values == ">=", inf, rhs_values) + for pos, lo, up in zip(positions, lower, upper): + h.changeRowBounds(int(pos), float(lo), float(up)) + + for name, values in diff.con_coef_updates.items(): + con = constraints[name] + csr, _ = con.to_matrix(var_label_index) + csr.sort_indices() + csr.eliminate_zeros() + n_rows = csr.shape[0] + con_positions, _ = _con_positions_and_mask(con) + assert len(con_positions) == n_rows + for row_idx in range(n_rows): + row_pos = int(con_positions[row_idx]) + start = csr.indptr[row_idx] + end = csr.indptr[row_idx + 1] + cols = csr.indices[start:end] + vals = csr.data[start:end] + for col, val in zip(cols, vals): + h.changeCoeff(row_pos, int(col), float(val)) + + if diff.obj_linear is not None: + n = len(diff.obj_linear.values) + positions = np.arange(n, dtype=np.int32) + costs = np.asarray(diff.obj_linear.values, dtype=np.float64) + h.changeColsCost(n, positions, costs) + + if diff.obj_sense is not None: + sense = ( + highspy.ObjSense.kMaximize + if diff.obj_sense == "max" + else highspy.ObjSense.kMinimize + ) + h.changeObjectiveSense(sense) + self.sense = diff.obj_sense + def _build_direct( self, explicit_coordinate_names: bool = False, @@ -1594,6 +1703,7 @@ class Gurobi(Solver["gurobipy.Env | dict[str, Any] | None"]): SolverFeature.MIP_DUAL_BOUND_REPORT, } ) + supports_persistent_update: ClassVar[bool] = True @classmethod @functools.cache @@ -1704,6 +1814,152 @@ def add_sos(s: xr.DataArray, sos_type: int, sos_dim: str) -> None: gm.update() return gm + _GUROBI_VTYPE_MAP: ClassVar[dict[str, str]] = { + "continuous": "C", + "binary": "B", + "integer": "I", + "semi_continuous": "S", + } + _GUROBI_SIGN_MAP: ClassVar[dict[str, str]] = { + "<=": "<", + ">=": ">", + "=": "=", + } + _GUROBI_SENSE_MAP: ClassVar[dict[str, int]] = {"min": 1, "max": -1} + + def apply_update( + self, + diff: ModelDiff, + var_label_index: Any, + con_label_index: Any, + ) -> None: + model = self.model + assert model is not None + gm = self.solver_model + + var_l2p = var_label_index.label_to_pos + con_l2p = con_label_index.label_to_pos + n_active_vars = var_label_index.n_active_vars + n_active_cons = con_label_index.n_active_cons + + var_payloads: list[tuple[np.ndarray, np.ndarray, str]] = [] + for name, da in diff.var_lb.items(): + var = model.variables[name] + labels = var.labels.values.ravel() + mask = labels != -1 + positions = var_l2p[labels[mask]] + if (positions < 0).any() or (positions >= n_active_vars).any(): + raise UnsupportedUpdate(f"var positions out of range for {name}") + var_payloads.append((positions, da.values.ravel()[mask], "LB")) + for name, da in diff.var_ub.items(): + var = model.variables[name] + labels = var.labels.values.ravel() + mask = labels != -1 + positions = var_l2p[labels[mask]] + if (positions < 0).any() or (positions >= n_active_vars).any(): + raise UnsupportedUpdate(f"var positions out of range for {name}") + var_payloads.append((positions, da.values.ravel()[mask], "UB")) + + type_payloads: list[tuple[np.ndarray, str]] = [] + for name, vtype in diff.var_type.items(): + if vtype not in self._GUROBI_VTYPE_MAP: + raise UnsupportedUpdate(f"unknown var type {vtype}") + var = model.variables[name] + labels = var.labels.values.ravel() + mask = labels != -1 + positions = var_l2p[labels[mask]] + if (positions < 0).any() or (positions >= n_active_vars).any(): + raise UnsupportedUpdate(f"var positions out of range for {name}") + type_payloads.append((positions, self._GUROBI_VTYPE_MAP[vtype])) + + rhs_payloads: list[tuple[np.ndarray, np.ndarray]] = [] + for name, da in diff.con_rhs.items(): + con = model.constraints[name] + labels = con.labels.values.ravel() + mask = labels != -1 + positions = con_l2p[labels[mask]] + if (positions < 0).any() or (positions >= n_active_cons).any(): + raise UnsupportedUpdate(f"con positions out of range for {name}") + rhs_payloads.append((positions, da.values.ravel()[mask])) + + sign_payloads: list[tuple[np.ndarray, np.ndarray]] = [] + for name, da in diff.con_sign.items(): + sign_strs = da.values.ravel() + con = model.constraints[name] + labels = con.labels.values.ravel() + mask = labels != -1 + sign_strs = sign_strs[mask] + mapped = np.empty(len(sign_strs), dtype=object) + for i, s in enumerate(sign_strs): + s = str(s) + if s not in self._GUROBI_SIGN_MAP: + raise UnsupportedUpdate(f"unknown sign {s!r}") + mapped[i] = self._GUROBI_SIGN_MAP[s] + positions = con_l2p[labels[mask]] + if (positions < 0).any() or (positions >= n_active_cons).any(): + raise UnsupportedUpdate(f"con positions out of range for {name}") + sign_payloads.append((positions, mapped)) + + coef_payloads: list[tuple[np.ndarray, np.ndarray, np.ndarray]] = [] + for name, values in diff.con_coef_updates.items(): + con = model.constraints[name] + csr, _ = con.to_matrix(var_label_index) + csr.sort_indices() + csr.eliminate_zeros() + if csr.data.shape != values.shape: + raise UnsupportedUpdate(f"coef shape mismatch for {name}") + row_pos_local = np.repeat( + np.arange(csr.shape[0], dtype=np.int64), np.diff(csr.indptr) + ) + active_labels = con.active_labels() + row_positions = con_l2p[active_labels[row_pos_local]] + col_positions = csr.indices.astype(np.int64) + if (row_positions < 0).any() or (row_positions >= n_active_cons).any(): + raise UnsupportedUpdate(f"con positions out of range for {name}") + if (col_positions < 0).any() or (col_positions >= n_active_vars).any(): + raise UnsupportedUpdate(f"var positions out of range for {name}") + coef_payloads.append((row_positions, col_positions, values)) + + if diff.obj_sense is not None and diff.obj_sense not in self._GUROBI_SENSE_MAP: + raise UnsupportedUpdate(f"unknown obj sense {diff.obj_sense!r}") + + gurobi_vars = gm.getVars() + gurobi_cons = gm.getConstrs() + if len(gurobi_vars) != n_active_vars: + raise UnsupportedUpdate("gurobi var count mismatch") + if len(gurobi_cons) != n_active_cons: + raise UnsupportedUpdate("gurobi con count mismatch") + + for positions, values, attr in var_payloads: + for pos, val in zip(positions, values): + gurobi_vars[int(pos)].setAttr(attr, float(val)) + + for positions, vtype_str in type_payloads: + for pos in positions: + gurobi_vars[int(pos)].setAttr("VType", vtype_str) + + for positions, values in rhs_payloads: + for pos, val in zip(positions, values): + gurobi_cons[int(pos)].setAttr("RHS", float(val)) + + for positions, senses in sign_payloads: + for pos, s in zip(positions, senses): + gurobi_cons[int(pos)].setAttr("Sense", s) + + for row_positions, col_positions, values in coef_payloads: + for r, c, v in zip(row_positions, col_positions, values): + gm.chgCoeff(gurobi_cons[int(r)], gurobi_vars[int(c)], float(v)) + + if diff.obj_linear is not None: + obj_values = diff.obj_linear.values + for pos in range(n_active_vars): + gurobi_vars[pos].setAttr("Obj", float(obj_values[pos])) + + if diff.obj_sense is not None: + gm.ModelSense = self._GUROBI_SENSE_MAP[diff.obj_sense] + + gm.update() + def _run_direct( self, solution_fn: Path | None = None, diff --git a/test/test_persistent_gurobi.py b/test/test_persistent_gurobi.py new file mode 100644 index 00000000..d2dd8bd9 --- /dev/null +++ b/test/test_persistent_gurobi.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import numpy as np +import pytest + +from linopy import Model +from linopy.persistent import RebuildReason +from linopy.solvers import Gurobi + +pytest.importorskip("gurobipy") + + +def _base_model() -> Model: + m = Model() + x = m.add_variables(0, 10, coords=[range(3)], name="x") + y = m.add_variables(0, 10, coords=[range(3)], name="y") + m.add_constraints(x + y >= 4, name="c1") + m.add_constraints(2 * x + y <= 20, name="c2") + m.add_objective(x.sum() + 2 * y.sum()) + return m + + +def _built(model: Model) -> Gurobi: + s = Gurobi(model=model, io_api="direct") + s.options = {"OutputFlag": 0} + s._build() + return s + + +def _solve_and_assign(solver: Gurobi, model: Model) -> float: + result = solver.solve(model, assign=True) + return float(result.solution.objective) + + +def test_var_lb_in_place() -> None: + m = _base_model() + s = _built(m) + s.solve(assign=True) + assert s._rebuilds == 0 + assert s._in_place_updates == 0 + base_obj = float(m.objective.value) + + m.variables["x"].lower.values[...] = 5.0 + obj = _solve_and_assign(s, m) + assert s._rebuilds == 0 + assert s._in_place_updates == 1 + assert s._last_rebuild_reason is RebuildReason.NONE + assert obj > base_obj + + +def test_var_ub_in_place() -> None: + m = _base_model() + s = _built(m) + s.solve(assign=True) + + m.variables["x"].upper.values[...] = 1.0 + _solve_and_assign(s, m) + assert s._in_place_updates == 1 + assert s._rebuilds == 0 + + +def test_rhs_only_in_place() -> None: + m = _base_model() + s = _built(m) + s.solve(assign=True) + base_obj = float(m.objective.value) + + c = m.constraints["c1"] + c.rhs = 8.0 + assert c._coef_dirty is False + obj = _solve_and_assign(s, m) + assert s._in_place_updates == 1 + assert s._rebuilds == 0 + assert obj > base_obj + + +def test_constraint_coef_change_in_place() -> None: + m = _base_model() + s = _built(m) + s.solve(assign=True) + base_obj = float(m.objective.value) + + c = m.constraints["c1"] + new_coeffs = c.coeffs * 2 + c.coeffs = new_coeffs + obj = _solve_and_assign(s, m) + assert s._in_place_updates == 1 + assert s._rebuilds == 0 + assert obj != base_obj + + +def test_objective_linear_change_in_place() -> None: + m = _base_model() + s = _built(m) + s.solve(assign=True) + base_obj = float(m.objective.value) + + x = m.variables["x"] + y = m.variables["y"] + m.objective.expression = 3 * x.sum() + 7 * y.sum() + obj = _solve_and_assign(s, m) + assert s._in_place_updates == 1 + assert s._rebuilds == 0 + assert obj != base_obj + + +def test_objective_sense_flip_in_place() -> None: + m = _base_model() + s = _built(m) + s.solve(assign=True) + min_obj = float(m.objective.value) + + m.objective.sense = "max" + max_obj = _solve_and_assign(s, m) + assert s._in_place_updates == 1 + assert s._rebuilds == 0 + assert max_obj > min_obj + + +def test_sparsity_change_triggers_rebuild() -> None: + m = _base_model() + s = _built(m) + s.solve(assign=True) + + x = m.variables["x"] + m.add_constraints(x <= 5, name="c3") + s.solve(m, assign=True) + assert s._rebuilds == 1 + assert s._last_rebuild_reason is RebuildReason.STRUCTURAL_CONTAINERS + + +def test_cross_model_in_place() -> None: + m1 = _base_model() + s = _built(m1) + s.solve(assign=True) + + m2 = _base_model() + m2.constraints["c1"].rhs = 8.0 + + s.solve(m2, assign=True) + assert s._in_place_updates == 1 + assert s._rebuilds == 0 + + fresh_obj = m2.objective.value + m3 = _base_model() + m3.constraints["c1"].rhs = 8.0 + s_fresh = _built(m3) + s_fresh.solve(assign=True) + assert np.isclose(float(fresh_obj), float(m3.objective.value)) From 80262939cf93c115e5ed7d057155d3be99c26add Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 19 May 2026 15:43:28 +0200 Subject: [PATCH 08/21] feat(solvers): HiGHS apply_update for persistent solves --- linopy/solvers.py | 29 ++---- test/test_persistent_highs.py | 161 ++++++++++++++++++++++++++++++++++ 2 files changed, 169 insertions(+), 21 deletions(-) create mode 100644 test/test_persistent_highs.py diff --git a/linopy/solvers.py b/linopy/solvers.py index 586a9f4c..9734e407 100644 --- a/linopy/solvers.py +++ b/linopy/solvers.py @@ -1320,17 +1320,12 @@ def _con_positions_and_mask(obj: Any) -> tuple[np.ndarray, np.ndarray]: h = self.solver_model - bounds_for_type_binary: dict[str, None] = {} - for name, vtype in diff.var_type.items(): - if vtype == "binary": - bounds_for_type_binary[name] = None - - updated_bounds: set[str] = set() - names_for_bounds = set(diff.var_lb) | set(diff.var_ub) | set(bounds_for_type_binary) + binary_names = {n for n, t in diff.var_type.items() if t == "binary"} + names_for_bounds = set(diff.var_lb) | set(diff.var_ub) | binary_names for name in names_for_bounds: var = variables[name] positions, mask = _container_positions_and_mask(var) - if name in bounds_for_type_binary: + if name in binary_names: lb = np.zeros(positions.size, dtype=np.float64) ub = np.ones(positions.size, dtype=np.float64) else: @@ -1339,7 +1334,6 @@ def _con_positions_and_mask(obj: Any) -> tuple[np.ndarray, np.ndarray]: lb = np.asarray(lb_src, dtype=np.float64) ub = np.asarray(ub_src, dtype=np.float64) h.changeColsBounds(positions.size, positions, lb, ub) - updated_bounds.add(name) type_map = { "continuous": highspy.HighsVarType.kContinuous, @@ -1350,9 +1344,7 @@ def _con_positions_and_mask(obj: Any) -> tuple[np.ndarray, np.ndarray]: for name, vtype in diff.var_type.items(): var = variables[name] positions, _ = _container_positions_and_mask(var) - integrality = np.full( - positions.size, int(type_map[vtype]), dtype=np.uint8 - ) + integrality = np.full(positions.size, int(type_map[vtype]), dtype=np.uint8) h.changeColsIntegrality(positions.size, positions, integrality) for name, rhs in diff.con_rhs.items(): @@ -1366,22 +1358,17 @@ def _con_positions_and_mask(obj: Any) -> tuple[np.ndarray, np.ndarray]: for pos, lo, up in zip(positions, lower, upper): h.changeRowBounds(int(pos), float(lo), float(up)) - for name, values in diff.con_coef_updates.items(): + for name in diff.con_coef_updates: con = constraints[name] csr, _ = con.to_matrix(var_label_index) csr.sort_indices() csr.eliminate_zeros() - n_rows = csr.shape[0] con_positions, _ = _con_positions_and_mask(con) - assert len(con_positions) == n_rows - for row_idx in range(n_rows): - row_pos = int(con_positions[row_idx]) + for row_idx, row_pos in enumerate(con_positions): start = csr.indptr[row_idx] end = csr.indptr[row_idx + 1] - cols = csr.indices[start:end] - vals = csr.data[start:end] - for col, val in zip(cols, vals): - h.changeCoeff(row_pos, int(col), float(val)) + for col, val in zip(csr.indices[start:end], csr.data[start:end]): + h.changeCoeff(int(row_pos), int(col), float(val)) if diff.obj_linear is not None: n = len(diff.obj_linear.values) diff --git a/test/test_persistent_highs.py b/test/test_persistent_highs.py new file mode 100644 index 00000000..d1620a30 --- /dev/null +++ b/test/test_persistent_highs.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import numpy as np +import pytest + +from linopy import Model +from linopy.persistent import RebuildReason +from linopy.solvers import Highs + +pytest.importorskip("highspy") + + +def _base_model() -> Model: + m = Model() + x = m.add_variables(0, 10, coords=[range(3)], name="x") + y = m.add_variables(0, 10, coords=[range(3)], name="y") + m.add_constraints(x + y >= 4, name="c1") + m.add_constraints(2 * x + y <= 20, name="c2") + m.add_objective(x.sum() + 2 * y.sum()) + return m + + +def _built(model: Model) -> Highs: + s = Highs(model=model, io_api="direct") + s.options = {"output_flag": False} + s._build() + return s + + +def _solve_and_assign(solver: Highs, model: Model) -> float: + result = solver.solve(model, assign=True) + return float(result.solution.objective) + + +def test_var_lb_in_place() -> None: + m = _base_model() + s = _built(m) + s.solve(assign=True) + base_obj = float(m.objective.value) + + m.variables["x"].lower.values[...] = 6.0 + obj = _solve_and_assign(s, m) + assert s._in_place_updates == 1 + assert s._rebuilds == 0 + assert s._last_rebuild_reason is RebuildReason.NONE + assert obj > base_obj + + +def test_var_ub_in_place() -> None: + m = _base_model() + s = _built(m) + s.solve(assign=True) + + m.variables["x"].upper.values[...] = 1.0 + _solve_and_assign(s, m) + assert s._in_place_updates == 1 + assert s._rebuilds == 0 + + +def test_rhs_only_in_place() -> None: + m = _base_model() + s = _built(m) + s.solve(assign=True) + base_obj = float(m.objective.value) + + c = m.constraints["c1"] + c.rhs = 8.0 + assert c._coef_dirty is False + obj = _solve_and_assign(s, m) + assert s._in_place_updates == 1 + assert s._rebuilds == 0 + assert obj > base_obj + + +def test_constraint_coef_change_in_place() -> None: + m = _base_model() + s = _built(m) + s.solve(assign=True) + base_obj = float(m.objective.value) + + c = m.constraints["c1"] + c.coeffs = c.coeffs * 2 + obj = _solve_and_assign(s, m) + assert s._in_place_updates == 1 + assert s._rebuilds == 0 + assert not np.isclose(obj, base_obj) + + +def test_objective_linear_change_in_place() -> None: + m = _base_model() + s = _built(m) + s.solve(assign=True) + base_obj = float(m.objective.value) + + x = m.variables["x"] + y = m.variables["y"] + m.objective.expression = 5 * x.sum() + 3 * y.sum() + obj = _solve_and_assign(s, m) + assert s._in_place_updates == 1 + assert s._rebuilds == 0 + assert not np.isclose(obj, base_obj) + + +def test_objective_sense_flip_in_place() -> None: + m = _base_model() + s = _built(m) + s.solve(assign=True) + min_obj = float(m.objective.value) + + m.objective.sense = "max" + max_obj = _solve_and_assign(s, m) + assert s._in_place_updates == 1 + assert s._rebuilds == 0 + assert max_obj > min_obj + + +def test_sign_flip_falls_back_to_rebuild() -> None: + m = _base_model() + s = _built(m) + s.solve(assign=True) + + c = m.constraints["c1"] + c.sign = "<=" + s.solve(m, assign=True) + assert s._rebuilds == 1 + assert s._last_rebuild_reason is RebuildReason.BACKEND_REJECTED + + +def test_sparsity_change_triggers_rebuild() -> None: + m = _base_model() + s = _built(m) + s.solve(assign=True) + + x = m.variables["x"] + m.add_constraints(x <= 5, name="c3") + s.solve(m, assign=True) + assert s._rebuilds == 1 + assert s._last_rebuild_reason in { + RebuildReason.STRUCTURAL_LABELS, + RebuildReason.STRUCTURAL_CONTAINERS, + } + + +def test_cross_model_in_place() -> None: + m1 = _base_model() + s = _built(m1) + s.solve(assign=True) + + m2 = _base_model() + m2.constraints["c1"].rhs = 8.0 + + s.solve(m2, assign=True) + assert s._in_place_updates == 1 + assert s._rebuilds == 0 + + cross_obj = float(m2.objective.value) + m3 = _base_model() + m3.constraints["c1"].rhs = 8.0 + s_fresh = _built(m3) + s_fresh.solve(assign=True) + assert np.isclose(cross_obj, float(m3.objective.value)) From b6601e3d4e000eda417be405d68e724bf1ce978d Mon Sep 17 00:00:00 2001 From: Fabian Date: Tue, 19 May 2026 15:49:53 +0200 Subject: [PATCH 09/21] test(persistent): cross-model, pickle, threading, failure-path coverage --- test/test_persistent_solver_extras.py | 296 ++++++++++++++++++++++++++ 1 file changed, 296 insertions(+) create mode 100644 test/test_persistent_solver_extras.py diff --git a/test/test_persistent_solver_extras.py b/test/test_persistent_solver_extras.py new file mode 100644 index 00000000..7c49bfc1 --- /dev/null +++ b/test/test_persistent_solver_extras.py @@ -0,0 +1,296 @@ +from __future__ import annotations + +import pickle +import threading +from typing import Any + +import numpy as np +import pytest + +from linopy import Model +from linopy.persistent import RebuildReason +from linopy.solvers import Gurobi, Highs, Solver + +_BACKENDS: dict[str, tuple[type[Solver], dict[str, Any]]] = { + "gurobi": (Gurobi, {"OutputFlag": 0}), + "highs": (Highs, {"output_flag": False}), +} + + +def _have(name: str) -> bool: + try: + if name == "gurobi": + import gurobipy # noqa: F401 + elif name == "highs": + import highspy # noqa: F401 + return True + except ImportError: + return False + + +SOLVER_PARAMS = [ + pytest.param( + "gurobi", + marks=pytest.mark.skipif(not _have("gurobi"), reason="gurobipy not installed"), + ), + pytest.param( + "highs", + marks=pytest.mark.skipif(not _have("highs"), reason="highspy not installed"), + ), +] + + +def _base_model() -> Model: + m = Model() + x = m.add_variables(0, 10, coords=[range(3)], name="x") + y = m.add_variables(0, 10, coords=[range(3)], name="y") + m.add_constraints(x + y >= 4, name="c1") + m.add_constraints(2 * x + y <= 20, name="c2") + m.add_objective(x.sum() + 2 * y.sum()) + return m + + +def _built(solver_name: str, model: Model) -> Solver: + cls, opts = _BACKENDS[solver_name] + s = cls(model=model, io_api="direct") + s.options = opts + s._build() + return s + + +@pytest.mark.parametrize("solver_name", SOLVER_PARAMS) +def test_noop_resolve_increments_in_place(solver_name: str) -> None: + m = _base_model() + s = _built(solver_name, m) + s.solve(assign=True) + first_obj = float(m.objective.value) + + s.solve(m, assign=True) + assert s._in_place_updates == 1 + assert s._rebuilds == 0 + assert np.isclose(float(m.objective.value), first_obj) + + +@pytest.mark.parametrize("solver_name", SOLVER_PARAMS) +def test_two_consecutive_solves_no_stale_state(solver_name: str) -> None: + m = _base_model() + s = _built(solver_name, m) + s.solve(assign=True) + first_status = s.status + + m.variables["x"].lower.values[...] = 5.0 + s.solve(m, assign=True) + assert s.status is not first_status + assert s.solution is not None + assert np.isclose(float(s.solution.objective), float(m.objective.value)) + + +@pytest.mark.parametrize("solver_name", SOLVER_PARAMS) +def test_cross_model_scenario_sweep(solver_name: str) -> None: + m1 = _base_model() + m2 = _base_model() + m2.constraints["c1"].rhs = 6.0 + m3 = _base_model() + m3.variables["x"].lower.values[...] = 2.0 + + s = _built(solver_name, m1) + s.solve(assign=True) + obj1 = float(m1.objective.value) + sol1 = m1.solution + + s.solve(m2, assign=True) + s.solve(m3, assign=True) + + assert s._rebuilds == 0 + assert s._in_place_updates >= 2 + + assert m1.objective._value == obj1 + np.testing.assert_array_equal(m1.solution.x.values, sol1.x.values) + assert m2.objective._value is not None + assert m3.objective._value is not None + + for mk in (m2, m3): + fresh = _base_model() + if mk is m2: + fresh.constraints["c1"].rhs = 6.0 + else: + fresh.variables["x"].lower.values[...] = 2.0 + s_fresh = _built(solver_name, fresh) + s_fresh.solve(assign=True) + assert np.isclose(float(mk.objective.value), float(fresh.objective.value)) + s_fresh.close() + + +@pytest.mark.parametrize("solver_name", SOLVER_PARAMS) +def test_cross_model_sparsity_change_rebuilds(solver_name: str) -> None: + def build(include_y_in_c1: bool) -> Model: + m = Model() + x = m.add_variables(0, 10, coords=[range(3)], name="x") + y = m.add_variables(0, 10, coords=[range(3)], name="y") + if include_y_in_c1: + m.add_constraints(x + y >= 4, name="c1") + else: + m.add_constraints(2 * x >= 4, name="c1") + m.add_constraints(2 * x + y <= 20, name="c2") + m.add_objective(x.sum() + 2 * y.sum()) + return m + + m1 = build(include_y_in_c1=True) + s = _built(solver_name, m1) + s.solve(assign=True) + + m2 = build(include_y_in_c1=False) + + s.solve(m2, assign=True) + assert s._rebuilds == 1 + assert s._last_rebuild_reason in { + RebuildReason.SPARSITY, + RebuildReason.STRUCTURAL_LABELS, + RebuildReason.STRUCTURAL_CONTAINERS, + } + + +@pytest.mark.parametrize("solver_name", SOLVER_PARAMS) +def test_cross_model_structural_mismatch_rebuilds(solver_name: str) -> None: + m1 = _base_model() + s = _built(solver_name, m1) + s.solve(assign=True) + + m2 = _base_model() + m2.add_variables(0, 5, coords=[range(3)], name="z") + + s.solve(m2, assign=True) + assert s._rebuilds == 1 + assert s.model is m2 + + +@pytest.mark.parametrize("solver_name", SOLVER_PARAMS) +def test_dirty_flag_ignored_across_models(solver_name: str) -> None: + m1 = _base_model() + s = _built(solver_name, m1) + s.solve(assign=True) + + m2 = _base_model() + c = m2.constraints["c1"] + c.coeffs = c.coeffs * 3 + c._coef_dirty = False + + s.solve(m2, assign=True) + assert s._rebuilds == 0 + assert s._in_place_updates == 1 + + fresh = _base_model() + cf = fresh.constraints["c1"] + cf.coeffs = cf.coeffs * 3 + s_fresh = _built(solver_name, fresh) + s_fresh.solve(assign=True) + assert np.isclose(float(m2.objective.value), float(fresh.objective.value)) + s_fresh.close() + + +@pytest.mark.parametrize("solver_name", SOLVER_PARAMS) +def test_solver_pickle_round_trip_drops_native(solver_name: str) -> None: + m = _base_model() + s = _built(solver_name, m) + s.solve(assign=True) + + state = s.__getstate__() + for key in ("solver_model", "env", "_env_stack", "snapshot", "_lock"): + assert key not in state + + restored = pickle.loads(pickle.dumps(s)) + assert restored.solver_model is None + assert restored.snapshot is None + assert restored._env_stack is None + assert isinstance(restored._lock, type(threading.Lock())) + + +@pytest.mark.parametrize("solver_name", SOLVER_PARAMS) +def test_model_pickle_round_trip_no_native_handle(solver_name: str) -> None: + m = _base_model() + s = _built(solver_name, m) + s.solve(assign=True) + + m2 = pickle.loads(pickle.dumps(m)) + s2 = _built(solver_name, m2) + assert s2.solver_model is not None + s2.solve(assign=True) + assert s2._rebuilds == 0 + assert np.isclose(float(m.objective.value), float(m2.objective.value)) + s2.close() + + +@pytest.mark.parametrize("solver_name", SOLVER_PARAMS) +def test_backend_exception_during_apply_rebuilds( + solver_name: str, monkeypatch: pytest.MonkeyPatch +) -> None: + m = _base_model() + s = _built(solver_name, m) + s.solve(assign=True) + + c = m.constraints["c1"] + c.coeffs = c.coeffs * 2 + assert c._coef_dirty is True + + def _boom(*args: Any, **kwargs: Any) -> None: + raise RuntimeError("simulated backend failure") + + monkeypatch.setattr(s, "apply_update", _boom) + + dirty_at_rebuild: list[bool] = [] + original_build = s._build + + def _spy_build(**kwargs: Any) -> None: + dirty_at_rebuild.append(m.constraints["c1"]._coef_dirty) + original_build(**kwargs) + + monkeypatch.setattr(s, "_build", _spy_build) + + s.solve(m, assign=True) + assert s._rebuilds == 1 + assert s._last_rebuild_reason is RebuildReason.BACKEND_REJECTED + assert dirty_at_rebuild == [True] + + +@pytest.mark.parametrize("solver_name", SOLVER_PARAMS) +def test_concurrent_solves_serialize(solver_name: str) -> None: + m = _base_model() + s = _built(solver_name, m) + s.solve(assign=True) + expected = float(m.objective.value) + + barrier = threading.Barrier(2) + results: list[float] = [] + errors: list[BaseException] = [] + + def _run() -> None: + try: + barrier.wait() + res = s.solve(m, assign=True) + results.append(float(res.solution.objective)) + except BaseException as e: + errors.append(e) + + threads = [threading.Thread(target=_run) for _ in range(2)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert not errors, errors + assert len(results) == 2 + for r in results: + assert np.isclose(r, expected) + + +@pytest.mark.parametrize("solver_name", SOLVER_PARAMS) +def test_solve_without_assign_does_not_mutate_model(solver_name: str) -> None: + m = _base_model() + s = _built(solver_name, m) + + assert m.objective._value is None + s.solve() + assert m.objective._value is None + + s.solve(assign=True) + assert m.objective._value is not None From 6922c7d8f064080137136ea27e06c2040882ec63 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 20 May 2026 12:42:52 +0200 Subject: [PATCH 10/21] refactor(persistent): row-block numpy snapshot/diff Replace xarray-based snapshot and CSR pattern compare with per-row canonicalised numpy buffers; new ContainerVarUpdate / ContainerRowUpdate payloads. Gurobi/HiGHS apply_update rewritten around batched setAttr / changeColsBounds / changeColsCost / changeColsIntegrality; coefficient writes touch only changed cells. Cross-model diff now ~matches same-model cost for bound/rhs/coef-value sweeps. --- linopy/persistent/__init__.py | 20 +- linopy/persistent/diff.py | 262 +++++++++++++++----- linopy/persistent/snapshot.py | 179 ++++++++------ linopy/solvers.py | 291 +++++++++-------------- test/test_persistent_snapshot_buffers.py | 126 ++++++++++ test/test_persistent_snapshot_diff.py | 37 ++- test/test_persistent_solver_extras.py | 59 +++++ 7 files changed, 645 insertions(+), 329 deletions(-) create mode 100644 test/test_persistent_snapshot_buffers.py diff --git a/linopy/persistent/__init__.py b/linopy/persistent/__init__.py index 5823ee2f..6fb1ca4c 100644 --- a/linopy/persistent/__init__.py +++ b/linopy/persistent/__init__.py @@ -2,12 +2,26 @@ from __future__ import annotations -from linopy.persistent.diff import ModelDiff, RebuildReason, compute_diff +from linopy.persistent.diff import ( + ContainerRowUpdate, + ContainerVarUpdate, + ModelDiff, + RebuildReason, + compute_diff, +) from linopy.persistent.errors import UnsupportedUpdate -from linopy.persistent.snapshot import CoefPattern, ModelSnapshot, StructuralKey +from linopy.persistent.snapshot import ( + ContainerConBuffers, + ContainerVarBuffers, + ModelSnapshot, + StructuralKey, +) __all__ = [ - "CoefPattern", + "ContainerConBuffers", + "ContainerRowUpdate", + "ContainerVarBuffers", + "ContainerVarUpdate", "ModelDiff", "ModelSnapshot", "RebuildReason", diff --git a/linopy/persistent/diff.py b/linopy/persistent/diff.py index 28b7a4d8..4af3df63 100644 --- a/linopy/persistent/diff.py +++ b/linopy/persistent/diff.py @@ -5,15 +5,12 @@ from typing import TYPE_CHECKING import numpy as np -import xarray as xr from linopy.persistent.snapshot import ( - CoefPattern, ModelSnapshot, - _canonical_csr, - _coord_snapshot, + _extract_con_buffers, + _extract_var_buffers, _objective_linear_vector, - _variable_type, ) if TYPE_CHECKING: @@ -31,30 +28,57 @@ class RebuildReason(enum.Enum): @dataclass -class ModelDiff: - rebuild_reason: RebuildReason = RebuildReason.NONE +class ContainerVarUpdate: + """ + In-place variable bounds / type update for one container. + + Bounds payloads share ``bounds_indices``. When only ``lower`` (or only + ``upper``) changes, both arrays are still populated from the new model so + backends with a single batched call (HiGHS ``changeColsBounds``) can be + fed directly. + """ + + bounds_indices: np.ndarray | None = None + lower: np.ndarray | None = None + upper: np.ndarray | None = None + type_change: str | None = None + + +@dataclass +class ContainerRowUpdate: + """ + Per-row constraint update. + + Holds views into the new model's canonicalised buffers; the orchestrator + diffs and applies under the same lock, so aliasing is bounded. + """ + + coef_row_indices: np.ndarray | None = None + coef_vars: np.ndarray | None = None + coef_values: np.ndarray | None = None + rhs_row_indices: np.ndarray | None = None + rhs_values: np.ndarray | None = None + rhs_signs: np.ndarray | None = None + sign_row_indices: np.ndarray | None = None + sign_values: np.ndarray | None = None - var_lb: dict[str, xr.DataArray] = field(default_factory=dict) - var_ub: dict[str, xr.DataArray] = field(default_factory=dict) - var_type: dict[str, str] = field(default_factory=dict) - con_rhs: dict[str, xr.DataArray] = field(default_factory=dict) - con_sign: dict[str, xr.DataArray] = field(default_factory=dict) - con_coef_updates: dict[str, np.ndarray] = field(default_factory=dict) - obj_linear: xr.DataArray | None = None +@dataclass +class ModelDiff: + rebuild_reason: RebuildReason = RebuildReason.NONE + vars: dict[str, ContainerVarUpdate] = field(default_factory=dict) + cons: dict[str, ContainerRowUpdate] = field(default_factory=dict) + obj_c_indices: np.ndarray | None = None + obj_c_values: np.ndarray | None = None obj_sense: str | None = None @property def is_empty(self) -> bool: return ( self.rebuild_reason is RebuildReason.NONE - and not self.var_lb - and not self.var_ub - and not self.var_type - and not self.con_rhs - and not self.con_sign - and not self.con_coef_updates - and self.obj_linear is None + and not self.vars + and not self.cons + and self.obj_c_indices is None and self.obj_sense is None ) @@ -62,15 +86,80 @@ def is_empty(self) -> bool: def rebuild_required(self) -> bool: return self.rebuild_reason is not RebuildReason.NONE + @property + def changed_variables(self) -> set[str]: + return set(self.vars) + + @property + def changed_constraints(self) -> set[str]: + return set(self.cons) + + @property + def n_coef_updates(self) -> int: + total = 0 + for upd in self.cons.values(): + if upd.coef_vars is not None: + total += int((upd.coef_vars != -1).sum()) + return total + + def summary(self) -> dict[str, int | bool | str | None]: + n_var_lb = sum(1 for u in self.vars.values() if u.lower is not None) + n_var_ub = sum(1 for u in self.vars.values() if u.upper is not None) + n_var_type = sum(1 for u in self.vars.values() if u.type_change is not None) + n_con_rhs = sum(1 for u in self.cons.values() if u.rhs_values is not None) + n_con_sign = sum(1 for u in self.cons.values() if u.sign_values is not None) + n_con_coef = sum(1 for u in self.cons.values() if u.coef_values is not None) + return { + "rebuild_reason": self.rebuild_reason.value, + "var_lb": n_var_lb, + "var_ub": n_var_ub, + "var_type": n_var_type, + "con_rhs": n_con_rhs, + "con_sign": n_con_sign, + "con_coef_updates": n_con_coef, + "n_coef_values": self.n_coef_updates, + "obj_linear_changed": self.obj_c_indices is not None, + "obj_sense_changed_to": self.obj_sense, + } -def _coords_equal(a: dict[str, np.ndarray], b: dict[str, np.ndarray]) -> bool: - if a.keys() != b.keys(): - return False - return all(np.array_equal(a[k], b[k]) for k in a) + def inspect_variable(self, name: str) -> dict[str, object]: + if name not in self.vars: + return {} + u = self.vars[name] + entry: dict[str, object] = {} + if u.lower is not None: + entry["lower"] = u.lower + if u.upper is not None: + entry["upper"] = u.upper + if u.type_change is not None: + entry["type"] = u.type_change + return entry + def inspect_constraint(self, name: str) -> dict[str, object]: + if name not in self.cons: + return {} + u = self.cons[name] + entry: dict[str, object] = {} + if u.rhs_values is not None: + entry["rhs"] = u.rhs_values + if u.sign_values is not None: + entry["sign"] = u.sign_values + if u.coef_values is not None: + entry["coef_values"] = u.coef_values + return entry -def _any_diff(a: xr.DataArray, b: xr.DataArray) -> bool: - return bool((a != b).any().item()) + def __repr__(self) -> str: + if self.is_empty: + return "ModelDiff(empty)" + if self.rebuild_required: + return f"ModelDiff(rebuild_required={self.rebuild_reason.value!r})" + s = self.summary() + parts = [ + f"{k}={v}" + for k, v in s.items() + if k != "rebuild_reason" and v not in (0, False, None) + ] + return "ModelDiff(" + ", ".join(parts) + ")" def compute_diff( @@ -96,42 +185,93 @@ def compute_diff( diff.rebuild_reason = RebuildReason.STRUCTURAL_LABELS return diff + var_l2p = var_label_index.label_to_pos + con_l2p = con_label_index.label_to_pos + for name, var in model.variables.items(): - if not _coords_equal(snapshot.var_coords[name], _coord_snapshot(var)): + snap_buf = snapshot.var_buffers[name] + new_buf = _extract_var_buffers(var) + if new_buf.lower.shape != snap_buf.lower.shape: diff.rebuild_reason = RebuildReason.COORD_REINDEX return diff - if _any_diff(snapshot.var_lb[name], var.lower): - diff.var_lb[name] = var.lower.copy(deep=True) - if _any_diff(snapshot.var_ub[name], var.upper): - diff.var_ub[name] = var.upper.copy(deep=True) - vtype = _variable_type(var) - if snapshot.var_type[name] != vtype: - diff.var_type[name] = vtype + if not np.array_equal(new_buf.active_labels, snap_buf.active_labels): + diff.rebuild_reason = RebuildReason.STRUCTURAL_LABELS + return diff + + lower_diff = new_buf.lower != snap_buf.lower + upper_diff = new_buf.upper != snap_buf.upper + type_changed = new_buf.type != snap_buf.type + + bound_mask = lower_diff | upper_diff + if not (bound_mask.any() or type_changed): + continue + + update = ContainerVarUpdate(type_change=new_buf.type if type_changed else None) + if bound_mask.any(): + local_idx = np.flatnonzero(bound_mask) + update.bounds_indices = var_l2p[ + new_buf.active_labels[local_idx] + ].astype(np.int32, copy=False) + update.lower = new_buf.lower[local_idx] + update.upper = new_buf.upper[local_idx] + diff.vars[name] = update for name, con in model.constraints.items(): - if not _coords_equal(snapshot.con_coords[name], _coord_snapshot(con)): - diff.rebuild_reason = RebuildReason.COORD_REINDEX + snap_buf = snapshot.con_buffers[name] + new_buf = _extract_con_buffers(con, var_l2p) + + if new_buf.coeffs.shape != snap_buf.coeffs.shape: + diff.rebuild_reason = RebuildReason.SPARSITY + return diff + if not np.array_equal(new_buf.active_labels, snap_buf.active_labels): + diff.rebuild_reason = RebuildReason.STRUCTURAL_LABELS return diff - if _any_diff(snapshot.con_rhs[name], con.rhs): - diff.con_rhs[name] = con.rhs.copy(deep=True) - if _any_diff(snapshot.con_sign[name], con.sign): - diff.con_sign[name] = con.sign.copy(deep=True) - - if same_model: - dirty_names = [n for n, c in model.constraints.items() if c._coef_dirty] - else: - dirty_names = list(con_names) - - for name in dirty_names: - con = model.constraints[name] - indptr, indices, data = _canonical_csr(con, var_label_index) - pattern = CoefPattern(indptr=indptr, indices=indices) - if pattern == snapshot.con_coef_pattern[name]: - diff.con_coef_updates[name] = data + + n_rows = new_buf.active_labels.size + if n_rows == 0: + continue + + skip_coef_compare = same_model and not con._coef_dirty + if skip_coef_compare: + row_value_changed = np.zeros(n_rows, dtype=bool) + row_struct_changed = np.zeros(n_rows, dtype=bool) else: + row_struct_changed = np.any(new_buf.vars != snap_buf.vars, axis=-1) + row_value_changed = np.any(new_buf.coeffs != snap_buf.coeffs, axis=-1) + + if row_struct_changed.any(): diff.rebuild_reason = RebuildReason.SPARSITY return diff + rhs_changed = new_buf.rhs != snap_buf.rhs + sign_changed = new_buf.sign != snap_buf.sign + + if not (row_value_changed.any() or rhs_changed.any() or sign_changed.any()): + continue + + update = ContainerRowUpdate() + if row_value_changed.any(): + idx = np.flatnonzero(row_value_changed) + update.coef_row_indices = con_l2p[ + new_buf.active_labels[idx] + ].astype(np.int32, copy=False) + update.coef_vars = new_buf.vars[idx] + update.coef_values = new_buf.coeffs[idx] + if rhs_changed.any(): + idx = np.flatnonzero(rhs_changed) + update.rhs_row_indices = con_l2p[ + new_buf.active_labels[idx] + ].astype(np.int32, copy=False) + update.rhs_values = new_buf.rhs[idx] + update.rhs_signs = new_buf.sign[idx] + if sign_changed.any(): + idx = np.flatnonzero(sign_changed) + update.sign_row_indices = con_l2p[ + new_buf.active_labels[idx] + ].astype(np.int32, copy=False) + update.sign_values = new_buf.sign[idx] + diff.cons[name] = update + obj_quad_present = model.objective.is_quadratic if obj_quad_present != snapshot.obj_quad_present: diff.rebuild_reason = RebuildReason.QUAD_OBJ @@ -140,13 +280,15 @@ def compute_diff( diff.rebuild_reason = RebuildReason.QUAD_OBJ return diff - obj_linear = _objective_linear_vector(model) - if not np.array_equal( - obj_linear.values, snapshot.obj_linear.values - ) or not np.array_equal( - obj_linear["vlabel"].values, snapshot.obj_linear["vlabel"].values - ): - diff.obj_linear = obj_linear.copy(deep=True) + obj_c = _objective_linear_vector(model) + if obj_c.shape != snapshot.obj_c.shape: + diff.rebuild_reason = RebuildReason.COORD_REINDEX + return diff + obj_diff_mask = obj_c != snapshot.obj_c + if obj_diff_mask.any(): + idx = np.flatnonzero(obj_diff_mask).astype(np.int32, copy=False) + diff.obj_c_indices = idx + diff.obj_c_values = obj_c[idx] if model.objective.sense != snapshot.obj_sense: diff.obj_sense = model.objective.sense diff --git a/linopy/persistent/snapshot.py b/linopy/persistent/snapshot.py index ea35c996..0090b600 100644 --- a/linopy/persistent/snapshot.py +++ b/linopy/persistent/snapshot.py @@ -4,7 +4,6 @@ from typing import TYPE_CHECKING import numpy as np -import xarray as xr from linopy import expressions @@ -14,6 +13,9 @@ from linopy.variables import Variable +_INT64_MAX = np.iinfo(np.int64).max + + def _variable_type(var: Variable) -> str: attrs = var.attrs if attrs.get("binary"): @@ -25,25 +27,10 @@ def _variable_type(var: Variable) -> str: return "continuous" -def _coord_snapshot(obj: Variable | ConstraintBase) -> dict[str, np.ndarray]: - return {str(name): np.asarray(idx) for name, idx in obj.indexes.items()} - - -def _canonical_csr( - constraint: ConstraintBase, label_index -) -> tuple[np.ndarray, np.ndarray, np.ndarray]: - csr, _ = constraint.to_matrix(label_index) - csr.sort_indices() - csr.eliminate_zeros() - indptr = csr.indptr.astype(np.int64) - indices = csr.indices.astype(np.int64) - return indptr, indices, csr.data - - -def _objective_linear_vector(model: Model) -> xr.DataArray: +def _objective_linear_vector(model: Model) -> np.ndarray: vlabels = model.variables.label_index.vlabels label_to_pos = model.variables.label_index.label_to_pos - result = np.zeros(len(vlabels)) + result = np.zeros(len(vlabels), dtype=np.float64) expr = model.objective.expression if isinstance(expr, expressions.QuadraticExpression): vars_2d = expr.data.vars.values @@ -57,22 +44,72 @@ def _objective_linear_vector(model: Model) -> xr.DataArray: coeffs = expr.data.coeffs.values.ravel() mask = var_labels != -1 np.add.at(result, label_to_pos[var_labels[mask]], coeffs[mask]) - return xr.DataArray(result, dims="vlabel", coords={"vlabel": vlabels}) + return result -@dataclass(frozen=True) -class CoefPattern: - indptr: np.ndarray - indices: np.ndarray - - def __eq__(self, other: object) -> bool: - return ( - isinstance(other, CoefPattern) - and np.array_equal(self.indptr, other.indptr) - and np.array_equal(self.indices, other.indices) +def _canonicalize_rows( + vars_arr: np.ndarray, coeffs_arr: np.ndarray +) -> tuple[np.ndarray, np.ndarray]: + """Sort each row jointly by var index. -1 sentinels sort to the right.""" + if vars_arr.size == 0: + return vars_arr.astype(np.int64, copy=False), coeffs_arr.astype( + np.float64, copy=False ) + sort_key = np.where(vars_arr == -1, _INT64_MAX, vars_arr).astype(np.int64) + order = np.argsort(sort_key, axis=1, kind="stable") + rows = np.arange(vars_arr.shape[0])[:, None] + return ( + vars_arr[rows, order].astype(np.int64, copy=False), + coeffs_arr[rows, order].astype(np.float64, copy=False), + ) + + +def _extract_var_buffers(var: Variable) -> ContainerVarBuffers: + labels_flat = var.labels.values.ravel() + mask = labels_flat != -1 + return ContainerVarBuffers( + lower=var.lower.values.ravel()[mask].astype(np.float64, copy=True), + upper=var.upper.values.ravel()[mask].astype(np.float64, copy=True), + type=_variable_type(var), + active_labels=labels_flat[mask].astype(np.int64, copy=True), + ) + + +def _extract_con_buffers( + con: ConstraintBase, var_l2p: np.ndarray +) -> ContainerConBuffers: + labels_flat = con.labels.values.ravel() + vars_vals = con.vars.values + coeffs_vals = con.coeffs.values + n_rows = len(labels_flat) + if n_rows > 0: + vars_2d = vars_vals.reshape(n_rows, -1) + coeffs_2d = coeffs_vals.reshape(vars_2d.shape) + else: + n_term = max(1, vars_vals.size) + vars_2d = vars_vals.reshape(0, n_term) + coeffs_2d = coeffs_vals.reshape(0, n_term) - __hash__ = None # type: ignore[assignment] + row_mask = (labels_flat != -1) & (vars_2d != -1).any(axis=1) + active_labels = labels_flat[row_mask].astype(np.int64, copy=True) + + vars_active = vars_2d[row_mask] + coeffs_active = coeffs_2d[row_mask].astype(np.float64, copy=True) + + valid = vars_active != -1 + col_indices = np.full(vars_active.shape, -1, dtype=np.int64) + col_indices[valid] = var_l2p[vars_active[valid]] + coeffs_clean = np.where(valid, coeffs_active, 0.0) + + vars_sorted, coeffs_sorted = _canonicalize_rows(col_indices, coeffs_clean) + + return ContainerConBuffers( + coeffs=coeffs_sorted, + vars=vars_sorted, + rhs=con.rhs.values.ravel()[row_mask].astype(np.float64, copy=True), + sign=con.sign.values.ravel()[row_mask].astype("U2", copy=True), + active_labels=active_labels, + ) @dataclass(frozen=True) @@ -94,21 +131,31 @@ def __eq__(self, other: object) -> bool: __hash__ = None # type: ignore[assignment] -@dataclass -class ModelSnapshot: - structural_key: StructuralKey +@dataclass(frozen=True) +class ContainerVarBuffers: + lower: np.ndarray + upper: np.ndarray + type: str + active_labels: np.ndarray + - var_lb: dict[str, xr.DataArray] = field(default_factory=dict) - var_ub: dict[str, xr.DataArray] = field(default_factory=dict) - var_type: dict[str, str] = field(default_factory=dict) - var_coords: dict[str, dict[str, np.ndarray]] = field(default_factory=dict) +@dataclass(frozen=True) +class ContainerConBuffers: + coeffs: np.ndarray + vars: np.ndarray + rhs: np.ndarray + sign: np.ndarray + active_labels: np.ndarray - con_rhs: dict[str, xr.DataArray] = field(default_factory=dict) - con_sign: dict[str, xr.DataArray] = field(default_factory=dict) - con_coords: dict[str, dict[str, np.ndarray]] = field(default_factory=dict) - con_coef_pattern: dict[str, CoefPattern] = field(default_factory=dict) - obj_linear: xr.DataArray = field(default_factory=lambda: xr.DataArray([])) +@dataclass +class ModelSnapshot: + structural_key: StructuralKey + var_buffers: dict[str, ContainerVarBuffers] = field(default_factory=dict) + con_buffers: dict[str, ContainerConBuffers] = field(default_factory=dict) + obj_c: np.ndarray = field( + default_factory=lambda: np.zeros(0, dtype=np.float64) + ) obj_quad_present: bool = False obj_sense: str = "min" @@ -116,6 +163,7 @@ class ModelSnapshot: def capture(cls, model: Model) -> ModelSnapshot: var_label_index = model.variables.label_index con_label_index = model.constraints.label_index + var_l2p = var_label_index.label_to_pos structural_key = StructuralKey( var_container_names=tuple(model.variables), @@ -124,42 +172,19 @@ def capture(cls, model: Model) -> ModelSnapshot: clabels=con_label_index.clabels, ) - var_lb: dict[str, xr.DataArray] = {} - var_ub: dict[str, xr.DataArray] = {} - var_type: dict[str, str] = {} - var_coords: dict[str, dict[str, np.ndarray]] = {} - for name, var in model.variables.items(): - var_lb[name] = var.lower.copy(deep=True) - var_ub[name] = var.upper.copy(deep=True) - var_type[name] = _variable_type(var) - var_coords[name] = _coord_snapshot(var) - - con_rhs: dict[str, xr.DataArray] = {} - con_sign: dict[str, xr.DataArray] = {} - con_coords: dict[str, dict[str, np.ndarray]] = {} - con_coef_pattern: dict[str, CoefPattern] = {} - for name, con in model.constraints.items(): - con_rhs[name] = con.rhs.copy(deep=True) - con_sign[name] = con.sign.copy(deep=True) - con_coords[name] = _coord_snapshot(con) - indptr, indices, _ = _canonical_csr(con, var_label_index) - con_coef_pattern[name] = CoefPattern(indptr=indptr, indices=indices) - - obj_linear = _objective_linear_vector(model).copy(deep=True) - obj_quad_present = model.objective.is_quadratic - obj_sense = model.objective.sense + var_buffers = { + name: _extract_var_buffers(var) for name, var in model.variables.items() + } + con_buffers = { + name: _extract_con_buffers(con, var_l2p) + for name, con in model.constraints.items() + } return cls( structural_key=structural_key, - var_lb=var_lb, - var_ub=var_ub, - var_type=var_type, - var_coords=var_coords, - con_rhs=con_rhs, - con_sign=con_sign, - con_coords=con_coords, - con_coef_pattern=con_coef_pattern, - obj_linear=obj_linear, - obj_quad_present=obj_quad_present, - obj_sense=obj_sense, + var_buffers=var_buffers, + con_buffers=con_buffers, + obj_c=_objective_linear_vector(model), + obj_quad_present=model.objective.is_quadratic, + obj_sense=model.objective.sense, ) diff --git a/linopy/solvers.py b/linopy/solvers.py index 9734e407..8cea9940 100644 --- a/linopy/solvers.py +++ b/linopy/solvers.py @@ -1296,85 +1296,78 @@ def apply_update( var_label_index: Any, con_label_index: Any, ) -> None: - if diff.con_sign: - raise UnsupportedUpdate( - "HiGHS does not support in-place constraint sign change" - ) + for upd in diff.cons.values(): + if upd.sign_values is not None: + raise UnsupportedUpdate( + "HiGHS does not support in-place constraint sign change" + ) variables = var_label_index._variables - constraints = con_label_index._constraints - var_pos = var_label_index.label_to_pos - con_pos = con_label_index.label_to_pos - - def _container_positions_and_mask(obj: Any) -> tuple[np.ndarray, np.ndarray]: - labels = obj.labels.values.ravel() - mask = labels != -1 - positions = var_pos[labels[mask]].astype(np.int32) - return positions, mask - - def _con_positions_and_mask(obj: Any) -> tuple[np.ndarray, np.ndarray]: - labels = obj.labels.values.ravel() - mask = labels != -1 - positions = con_pos[labels[mask]].astype(np.int32) - return positions, mask - h = self.solver_model - binary_names = {n for n, t in diff.var_type.items() if t == "binary"} - names_for_bounds = set(diff.var_lb) | set(diff.var_ub) | binary_names - for name in names_for_bounds: - var = variables[name] - positions, mask = _container_positions_and_mask(var) - if name in binary_names: - lb = np.zeros(positions.size, dtype=np.float64) - ub = np.ones(positions.size, dtype=np.float64) - else: - lb_src = diff.var_lb.get(name, var.lower).values.ravel()[mask] - ub_src = diff.var_ub.get(name, var.upper).values.ravel()[mask] - lb = np.asarray(lb_src, dtype=np.float64) - ub = np.asarray(ub_src, dtype=np.float64) - h.changeColsBounds(positions.size, positions, lb, ub) - type_map = { "continuous": highspy.HighsVarType.kContinuous, "binary": highspy.HighsVarType.kInteger, "integer": highspy.HighsVarType.kInteger, "semi_continuous": highspy.HighsVarType.kSemiContinuous, } - for name, vtype in diff.var_type.items(): + + for name, upd in diff.vars.items(): var = variables[name] - positions, _ = _container_positions_and_mask(var) - integrality = np.full(positions.size, int(type_map[vtype]), dtype=np.uint8) - h.changeColsIntegrality(positions.size, positions, integrality) - - for name, rhs in diff.con_rhs.items(): - con = constraints[name] - positions, mask = _con_positions_and_mask(con) - rhs_values = np.asarray(rhs.values.ravel()[mask], dtype=np.float64) - sign_values = con.sign.values.ravel()[mask] - inf = np.inf - lower = np.where(sign_values == "<=", -inf, rhs_values) - upper = np.where(sign_values == ">=", inf, rhs_values) - for pos, lo, up in zip(positions, lower, upper): - h.changeRowBounds(int(pos), float(lo), float(up)) - - for name in diff.con_coef_updates: - con = constraints[name] - csr, _ = con.to_matrix(var_label_index) - csr.sort_indices() - csr.eliminate_zeros() - con_positions, _ = _con_positions_and_mask(con) - for row_idx, row_pos in enumerate(con_positions): - start = csr.indptr[row_idx] - end = csr.indptr[row_idx + 1] - for col, val in zip(csr.indices[start:end], csr.data[start:end]): - h.changeCoeff(int(row_pos), int(col), float(val)) - - if diff.obj_linear is not None: - n = len(diff.obj_linear.values) - positions = np.arange(n, dtype=np.int32) - costs = np.asarray(diff.obj_linear.values, dtype=np.float64) - h.changeColsCost(n, positions, costs) + if upd.type_change == "binary": + labels = var.labels.values.ravel() + mask = labels != -1 + container_positions = var_label_index.label_to_pos[labels[mask]].astype( + np.int32 + ) + lb = np.zeros(container_positions.size, dtype=np.float64) + ub = np.ones(container_positions.size, dtype=np.float64) + h.changeColsBounds(container_positions.size, container_positions, lb, ub) + elif upd.bounds_indices is not None: + indices = upd.bounds_indices + lower = np.asarray(upd.lower, dtype=np.float64) + upper = np.asarray(upd.upper, dtype=np.float64) + h.changeColsBounds(indices.size, indices, lower, upper) + + if upd.type_change is not None: + labels = var.labels.values.ravel() + mask = labels != -1 + container_positions = var_label_index.label_to_pos[labels[mask]].astype( + np.int32 + ) + integrality = np.full( + container_positions.size, + int(type_map[upd.type_change]), + dtype=np.uint8, + ) + h.changeColsIntegrality( + container_positions.size, container_positions, integrality + ) + + for name, upd in diff.cons.items(): + if upd.rhs_values is not None: + positions = upd.rhs_row_indices + rhs_values = np.asarray(upd.rhs_values, dtype=np.float64) + sign_for_rows = upd.rhs_signs + inf = np.inf + lower = np.where(sign_for_rows == "<=", -inf, rhs_values) + upper = np.where(sign_for_rows == ">=", inf, rhs_values) + for pos, lo, up in zip(positions, lower, upper): + h.changeRowBounds(int(pos), float(lo), float(up)) + + if upd.coef_values is not None: + rows = upd.coef_row_indices + for r, var_row, val_row in zip( + rows, upd.coef_vars, upd.coef_values + ): + valid = var_row != -1 + for c, v in zip(var_row[valid], val_row[valid]): + h.changeCoeff(int(r), int(c), float(v)) + + if diff.obj_c_indices is not None: + indices = diff.obj_c_indices + costs = np.asarray(diff.obj_c_values, dtype=np.float64) + h.changeColsCost(indices.size, indices, costs) if diff.obj_sense is not None: sense = ( @@ -1820,96 +1813,10 @@ def apply_update( var_label_index: Any, con_label_index: Any, ) -> None: - model = self.model - assert model is not None gm = self.solver_model - - var_l2p = var_label_index.label_to_pos - con_l2p = con_label_index.label_to_pos n_active_vars = var_label_index.n_active_vars n_active_cons = con_label_index.n_active_cons - var_payloads: list[tuple[np.ndarray, np.ndarray, str]] = [] - for name, da in diff.var_lb.items(): - var = model.variables[name] - labels = var.labels.values.ravel() - mask = labels != -1 - positions = var_l2p[labels[mask]] - if (positions < 0).any() or (positions >= n_active_vars).any(): - raise UnsupportedUpdate(f"var positions out of range for {name}") - var_payloads.append((positions, da.values.ravel()[mask], "LB")) - for name, da in diff.var_ub.items(): - var = model.variables[name] - labels = var.labels.values.ravel() - mask = labels != -1 - positions = var_l2p[labels[mask]] - if (positions < 0).any() or (positions >= n_active_vars).any(): - raise UnsupportedUpdate(f"var positions out of range for {name}") - var_payloads.append((positions, da.values.ravel()[mask], "UB")) - - type_payloads: list[tuple[np.ndarray, str]] = [] - for name, vtype in diff.var_type.items(): - if vtype not in self._GUROBI_VTYPE_MAP: - raise UnsupportedUpdate(f"unknown var type {vtype}") - var = model.variables[name] - labels = var.labels.values.ravel() - mask = labels != -1 - positions = var_l2p[labels[mask]] - if (positions < 0).any() or (positions >= n_active_vars).any(): - raise UnsupportedUpdate(f"var positions out of range for {name}") - type_payloads.append((positions, self._GUROBI_VTYPE_MAP[vtype])) - - rhs_payloads: list[tuple[np.ndarray, np.ndarray]] = [] - for name, da in diff.con_rhs.items(): - con = model.constraints[name] - labels = con.labels.values.ravel() - mask = labels != -1 - positions = con_l2p[labels[mask]] - if (positions < 0).any() or (positions >= n_active_cons).any(): - raise UnsupportedUpdate(f"con positions out of range for {name}") - rhs_payloads.append((positions, da.values.ravel()[mask])) - - sign_payloads: list[tuple[np.ndarray, np.ndarray]] = [] - for name, da in diff.con_sign.items(): - sign_strs = da.values.ravel() - con = model.constraints[name] - labels = con.labels.values.ravel() - mask = labels != -1 - sign_strs = sign_strs[mask] - mapped = np.empty(len(sign_strs), dtype=object) - for i, s in enumerate(sign_strs): - s = str(s) - if s not in self._GUROBI_SIGN_MAP: - raise UnsupportedUpdate(f"unknown sign {s!r}") - mapped[i] = self._GUROBI_SIGN_MAP[s] - positions = con_l2p[labels[mask]] - if (positions < 0).any() or (positions >= n_active_cons).any(): - raise UnsupportedUpdate(f"con positions out of range for {name}") - sign_payloads.append((positions, mapped)) - - coef_payloads: list[tuple[np.ndarray, np.ndarray, np.ndarray]] = [] - for name, values in diff.con_coef_updates.items(): - con = model.constraints[name] - csr, _ = con.to_matrix(var_label_index) - csr.sort_indices() - csr.eliminate_zeros() - if csr.data.shape != values.shape: - raise UnsupportedUpdate(f"coef shape mismatch for {name}") - row_pos_local = np.repeat( - np.arange(csr.shape[0], dtype=np.int64), np.diff(csr.indptr) - ) - active_labels = con.active_labels() - row_positions = con_l2p[active_labels[row_pos_local]] - col_positions = csr.indices.astype(np.int64) - if (row_positions < 0).any() or (row_positions >= n_active_cons).any(): - raise UnsupportedUpdate(f"con positions out of range for {name}") - if (col_positions < 0).any() or (col_positions >= n_active_vars).any(): - raise UnsupportedUpdate(f"var positions out of range for {name}") - coef_payloads.append((row_positions, col_positions, values)) - - if diff.obj_sense is not None and diff.obj_sense not in self._GUROBI_SENSE_MAP: - raise UnsupportedUpdate(f"unknown obj sense {diff.obj_sense!r}") - gurobi_vars = gm.getVars() gurobi_cons = gm.getConstrs() if len(gurobi_vars) != n_active_vars: @@ -1917,32 +1824,64 @@ def apply_update( if len(gurobi_cons) != n_active_cons: raise UnsupportedUpdate("gurobi con count mismatch") - for positions, values, attr in var_payloads: - for pos, val in zip(positions, values): - gurobi_vars[int(pos)].setAttr(attr, float(val)) - - for positions, vtype_str in type_payloads: - for pos in positions: - gurobi_vars[int(pos)].setAttr("VType", vtype_str) - - for positions, values in rhs_payloads: - for pos, val in zip(positions, values): - gurobi_cons[int(pos)].setAttr("RHS", float(val)) - - for positions, senses in sign_payloads: - for pos, s in zip(positions, senses): - gurobi_cons[int(pos)].setAttr("Sense", s) + variables = var_label_index._variables + var_l2p = var_label_index.label_to_pos - for row_positions, col_positions, values in coef_payloads: - for r, c, v in zip(row_positions, col_positions, values): - gm.chgCoeff(gurobi_cons[int(r)], gurobi_vars[int(c)], float(v)) + for name, upd in diff.vars.items(): + if upd.bounds_indices is not None: + indices = upd.bounds_indices + var_subset = [gurobi_vars[int(i)] for i in indices] + if upd.lower is not None: + gm.setAttr("LB", var_subset, upd.lower.tolist()) + if upd.upper is not None: + gm.setAttr("UB", var_subset, upd.upper.tolist()) + if upd.type_change is not None: + vtype = self._GUROBI_VTYPE_MAP.get(upd.type_change) + if vtype is None: + raise UnsupportedUpdate(f"unknown var type {upd.type_change}") + var = variables[name] + labels = var.labels.values.ravel() + mask = labels != -1 + container_positions = var_l2p[labels[mask]] + container_subset = [ + gurobi_vars[int(p)] for p in container_positions + ] + gm.setAttr("VType", container_subset, [vtype] * len(container_subset)) + + for name, upd in diff.cons.items(): + if upd.rhs_values is not None: + rows = upd.rhs_row_indices + con_subset = [gurobi_cons[int(r)] for r in rows] + gm.setAttr("RHS", con_subset, upd.rhs_values.tolist()) + if upd.sign_values is not None: + rows = upd.sign_row_indices + con_subset = [gurobi_cons[int(r)] for r in rows] + senses = [] + for s in upd.sign_values: + s_str = str(s) + if s_str not in self._GUROBI_SIGN_MAP: + raise UnsupportedUpdate(f"unknown sign {s_str!r}") + senses.append(self._GUROBI_SIGN_MAP[s_str]) + gm.setAttr("Sense", con_subset, senses) + if upd.coef_values is not None: + rows = upd.coef_row_indices + for r, var_row, val_row in zip( + rows, upd.coef_vars, upd.coef_values + ): + valid = var_row != -1 + for c, v in zip(var_row[valid], val_row[valid]): + gm.chgCoeff( + gurobi_cons[int(r)], gurobi_vars[int(c)], float(v) + ) - if diff.obj_linear is not None: - obj_values = diff.obj_linear.values - for pos in range(n_active_vars): - gurobi_vars[pos].setAttr("Obj", float(obj_values[pos])) + if diff.obj_c_indices is not None: + indices = diff.obj_c_indices + var_subset = [gurobi_vars[int(i)] for i in indices] + gm.setAttr("Obj", var_subset, diff.obj_c_values.tolist()) if diff.obj_sense is not None: + if diff.obj_sense not in self._GUROBI_SENSE_MAP: + raise UnsupportedUpdate(f"unknown obj sense {diff.obj_sense!r}") gm.ModelSense = self._GUROBI_SENSE_MAP[diff.obj_sense] gm.update() diff --git a/test/test_persistent_snapshot_buffers.py b/test/test_persistent_snapshot_buffers.py new file mode 100644 index 00000000..31bc0e83 --- /dev/null +++ b/test/test_persistent_snapshot_buffers.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +import numpy as np +import pytest + +from linopy import Model +from linopy.persistent import ModelSnapshot, RebuildReason, compute_diff +from linopy.persistent.snapshot import ( + _canonicalize_rows, + _extract_con_buffers, +) + + +def test_canonicalize_rows_sorts_by_var_label() -> None: + vars_in = np.array([[5, 2, 9], [1, 3, 0]], dtype=np.int64) + coeffs_in = np.array([[0.5, 0.2, 0.9], [0.1, 0.3, 0.0]], dtype=np.float64) + vars_out, coeffs_out = _canonicalize_rows(vars_in, coeffs_in) + np.testing.assert_array_equal(vars_out, [[2, 5, 9], [0, 1, 3]]) + np.testing.assert_array_equal(coeffs_out, [[0.2, 0.5, 0.9], [0.0, 0.1, 0.3]]) + + +def test_canonicalize_rows_minus_one_to_right() -> None: + vars_in = np.array([[5, -1, 2], [-1, 0, -1]], dtype=np.int64) + coeffs_in = np.array([[0.5, 0.0, 0.2], [0.0, 0.1, 0.0]], dtype=np.float64) + vars_out, coeffs_out = _canonicalize_rows(vars_in, coeffs_in) + np.testing.assert_array_equal(vars_out[:, 0], [2, 0]) + assert (vars_out[:, -1] == -1).all() + + +def test_canonicalize_empty_buffers_round_trip() -> None: + vars_in = np.empty((0, 3), dtype=np.int64) + coeffs_in = np.empty((0, 3), dtype=np.float64) + vars_out, coeffs_out = _canonicalize_rows(vars_in, coeffs_in) + assert vars_out.shape == (0, 3) + assert coeffs_out.shape == (0, 3) + + +def _build_permuted_pair() -> tuple[Model, Model]: + m1 = Model() + x1 = m1.add_variables(0, 10, coords=[range(3)], name="x") + y1 = m1.add_variables(0, 5, coords=[range(2)], name="y") + m1.add_constraints(2 * x1 + 3 * y1.sum() >= 4, name="c1") + m1.add_objective(x1.sum()) + + m2 = Model() + x2 = m2.add_variables(0, 10, coords=[range(3)], name="x") + y2 = m2.add_variables(0, 5, coords=[range(2)], name="y") + m2.add_constraints(3 * y2.sum() + 2 * x2 >= 4, name="c1") + m2.add_objective(x2.sum()) + return m1, m2 + + +def test_permuted_term_order_produces_equal_buffers() -> None: + m1, m2 = _build_permuted_pair() + s1 = ModelSnapshot.capture(m1) + s2 = ModelSnapshot.capture(m2) + np.testing.assert_array_equal(s1.con_buffers["c1"].vars, s2.con_buffers["c1"].vars) + np.testing.assert_array_equal( + s1.con_buffers["c1"].coeffs, s2.con_buffers["c1"].coeffs + ) + + +def test_active_labels_match_label_index(baseline_model: Model) -> None: + snap = ModelSnapshot.capture(baseline_model) + expected = baseline_model.constraints.label_index.clabels + concatenated = np.concatenate( + [buf.active_labels for buf in snap.con_buffers.values()] + ) + np.testing.assert_array_equal(concatenated, expected) + + +@pytest.fixture +def baseline_model() -> Model: + m = Model() + x = m.add_variables(0, 10, coords=[range(3)], name="x") + y = m.add_variables(0, 5, coords=[range(2)], name="y") + m.add_constraints(2 * x >= 4, name="c1") + m.add_constraints(x.sum() + y.sum() <= 20, name="c2") + m.add_objective(x.sum()) + return m + + +def test_shape_mismatch_triggers_sparsity_rebuild(baseline_model: Model) -> None: + snap = ModelSnapshot.capture(baseline_model) + # Mutate to widen the term dim of c1 via lhs replacement + x = baseline_model.variables["x"] + y = baseline_model.variables["y"] + baseline_model.constraints["c1"].lhs = 2 * x + 0 * y.sum() + diff = compute_diff(snap, baseline_model) + assert diff.rebuild_reason in { + RebuildReason.SPARSITY, + RebuildReason.STRUCTURAL_LABELS, + } + + +def test_zero_row_container_capture() -> None: + m = Model() + m.add_variables(0, 10, coords=[range(2)], name="x") + m.add_objective(0.0 * m.variables["x"].sum()) + snap = ModelSnapshot.capture(m) + assert snap.con_buffers == {} + diff = compute_diff(snap, m) + assert diff.is_empty + + +def test_con_buffers_rhs_and_sign_dtypes(baseline_model: Model) -> None: + snap = ModelSnapshot.capture(baseline_model) + buf = snap.con_buffers["c1"] + assert buf.rhs.dtype == np.float64 + assert buf.sign.dtype.kind == "U" + assert buf.coeffs.dtype == np.float64 + assert buf.vars.dtype == np.int64 + + +def test_masked_rows_excluded_from_active_labels() -> None: + m = Model() + x = m.add_variables(0, 10, coords=[range(4)], name="x") + mask = np.array([True, False, True, True]) + m.add_constraints(2 * x >= 1, mask=mask, name="c1") + m.add_objective(x.sum()) + snap = ModelSnapshot.capture(m) + buf = snap.con_buffers["c1"] + assert buf.active_labels.size == 3 + var_l2p = m.variables.label_index.label_to_pos + rebuilt = _extract_con_buffers(m.constraints["c1"], var_l2p) + np.testing.assert_array_equal(rebuilt.active_labels, buf.active_labels) diff --git a/test/test_persistent_snapshot_diff.py b/test/test_persistent_snapshot_diff.py index 53bff0ad..20501a67 100644 --- a/test/test_persistent_snapshot_diff.py +++ b/test/test_persistent_snapshot_diff.py @@ -5,7 +5,8 @@ from linopy import Model from linopy.persistent import ( - CoefPattern, + ContainerConBuffers, + ContainerVarBuffers, ModelDiff, ModelSnapshot, RebuildReason, @@ -37,7 +38,8 @@ def test_capture_structural_key(baseline: Model) -> None: np.testing.assert_array_equal( snap.structural_key.clabels, baseline.constraints.label_index.clabels ) - assert isinstance(snap.con_coef_pattern["c1"], CoefPattern) + assert isinstance(snap.var_buffers["x"], ContainerVarBuffers) + assert isinstance(snap.con_buffers["c1"], ContainerConBuffers) def test_is_empty_on_unmutated(baseline: Model) -> None: @@ -53,8 +55,11 @@ def test_bounds_only_mutation(baseline: Model) -> None: baseline.variables["x"].lower = 1 diff = compute_diff(snap, baseline) assert diff.rebuild_reason is RebuildReason.NONE - assert "x" in diff.var_lb - assert "x" not in diff.var_ub + assert "x" in diff.vars + assert "y" not in diff.vars + upd = diff.vars["x"] + assert upd.lower is not None + np.testing.assert_array_equal(upd.lower, np.ones(3)) def test_rhs_only_mutation(baseline: Model) -> None: @@ -62,8 +67,10 @@ def test_rhs_only_mutation(baseline: Model) -> None: baseline.constraints["c1"].rhs = 9 diff = compute_diff(snap, baseline) assert diff.rebuild_reason is RebuildReason.NONE - assert "c1" in diff.con_rhs - assert not diff.con_coef_updates + assert "c1" in diff.cons + upd = diff.cons["c1"] + assert upd.rhs_values is not None + assert upd.coef_values is None def test_objective_linear_change(baseline: Model) -> None: @@ -73,7 +80,8 @@ def test_objective_linear_change(baseline: Model) -> None: baseline.add_objective(3 * x.sum() + 2 * y.sum(), overwrite=True) diff = compute_diff(snap, baseline) assert diff.rebuild_reason is RebuildReason.NONE - assert diff.obj_linear is not None + assert diff.obj_c_indices is not None + assert diff.obj_c_values is not None def test_objective_sense_flip(baseline: Model) -> None: @@ -111,9 +119,11 @@ def test_coef_value_change_same_sparsity(baseline: Model) -> None: c.coeffs = c.coeffs * 3 diff = compute_diff(snap, baseline) assert diff.rebuild_reason is RebuildReason.NONE - assert "c1" in diff.con_coef_updates - values = diff.con_coef_updates["c1"] - np.testing.assert_array_equal(values, np.full_like(values, 6.0)) + assert "c1" in diff.cons + upd = diff.cons["c1"] + assert upd.coef_values is not None + valid = upd.coef_vars != -1 + np.testing.assert_array_equal(upd.coef_values[valid], np.full(valid.sum(), 6.0)) def test_coef_sparsity_change(baseline: Model) -> None: @@ -128,7 +138,7 @@ def test_deep_copy_invariant(baseline: Model) -> None: snap = ModelSnapshot.capture(baseline) baseline.variables["x"].lower.values[...] = 99 diff = compute_diff(snap, baseline) - assert "x" in diff.var_lb + assert "x" in diff.vars def test_same_model_false_ignores_dirty_flag(baseline: Model) -> None: @@ -137,9 +147,10 @@ def test_same_model_false_ignores_dirty_flag(baseline: Model) -> None: c.coeffs = c.coeffs * 5 c._coef_dirty = False diff_fast = compute_diff(snap, baseline, same_model=True) - assert "c1" not in diff_fast.con_coef_updates + assert "c1" not in diff_fast.cons or diff_fast.cons["c1"].coef_values is None diff_full = compute_diff(snap, baseline, same_model=False) - assert "c1" in diff_full.con_coef_updates + assert "c1" in diff_full.cons + assert diff_full.cons["c1"].coef_values is not None def test_modeldiff_default_is_empty() -> None: diff --git a/test/test_persistent_solver_extras.py b/test/test_persistent_solver_extras.py index 7c49bfc1..37900dbb 100644 --- a/test/test_persistent_solver_extras.py +++ b/test/test_persistent_solver_extras.py @@ -283,6 +283,65 @@ def _run() -> None: assert np.isclose(r, expected) +_SCENARIO_PARAMS = [ + "bound_only", + "rhs_only", + "single_cell_coef", + "multi_row_coef", + "mixed", +] + + +def _apply_scenario(model: Model, scenario: str) -> None: + if scenario == "bound_only": + model.variables["x"].lower.values[...] = 3.0 + elif scenario == "rhs_only": + model.constraints["c1"].rhs = 7.0 + elif scenario == "single_cell_coef": + c = model.constraints["c1"] + new = c.coeffs.copy() + new.values[0, 0] = 5.0 + c.coeffs = new + elif scenario == "multi_row_coef": + c = model.constraints["c2"] + c.coeffs = c.coeffs * 2 + elif scenario == "mixed": + model.variables["x"].lower.values[...] = 1.0 + model.constraints["c1"].rhs = 6.0 + c = model.constraints["c2"] + new = c.coeffs.copy() + new.values[0, 0] = 4.0 + c.coeffs = new + else: + raise ValueError(scenario) + + +@pytest.mark.parametrize("solver_name", SOLVER_PARAMS) +@pytest.mark.parametrize("scenario", _SCENARIO_PARAMS) +@pytest.mark.parametrize("same_model", [True, False]) +def test_scenario_sweep_in_place( + solver_name: str, scenario: str, same_model: bool +) -> None: + m = _base_model() + s = _built(solver_name, m) + s.solve(assign=True) + + target = m if same_model else _base_model() + _apply_scenario(target, scenario) + s.solve(target, assign=True) + + assert s._rebuilds == 0 + assert s._in_place_updates == 1 + assert s._last_rebuild_reason is RebuildReason.NONE + + fresh = _base_model() + _apply_scenario(fresh, scenario) + s_fresh = _built(solver_name, fresh) + s_fresh.solve(assign=True) + assert np.isclose(float(target.objective.value), float(fresh.objective.value)) + s_fresh.close() + + @pytest.mark.parametrize("solver_name", SOLVER_PARAMS) def test_solve_without_assign_does_not_mutate_model(solver_name: str) -> None: m = _base_model() From 0c306886fb1d21cc9f927107a83373e445438b23 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 20 May 2026 12:59:07 +0200 Subject: [PATCH 11/21] feat(persistent): opt-in coord-equality via ignore_dims compute_diff/Solver.solve/Solver.update grow an ignore_dims kwarg. None (default) keeps the current no-coord-check behaviour; any iterable opts into per-container coord-equality on every dim not in the set, supporting rolling-horizon workflows where e.g. the snapshot dim is expected to drift. --- linopy/persistent/diff.py | 37 ++++++++++++++++++++++++++- linopy/persistent/snapshot.py | 14 ++++++++++ linopy/solvers.py | 29 ++++++++++++++++----- test/test_persistent_snapshot_diff.py | 22 ++++++++++++++++ 4 files changed, 95 insertions(+), 7 deletions(-) diff --git a/linopy/persistent/diff.py b/linopy/persistent/diff.py index 4af3df63..51a8f682 100644 --- a/linopy/persistent/diff.py +++ b/linopy/persistent/diff.py @@ -6,8 +6,11 @@ import numpy as np +from collections.abc import Iterable + from linopy.persistent.snapshot import ( ModelSnapshot, + _coord_snapshot, _extract_con_buffers, _extract_var_buffers, _objective_linear_vector, @@ -162,9 +165,31 @@ def __repr__(self) -> str: return "ModelDiff(" + ", ".join(parts) + ")" +def _coords_equal( + a: dict[str, np.ndarray], b: dict[str, np.ndarray], ignored: frozenset[str] +) -> bool: + keys_a = set(a) - ignored + keys_b = set(b) - ignored + if keys_a != keys_b: + return False + return all(np.array_equal(a[k], b[k]) for k in keys_a) + + def compute_diff( - snapshot: ModelSnapshot, model: Model, same_model: bool = True + snapshot: ModelSnapshot, + model: Model, + same_model: bool = True, + ignore_dims: Iterable[str] | None = None, ) -> ModelDiff: + """Compute a ``ModelDiff`` between ``snapshot`` and ``model``. + + Coordinate values are not compared by default. Pass ``ignore_dims`` + (e.g. ``ignore_dims=()`` or ``ignore_dims={"snapshot"}``) to opt into + per-container coord-equality on every dim *not* in the set — a mismatch + triggers ``RebuildReason.COORD_REINDEX``. + """ + check_coords = ignore_dims is not None + ignored = frozenset(ignore_dims) if ignore_dims is not None else frozenset() diff = ModelDiff() var_names = tuple(model.variables) @@ -197,6 +222,11 @@ def compute_diff( if not np.array_equal(new_buf.active_labels, snap_buf.active_labels): diff.rebuild_reason = RebuildReason.STRUCTURAL_LABELS return diff + if check_coords and not _coords_equal( + snapshot.var_coords[name], _coord_snapshot(var), ignored + ): + diff.rebuild_reason = RebuildReason.COORD_REINDEX + return diff lower_diff = new_buf.lower != snap_buf.lower upper_diff = new_buf.upper != snap_buf.upper @@ -226,6 +256,11 @@ def compute_diff( if not np.array_equal(new_buf.active_labels, snap_buf.active_labels): diff.rebuild_reason = RebuildReason.STRUCTURAL_LABELS return diff + if check_coords and not _coords_equal( + snapshot.con_coords[name], _coord_snapshot(con), ignored + ): + diff.rebuild_reason = RebuildReason.COORD_REINDEX + return diff n_rows = new_buf.active_labels.size if n_rows == 0: diff --git a/linopy/persistent/snapshot.py b/linopy/persistent/snapshot.py index 0090b600..ffa74444 100644 --- a/linopy/persistent/snapshot.py +++ b/linopy/persistent/snapshot.py @@ -148,11 +148,17 @@ class ContainerConBuffers: active_labels: np.ndarray +def _coord_snapshot(obj: Variable | ConstraintBase) -> dict[str, np.ndarray]: + return {str(name): np.asarray(idx) for name, idx in obj.indexes.items()} + + @dataclass class ModelSnapshot: structural_key: StructuralKey var_buffers: dict[str, ContainerVarBuffers] = field(default_factory=dict) con_buffers: dict[str, ContainerConBuffers] = field(default_factory=dict) + var_coords: dict[str, dict[str, np.ndarray]] = field(default_factory=dict) + con_coords: dict[str, dict[str, np.ndarray]] = field(default_factory=dict) obj_c: np.ndarray = field( default_factory=lambda: np.zeros(0, dtype=np.float64) ) @@ -179,11 +185,19 @@ def capture(cls, model: Model) -> ModelSnapshot: name: _extract_con_buffers(con, var_l2p) for name, con in model.constraints.items() } + var_coords = { + name: _coord_snapshot(var) for name, var in model.variables.items() + } + con_coords = { + name: _coord_snapshot(con) for name, con in model.constraints.items() + } return cls( structural_key=structural_key, var_buffers=var_buffers, con_buffers=con_buffers, + var_coords=var_coords, + con_coords=con_coords, obj_c=_objective_linear_vector(model), obj_quad_present=model.objective.is_quadratic, obj_sense=model.objective.sense, diff --git a/linopy/solvers.py b/linopy/solvers.py index 8cea9940..a406bce2 100644 --- a/linopy/solvers.py +++ b/linopy/solvers.py @@ -19,7 +19,7 @@ import warnings from abc import ABC from collections import namedtuple -from collections.abc import Callable, Generator, Iterator, Sequence +from collections.abc import Callable, Generator, Iterable, Iterator, Sequence from dataclasses import dataclass, field from enum import Enum, auto from importlib.metadata import PackageNotFoundError @@ -625,6 +625,7 @@ def solve( self, model: Model | None = None, assign: bool = False, + ignore_dims: Iterable[str] | None = None, **run_kwargs: Any, ) -> Result: """ @@ -634,6 +635,10 @@ def solve( apply in place or rebuild before running. Requires ``io_api='direct'``. With ``assign=True`` the Result is written back to the target Model via :meth:`Model.assign_result`. + + Pass ``ignore_dims`` (e.g. ``{"snapshot"}``) to opt into per-container + coordinate-equality checking on every dim *not* in the set. Default + (``None``) skips the coord check entirely. """ if model is not None: if self.io_api != "direct": @@ -643,7 +648,7 @@ def solve( self.model = model self._build() else: - self._update_locked(model, apply=True) + self._update_locked(model, apply=True, ignore_dims=ignore_dims) target = model else: target = self.model # type: ignore[assignment] @@ -666,22 +671,34 @@ def solve( target.assign_result(result, solver=self) return result - def update(self, model: Model, apply: bool = True) -> ModelDiff: + def update( + self, + model: Model, + apply: bool = True, + ignore_dims: Iterable[str] | None = None, + ) -> ModelDiff: if self.io_api != "direct": raise ValueError("update requires io_api='direct'") if self.snapshot is None or self.solver_model is None: raise RuntimeError("Solver has not been built") with self._lock: - return self._update_locked(model, apply=apply) + return self._update_locked(model, apply=apply, ignore_dims=ignore_dims) - def _update_locked(self, model: Model, apply: bool) -> ModelDiff: + def _update_locked( + self, + model: Model, + apply: bool, + ignore_dims: Iterable[str] | None = None, + ) -> ModelDiff: assert self.snapshot is not None if apply and not type(self).supports_persistent_update: diff = ModelDiff(rebuild_reason=RebuildReason.BACKEND_REJECTED) self._rebuild(model, RebuildReason.BACKEND_REJECTED) return diff same_model = model is self.model - diff = compute_diff(self.snapshot, model, same_model=same_model) + diff = compute_diff( + self.snapshot, model, same_model=same_model, ignore_dims=ignore_dims + ) if not apply: return diff if diff.rebuild_required: diff --git a/test/test_persistent_snapshot_diff.py b/test/test_persistent_snapshot_diff.py index 20501a67..aa024f5b 100644 --- a/test/test_persistent_snapshot_diff.py +++ b/test/test_persistent_snapshot_diff.py @@ -1,6 +1,7 @@ from __future__ import annotations import numpy as np +import pandas as pd import pytest from linopy import Model @@ -157,3 +158,24 @@ def test_modeldiff_default_is_empty() -> None: d = ModelDiff() assert d.is_empty assert not d.rebuild_required + + +def test_ignore_dims_detects_coord_change() -> None: + m1 = Model() + m1.add_variables(0, 10, coords=[pd.Index([0, 1, 2], name="t")], name="x") + m1.add_constraints(m1.variables["x"] >= 0, name="c1") + m1.add_objective(m1.variables["x"].sum()) + snap = ModelSnapshot.capture(m1) + + m2 = Model() + m2.add_variables(0, 10, coords=[pd.Index([10, 11, 12], name="t")], name="x") + m2.add_constraints(m2.variables["x"] >= 0, name="c1") + m2.add_objective(m2.variables["x"].sum()) + + assert compute_diff(snap, m2).rebuild_reason is RebuildReason.NONE + assert compute_diff(snap, m2, ignore_dims=()).rebuild_reason is ( + RebuildReason.COORD_REINDEX + ) + assert compute_diff(snap, m2, ignore_dims={"t"}).rebuild_reason is ( + RebuildReason.NONE + ) From 8dbb8be63c7cbc50d618f07f82aec77e87f74f5f Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 20 May 2026 16:26:51 +0200 Subject: [PATCH 12/21] feat(persistent): lazy-build Solver, ModelDiff constructors, disallow_rebuild - Solver.from_name now accepts model=None; the first solve(m, ...) builds. - compute_diff folded into ModelDiff.from_snapshot classmethod; new ModelDiff.from_models diffs two linopy models directly. - Solver.solve grows disallow_rebuild=True, which raises RebuildRequiredError instead of falling back to a rebuild. --- linopy/persistent/__init__.py | 5 +-- linopy/persistent/diff.py | 47 ++++++++++++++++++----- linopy/persistent/errors.py | 11 ++++++ linopy/solvers.py | 40 ++++++++++++++++--- test/test_persistent_snapshot_buffers.py | 6 +-- test/test_persistent_snapshot_diff.py | 49 ++++++++++++++++-------- test/test_persistent_solver_extras.py | 28 ++++++++++++++ 7 files changed, 148 insertions(+), 38 deletions(-) diff --git a/linopy/persistent/__init__.py b/linopy/persistent/__init__.py index 6fb1ca4c..ddd936e4 100644 --- a/linopy/persistent/__init__.py +++ b/linopy/persistent/__init__.py @@ -7,9 +7,8 @@ ContainerVarUpdate, ModelDiff, RebuildReason, - compute_diff, ) -from linopy.persistent.errors import UnsupportedUpdate +from linopy.persistent.errors import RebuildRequiredError, UnsupportedUpdate from linopy.persistent.snapshot import ( ContainerConBuffers, ContainerVarBuffers, @@ -25,7 +24,7 @@ "ModelDiff", "ModelSnapshot", "RebuildReason", + "RebuildRequiredError", "StructuralKey", "UnsupportedUpdate", - "compute_diff", ] diff --git a/linopy/persistent/diff.py b/linopy/persistent/diff.py index 51a8f682..026ae98e 100644 --- a/linopy/persistent/diff.py +++ b/linopy/persistent/diff.py @@ -164,6 +164,40 @@ def __repr__(self) -> str: ] return "ModelDiff(" + ", ".join(parts) + ")" + @classmethod + def from_snapshot( + cls, + snapshot: ModelSnapshot, + model: Model, + same_model: bool = True, + ignore_dims: Iterable[str] | None = None, + ) -> ModelDiff: + """Diff ``model`` against a captured ``snapshot``. + + Coordinate values are not compared by default. Pass ``ignore_dims`` + (e.g. ``ignore_dims=()`` or ``ignore_dims={"snapshot"}``) to opt into + per-container coord-equality on every dim *not* in the set — a + mismatch triggers ``RebuildReason.COORD_REINDEX``. + """ + return _compute_diff(snapshot, model, same_model, ignore_dims) + + @classmethod + def from_models( + cls, + model_a: Model, + model_b: Model, + ignore_dims: Iterable[str] | None = None, + ) -> ModelDiff: + """Diff two linopy models directly. + + ``model_a`` is the baseline (snapshotted internally), ``model_b`` is + the target. ``same_model`` is forced to ``False`` so the coefficient + compare runs unconditionally — no ``_coef_dirty`` shortcut applies + between independently-built models. + """ + snapshot = ModelSnapshot.capture(model_a) + return _compute_diff(snapshot, model_b, same_model=False, ignore_dims=ignore_dims) + def _coords_equal( a: dict[str, np.ndarray], b: dict[str, np.ndarray], ignored: frozenset[str] @@ -175,19 +209,12 @@ def _coords_equal( return all(np.array_equal(a[k], b[k]) for k in keys_a) -def compute_diff( +def _compute_diff( snapshot: ModelSnapshot, model: Model, - same_model: bool = True, - ignore_dims: Iterable[str] | None = None, + same_model: bool, + ignore_dims: Iterable[str] | None, ) -> ModelDiff: - """Compute a ``ModelDiff`` between ``snapshot`` and ``model``. - - Coordinate values are not compared by default. Pass ``ignore_dims`` - (e.g. ``ignore_dims=()`` or ``ignore_dims={"snapshot"}``) to opt into - per-container coord-equality on every dim *not* in the set — a mismatch - triggers ``RebuildReason.COORD_REINDEX``. - """ check_coords = ignore_dims is not None ignored = frozenset(ignore_dims) if ignore_dims is not None else frozenset() diff = ModelDiff() diff --git a/linopy/persistent/errors.py b/linopy/persistent/errors.py index 839adedb..8c271e58 100644 --- a/linopy/persistent/errors.py +++ b/linopy/persistent/errors.py @@ -3,3 +3,14 @@ class UnsupportedUpdate(Exception): pass + + +class RebuildRequiredError(RuntimeError): + """Raised when an in-place update is required but a rebuild is needed. + + Carries the :class:`RebuildReason` that forced the rebuild attempt. + """ + + def __init__(self, reason: object, message: str | None = None) -> None: + self.reason = reason + super().__init__(message or f"rebuild required: {reason}") diff --git a/linopy/solvers.py b/linopy/solvers.py index a406bce2..ce4a42b8 100644 --- a/linopy/solvers.py +++ b/linopy/solvers.py @@ -50,8 +50,8 @@ ModelDiff, ModelSnapshot, RebuildReason, + RebuildRequiredError, UnsupportedUpdate, - compute_diff, ) @@ -507,15 +507,22 @@ def supports(cls, feature: SolverFeature) -> bool: @staticmethod def from_name( name: str, - model: Model, + model: Model | None = None, io_api: str | None = None, options: dict[str, Any] | None = None, **build_kwargs: Any, ) -> Solver: - """Construct and build the solver subclass registered as ``name``.""" + """Construct the solver subclass registered as ``name``. + + With ``model`` supplied, the solver is built immediately. Without it, + an unbuilt instance is returned and the first ``solve(model, ...)`` + call performs the build. + """ cls = _solver_class_for(name) if cls is None: raise ValueError(f"unknown solver: {name}") + if model is None: + return cls(model=None, io_api=io_api, options=options or {}) return cls.from_model( model, io_api=io_api, options=options or {}, **build_kwargs ) @@ -626,6 +633,7 @@ def solve( model: Model | None = None, assign: bool = False, ignore_dims: Iterable[str] | None = None, + disallow_rebuild: bool = False, **run_kwargs: Any, ) -> Result: """ @@ -639,6 +647,12 @@ def solve( Pass ``ignore_dims`` (e.g. ``{"snapshot"}``) to opt into per-container coordinate-equality checking on every dim *not* in the set. Default (``None``) skips the coord check entirely. + + Pass ``disallow_rebuild=True`` to guarantee that an existing solver + model is updated in place — any condition that would force a rebuild + (structural change, sparsity change, backend rejection, …) raises + :class:`RebuildRequiredError` instead. The initial build on the first + ``solve(model, ...)`` is still allowed. """ if model is not None: if self.io_api != "direct": @@ -648,7 +662,12 @@ def solve( self.model = model self._build() else: - self._update_locked(model, apply=True, ignore_dims=ignore_dims) + self._update_locked( + model, + apply=True, + ignore_dims=ignore_dims, + disallow_rebuild=disallow_rebuild, + ) target = model else: target = self.model # type: ignore[assignment] @@ -689,19 +708,24 @@ def _update_locked( model: Model, apply: bool, ignore_dims: Iterable[str] | None = None, + disallow_rebuild: bool = False, ) -> ModelDiff: assert self.snapshot is not None if apply and not type(self).supports_persistent_update: + if disallow_rebuild: + raise RebuildRequiredError(RebuildReason.BACKEND_REJECTED) diff = ModelDiff(rebuild_reason=RebuildReason.BACKEND_REJECTED) self._rebuild(model, RebuildReason.BACKEND_REJECTED) return diff same_model = model is self.model - diff = compute_diff( + diff = ModelDiff.from_snapshot( self.snapshot, model, same_model=same_model, ignore_dims=ignore_dims ) if not apply: return diff if diff.rebuild_required: + if disallow_rebuild: + raise RebuildRequiredError(diff.rebuild_reason) self._rebuild(model, diff.rebuild_reason) return diff try: @@ -710,7 +734,11 @@ def _update_locked( model.variables.label_index, model.constraints.label_index, ) - except Exception: + except Exception as exc: + if disallow_rebuild: + raise RebuildRequiredError( + RebuildReason.BACKEND_REJECTED, str(exc) + ) from exc self._last_rebuild_reason = RebuildReason.BACKEND_REJECTED self._rebuild(model, RebuildReason.BACKEND_REJECTED) return diff diff --git a/test/test_persistent_snapshot_buffers.py b/test/test_persistent_snapshot_buffers.py index 31bc0e83..d10f8a4a 100644 --- a/test/test_persistent_snapshot_buffers.py +++ b/test/test_persistent_snapshot_buffers.py @@ -4,7 +4,7 @@ import pytest from linopy import Model -from linopy.persistent import ModelSnapshot, RebuildReason, compute_diff +from linopy.persistent import ModelDiff, ModelSnapshot, RebuildReason from linopy.persistent.snapshot import ( _canonicalize_rows, _extract_con_buffers, @@ -86,7 +86,7 @@ def test_shape_mismatch_triggers_sparsity_rebuild(baseline_model: Model) -> None x = baseline_model.variables["x"] y = baseline_model.variables["y"] baseline_model.constraints["c1"].lhs = 2 * x + 0 * y.sum() - diff = compute_diff(snap, baseline_model) + diff = ModelDiff.from_snapshot(snap, baseline_model) assert diff.rebuild_reason in { RebuildReason.SPARSITY, RebuildReason.STRUCTURAL_LABELS, @@ -99,7 +99,7 @@ def test_zero_row_container_capture() -> None: m.add_objective(0.0 * m.variables["x"].sum()) snap = ModelSnapshot.capture(m) assert snap.con_buffers == {} - diff = compute_diff(snap, m) + diff = ModelDiff.from_snapshot(snap, m) assert diff.is_empty diff --git a/test/test_persistent_snapshot_diff.py b/test/test_persistent_snapshot_diff.py index aa024f5b..e164d6b7 100644 --- a/test/test_persistent_snapshot_diff.py +++ b/test/test_persistent_snapshot_diff.py @@ -12,7 +12,6 @@ ModelSnapshot, RebuildReason, StructuralKey, - compute_diff, ) @@ -45,7 +44,7 @@ def test_capture_structural_key(baseline: Model) -> None: def test_is_empty_on_unmutated(baseline: Model) -> None: snap = ModelSnapshot.capture(baseline) - diff = compute_diff(snap, baseline) + diff = ModelDiff.from_snapshot(snap, baseline) assert diff.is_empty assert diff.rebuild_reason is RebuildReason.NONE assert not diff.rebuild_required @@ -54,7 +53,7 @@ def test_is_empty_on_unmutated(baseline: Model) -> None: def test_bounds_only_mutation(baseline: Model) -> None: snap = ModelSnapshot.capture(baseline) baseline.variables["x"].lower = 1 - diff = compute_diff(snap, baseline) + diff = ModelDiff.from_snapshot(snap, baseline) assert diff.rebuild_reason is RebuildReason.NONE assert "x" in diff.vars assert "y" not in diff.vars @@ -66,7 +65,7 @@ def test_bounds_only_mutation(baseline: Model) -> None: def test_rhs_only_mutation(baseline: Model) -> None: snap = ModelSnapshot.capture(baseline) baseline.constraints["c1"].rhs = 9 - diff = compute_diff(snap, baseline) + diff = ModelDiff.from_snapshot(snap, baseline) assert diff.rebuild_reason is RebuildReason.NONE assert "c1" in diff.cons upd = diff.cons["c1"] @@ -79,7 +78,7 @@ def test_objective_linear_change(baseline: Model) -> None: x = baseline.variables["x"] y = baseline.variables["y"] baseline.add_objective(3 * x.sum() + 2 * y.sum(), overwrite=True) - diff = compute_diff(snap, baseline) + diff = ModelDiff.from_snapshot(snap, baseline) assert diff.rebuild_reason is RebuildReason.NONE assert diff.obj_c_indices is not None assert diff.obj_c_values is not None @@ -88,7 +87,7 @@ def test_objective_linear_change(baseline: Model) -> None: def test_objective_sense_flip(baseline: Model) -> None: snap = ModelSnapshot.capture(baseline) baseline.objective.sense = "max" - diff = compute_diff(snap, baseline) + diff = ModelDiff.from_snapshot(snap, baseline) assert diff.rebuild_reason is RebuildReason.NONE assert diff.obj_sense == "max" @@ -97,7 +96,7 @@ def test_add_constraints_is_structural(baseline: Model) -> None: snap = ModelSnapshot.capture(baseline) x = baseline.variables["x"] baseline.add_constraints(x.sum() <= 99, name="c3") - diff = compute_diff(snap, baseline) + diff = ModelDiff.from_snapshot(snap, baseline) assert diff.rebuild_reason in ( RebuildReason.STRUCTURAL_LABELS, RebuildReason.STRUCTURAL_CONTAINERS, @@ -107,7 +106,7 @@ def test_add_constraints_is_structural(baseline: Model) -> None: def test_remove_variables_is_structural(baseline: Model) -> None: snap = ModelSnapshot.capture(baseline) baseline.remove_variables("y") - diff = compute_diff(snap, baseline) + diff = ModelDiff.from_snapshot(snap, baseline) assert diff.rebuild_reason in ( RebuildReason.STRUCTURAL_LABELS, RebuildReason.STRUCTURAL_CONTAINERS, @@ -118,7 +117,7 @@ def test_coef_value_change_same_sparsity(baseline: Model) -> None: snap = ModelSnapshot.capture(baseline) c = baseline.constraints["c1"] c.coeffs = c.coeffs * 3 - diff = compute_diff(snap, baseline) + diff = ModelDiff.from_snapshot(snap, baseline) assert diff.rebuild_reason is RebuildReason.NONE assert "c1" in diff.cons upd = diff.cons["c1"] @@ -131,14 +130,14 @@ def test_coef_sparsity_change(baseline: Model) -> None: snap = ModelSnapshot.capture(baseline) x = baseline.variables["x"] baseline.constraints["c2"].lhs = 2 * x.sum() - diff = compute_diff(snap, baseline) + diff = ModelDiff.from_snapshot(snap, baseline) assert diff.rebuild_reason is RebuildReason.SPARSITY def test_deep_copy_invariant(baseline: Model) -> None: snap = ModelSnapshot.capture(baseline) baseline.variables["x"].lower.values[...] = 99 - diff = compute_diff(snap, baseline) + diff = ModelDiff.from_snapshot(snap, baseline) assert "x" in diff.vars @@ -147,9 +146,9 @@ def test_same_model_false_ignores_dirty_flag(baseline: Model) -> None: c = baseline.constraints["c1"] c.coeffs = c.coeffs * 5 c._coef_dirty = False - diff_fast = compute_diff(snap, baseline, same_model=True) + diff_fast = ModelDiff.from_snapshot(snap, baseline, same_model=True) assert "c1" not in diff_fast.cons or diff_fast.cons["c1"].coef_values is None - diff_full = compute_diff(snap, baseline, same_model=False) + diff_full = ModelDiff.from_snapshot(snap, baseline, same_model=False) assert "c1" in diff_full.cons assert diff_full.cons["c1"].coef_values is not None @@ -160,6 +159,24 @@ def test_modeldiff_default_is_empty() -> None: assert not d.rebuild_required +def test_from_models_diffs_two_models() -> None: + m1 = Model() + x1 = m1.add_variables(0, 10, coords=[range(3)], name="x") + m1.add_constraints(2 * x1 >= 4, name="c1") + m1.add_objective(x1.sum()) + + m2 = Model() + x2 = m2.add_variables(0, 10, coords=[range(3)], name="x") + m2.add_constraints(2 * x2 >= 7, name="c1") + m2.add_objective(x2.sum()) + + diff = ModelDiff.from_models(m1, m2) + assert diff.rebuild_reason is RebuildReason.NONE + assert "c1" in diff.cons + assert diff.cons["c1"].rhs_values is not None + np.testing.assert_array_equal(diff.cons["c1"].rhs_values, np.full(3, 7.0)) + + def test_ignore_dims_detects_coord_change() -> None: m1 = Model() m1.add_variables(0, 10, coords=[pd.Index([0, 1, 2], name="t")], name="x") @@ -172,10 +189,10 @@ def test_ignore_dims_detects_coord_change() -> None: m2.add_constraints(m2.variables["x"] >= 0, name="c1") m2.add_objective(m2.variables["x"].sum()) - assert compute_diff(snap, m2).rebuild_reason is RebuildReason.NONE - assert compute_diff(snap, m2, ignore_dims=()).rebuild_reason is ( + assert ModelDiff.from_snapshot(snap, m2).rebuild_reason is RebuildReason.NONE + assert ModelDiff.from_snapshot(snap, m2, ignore_dims=()).rebuild_reason is ( RebuildReason.COORD_REINDEX ) - assert compute_diff(snap, m2, ignore_dims={"t"}).rebuild_reason is ( + assert ModelDiff.from_snapshot(snap, m2, ignore_dims={"t"}).rebuild_reason is ( RebuildReason.NONE ) diff --git a/test/test_persistent_solver_extras.py b/test/test_persistent_solver_extras.py index 37900dbb..92f238a4 100644 --- a/test/test_persistent_solver_extras.py +++ b/test/test_persistent_solver_extras.py @@ -342,6 +342,34 @@ def test_scenario_sweep_in_place( s_fresh.close() +@pytest.mark.parametrize("solver_name", SOLVER_PARAMS) +def test_disallow_rebuild_raises_on_structural_change(solver_name: str) -> None: + from linopy.persistent import RebuildRequiredError + + m = _base_model() + s = _built(solver_name, m) + s.solve(assign=True) + + m2 = _base_model() + m2.add_variables(0, 5, coords=[range(3)], name="z") + + with pytest.raises(RebuildRequiredError): + s.solve(m2, disallow_rebuild=True, assign=True) + assert s._rebuilds == 0 + + +@pytest.mark.parametrize("solver_name", SOLVER_PARAMS) +def test_disallow_rebuild_passes_when_update_works(solver_name: str) -> None: + m = _base_model() + s = _built(solver_name, m) + s.solve(assign=True) + + m.constraints["c1"].rhs = 6.0 + s.solve(m, disallow_rebuild=True, assign=True) + assert s._in_place_updates == 1 + assert s._rebuilds == 0 + + @pytest.mark.parametrize("solver_name", SOLVER_PARAMS) def test_solve_without_assign_does_not_mutate_model(solver_name: str) -> None: m = _base_model() From 595bab004be37fdbcb1bc86c373814c9085397c3 Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 21 May 2026 08:57:52 +0200 Subject: [PATCH 13/21] feat(persistent): opt-in update tracking, snapshot-free ModelDiff.from_models - Add `track_updates` flag (default False) to Solver; skip ModelSnapshot capture when disabled. Raise UpdatesDisabledError on solve(model)/update() if a built solver was constructed without tracking. - Rewrite ModelDiff.from_models to build directly from two models without capturing snapshots; share helpers with from_snapshot. - Update persistent tests to opt into track_updates=True; add coverage for the disabled path. --- linopy/persistent/__init__.py | 7 +- linopy/persistent/diff.py | 384 ++++++++++++-------- linopy/persistent/errors.py | 7 + linopy/solvers.py | 72 +++- test/test_persistent_gurobi.py | 2 +- test/test_persistent_highs.py | 2 +- test/test_persistent_solver_extras.py | 41 ++- test/test_persistent_solver_orchestrator.py | 2 +- 8 files changed, 359 insertions(+), 158 deletions(-) diff --git a/linopy/persistent/__init__.py b/linopy/persistent/__init__.py index ddd936e4..1ff0e8c0 100644 --- a/linopy/persistent/__init__.py +++ b/linopy/persistent/__init__.py @@ -8,7 +8,11 @@ ModelDiff, RebuildReason, ) -from linopy.persistent.errors import RebuildRequiredError, UnsupportedUpdate +from linopy.persistent.errors import ( + RebuildRequiredError, + UnsupportedUpdate, + UpdatesDisabledError, +) from linopy.persistent.snapshot import ( ContainerConBuffers, ContainerVarBuffers, @@ -27,4 +31,5 @@ "RebuildRequiredError", "StructuralKey", "UnsupportedUpdate", + "UpdatesDisabledError", ] diff --git a/linopy/persistent/diff.py b/linopy/persistent/diff.py index 026ae98e..9df01ebd 100644 --- a/linopy/persistent/diff.py +++ b/linopy/persistent/diff.py @@ -9,6 +9,8 @@ from collections.abc import Iterable from linopy.persistent.snapshot import ( + ContainerConBuffers, + ContainerVarBuffers, ModelSnapshot, _coord_snapshot, _extract_con_buffers, @@ -17,7 +19,9 @@ ) if TYPE_CHECKING: + from linopy.constraints import ConstraintBase from linopy.model import Model + from linopy.variables import Variable class RebuildReason(enum.Enum): @@ -179,7 +183,60 @@ def from_snapshot( per-container coord-equality on every dim *not* in the set — a mismatch triggers ``RebuildReason.COORD_REINDEX``. """ - return _compute_diff(snapshot, model, same_model, ignore_dims) + check_coords = ignore_dims is not None + ignored = frozenset(ignore_dims) if ignore_dims is not None else frozenset() + diff = cls() + + var_names = tuple(model.variables) + con_names = tuple(model.constraints) + if ( + snapshot.structural_key.var_container_names != var_names + or snapshot.structural_key.con_container_names != con_names + ): + diff.rebuild_reason = RebuildReason.STRUCTURAL_CONTAINERS + return diff + + var_label_index = model.variables.label_index + con_label_index = model.constraints.label_index + if not np.array_equal(snapshot.structural_key.vlabels, var_label_index.vlabels): + diff.rebuild_reason = RebuildReason.STRUCTURAL_LABELS + return diff + if not np.array_equal(snapshot.structural_key.clabels, con_label_index.clabels): + diff.rebuild_reason = RebuildReason.STRUCTURAL_LABELS + return diff + + var_l2p = var_label_index.label_to_pos + con_l2p = con_label_index.label_to_pos + + for name, var in model.variables.items(): + base_coords = snapshot.var_coords[name] if check_coords else None + reason = _diff_var_container( + diff, name, var, snapshot.var_buffers[name], + base_coords, var_l2p, ignored, check_coords, + ) + if reason is not None: + diff.rebuild_reason = reason + return diff + + for name, con in model.constraints.items(): + base_coords = snapshot.con_coords[name] if check_coords else None + skip_coef_compare = same_model and not con._coef_dirty + reason = _diff_con_container( + diff, name, con, snapshot.con_buffers[name], + base_coords, var_l2p, con_l2p, ignored, check_coords, + skip_coef_compare, + ) + if reason is not None: + diff.rebuild_reason = reason + return diff + + reason = _diff_objective( + diff, model, + snapshot.obj_c, snapshot.obj_quad_present, snapshot.obj_sense, + ) + if reason is not None: + diff.rebuild_reason = reason + return diff @classmethod def from_models( @@ -188,15 +245,73 @@ def from_models( model_b: Model, ignore_dims: Iterable[str] | None = None, ) -> ModelDiff: - """Diff two linopy models directly. + """Diff two linopy models directly, without capturing a snapshot. - ``model_a`` is the baseline (snapshotted internally), ``model_b`` is - the target. ``same_model`` is forced to ``False`` so the coefficient - compare runs unconditionally — no ``_coef_dirty`` shortcut applies - between independently-built models. + ``model_a`` is the baseline, ``model_b`` is the target. The + coefficient comparison runs unconditionally — no ``_coef_dirty`` + shortcut applies between independently-built models. """ - snapshot = ModelSnapshot.capture(model_a) - return _compute_diff(snapshot, model_b, same_model=False, ignore_dims=ignore_dims) + check_coords = ignore_dims is not None + ignored = frozenset(ignore_dims) if ignore_dims is not None else frozenset() + diff = cls() + + var_names_a = tuple(model_a.variables) + con_names_a = tuple(model_a.constraints) + if ( + var_names_a != tuple(model_b.variables) + or con_names_a != tuple(model_b.constraints) + ): + diff.rebuild_reason = RebuildReason.STRUCTURAL_CONTAINERS + return diff + + var_idx_a = model_a.variables.label_index + con_idx_a = model_a.constraints.label_index + var_idx_b = model_b.variables.label_index + con_idx_b = model_b.constraints.label_index + if not np.array_equal(var_idx_a.vlabels, var_idx_b.vlabels): + diff.rebuild_reason = RebuildReason.STRUCTURAL_LABELS + return diff + if not np.array_equal(con_idx_a.clabels, con_idx_b.clabels): + diff.rebuild_reason = RebuildReason.STRUCTURAL_LABELS + return diff + + var_l2p = var_idx_b.label_to_pos + con_l2p = con_idx_b.label_to_pos + + for name, var_b in model_b.variables.items(): + var_a = model_a.variables[name] + base_buf = _extract_var_buffers(var_a) + base_coords = _coord_snapshot(var_a) if check_coords else None + reason = _diff_var_container( + diff, name, var_b, base_buf, + base_coords, var_l2p, ignored, check_coords, + ) + if reason is not None: + diff.rebuild_reason = reason + return diff + + for name, con_b in model_b.constraints.items(): + con_a = model_a.constraints[name] + base_buf = _extract_con_buffers(con_a, var_l2p) + base_coords = _coord_snapshot(con_a) if check_coords else None + reason = _diff_con_container( + diff, name, con_b, base_buf, + base_coords, var_l2p, con_l2p, ignored, check_coords, + skip_coef_compare=False, + ) + if reason is not None: + diff.rebuild_reason = reason + return diff + + reason = _diff_objective( + diff, model_b, + _objective_linear_vector(model_a), + model_a.objective.is_quadratic, + model_a.objective.sense, + ) + if reason is not None: + diff.rebuild_reason = reason + return diff def _coords_equal( @@ -209,150 +324,131 @@ def _coords_equal( return all(np.array_equal(a[k], b[k]) for k in keys_a) -def _compute_diff( - snapshot: ModelSnapshot, +def _diff_var_container( + diff: ModelDiff, + name: str, + var: Variable, + base_buf: ContainerVarBuffers, + base_coords: dict[str, np.ndarray] | None, + var_l2p: np.ndarray, + ignored: frozenset[str], + check_coords: bool, +) -> RebuildReason | None: + new_buf = _extract_var_buffers(var) + if new_buf.lower.shape != base_buf.lower.shape: + return RebuildReason.COORD_REINDEX + if not np.array_equal(new_buf.active_labels, base_buf.active_labels): + return RebuildReason.STRUCTURAL_LABELS + if check_coords and not _coords_equal(base_coords, _coord_snapshot(var), ignored): + return RebuildReason.COORD_REINDEX + + lower_diff = new_buf.lower != base_buf.lower + upper_diff = new_buf.upper != base_buf.upper + type_changed = new_buf.type != base_buf.type + + bound_mask = lower_diff | upper_diff + if not (bound_mask.any() or type_changed): + return None + + update = ContainerVarUpdate(type_change=new_buf.type if type_changed else None) + if bound_mask.any(): + local_idx = np.flatnonzero(bound_mask) + update.bounds_indices = var_l2p[ + new_buf.active_labels[local_idx] + ].astype(np.int32, copy=False) + update.lower = new_buf.lower[local_idx] + update.upper = new_buf.upper[local_idx] + diff.vars[name] = update + return None + + +def _diff_con_container( + diff: ModelDiff, + name: str, + con: ConstraintBase, + base_buf: ContainerConBuffers, + base_coords: dict[str, np.ndarray] | None, + var_l2p: np.ndarray, + con_l2p: np.ndarray, + ignored: frozenset[str], + check_coords: bool, + skip_coef_compare: bool, +) -> RebuildReason | None: + new_buf = _extract_con_buffers(con, var_l2p) + if new_buf.coeffs.shape != base_buf.coeffs.shape: + return RebuildReason.SPARSITY + if not np.array_equal(new_buf.active_labels, base_buf.active_labels): + return RebuildReason.STRUCTURAL_LABELS + if check_coords and not _coords_equal(base_coords, _coord_snapshot(con), ignored): + return RebuildReason.COORD_REINDEX + + n_rows = new_buf.active_labels.size + if n_rows == 0: + return None + + if skip_coef_compare: + row_value_changed = np.zeros(n_rows, dtype=bool) + row_struct_changed = np.zeros(n_rows, dtype=bool) + else: + row_struct_changed = np.any(new_buf.vars != base_buf.vars, axis=-1) + row_value_changed = np.any(new_buf.coeffs != base_buf.coeffs, axis=-1) + + if row_struct_changed.any(): + return RebuildReason.SPARSITY + + rhs_changed = new_buf.rhs != base_buf.rhs + sign_changed = new_buf.sign != base_buf.sign + + if not (row_value_changed.any() or rhs_changed.any() or sign_changed.any()): + return None + + update = ContainerRowUpdate() + if row_value_changed.any(): + idx = np.flatnonzero(row_value_changed) + update.coef_row_indices = con_l2p[ + new_buf.active_labels[idx] + ].astype(np.int32, copy=False) + update.coef_vars = new_buf.vars[idx] + update.coef_values = new_buf.coeffs[idx] + if rhs_changed.any(): + idx = np.flatnonzero(rhs_changed) + update.rhs_row_indices = con_l2p[ + new_buf.active_labels[idx] + ].astype(np.int32, copy=False) + update.rhs_values = new_buf.rhs[idx] + update.rhs_signs = new_buf.sign[idx] + if sign_changed.any(): + idx = np.flatnonzero(sign_changed) + update.sign_row_indices = con_l2p[ + new_buf.active_labels[idx] + ].astype(np.int32, copy=False) + update.sign_values = new_buf.sign[idx] + diff.cons[name] = update + return None + + +def _diff_objective( + diff: ModelDiff, model: Model, - same_model: bool, - ignore_dims: Iterable[str] | None, -) -> ModelDiff: - check_coords = ignore_dims is not None - ignored = frozenset(ignore_dims) if ignore_dims is not None else frozenset() - diff = ModelDiff() - - var_names = tuple(model.variables) - con_names = tuple(model.constraints) - if ( - snapshot.structural_key.var_container_names != var_names - or snapshot.structural_key.con_container_names != con_names - ): - diff.rebuild_reason = RebuildReason.STRUCTURAL_CONTAINERS - return diff - - var_label_index = model.variables.label_index - con_label_index = model.constraints.label_index - if not np.array_equal(snapshot.structural_key.vlabels, var_label_index.vlabels): - diff.rebuild_reason = RebuildReason.STRUCTURAL_LABELS - return diff - if not np.array_equal(snapshot.structural_key.clabels, con_label_index.clabels): - diff.rebuild_reason = RebuildReason.STRUCTURAL_LABELS - return diff - - var_l2p = var_label_index.label_to_pos - con_l2p = con_label_index.label_to_pos - - for name, var in model.variables.items(): - snap_buf = snapshot.var_buffers[name] - new_buf = _extract_var_buffers(var) - if new_buf.lower.shape != snap_buf.lower.shape: - diff.rebuild_reason = RebuildReason.COORD_REINDEX - return diff - if not np.array_equal(new_buf.active_labels, snap_buf.active_labels): - diff.rebuild_reason = RebuildReason.STRUCTURAL_LABELS - return diff - if check_coords and not _coords_equal( - snapshot.var_coords[name], _coord_snapshot(var), ignored - ): - diff.rebuild_reason = RebuildReason.COORD_REINDEX - return diff - - lower_diff = new_buf.lower != snap_buf.lower - upper_diff = new_buf.upper != snap_buf.upper - type_changed = new_buf.type != snap_buf.type - - bound_mask = lower_diff | upper_diff - if not (bound_mask.any() or type_changed): - continue - - update = ContainerVarUpdate(type_change=new_buf.type if type_changed else None) - if bound_mask.any(): - local_idx = np.flatnonzero(bound_mask) - update.bounds_indices = var_l2p[ - new_buf.active_labels[local_idx] - ].astype(np.int32, copy=False) - update.lower = new_buf.lower[local_idx] - update.upper = new_buf.upper[local_idx] - diff.vars[name] = update - - for name, con in model.constraints.items(): - snap_buf = snapshot.con_buffers[name] - new_buf = _extract_con_buffers(con, var_l2p) - - if new_buf.coeffs.shape != snap_buf.coeffs.shape: - diff.rebuild_reason = RebuildReason.SPARSITY - return diff - if not np.array_equal(new_buf.active_labels, snap_buf.active_labels): - diff.rebuild_reason = RebuildReason.STRUCTURAL_LABELS - return diff - if check_coords and not _coords_equal( - snapshot.con_coords[name], _coord_snapshot(con), ignored - ): - diff.rebuild_reason = RebuildReason.COORD_REINDEX - return diff - - n_rows = new_buf.active_labels.size - if n_rows == 0: - continue - - skip_coef_compare = same_model and not con._coef_dirty - if skip_coef_compare: - row_value_changed = np.zeros(n_rows, dtype=bool) - row_struct_changed = np.zeros(n_rows, dtype=bool) - else: - row_struct_changed = np.any(new_buf.vars != snap_buf.vars, axis=-1) - row_value_changed = np.any(new_buf.coeffs != snap_buf.coeffs, axis=-1) - - if row_struct_changed.any(): - diff.rebuild_reason = RebuildReason.SPARSITY - return diff - - rhs_changed = new_buf.rhs != snap_buf.rhs - sign_changed = new_buf.sign != snap_buf.sign - - if not (row_value_changed.any() or rhs_changed.any() or sign_changed.any()): - continue - - update = ContainerRowUpdate() - if row_value_changed.any(): - idx = np.flatnonzero(row_value_changed) - update.coef_row_indices = con_l2p[ - new_buf.active_labels[idx] - ].astype(np.int32, copy=False) - update.coef_vars = new_buf.vars[idx] - update.coef_values = new_buf.coeffs[idx] - if rhs_changed.any(): - idx = np.flatnonzero(rhs_changed) - update.rhs_row_indices = con_l2p[ - new_buf.active_labels[idx] - ].astype(np.int32, copy=False) - update.rhs_values = new_buf.rhs[idx] - update.rhs_signs = new_buf.sign[idx] - if sign_changed.any(): - idx = np.flatnonzero(sign_changed) - update.sign_row_indices = con_l2p[ - new_buf.active_labels[idx] - ].astype(np.int32, copy=False) - update.sign_values = new_buf.sign[idx] - diff.cons[name] = update - + base_obj_c: np.ndarray, + base_obj_quad: bool, + base_obj_sense: str, +) -> RebuildReason | None: obj_quad_present = model.objective.is_quadratic - if obj_quad_present != snapshot.obj_quad_present: - diff.rebuild_reason = RebuildReason.QUAD_OBJ - return diff + if obj_quad_present != base_obj_quad: + return RebuildReason.QUAD_OBJ if obj_quad_present: - diff.rebuild_reason = RebuildReason.QUAD_OBJ - return diff + return RebuildReason.QUAD_OBJ obj_c = _objective_linear_vector(model) - if obj_c.shape != snapshot.obj_c.shape: - diff.rebuild_reason = RebuildReason.COORD_REINDEX - return diff - obj_diff_mask = obj_c != snapshot.obj_c + if obj_c.shape != base_obj_c.shape: + return RebuildReason.COORD_REINDEX + obj_diff_mask = obj_c != base_obj_c if obj_diff_mask.any(): idx = np.flatnonzero(obj_diff_mask).astype(np.int32, copy=False) diff.obj_c_indices = idx diff.obj_c_values = obj_c[idx] - if model.objective.sense != snapshot.obj_sense: + if model.objective.sense != base_obj_sense: diff.obj_sense = model.objective.sense - - return diff + return None diff --git a/linopy/persistent/errors.py b/linopy/persistent/errors.py index 8c271e58..2a626346 100644 --- a/linopy/persistent/errors.py +++ b/linopy/persistent/errors.py @@ -14,3 +14,10 @@ class RebuildRequiredError(RuntimeError): def __init__(self, reason: object, message: str | None = None) -> None: self.reason = reason super().__init__(message or f"rebuild required: {reason}") + + +class UpdatesDisabledError(RuntimeError): + """Raised when an in-place update is requested on a solver built with + ``track_updates=False``. Reconstruct the solver with ``track_updates=True`` + to enable diff-based updates. + """ diff --git a/linopy/solvers.py b/linopy/solvers.py index ce4a42b8..29ea1720 100644 --- a/linopy/solvers.py +++ b/linopy/solvers.py @@ -52,6 +52,7 @@ RebuildReason, RebuildRequiredError, UnsupportedUpdate, + UpdatesDisabledError, ) @@ -393,11 +394,22 @@ class Solver(ABC, Generic[EnvType]): Subclasses provide ``_build_direct`` / ``_run_direct`` (when supporting the direct API) and ``_run_file`` (when supporting LP/MPS files). Construction goes via :meth:`Solver.from_name` or :meth:`Solver.from_model`. + + ``track_updates`` toggles persistent-update support: + + * ``False`` (default) — one-shot mode. No :class:`ModelSnapshot` is + captured at build time; any later ``solve(model=...)`` or + ``update(model)`` raises :class:`UpdatesDisabledError`. Use for + throw-away solver instances and high-level ``Model.solve(...)``. + * ``True`` — long-lived mode. A snapshot is captured at build time and + re-captured after each successful in-place update, enabling + diff-based ``solve(model=...)`` / ``update(model)`` across iterations. """ model: Model | None = None io_api: str | None = None options: dict[str, Any] = field(default_factory=dict) + track_updates: bool = False # Runtime state — never set via constructor. status: Status | None = field(init=False, default=None, repr=False) @@ -510,6 +522,7 @@ def from_name( model: Model | None = None, io_api: str | None = None, options: dict[str, Any] | None = None, + track_updates: bool = False, **build_kwargs: Any, ) -> Solver: """Construct the solver subclass registered as ``name``. @@ -517,14 +530,30 @@ def from_name( With ``model`` supplied, the solver is built immediately. Without it, an unbuilt instance is returned and the first ``solve(model, ...)`` call performs the build. + + ``track_updates=False`` (default) is the one-shot mode: no + :class:`ModelSnapshot` is captured at build time, and any subsequent + ``solver.solve(model=...)`` / ``solver.update(model)`` raises + :class:`UpdatesDisabledError`. Pass ``track_updates=True`` for + long-lived solvers that want in-place diff-based updates across + iterations. """ cls = _solver_class_for(name) if cls is None: raise ValueError(f"unknown solver: {name}") if model is None: - return cls(model=None, io_api=io_api, options=options or {}) + return cls( + model=None, + io_api=io_api, + options=options or {}, + track_updates=track_updates, + ) return cls.from_model( - model, io_api=io_api, options=options or {}, **build_kwargs + model, + io_api=io_api, + options=options or {}, + track_updates=track_updates, + **build_kwargs, ) @classmethod @@ -533,10 +562,19 @@ def from_model( model: Model, io_api: str | None = None, options: dict[str, Any] | None = None, + track_updates: bool = False, **build_kwargs: Any, ) -> Solver: - """Instantiate and build the solver against ``model``.""" - instance = cls(model=model, io_api=io_api, options=options or {}) + """Instantiate and build the solver against ``model``. + + See :meth:`from_name` for ``track_updates`` semantics. + """ + instance = cls( + model=model, + io_api=io_api, + options=options or {}, + track_updates=track_updates, + ) instance._build(**build_kwargs) return instance @@ -557,8 +595,9 @@ def _build(self, **build_kwargs: Any) -> None: self.model._check_sos_unmasked() if self.io_api == "direct": self._build_direct(**build_kwargs) - self.snapshot = ModelSnapshot.capture(self.model) - _clear_coef_dirty(self.model.constraints) + if self.track_updates: + self.snapshot = ModelSnapshot.capture(self.model) + _clear_coef_dirty(self.model.constraints) else: self._build_file(**build_kwargs) @@ -640,7 +679,10 @@ def solve( Run the prepared solver and return a :class:`Result`. With ``model`` supplied, diff against the held snapshot and either - apply in place or rebuild before running. Requires ``io_api='direct'``. + apply in place or rebuild before running. Requires ``io_api='direct'`` + and ``track_updates=True`` at construction time; otherwise resolving + with a model raises :class:`UpdatesDisabledError` (the initial build + on the first ``solve(model, ...)`` is still allowed). With ``assign=True`` the Result is written back to the target Model via :meth:`Model.assign_result`. @@ -662,6 +704,13 @@ def solve( self.model = model self._build() else: + if not self.track_updates: + raise UpdatesDisabledError( + "Solver was constructed with track_updates=False; " + "in-place updates are not available. Reconstruct " + "with Solver.from_name(..., track_updates=True) " + "to enable diff-based updates across solves." + ) self._update_locked( model, apply=True, @@ -698,8 +747,15 @@ def update( ) -> ModelDiff: if self.io_api != "direct": raise ValueError("update requires io_api='direct'") - if self.snapshot is None or self.solver_model is None: + if self.solver_model is None: raise RuntimeError("Solver has not been built") + if not self.track_updates: + raise UpdatesDisabledError( + "Solver was constructed with track_updates=False; " + "in-place updates are not available. Reconstruct with " + "Solver.from_name(..., track_updates=True) to enable " + "diff-based updates." + ) with self._lock: return self._update_locked(model, apply=apply, ignore_dims=ignore_dims) diff --git a/test/test_persistent_gurobi.py b/test/test_persistent_gurobi.py index d2dd8bd9..f108bfd2 100644 --- a/test/test_persistent_gurobi.py +++ b/test/test_persistent_gurobi.py @@ -21,7 +21,7 @@ def _base_model() -> Model: def _built(model: Model) -> Gurobi: - s = Gurobi(model=model, io_api="direct") + s = Gurobi(model=model, io_api="direct", track_updates=True) s.options = {"OutputFlag": 0} s._build() return s diff --git a/test/test_persistent_highs.py b/test/test_persistent_highs.py index d1620a30..77325ddc 100644 --- a/test/test_persistent_highs.py +++ b/test/test_persistent_highs.py @@ -21,7 +21,7 @@ def _base_model() -> Model: def _built(model: Model) -> Highs: - s = Highs(model=model, io_api="direct") + s = Highs(model=model, io_api="direct", track_updates=True) s.options = {"output_flag": False} s._build() return s diff --git a/test/test_persistent_solver_extras.py b/test/test_persistent_solver_extras.py index 92f238a4..479bcc7e 100644 --- a/test/test_persistent_solver_extras.py +++ b/test/test_persistent_solver_extras.py @@ -8,7 +8,7 @@ import pytest from linopy import Model -from linopy.persistent import RebuildReason +from linopy.persistent import RebuildReason, UpdatesDisabledError from linopy.solvers import Gurobi, Highs, Solver _BACKENDS: dict[str, tuple[type[Solver], dict[str, Any]]] = { @@ -52,7 +52,7 @@ def _base_model() -> Model: def _built(solver_name: str, model: Model) -> Solver: cls, opts = _BACKENDS[solver_name] - s = cls(model=model, io_api="direct") + s = cls(model=model, io_api="direct", track_updates=True) s.options = opts s._build() return s @@ -381,3 +381,40 @@ def test_solve_without_assign_does_not_mutate_model(solver_name: str) -> None: s.solve(assign=True) assert m.objective._value is not None + + +@pytest.mark.parametrize("solver_name", SOLVER_PARAMS) +def test_track_updates_false_skips_snapshot(solver_name: str) -> None: + cls, opts = _BACKENDS[solver_name] + m = _base_model() + s = cls(model=m, io_api="direct", track_updates=False) + s.options = opts + s._build() + assert s.snapshot is None + s.solve(assign=True) + assert s.snapshot is None + + +@pytest.mark.parametrize("solver_name", SOLVER_PARAMS) +def test_track_updates_false_rejects_resolve_with_model(solver_name: str) -> None: + cls, opts = _BACKENDS[solver_name] + m = _base_model() + s = cls(model=m, io_api="direct", track_updates=False) + s.options = opts + s._build() + s.solve(assign=True) + + m.variables["x"].lower.values[...] = 6.0 + with pytest.raises(UpdatesDisabledError, match="track_updates=False"): + s.solve(m, assign=True) + + +@pytest.mark.parametrize("solver_name", SOLVER_PARAMS) +def test_track_updates_false_rejects_update(solver_name: str) -> None: + cls, opts = _BACKENDS[solver_name] + m = _base_model() + s = cls(model=m, io_api="direct", track_updates=False) + s.options = opts + s._build() + with pytest.raises(UpdatesDisabledError, match="track_updates=False"): + s.update(m) diff --git a/test/test_persistent_solver_orchestrator.py b/test/test_persistent_solver_orchestrator.py index dbe40a48..4fcdb58f 100644 --- a/test/test_persistent_solver_orchestrator.py +++ b/test/test_persistent_solver_orchestrator.py @@ -66,7 +66,7 @@ def other_model() -> Model: def _built(model: Model) -> FakeSolver: - s = FakeSolver(model=model, io_api="direct") + s = FakeSolver(model=model, io_api="direct", track_updates=True) s._build() return s From 67079178d473c02ccf949250ab89882d5a8762c6 Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 21 May 2026 09:37:01 +0200 Subject: [PATCH 14/21] feat(persistent): wire ModelDiff.from_models for track_updates=False Cross-instance resolves now diff via from_models against the previously built model, with no snapshot. Same-instance mutation still raises UpdatesDisabledError. Snapshot recapture is skipped in this mode. Add cross-instance solve/update tests for the no-snapshot path. --- linopy/solvers.py | 48 ++++++++++++++++----------- test/test_persistent_solver_extras.py | 36 ++++++++++++++++++++ 2 files changed, 64 insertions(+), 20 deletions(-) diff --git a/linopy/solvers.py b/linopy/solvers.py index 29ea1720..f3a92282 100644 --- a/linopy/solvers.py +++ b/linopy/solvers.py @@ -678,11 +678,13 @@ def solve( """ Run the prepared solver and return a :class:`Result`. - With ``model`` supplied, diff against the held snapshot and either - apply in place or rebuild before running. Requires ``io_api='direct'`` - and ``track_updates=True`` at construction time; otherwise resolving - with a model raises :class:`UpdatesDisabledError` (the initial build - on the first ``solve(model, ...)`` is still allowed). + With ``model`` supplied, diff against the previous build and either + apply in place or rebuild before running. Requires ``io_api='direct'``. + Diffing uses :meth:`ModelDiff.from_snapshot` when ``track_updates=True`` + (in-place mutations of the build-time Model are detected) and + :meth:`ModelDiff.from_models` otherwise (only cross-instance resolves + are supported — passing the same Model instance after in-place + mutation raises :class:`UpdatesDisabledError`). With ``assign=True`` the Result is written back to the target Model via :meth:`Model.assign_result`. @@ -704,12 +706,13 @@ def solve( self.model = model self._build() else: - if not self.track_updates: + if not self.track_updates and model is self.model: raise UpdatesDisabledError( "Solver was constructed with track_updates=False; " - "in-place updates are not available. Reconstruct " - "with Solver.from_name(..., track_updates=True) " - "to enable diff-based updates across solves." + "in-place mutations of the build-time Model cannot " + "be detected without a snapshot. Pass a freshly " + "built Model instance, or reconstruct the solver " + "with Solver.from_name(..., track_updates=True)." ) self._update_locked( model, @@ -749,12 +752,13 @@ def update( raise ValueError("update requires io_api='direct'") if self.solver_model is None: raise RuntimeError("Solver has not been built") - if not self.track_updates: + if not self.track_updates and model is self.model: raise UpdatesDisabledError( "Solver was constructed with track_updates=False; " - "in-place updates are not available. Reconstruct with " - "Solver.from_name(..., track_updates=True) to enable " - "diff-based updates." + "in-place mutations of the build-time Model cannot be " + "detected without a snapshot. Pass a freshly built Model " + "instance, or reconstruct the solver with " + "Solver.from_name(..., track_updates=True)." ) with self._lock: return self._update_locked(model, apply=apply, ignore_dims=ignore_dims) @@ -766,17 +770,20 @@ def _update_locked( ignore_dims: Iterable[str] | None = None, disallow_rebuild: bool = False, ) -> ModelDiff: - assert self.snapshot is not None if apply and not type(self).supports_persistent_update: if disallow_rebuild: raise RebuildRequiredError(RebuildReason.BACKEND_REJECTED) diff = ModelDiff(rebuild_reason=RebuildReason.BACKEND_REJECTED) self._rebuild(model, RebuildReason.BACKEND_REJECTED) return diff - same_model = model is self.model - diff = ModelDiff.from_snapshot( - self.snapshot, model, same_model=same_model, ignore_dims=ignore_dims - ) + if self.snapshot is not None: + same_model = model is self.model + diff = ModelDiff.from_snapshot( + self.snapshot, model, same_model=same_model, ignore_dims=ignore_dims + ) + else: + assert self.model is not None + diff = ModelDiff.from_models(self.model, model, ignore_dims=ignore_dims) if not apply: return diff if diff.rebuild_required: @@ -799,8 +806,9 @@ def _update_locked( self._rebuild(model, RebuildReason.BACKEND_REJECTED) return diff self.model = model - self.snapshot = ModelSnapshot.capture(model) - _clear_coef_dirty(model.constraints) + if self.track_updates: + self.snapshot = ModelSnapshot.capture(model) + _clear_coef_dirty(model.constraints) self._in_place_updates += 1 self._last_rebuild_reason = RebuildReason.NONE return diff diff --git a/test/test_persistent_solver_extras.py b/test/test_persistent_solver_extras.py index 479bcc7e..4c642a06 100644 --- a/test/test_persistent_solver_extras.py +++ b/test/test_persistent_solver_extras.py @@ -418,3 +418,39 @@ def test_track_updates_false_rejects_update(solver_name: str) -> None: s._build() with pytest.raises(UpdatesDisabledError, match="track_updates=False"): s.update(m) + + +@pytest.mark.parametrize("solver_name", SOLVER_PARAMS) +def test_track_updates_false_cross_instance_resolve(solver_name: str) -> None: + cls, opts = _BACKENDS[solver_name] + m1 = _base_model() + s = cls(model=m1, io_api="direct", track_updates=False) + s.options = opts + s._build() + s.solve(assign=True) + base_obj = float(m1.objective.value) + + m2 = _base_model() + m2.constraints["c1"].rhs = 8.0 + result = s.solve(m2, assign=True) + assert s._in_place_updates == 1 + assert s._rebuilds == 0 + assert s.snapshot is None + assert s.model is m2 + assert float(result.solution.objective) > base_obj + + +@pytest.mark.parametrize("solver_name", SOLVER_PARAMS) +def test_track_updates_false_cross_instance_update(solver_name: str) -> None: + cls, opts = _BACKENDS[solver_name] + m1 = _base_model() + s = cls(model=m1, io_api="direct", track_updates=False) + s.options = opts + s._build() + s.solve(assign=True) + + m2 = _base_model() + m2.constraints["c1"].rhs = 8.0 + diff = s.update(m2, apply=False) + assert diff.summary()["con_rhs"] == 1 + assert s.snapshot is None From 2ae0bef4d8aa5935ffa9a5afea46603453d1fd77 Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 21 May 2026 09:54:08 +0200 Subject: [PATCH 15/21] refactor(persistent): cleanups, VarKind enum, fold _clear_coef_dirty Collapse _diff_objective QUAD_OBJ branches; cache n_coef_updates; short-circuit _canonicalize_rows when rows already sorted; tighten buffer extraction. Introduce VarKind enum used across snapshot/diff and HiGHS/Gurobi apply_update; reuse linopy.constants sign tokens. Move _clear_coef_dirty into ModelSnapshot.capture. --- linopy/persistent/__init__.py | 2 ++ linopy/persistent/diff.py | 41 +++++----------------- linopy/persistent/snapshot.py | 48 ++++++++++++++++---------- linopy/solvers.py | 65 ++++++++++++----------------------- 4 files changed, 61 insertions(+), 95 deletions(-) diff --git a/linopy/persistent/__init__.py b/linopy/persistent/__init__.py index 1ff0e8c0..81e0f816 100644 --- a/linopy/persistent/__init__.py +++ b/linopy/persistent/__init__.py @@ -18,6 +18,7 @@ ContainerVarBuffers, ModelSnapshot, StructuralKey, + VarKind, ) __all__ = [ @@ -32,4 +33,5 @@ "StructuralKey", "UnsupportedUpdate", "UpdatesDisabledError", + "VarKind", ] diff --git a/linopy/persistent/diff.py b/linopy/persistent/diff.py index 9df01ebd..7bcd1951 100644 --- a/linopy/persistent/diff.py +++ b/linopy/persistent/diff.py @@ -12,6 +12,7 @@ ContainerConBuffers, ContainerVarBuffers, ModelSnapshot, + VarKind, _coord_snapshot, _extract_con_buffers, _extract_var_buffers, @@ -36,30 +37,14 @@ class RebuildReason(enum.Enum): @dataclass class ContainerVarUpdate: - """ - In-place variable bounds / type update for one container. - - Bounds payloads share ``bounds_indices``. When only ``lower`` (or only - ``upper``) changes, both arrays are still populated from the new model so - backends with a single batched call (HiGHS ``changeColsBounds``) can be - fed directly. - """ - bounds_indices: np.ndarray | None = None lower: np.ndarray | None = None upper: np.ndarray | None = None - type_change: str | None = None + type_change: VarKind | None = None @dataclass class ContainerRowUpdate: - """ - Per-row constraint update. - - Holds views into the new model's canonicalised buffers; the orchestrator - diffs and applies under the same lock, so aliasing is bounded. - """ - coef_row_indices: np.ndarray | None = None coef_vars: np.ndarray | None = None coef_values: np.ndarray | None = None @@ -78,6 +63,7 @@ class ModelDiff: obj_c_indices: np.ndarray | None = None obj_c_values: np.ndarray | None = None obj_sense: str | None = None + n_coef_updates: int = 0 @property def is_empty(self) -> bool: @@ -101,14 +87,6 @@ def changed_variables(self) -> set[str]: def changed_constraints(self) -> set[str]: return set(self.cons) - @property - def n_coef_updates(self) -> int: - total = 0 - for upd in self.cons.values(): - if upd.coef_vars is not None: - total += int((upd.coef_vars != -1).sum()) - return total - def summary(self) -> dict[str, int | bool | str | None]: n_var_lb = sum(1 for u in self.vars.values() if u.lower is not None) n_var_ub = sum(1 for u in self.vars.values() if u.upper is not None) @@ -317,11 +295,10 @@ def from_models( def _coords_equal( a: dict[str, np.ndarray], b: dict[str, np.ndarray], ignored: frozenset[str] ) -> bool: - keys_a = set(a) - ignored - keys_b = set(b) - ignored - if keys_a != keys_b: + keys = a.keys() - ignored + if keys != b.keys() - ignored: return False - return all(np.array_equal(a[k], b[k]) for k in keys_a) + return all(np.array_equal(a[k], b[k]) for k in keys) def _diff_var_container( @@ -410,6 +387,7 @@ def _diff_con_container( ].astype(np.int32, copy=False) update.coef_vars = new_buf.vars[idx] update.coef_values = new_buf.coeffs[idx] + diff.n_coef_updates += int((update.coef_vars != -1).sum()) if rhs_changed.any(): idx = np.flatnonzero(rhs_changed) update.rhs_row_indices = con_l2p[ @@ -434,10 +412,7 @@ def _diff_objective( base_obj_quad: bool, base_obj_sense: str, ) -> RebuildReason | None: - obj_quad_present = model.objective.is_quadratic - if obj_quad_present != base_obj_quad: - return RebuildReason.QUAD_OBJ - if obj_quad_present: + if model.objective.is_quadratic or base_obj_quad: return RebuildReason.QUAD_OBJ obj_c = _objective_linear_vector(model) diff --git a/linopy/persistent/snapshot.py b/linopy/persistent/snapshot.py index ffa74444..80569c6e 100644 --- a/linopy/persistent/snapshot.py +++ b/linopy/persistent/snapshot.py @@ -1,5 +1,6 @@ from __future__ import annotations +import enum from dataclasses import dataclass, field from typing import TYPE_CHECKING @@ -16,15 +17,22 @@ _INT64_MAX = np.iinfo(np.int64).max -def _variable_type(var: Variable) -> str: +class VarKind(enum.Enum): + CONTINUOUS = "continuous" + BINARY = "binary" + INTEGER = "integer" + SEMI_CONTINUOUS = "semi_continuous" + + +def _variable_type(var: Variable) -> VarKind: attrs = var.attrs if attrs.get("binary"): - return "binary" + return VarKind.BINARY if attrs.get("integer"): - return "integer" + return VarKind.INTEGER if attrs.get("semi_continuous"): - return "semi_continuous" - return "continuous" + return VarKind.SEMI_CONTINUOUS + return VarKind.CONTINUOUS def _objective_linear_vector(model: Model) -> np.ndarray: @@ -51,27 +59,26 @@ def _canonicalize_rows( vars_arr: np.ndarray, coeffs_arr: np.ndarray ) -> tuple[np.ndarray, np.ndarray]: """Sort each row jointly by var index. -1 sentinels sort to the right.""" - if vars_arr.size == 0: - return vars_arr.astype(np.int64, copy=False), coeffs_arr.astype( - np.float64, copy=False - ) - sort_key = np.where(vars_arr == -1, _INT64_MAX, vars_arr).astype(np.int64) + vars_i64 = np.ascontiguousarray(vars_arr, dtype=np.int64) + coeffs_f64 = np.ascontiguousarray(coeffs_arr, dtype=np.float64) + if vars_i64.size == 0: + return vars_i64, coeffs_f64 + sort_key = np.where(vars_i64 == -1, _INT64_MAX, vars_i64) + if vars_i64.shape[1] <= 1 or np.all(np.diff(sort_key, axis=1) >= 0): + return vars_i64, coeffs_f64 order = np.argsort(sort_key, axis=1, kind="stable") - rows = np.arange(vars_arr.shape[0])[:, None] - return ( - vars_arr[rows, order].astype(np.int64, copy=False), - coeffs_arr[rows, order].astype(np.float64, copy=False), - ) + rows = np.arange(vars_i64.shape[0])[:, None] + return vars_i64[rows, order], coeffs_f64[rows, order] def _extract_var_buffers(var: Variable) -> ContainerVarBuffers: labels_flat = var.labels.values.ravel() mask = labels_flat != -1 return ContainerVarBuffers( - lower=var.lower.values.ravel()[mask].astype(np.float64, copy=True), - upper=var.upper.values.ravel()[mask].astype(np.float64, copy=True), + lower=np.ascontiguousarray(var.lower.values.ravel()[mask], dtype=np.float64), + upper=np.ascontiguousarray(var.upper.values.ravel()[mask], dtype=np.float64), type=_variable_type(var), - active_labels=labels_flat[mask].astype(np.int64, copy=True), + active_labels=np.ascontiguousarray(labels_flat[mask], dtype=np.int64), ) @@ -135,7 +142,7 @@ def __eq__(self, other: object) -> bool: class ContainerVarBuffers: lower: np.ndarray upper: np.ndarray - type: str + type: VarKind active_labels: np.ndarray @@ -192,6 +199,9 @@ def capture(cls, model: Model) -> ModelSnapshot: name: _coord_snapshot(con) for name, con in model.constraints.items() } + for con in model.constraints.data.values(): + con._coef_dirty = False + return cls( structural_key=structural_key, var_buffers=var_buffers, diff --git a/linopy/solvers.py b/linopy/solvers.py index f3a92282..52d82991 100644 --- a/linopy/solvers.py +++ b/linopy/solvers.py @@ -37,6 +37,9 @@ import linopy.io from linopy.common import count_initial_letters, values_to_lookup_array from linopy.constants import ( + EQUAL, + GREATER_EQUAL, + LESS_EQUAL, SOS_DIM_ATTR, SOS_TYPE_ATTR, Result, @@ -53,6 +56,7 @@ RebuildRequiredError, UnsupportedUpdate, UpdatesDisabledError, + VarKind, ) @@ -114,11 +118,6 @@ def _solution_from_labels( return values_to_lookup_array(np.asarray(values, dtype=float), labels, size=size) -def _clear_coef_dirty(constraints: Any) -> None: - for c in constraints.data.values(): - c._coef_dirty = False - - class SolverFeature(Enum): """Enumeration of all solver capabilities tracked by linopy.""" @@ -462,7 +461,6 @@ def apply_update( @property def solver_options(self) -> dict[str, Any]: - """Back-compat alias for ``self.options``.""" return self.options @classmethod @@ -529,14 +527,7 @@ def from_name( With ``model`` supplied, the solver is built immediately. Without it, an unbuilt instance is returned and the first ``solve(model, ...)`` - call performs the build. - - ``track_updates=False`` (default) is the one-shot mode: no - :class:`ModelSnapshot` is captured at build time, and any subsequent - ``solver.solve(model=...)`` / ``solver.update(model)`` raises - :class:`UpdatesDisabledError`. Pass ``track_updates=True`` for - long-lived solvers that want in-place diff-based updates across - iterations. + call performs the build. See :class:`Solver` for ``track_updates``. """ cls = _solver_class_for(name) if cls is None: @@ -565,10 +556,7 @@ def from_model( track_updates: bool = False, **build_kwargs: Any, ) -> Solver: - """Instantiate and build the solver against ``model``. - - See :meth:`from_name` for ``track_updates`` semantics. - """ + """Instantiate and build the solver against ``model``.""" instance = cls( model=model, io_api=io_api, @@ -597,7 +585,6 @@ def _build(self, **build_kwargs: Any) -> None: self._build_direct(**build_kwargs) if self.track_updates: self.snapshot = ModelSnapshot.capture(self.model) - _clear_coef_dirty(self.model.constraints) else: self._build_file(**build_kwargs) @@ -680,11 +667,6 @@ def solve( With ``model`` supplied, diff against the previous build and either apply in place or rebuild before running. Requires ``io_api='direct'``. - Diffing uses :meth:`ModelDiff.from_snapshot` when ``track_updates=True`` - (in-place mutations of the build-time Model are detected) and - :meth:`ModelDiff.from_models` otherwise (only cross-instance resolves - are supported — passing the same Model instance after in-place - mutation raises :class:`UpdatesDisabledError`). With ``assign=True`` the Result is written back to the target Model via :meth:`Model.assign_result`. @@ -808,7 +790,6 @@ def _update_locked( self.model = model if self.track_updates: self.snapshot = ModelSnapshot.capture(model) - _clear_coef_dirty(model.constraints) self._in_place_updates += 1 self._last_rebuild_reason = RebuildReason.NONE return diff @@ -1415,15 +1396,15 @@ def apply_update( h = self.solver_model type_map = { - "continuous": highspy.HighsVarType.kContinuous, - "binary": highspy.HighsVarType.kInteger, - "integer": highspy.HighsVarType.kInteger, - "semi_continuous": highspy.HighsVarType.kSemiContinuous, + VarKind.CONTINUOUS: highspy.HighsVarType.kContinuous, + VarKind.BINARY: highspy.HighsVarType.kInteger, + VarKind.INTEGER: highspy.HighsVarType.kInteger, + VarKind.SEMI_CONTINUOUS: highspy.HighsVarType.kSemiContinuous, } for name, upd in diff.vars.items(): var = variables[name] - if upd.type_change == "binary": + if upd.type_change is VarKind.BINARY: labels = var.labels.values.ravel() mask = labels != -1 container_positions = var_label_index.label_to_pos[labels[mask]].astype( @@ -1459,8 +1440,8 @@ def apply_update( rhs_values = np.asarray(upd.rhs_values, dtype=np.float64) sign_for_rows = upd.rhs_signs inf = np.inf - lower = np.where(sign_for_rows == "<=", -inf, rhs_values) - upper = np.where(sign_for_rows == ">=", inf, rhs_values) + lower = np.where(sign_for_rows == LESS_EQUAL, -inf, rhs_values) + upper = np.where(sign_for_rows == GREATER_EQUAL, inf, rhs_values) for pos, lo, up in zip(positions, lower, upper): h.changeRowBounds(int(pos), float(lo), float(up)) @@ -1903,16 +1884,16 @@ def add_sos(s: xr.DataArray, sos_type: int, sos_dim: str) -> None: gm.update() return gm - _GUROBI_VTYPE_MAP: ClassVar[dict[str, str]] = { - "continuous": "C", - "binary": "B", - "integer": "I", - "semi_continuous": "S", + _GUROBI_VTYPE_MAP: ClassVar[dict[VarKind, str]] = { + VarKind.CONTINUOUS: "C", + VarKind.BINARY: "B", + VarKind.INTEGER: "I", + VarKind.SEMI_CONTINUOUS: "S", } _GUROBI_SIGN_MAP: ClassVar[dict[str, str]] = { - "<=": "<", - ">=": ">", - "=": "=", + LESS_EQUAL: "<", + GREATER_EQUAL: ">", + EQUAL: "=", } _GUROBI_SENSE_MAP: ClassVar[dict[str, int]] = {"min": 1, "max": -1} @@ -1945,9 +1926,7 @@ def apply_update( if upd.upper is not None: gm.setAttr("UB", var_subset, upd.upper.tolist()) if upd.type_change is not None: - vtype = self._GUROBI_VTYPE_MAP.get(upd.type_change) - if vtype is None: - raise UnsupportedUpdate(f"unknown var type {upd.type_change}") + vtype = self._GUROBI_VTYPE_MAP[upd.type_change] var = variables[name] labels = var.labels.values.ravel() mask = labels != -1 From 33fc991ce860353840baac241027fede279b1921 Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 21 May 2026 12:48:34 +0200 Subject: [PATCH 16/21] refactor(persistent): CSR-backed ContainerConBuffers Source con buffers from Constraint.to_matrix_with_rhs, replacing the dense (n_rows, max_n_term) arrays with CSR (indptr, indices, data). Sign dtype adopts 'U1' across the persistent layer and apply_update in HiGHS/Gurobi consumes CSR-slice payloads instead of -1 masks. Deletes _canonicalize_rows and the _INT64_MAX sentinel. --- linopy/persistent/diff.py | 82 ++++++++++++++++-------- linopy/persistent/errors.py | 6 +- linopy/persistent/snapshot.py | 68 ++++---------------- linopy/solvers.py | 45 +++++++------ test/test_persistent_snapshot_buffers.py | 76 +++++++++++----------- test/test_persistent_snapshot_diff.py | 11 ++-- 6 files changed, 139 insertions(+), 149 deletions(-) diff --git a/linopy/persistent/diff.py b/linopy/persistent/diff.py index 7bcd1951..1ea02dcf 100644 --- a/linopy/persistent/diff.py +++ b/linopy/persistent/diff.py @@ -1,13 +1,12 @@ from __future__ import annotations import enum +from collections.abc import Iterable from dataclasses import dataclass, field from typing import TYPE_CHECKING import numpy as np -from collections.abc import Iterable - from linopy.persistent.snapshot import ( ContainerConBuffers, ContainerVarBuffers, @@ -22,7 +21,7 @@ if TYPE_CHECKING: from linopy.constraints import ConstraintBase from linopy.model import Model - from linopy.variables import Variable + from linopy.variables import Variable, VariableLabelIndex class RebuildReason(enum.Enum): @@ -46,8 +45,9 @@ class ContainerVarUpdate: @dataclass class ContainerRowUpdate: coef_row_indices: np.ndarray | None = None - coef_vars: np.ndarray | None = None - coef_values: np.ndarray | None = None + coef_indptr: np.ndarray | None = None + coef_indices: np.ndarray | None = None + coef_data: np.ndarray | None = None rhs_row_indices: np.ndarray | None = None rhs_values: np.ndarray | None = None rhs_signs: np.ndarray | None = None @@ -93,7 +93,7 @@ def summary(self) -> dict[str, int | bool | str | None]: n_var_type = sum(1 for u in self.vars.values() if u.type_change is not None) n_con_rhs = sum(1 for u in self.cons.values() if u.rhs_values is not None) n_con_sign = sum(1 for u in self.cons.values() if u.sign_values is not None) - n_con_coef = sum(1 for u in self.cons.values() if u.coef_values is not None) + n_con_coef = sum(1 for u in self.cons.values() if u.coef_data is not None) return { "rebuild_reason": self.rebuild_reason.value, "var_lb": n_var_lb, @@ -129,8 +129,16 @@ def inspect_constraint(self, name: str) -> dict[str, object]: entry["rhs"] = u.rhs_values if u.sign_values is not None: entry["sign"] = u.sign_values - if u.coef_values is not None: - entry["coef_values"] = u.coef_values + if u.coef_data is not None: + indptr = u.coef_indptr + indices = u.coef_indices + data = u.coef_data + assert indptr is not None and indices is not None + entry["coef_rows"] = [ + (indices[indptr[i] : indptr[i + 1]], + data[indptr[i] : indptr[i + 1]]) + for i in range(len(indptr) - 1) + ] return entry def __repr__(self) -> str: @@ -154,7 +162,8 @@ def from_snapshot( same_model: bool = True, ignore_dims: Iterable[str] | None = None, ) -> ModelDiff: - """Diff ``model`` against a captured ``snapshot``. + """ + Diff ``model`` against a captured ``snapshot``. Coordinate values are not compared by default. Pass ``ignore_dims`` (e.g. ``ignore_dims=()`` or ``ignore_dims={"snapshot"}``) to opt into @@ -201,7 +210,7 @@ def from_snapshot( skip_coef_compare = same_model and not con._coef_dirty reason = _diff_con_container( diff, name, con, snapshot.con_buffers[name], - base_coords, var_l2p, con_l2p, ignored, check_coords, + base_coords, var_label_index, con_l2p, ignored, check_coords, skip_coef_compare, ) if reason is not None: @@ -223,7 +232,8 @@ def from_models( model_b: Model, ignore_dims: Iterable[str] | None = None, ) -> ModelDiff: - """Diff two linopy models directly, without capturing a snapshot. + """ + Diff two linopy models directly, without capturing a snapshot. ``model_a`` is the baseline, ``model_b`` is the target. The coefficient comparison runs unconditionally — no ``_coef_dirty`` @@ -270,11 +280,11 @@ def from_models( for name, con_b in model_b.constraints.items(): con_a = model_a.constraints[name] - base_buf = _extract_con_buffers(con_a, var_l2p) + base_buf = _extract_con_buffers(con_a, var_idx_a) base_coords = _coord_snapshot(con_a) if check_coords else None reason = _diff_con_container( diff, name, con_b, base_buf, - base_coords, var_l2p, con_l2p, ignored, check_coords, + base_coords, var_idx_b, con_l2p, ignored, check_coords, skip_coef_compare=False, ) if reason is not None: @@ -345,19 +355,23 @@ def _diff_con_container( con: ConstraintBase, base_buf: ContainerConBuffers, base_coords: dict[str, np.ndarray] | None, - var_l2p: np.ndarray, + var_label_index: VariableLabelIndex, con_l2p: np.ndarray, ignored: frozenset[str], check_coords: bool, skip_coef_compare: bool, ) -> RebuildReason | None: - new_buf = _extract_con_buffers(con, var_l2p) - if new_buf.coeffs.shape != base_buf.coeffs.shape: - return RebuildReason.SPARSITY + new_buf = _extract_con_buffers(con, var_label_index) + if new_buf.indptr.shape != base_buf.indptr.shape: + return RebuildReason.COORD_REINDEX if not np.array_equal(new_buf.active_labels, base_buf.active_labels): return RebuildReason.STRUCTURAL_LABELS if check_coords and not _coords_equal(base_coords, _coord_snapshot(con), ignored): return RebuildReason.COORD_REINDEX + if not np.array_equal(new_buf.indptr, base_buf.indptr): + return RebuildReason.SPARSITY + if not np.array_equal(new_buf.indices, base_buf.indices): + return RebuildReason.SPARSITY n_rows = new_buf.active_labels.size if n_rows == 0: @@ -365,13 +379,15 @@ def _diff_con_container( if skip_coef_compare: row_value_changed = np.zeros(n_rows, dtype=bool) - row_struct_changed = np.zeros(n_rows, dtype=bool) else: - row_struct_changed = np.any(new_buf.vars != base_buf.vars, axis=-1) - row_value_changed = np.any(new_buf.coeffs != base_buf.coeffs, axis=-1) - - if row_struct_changed.any(): - return RebuildReason.SPARSITY + data_diff = new_buf.data != base_buf.data + if data_diff.any(): + nnz_per_row = np.diff(new_buf.indptr) + row_idx_per_nnz = np.repeat(np.arange(n_rows), nnz_per_row) + row_value_changed = np.zeros(n_rows, dtype=bool) + row_value_changed[row_idx_per_nnz[data_diff]] = True + else: + row_value_changed = np.zeros(n_rows, dtype=bool) rhs_changed = new_buf.rhs != base_buf.rhs sign_changed = new_buf.sign != base_buf.sign @@ -385,9 +401,23 @@ def _diff_con_container( update.coef_row_indices = con_l2p[ new_buf.active_labels[idx] ].astype(np.int32, copy=False) - update.coef_vars = new_buf.vars[idx] - update.coef_values = new_buf.coeffs[idx] - diff.n_coef_updates += int((update.coef_vars != -1).sum()) + new_indptr = new_buf.indptr + nnz_per_changed = (new_indptr[idx + 1] - new_indptr[idx]).astype(np.int32) + payload_indptr = np.empty(len(idx) + 1, dtype=np.int32) + payload_indptr[0] = 0 + np.cumsum(nnz_per_changed, out=payload_indptr[1:]) + total_nnz = int(payload_indptr[-1]) + payload_indices = np.empty(total_nnz, dtype=new_buf.indices.dtype) + payload_data = np.empty(total_nnz, dtype=np.float64) + for j, i in enumerate(idx): + s, e = int(new_indptr[i]), int(new_indptr[i + 1]) + ps, pe = int(payload_indptr[j]), int(payload_indptr[j + 1]) + payload_indices[ps:pe] = new_buf.indices[s:e] + payload_data[ps:pe] = new_buf.data[s:e] + update.coef_indptr = payload_indptr + update.coef_indices = payload_indices + update.coef_data = payload_data + diff.n_coef_updates += total_nnz if rhs_changed.any(): idx = np.flatnonzero(rhs_changed) update.rhs_row_indices = con_l2p[ diff --git a/linopy/persistent/errors.py b/linopy/persistent/errors.py index 2a626346..c6159207 100644 --- a/linopy/persistent/errors.py +++ b/linopy/persistent/errors.py @@ -6,7 +6,8 @@ class UnsupportedUpdate(Exception): class RebuildRequiredError(RuntimeError): - """Raised when an in-place update is required but a rebuild is needed. + """ + Raised when an in-place update is required but a rebuild is needed. Carries the :class:`RebuildReason` that forced the rebuild attempt. """ @@ -17,7 +18,8 @@ def __init__(self, reason: object, message: str | None = None) -> None: class UpdatesDisabledError(RuntimeError): - """Raised when an in-place update is requested on a solver built with + """ + Raised when an in-place update is requested on a solver built with ``track_updates=False``. Reconstruct the solver with ``track_updates=True`` to enable diff-based updates. """ diff --git a/linopy/persistent/snapshot.py b/linopy/persistent/snapshot.py index 80569c6e..8820bbab 100644 --- a/linopy/persistent/snapshot.py +++ b/linopy/persistent/snapshot.py @@ -11,10 +11,7 @@ if TYPE_CHECKING: from linopy.constraints import ConstraintBase from linopy.model import Model - from linopy.variables import Variable - - -_INT64_MAX = np.iinfo(np.int64).max + from linopy.variables import Variable, VariableLabelIndex class VarKind(enum.Enum): @@ -55,22 +52,6 @@ def _objective_linear_vector(model: Model) -> np.ndarray: return result -def _canonicalize_rows( - vars_arr: np.ndarray, coeffs_arr: np.ndarray -) -> tuple[np.ndarray, np.ndarray]: - """Sort each row jointly by var index. -1 sentinels sort to the right.""" - vars_i64 = np.ascontiguousarray(vars_arr, dtype=np.int64) - coeffs_f64 = np.ascontiguousarray(coeffs_arr, dtype=np.float64) - if vars_i64.size == 0: - return vars_i64, coeffs_f64 - sort_key = np.where(vars_i64 == -1, _INT64_MAX, vars_i64) - if vars_i64.shape[1] <= 1 or np.all(np.diff(sort_key, axis=1) >= 0): - return vars_i64, coeffs_f64 - order = np.argsort(sort_key, axis=1, kind="stable") - rows = np.arange(vars_i64.shape[0])[:, None] - return vars_i64[rows, order], coeffs_f64[rows, order] - - def _extract_var_buffers(var: Variable) -> ContainerVarBuffers: labels_flat = var.labels.values.ravel() mask = labels_flat != -1 @@ -83,39 +64,16 @@ def _extract_var_buffers(var: Variable) -> ContainerVarBuffers: def _extract_con_buffers( - con: ConstraintBase, var_l2p: np.ndarray + con: ConstraintBase, var_label_index: VariableLabelIndex ) -> ContainerConBuffers: - labels_flat = con.labels.values.ravel() - vars_vals = con.vars.values - coeffs_vals = con.coeffs.values - n_rows = len(labels_flat) - if n_rows > 0: - vars_2d = vars_vals.reshape(n_rows, -1) - coeffs_2d = coeffs_vals.reshape(vars_2d.shape) - else: - n_term = max(1, vars_vals.size) - vars_2d = vars_vals.reshape(0, n_term) - coeffs_2d = coeffs_vals.reshape(0, n_term) - - row_mask = (labels_flat != -1) & (vars_2d != -1).any(axis=1) - active_labels = labels_flat[row_mask].astype(np.int64, copy=True) - - vars_active = vars_2d[row_mask] - coeffs_active = coeffs_2d[row_mask].astype(np.float64, copy=True) - - valid = vars_active != -1 - col_indices = np.full(vars_active.shape, -1, dtype=np.int64) - col_indices[valid] = var_l2p[vars_active[valid]] - coeffs_clean = np.where(valid, coeffs_active, 0.0) - - vars_sorted, coeffs_sorted = _canonicalize_rows(col_indices, coeffs_clean) - + csr, con_labels, b, sense = con.to_matrix_with_rhs(var_label_index) return ContainerConBuffers( - coeffs=coeffs_sorted, - vars=vars_sorted, - rhs=con.rhs.values.ravel()[row_mask].astype(np.float64, copy=True), - sign=con.sign.values.ravel()[row_mask].astype("U2", copy=True), - active_labels=active_labels, + indptr=csr.indptr.astype(np.int32, copy=True), + indices=csr.indices.astype(np.int32, copy=True), + data=csr.data.astype(np.float64, copy=True), + rhs=np.asarray(b, dtype=np.float64).copy(), + sign=np.asarray(sense, dtype="U1").copy(), + active_labels=np.asarray(con_labels, dtype=np.int64).copy(), ) @@ -148,8 +106,9 @@ class ContainerVarBuffers: @dataclass(frozen=True) class ContainerConBuffers: - coeffs: np.ndarray - vars: np.ndarray + indptr: np.ndarray + indices: np.ndarray + data: np.ndarray rhs: np.ndarray sign: np.ndarray active_labels: np.ndarray @@ -176,7 +135,6 @@ class ModelSnapshot: def capture(cls, model: Model) -> ModelSnapshot: var_label_index = model.variables.label_index con_label_index = model.constraints.label_index - var_l2p = var_label_index.label_to_pos structural_key = StructuralKey( var_container_names=tuple(model.variables), @@ -189,7 +147,7 @@ def capture(cls, model: Model) -> ModelSnapshot: name: _extract_var_buffers(var) for name, var in model.variables.items() } con_buffers = { - name: _extract_con_buffers(con, var_l2p) + name: _extract_con_buffers(con, var_label_index) for name, con in model.constraints.items() } var_coords = { diff --git a/linopy/solvers.py b/linopy/solvers.py index 52d82991..59236572 100644 --- a/linopy/solvers.py +++ b/linopy/solvers.py @@ -38,8 +38,6 @@ from linopy.common import count_initial_letters, values_to_lookup_array from linopy.constants import ( EQUAL, - GREATER_EQUAL, - LESS_EQUAL, SOS_DIM_ATTR, SOS_TYPE_ATTR, Result, @@ -48,6 +46,8 @@ SolverStatus, Status, TerminationCondition, + short_GREATER_EQUAL, + short_LESS_EQUAL, ) from linopy.persistent import ( ModelDiff, @@ -523,7 +523,8 @@ def from_name( track_updates: bool = False, **build_kwargs: Any, ) -> Solver: - """Construct the solver subclass registered as ``name``. + """ + Construct the solver subclass registered as ``name``. With ``model`` supplied, the solver is built immediately. Without it, an unbuilt instance is returned and the first ``solve(model, ...)`` @@ -1440,19 +1441,19 @@ def apply_update( rhs_values = np.asarray(upd.rhs_values, dtype=np.float64) sign_for_rows = upd.rhs_signs inf = np.inf - lower = np.where(sign_for_rows == LESS_EQUAL, -inf, rhs_values) - upper = np.where(sign_for_rows == GREATER_EQUAL, inf, rhs_values) + lower = np.where(sign_for_rows == short_LESS_EQUAL, -inf, rhs_values) + upper = np.where(sign_for_rows == short_GREATER_EQUAL, inf, rhs_values) for pos, lo, up in zip(positions, lower, upper): h.changeRowBounds(int(pos), float(lo), float(up)) - if upd.coef_values is not None: + if upd.coef_data is not None: rows = upd.coef_row_indices - for r, var_row, val_row in zip( - rows, upd.coef_vars, upd.coef_values - ): - valid = var_row != -1 - for c, v in zip(var_row[valid], val_row[valid]): - h.changeCoeff(int(r), int(c), float(v)) + indptr = upd.coef_indptr + indices = upd.coef_indices + data = upd.coef_data + for i, r in enumerate(rows): + for j in range(int(indptr[i]), int(indptr[i + 1])): + h.changeCoeff(int(r), int(indices[j]), float(data[j])) if diff.obj_c_indices is not None: indices = diff.obj_c_indices @@ -1891,8 +1892,8 @@ def add_sos(s: xr.DataArray, sos_type: int, sos_dim: str) -> None: VarKind.SEMI_CONTINUOUS: "S", } _GUROBI_SIGN_MAP: ClassVar[dict[str, str]] = { - LESS_EQUAL: "<", - GREATER_EQUAL: ">", + short_LESS_EQUAL: "<", + short_GREATER_EQUAL: ">", EQUAL: "=", } _GUROBI_SENSE_MAP: ClassVar[dict[str, int]] = {"min": 1, "max": -1} @@ -1951,15 +1952,17 @@ def apply_update( raise UnsupportedUpdate(f"unknown sign {s_str!r}") senses.append(self._GUROBI_SIGN_MAP[s_str]) gm.setAttr("Sense", con_subset, senses) - if upd.coef_values is not None: + if upd.coef_data is not None: rows = upd.coef_row_indices - for r, var_row, val_row in zip( - rows, upd.coef_vars, upd.coef_values - ): - valid = var_row != -1 - for c, v in zip(var_row[valid], val_row[valid]): + indptr = upd.coef_indptr + indices = upd.coef_indices + data = upd.coef_data + for i, r in enumerate(rows): + for j in range(int(indptr[i]), int(indptr[i + 1])): gm.chgCoeff( - gurobi_cons[int(r)], gurobi_vars[int(c)], float(v) + gurobi_cons[int(r)], + gurobi_vars[int(indices[j])], + float(data[j]), ) if diff.obj_c_indices is not None: diff --git a/test/test_persistent_snapshot_buffers.py b/test/test_persistent_snapshot_buffers.py index d10f8a4a..9e608b28 100644 --- a/test/test_persistent_snapshot_buffers.py +++ b/test/test_persistent_snapshot_buffers.py @@ -5,34 +5,7 @@ from linopy import Model from linopy.persistent import ModelDiff, ModelSnapshot, RebuildReason -from linopy.persistent.snapshot import ( - _canonicalize_rows, - _extract_con_buffers, -) - - -def test_canonicalize_rows_sorts_by_var_label() -> None: - vars_in = np.array([[5, 2, 9], [1, 3, 0]], dtype=np.int64) - coeffs_in = np.array([[0.5, 0.2, 0.9], [0.1, 0.3, 0.0]], dtype=np.float64) - vars_out, coeffs_out = _canonicalize_rows(vars_in, coeffs_in) - np.testing.assert_array_equal(vars_out, [[2, 5, 9], [0, 1, 3]]) - np.testing.assert_array_equal(coeffs_out, [[0.2, 0.5, 0.9], [0.0, 0.1, 0.3]]) - - -def test_canonicalize_rows_minus_one_to_right() -> None: - vars_in = np.array([[5, -1, 2], [-1, 0, -1]], dtype=np.int64) - coeffs_in = np.array([[0.5, 0.0, 0.2], [0.0, 0.1, 0.0]], dtype=np.float64) - vars_out, coeffs_out = _canonicalize_rows(vars_in, coeffs_in) - np.testing.assert_array_equal(vars_out[:, 0], [2, 0]) - assert (vars_out[:, -1] == -1).all() - - -def test_canonicalize_empty_buffers_round_trip() -> None: - vars_in = np.empty((0, 3), dtype=np.int64) - coeffs_in = np.empty((0, 3), dtype=np.float64) - vars_out, coeffs_out = _canonicalize_rows(vars_in, coeffs_in) - assert vars_out.shape == (0, 3) - assert coeffs_out.shape == (0, 3) +from linopy.persistent.snapshot import _extract_con_buffers def _build_permuted_pair() -> tuple[Model, Model]: @@ -54,10 +27,11 @@ def test_permuted_term_order_produces_equal_buffers() -> None: m1, m2 = _build_permuted_pair() s1 = ModelSnapshot.capture(m1) s2 = ModelSnapshot.capture(m2) - np.testing.assert_array_equal(s1.con_buffers["c1"].vars, s2.con_buffers["c1"].vars) - np.testing.assert_array_equal( - s1.con_buffers["c1"].coeffs, s2.con_buffers["c1"].coeffs - ) + b1 = s1.con_buffers["c1"] + b2 = s2.con_buffers["c1"] + np.testing.assert_array_equal(b1.indptr, b2.indptr) + np.testing.assert_array_equal(b1.indices, b2.indices) + np.testing.assert_array_equal(b1.data, b2.data) def test_active_labels_match_label_index(baseline_model: Model) -> None: @@ -82,7 +56,6 @@ def baseline_model() -> Model: def test_shape_mismatch_triggers_sparsity_rebuild(baseline_model: Model) -> None: snap = ModelSnapshot.capture(baseline_model) - # Mutate to widen the term dim of c1 via lhs replacement x = baseline_model.variables["x"] y = baseline_model.variables["y"] baseline_model.constraints["c1"].lhs = 2 * x + 0 * y.sum() @@ -103,13 +76,14 @@ def test_zero_row_container_capture() -> None: assert diff.is_empty -def test_con_buffers_rhs_and_sign_dtypes(baseline_model: Model) -> None: +def test_con_buffers_dtypes(baseline_model: Model) -> None: snap = ModelSnapshot.capture(baseline_model) buf = snap.con_buffers["c1"] assert buf.rhs.dtype == np.float64 - assert buf.sign.dtype.kind == "U" - assert buf.coeffs.dtype == np.float64 - assert buf.vars.dtype == np.int64 + assert buf.sign.dtype == np.dtype("U1") + assert buf.data.dtype == np.float64 + assert buf.indices.dtype == np.int32 + assert buf.indptr.dtype == np.int32 def test_masked_rows_excluded_from_active_labels() -> None: @@ -121,6 +95,30 @@ def test_masked_rows_excluded_from_active_labels() -> None: snap = ModelSnapshot.capture(m) buf = snap.con_buffers["c1"] assert buf.active_labels.size == 3 - var_l2p = m.variables.label_index.label_to_pos - rebuilt = _extract_con_buffers(m.constraints["c1"], var_l2p) + rebuilt = _extract_con_buffers(m.constraints["c1"], m.variables.label_index) np.testing.assert_array_equal(rebuilt.active_labels, buf.active_labels) + + +def test_csr_capture_deterministic(baseline_model: Model) -> None: + s1 = ModelSnapshot.capture(baseline_model) + s2 = ModelSnapshot.capture(baseline_model) + for name in s1.con_buffers: + b1, b2 = s1.con_buffers[name], s2.con_buffers[name] + np.testing.assert_array_equal(b1.indptr, b2.indptr) + np.testing.assert_array_equal(b1.indices, b2.indices) + np.testing.assert_array_equal(b1.data, b2.data) + + +def test_duplicate_variable_terms_summed() -> None: + m1 = Model() + x1 = m1.add_variables(0, 10, coords=[range(3)], name="x") + m1.add_constraints(2 * x1 + 3 * x1 >= 1, name="c1") + m1.add_objective(x1.sum()) + + m2 = Model() + x2 = m2.add_variables(0, 10, coords=[range(3)], name="x") + m2.add_constraints(5 * x2 >= 1, name="c1") + m2.add_objective(x2.sum()) + + diff = ModelDiff.from_models(m1, m2) + assert diff.is_empty diff --git a/test/test_persistent_snapshot_diff.py b/test/test_persistent_snapshot_diff.py index e164d6b7..9f36685f 100644 --- a/test/test_persistent_snapshot_diff.py +++ b/test/test_persistent_snapshot_diff.py @@ -70,7 +70,7 @@ def test_rhs_only_mutation(baseline: Model) -> None: assert "c1" in diff.cons upd = diff.cons["c1"] assert upd.rhs_values is not None - assert upd.coef_values is None + assert upd.coef_data is None def test_objective_linear_change(baseline: Model) -> None: @@ -121,9 +121,8 @@ def test_coef_value_change_same_sparsity(baseline: Model) -> None: assert diff.rebuild_reason is RebuildReason.NONE assert "c1" in diff.cons upd = diff.cons["c1"] - assert upd.coef_values is not None - valid = upd.coef_vars != -1 - np.testing.assert_array_equal(upd.coef_values[valid], np.full(valid.sum(), 6.0)) + assert upd.coef_data is not None + np.testing.assert_array_equal(upd.coef_data, np.full(upd.coef_data.size, 6.0)) def test_coef_sparsity_change(baseline: Model) -> None: @@ -147,10 +146,10 @@ def test_same_model_false_ignores_dirty_flag(baseline: Model) -> None: c.coeffs = c.coeffs * 5 c._coef_dirty = False diff_fast = ModelDiff.from_snapshot(snap, baseline, same_model=True) - assert "c1" not in diff_fast.cons or diff_fast.cons["c1"].coef_values is None + assert "c1" not in diff_fast.cons or diff_fast.cons["c1"].coef_data is None diff_full = ModelDiff.from_snapshot(snap, baseline, same_model=False) assert "c1" in diff_full.cons - assert diff_full.cons["c1"].coef_values is not None + assert diff_full.cons["c1"].coef_data is not None def test_modeldiff_default_is_empty() -> None: From 3bffb6e8e76959f4e5947c8169960e42056389bd Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 21 May 2026 15:26:36 +0200 Subject: [PATCH 17/21] refactor(persistent): flat-native ModelDiff storage Replace per-container ContainerVarUpdate/ContainerRowUpdate dicts with flat arrays (var_bounds_*, var_type_*, con_coef_* COO, con_rhs_*, con_sign_*) plus VarSlice/ConSlice per-container offsets for diagnostics. Add con_rhs_as_bounds() for ranged-row solvers. Backend apply_update bodies collapse to flat-array calls; remove duplicated label->position resolution. --- linopy/persistent/__init__.py | 8 +- linopy/persistent/diff.py | 436 +++++++++++++++++++------- linopy/solvers.py | 203 ++++++------ test/test_persistent_snapshot_diff.py | 40 +-- test/test_persistent_solver_extras.py | 3 +- 5 files changed, 444 insertions(+), 246 deletions(-) diff --git a/linopy/persistent/__init__.py b/linopy/persistent/__init__.py index 81e0f816..9dbdf0cb 100644 --- a/linopy/persistent/__init__.py +++ b/linopy/persistent/__init__.py @@ -3,10 +3,10 @@ from __future__ import annotations from linopy.persistent.diff import ( - ContainerRowUpdate, - ContainerVarUpdate, + ConSlice, ModelDiff, RebuildReason, + VarSlice, ) from linopy.persistent.errors import ( RebuildRequiredError, @@ -22,10 +22,9 @@ ) __all__ = [ + "ConSlice", "ContainerConBuffers", - "ContainerRowUpdate", "ContainerVarBuffers", - "ContainerVarUpdate", "ModelDiff", "ModelSnapshot", "RebuildReason", @@ -34,4 +33,5 @@ "UnsupportedUpdate", "UpdatesDisabledError", "VarKind", + "VarSlice", ] diff --git a/linopy/persistent/diff.py b/linopy/persistent/diff.py index 1ea02dcf..0731abc7 100644 --- a/linopy/persistent/diff.py +++ b/linopy/persistent/diff.py @@ -7,6 +7,7 @@ import numpy as np +from linopy.constants import short_GREATER_EQUAL, short_LESS_EQUAL from linopy.persistent.snapshot import ( ContainerConBuffers, ContainerVarBuffers, @@ -24,6 +25,12 @@ from linopy.variables import Variable, VariableLabelIndex +_EMPTY_I32 = np.empty(0, dtype=np.int32) +_EMPTY_F64 = np.empty(0, dtype=np.float64) +_EMPTY_U1 = np.empty(0, dtype="U1") +_EMPTY_KIND: np.ndarray = np.empty(0, dtype=object) + + class RebuildReason(enum.Enum): NONE = "none" STRUCTURAL_LABELS = "vlabels/clabels mismatch" @@ -34,43 +41,213 @@ class RebuildReason(enum.Enum): BACKEND_REJECTED = "backend raised UnsupportedUpdate" -@dataclass -class ContainerVarUpdate: - bounds_indices: np.ndarray | None = None - lower: np.ndarray | None = None - upper: np.ndarray | None = None - type_change: VarKind | None = None +@dataclass(frozen=True) +class VarSlice: + bounds: slice + type: slice + + +@dataclass(frozen=True) +class ConSlice: + coef: slice + rhs: slice + sign: slice + + +def _empty_slice() -> slice: + return slice(0, 0) @dataclass -class ContainerRowUpdate: - coef_row_indices: np.ndarray | None = None - coef_indptr: np.ndarray | None = None - coef_indices: np.ndarray | None = None - coef_data: np.ndarray | None = None - rhs_row_indices: np.ndarray | None = None - rhs_values: np.ndarray | None = None - rhs_signs: np.ndarray | None = None - sign_row_indices: np.ndarray | None = None - sign_values: np.ndarray | None = None +class _DiffBuilder: + var_bounds_idx: list[np.ndarray] = field(default_factory=list) + var_bounds_lo: list[np.ndarray] = field(default_factory=list) + var_bounds_up: list[np.ndarray] = field(default_factory=list) + var_type_pos: list[np.ndarray] = field(default_factory=list) + var_type_kinds: list[np.ndarray] = field(default_factory=list) + + con_coef_rows: list[np.ndarray] = field(default_factory=list) + con_coef_cols: list[np.ndarray] = field(default_factory=list) + con_coef_vals: list[np.ndarray] = field(default_factory=list) + + con_rhs_idx: list[np.ndarray] = field(default_factory=list) + con_rhs_vals: list[np.ndarray] = field(default_factory=list) + con_rhs_signs: list[np.ndarray] = field(default_factory=list) + + con_sign_idx: list[np.ndarray] = field(default_factory=list) + con_sign_vals: list[np.ndarray] = field(default_factory=list) + + var_slices: dict[str, VarSlice] = field(default_factory=dict) + con_slices: dict[str, ConSlice] = field(default_factory=dict) + + obj_c_indices: np.ndarray | None = None + obj_c_values: np.ndarray | None = None + obj_sense: str | None = None + + _vb_cur: int = 0 + _vt_cur: int = 0 + _cc_cur: int = 0 + _cr_cur: int = 0 + _cs_cur: int = 0 + + def push_var( + self, + name: str, + bounds_idx: np.ndarray | None, + lower: np.ndarray | None, + upper: np.ndarray | None, + type_positions: np.ndarray | None, + type_kind: VarKind | None, + ) -> None: + b_start = self._vb_cur + if bounds_idx is not None: + self.var_bounds_idx.append(bounds_idx) + self.var_bounds_lo.append(lower) + self.var_bounds_up.append(upper) + self._vb_cur += bounds_idx.size + t_start = self._vt_cur + if type_positions is not None: + self.var_type_pos.append(type_positions) + self.var_type_kinds.append( + np.full(type_positions.size, type_kind, dtype=object) + ) + self._vt_cur += type_positions.size + self.var_slices[name] = VarSlice( + bounds=slice(b_start, self._vb_cur), + type=slice(t_start, self._vt_cur), + ) + + def push_con( + self, + name: str, + coef_rows: np.ndarray | None, + coef_cols: np.ndarray | None, + coef_vals: np.ndarray | None, + rhs_idx: np.ndarray | None, + rhs_vals: np.ndarray | None, + rhs_signs: np.ndarray | None, + sign_idx: np.ndarray | None, + sign_vals: np.ndarray | None, + ) -> None: + c_start = self._cc_cur + if coef_rows is not None: + self.con_coef_rows.append(coef_rows) + self.con_coef_cols.append(coef_cols) + self.con_coef_vals.append(coef_vals) + self._cc_cur += coef_rows.size + r_start = self._cr_cur + if rhs_idx is not None: + self.con_rhs_idx.append(rhs_idx) + self.con_rhs_vals.append(rhs_vals) + self.con_rhs_signs.append(rhs_signs) + self._cr_cur += rhs_idx.size + s_start = self._cs_cur + if sign_idx is not None: + self.con_sign_idx.append(sign_idx) + self.con_sign_vals.append(sign_vals) + self._cs_cur += sign_idx.size + self.con_slices[name] = ConSlice( + coef=slice(c_start, self._cc_cur), + rhs=slice(r_start, self._cr_cur), + sign=slice(s_start, self._cs_cur), + ) + + def set_objective( + self, + c_indices: np.ndarray | None, + c_values: np.ndarray | None, + sense: str | None, + ) -> None: + self.obj_c_indices = c_indices + self.obj_c_values = c_values + self.obj_sense = sense + + def finalize(self, diff: ModelDiff) -> None: + diff.obj_c_indices = self.obj_c_indices + diff.obj_c_values = self.obj_c_values + diff.obj_sense = self.obj_sense + diff.var_bounds_indices = _cat(self.var_bounds_idx, np.int32) + diff.var_bounds_lower = _cat(self.var_bounds_lo, np.float64) + diff.var_bounds_upper = _cat(self.var_bounds_up, np.float64) + diff.var_type_positions = _cat(self.var_type_pos, np.int32) + diff.var_type_kinds = _cat_obj(self.var_type_kinds) + diff.con_coef_rows = _cat(self.con_coef_rows, np.int32) + diff.con_coef_cols = _cat(self.con_coef_cols, np.int32) + diff.con_coef_vals = _cat(self.con_coef_vals, np.float64) + diff.con_rhs_indices = _cat(self.con_rhs_idx, np.int32) + diff.con_rhs_values = _cat(self.con_rhs_vals, np.float64) + diff.con_rhs_signs = _cat_str(self.con_rhs_signs) + diff.con_sign_indices = _cat(self.con_sign_idx, np.int32) + diff.con_sign_values = _cat_str(self.con_sign_vals) + diff.var_slices = { + n: s + for n, s in self.var_slices.items() + if s.bounds.stop > s.bounds.start or s.type.stop > s.type.start + } + diff.con_slices = { + n: s + for n, s in self.con_slices.items() + if s.coef.stop > s.coef.start + or s.rhs.stop > s.rhs.start + or s.sign.stop > s.sign.start + } + + +def _cat(parts: list[np.ndarray], dtype: type) -> np.ndarray: + if not parts: + return np.empty(0, dtype=dtype) + return np.concatenate(parts).astype(dtype, copy=False) + + +def _cat_obj(parts: list[np.ndarray]) -> np.ndarray: + if not parts: + return _EMPTY_KIND + return np.concatenate(parts) + + +def _cat_str(parts: list[np.ndarray]) -> np.ndarray: + if not parts: + return _EMPTY_U1 + return np.concatenate(parts) @dataclass class ModelDiff: rebuild_reason: RebuildReason = RebuildReason.NONE - vars: dict[str, ContainerVarUpdate] = field(default_factory=dict) - cons: dict[str, ContainerRowUpdate] = field(default_factory=dict) + + var_bounds_indices: np.ndarray = field(default_factory=lambda: _EMPTY_I32) + var_bounds_lower: np.ndarray = field(default_factory=lambda: _EMPTY_F64) + var_bounds_upper: np.ndarray = field(default_factory=lambda: _EMPTY_F64) + var_type_positions: np.ndarray = field(default_factory=lambda: _EMPTY_I32) + var_type_kinds: np.ndarray = field(default_factory=lambda: _EMPTY_KIND) + + con_coef_rows: np.ndarray = field(default_factory=lambda: _EMPTY_I32) + con_coef_cols: np.ndarray = field(default_factory=lambda: _EMPTY_I32) + con_coef_vals: np.ndarray = field(default_factory=lambda: _EMPTY_F64) + + con_rhs_indices: np.ndarray = field(default_factory=lambda: _EMPTY_I32) + con_rhs_values: np.ndarray = field(default_factory=lambda: _EMPTY_F64) + con_rhs_signs: np.ndarray = field(default_factory=lambda: _EMPTY_U1) + + con_sign_indices: np.ndarray = field(default_factory=lambda: _EMPTY_I32) + con_sign_values: np.ndarray = field(default_factory=lambda: _EMPTY_U1) + obj_c_indices: np.ndarray | None = None obj_c_values: np.ndarray | None = None obj_sense: str | None = None - n_coef_updates: int = 0 + + var_slices: dict[str, VarSlice] = field(default_factory=dict) + con_slices: dict[str, ConSlice] = field(default_factory=dict) @property def is_empty(self) -> bool: return ( self.rebuild_reason is RebuildReason.NONE - and not self.vars - and not self.cons + and self.var_bounds_indices.size == 0 + and self.var_type_positions.size == 0 + and self.con_coef_rows.size == 0 + and self.con_rhs_indices.size == 0 + and self.con_sign_indices.size == 0 and self.obj_c_indices is None and self.obj_sense is None ) @@ -81,64 +258,66 @@ def rebuild_required(self) -> bool: @property def changed_variables(self) -> set[str]: - return set(self.vars) + return set(self.var_slices) @property def changed_constraints(self) -> set[str]: - return set(self.cons) + return set(self.con_slices) + + @property + def n_coef_updates(self) -> int: + return int(self.con_coef_vals.size) + + def con_rhs_as_bounds(self) -> tuple[np.ndarray, np.ndarray]: + """Return (lower, upper) row-bounds form of the RHS updates.""" + vals = self.con_rhs_values + signs = self.con_rhs_signs + lower = np.where(signs == short_LESS_EQUAL, -np.inf, vals) + upper = np.where(signs == short_GREATER_EQUAL, np.inf, vals) + return lower, upper def summary(self) -> dict[str, int | bool | str | None]: - n_var_lb = sum(1 for u in self.vars.values() if u.lower is not None) - n_var_ub = sum(1 for u in self.vars.values() if u.upper is not None) - n_var_type = sum(1 for u in self.vars.values() if u.type_change is not None) - n_con_rhs = sum(1 for u in self.cons.values() if u.rhs_values is not None) - n_con_sign = sum(1 for u in self.cons.values() if u.sign_values is not None) - n_con_coef = sum(1 for u in self.cons.values() if u.coef_data is not None) return { "rebuild_reason": self.rebuild_reason.value, - "var_lb": n_var_lb, - "var_ub": n_var_ub, - "var_type": n_var_type, - "con_rhs": n_con_rhs, - "con_sign": n_con_sign, - "con_coef_updates": n_con_coef, - "n_coef_values": self.n_coef_updates, + "var_bounds": int(self.var_bounds_indices.size), + "var_type": int(self.var_type_positions.size), + "con_rhs": int(self.con_rhs_indices.size), + "con_sign": int(self.con_sign_indices.size), + "con_coef_updates": int(self.con_coef_vals.size), "obj_linear_changed": self.obj_c_indices is not None, "obj_sense_changed_to": self.obj_sense, } def inspect_variable(self, name: str) -> dict[str, object]: - if name not in self.vars: + sl = self.var_slices.get(name) + if sl is None: return {} - u = self.vars[name] entry: dict[str, object] = {} - if u.lower is not None: - entry["lower"] = u.lower - if u.upper is not None: - entry["upper"] = u.upper - if u.type_change is not None: - entry["type"] = u.type_change + if sl.bounds.stop > sl.bounds.start: + entry["bounds_indices"] = self.var_bounds_indices[sl.bounds] + entry["lower"] = self.var_bounds_lower[sl.bounds] + entry["upper"] = self.var_bounds_upper[sl.bounds] + if sl.type.stop > sl.type.start: + entry["type_positions"] = self.var_type_positions[sl.type] + entry["type_kinds"] = self.var_type_kinds[sl.type] return entry def inspect_constraint(self, name: str) -> dict[str, object]: - if name not in self.cons: + sl = self.con_slices.get(name) + if sl is None: return {} - u = self.cons[name] entry: dict[str, object] = {} - if u.rhs_values is not None: - entry["rhs"] = u.rhs_values - if u.sign_values is not None: - entry["sign"] = u.sign_values - if u.coef_data is not None: - indptr = u.coef_indptr - indices = u.coef_indices - data = u.coef_data - assert indptr is not None and indices is not None - entry["coef_rows"] = [ - (indices[indptr[i] : indptr[i + 1]], - data[indptr[i] : indptr[i + 1]]) - for i in range(len(indptr) - 1) - ] + if sl.coef.stop > sl.coef.start: + entry["coef_rows"] = self.con_coef_rows[sl.coef] + entry["coef_cols"] = self.con_coef_cols[sl.coef] + entry["coef_vals"] = self.con_coef_vals[sl.coef] + if sl.rhs.stop > sl.rhs.start: + entry["rhs_indices"] = self.con_rhs_indices[sl.rhs] + entry["rhs_values"] = self.con_rhs_values[sl.rhs] + entry["rhs_signs"] = self.con_rhs_signs[sl.rhs] + if sl.sign.stop > sl.sign.start: + entry["sign_indices"] = self.con_sign_indices[sl.sign] + entry["sign_values"] = self.con_sign_values[sl.sign] return entry def __repr__(self) -> str: @@ -194,11 +373,12 @@ def from_snapshot( var_l2p = var_label_index.label_to_pos con_l2p = con_label_index.label_to_pos + builder = _DiffBuilder() for name, var in model.variables.items(): base_coords = snapshot.var_coords[name] if check_coords else None reason = _diff_var_container( - diff, name, var, snapshot.var_buffers[name], + builder, name, var, snapshot.var_buffers[name], base_coords, var_l2p, ignored, check_coords, ) if reason is not None: @@ -209,7 +389,7 @@ def from_snapshot( base_coords = snapshot.con_coords[name] if check_coords else None skip_coef_compare = same_model and not con._coef_dirty reason = _diff_con_container( - diff, name, con, snapshot.con_buffers[name], + builder, name, con, snapshot.con_buffers[name], base_coords, var_label_index, con_l2p, ignored, check_coords, skip_coef_compare, ) @@ -218,11 +398,14 @@ def from_snapshot( return diff reason = _diff_objective( - diff, model, + builder, model, snapshot.obj_c, snapshot.obj_quad_present, snapshot.obj_sense, ) if reason is not None: diff.rebuild_reason = reason + return diff + + builder.finalize(diff) return diff @classmethod @@ -265,13 +448,14 @@ def from_models( var_l2p = var_idx_b.label_to_pos con_l2p = con_idx_b.label_to_pos + builder = _DiffBuilder() for name, var_b in model_b.variables.items(): var_a = model_a.variables[name] base_buf = _extract_var_buffers(var_a) base_coords = _coord_snapshot(var_a) if check_coords else None reason = _diff_var_container( - diff, name, var_b, base_buf, + builder, name, var_b, base_buf, base_coords, var_l2p, ignored, check_coords, ) if reason is not None: @@ -283,7 +467,7 @@ def from_models( base_buf = _extract_con_buffers(con_a, var_idx_a) base_coords = _coord_snapshot(con_a) if check_coords else None reason = _diff_con_container( - diff, name, con_b, base_buf, + builder, name, con_b, base_buf, base_coords, var_idx_b, con_l2p, ignored, check_coords, skip_coef_compare=False, ) @@ -292,13 +476,16 @@ def from_models( return diff reason = _diff_objective( - diff, model_b, + builder, model_b, _objective_linear_vector(model_a), model_a.objective.is_quadratic, model_a.objective.sense, ) if reason is not None: diff.rebuild_reason = reason + return diff + + builder.finalize(diff) return diff @@ -311,8 +498,16 @@ def _coords_equal( return all(np.array_equal(a[k], b[k]) for k in keys) +def _active_container_positions( + var: Variable, var_l2p: np.ndarray +) -> np.ndarray: + labels = var.labels.values.ravel() + active = labels[labels != -1] + return var_l2p[active].astype(np.int32, copy=False) + + def _diff_var_container( - diff: ModelDiff, + builder: _DiffBuilder, name: str, var: Variable, base_buf: ContainerVarBuffers, @@ -337,20 +532,27 @@ def _diff_var_container( if not (bound_mask.any() or type_changed): return None - update = ContainerVarUpdate(type_change=new_buf.type if type_changed else None) + bounds_idx = lower = upper = None if bound_mask.any(): local_idx = np.flatnonzero(bound_mask) - update.bounds_indices = var_l2p[ + bounds_idx = var_l2p[ new_buf.active_labels[local_idx] ].astype(np.int32, copy=False) - update.lower = new_buf.lower[local_idx] - update.upper = new_buf.upper[local_idx] - diff.vars[name] = update + lower = new_buf.lower[local_idx].astype(np.float64, copy=False) + upper = new_buf.upper[local_idx].astype(np.float64, copy=False) + + type_positions = None + type_kind: VarKind | None = None + if type_changed: + type_positions = _active_container_positions(var, var_l2p) + type_kind = new_buf.type + + builder.push_var(name, bounds_idx, lower, upper, type_positions, type_kind) return None def _diff_con_container( - diff: ModelDiff, + builder: _DiffBuilder, name: str, con: ConstraintBase, base_buf: ContainerConBuffers, @@ -379,6 +581,7 @@ def _diff_con_container( if skip_coef_compare: row_value_changed = np.zeros(n_rows, dtype=bool) + data_diff = None else: data_diff = new_buf.data != base_buf.data if data_diff.any(): @@ -395,48 +598,65 @@ def _diff_con_container( if not (row_value_changed.any() or rhs_changed.any() or sign_changed.any()): return None - update = ContainerRowUpdate() + coef_rows = coef_cols = coef_vals = None if row_value_changed.any(): - idx = np.flatnonzero(row_value_changed) - update.coef_row_indices = con_l2p[ - new_buf.active_labels[idx] - ].astype(np.int32, copy=False) - new_indptr = new_buf.indptr - nnz_per_changed = (new_indptr[idx + 1] - new_indptr[idx]).astype(np.int32) - payload_indptr = np.empty(len(idx) + 1, dtype=np.int32) - payload_indptr[0] = 0 - np.cumsum(nnz_per_changed, out=payload_indptr[1:]) - total_nnz = int(payload_indptr[-1]) - payload_indices = np.empty(total_nnz, dtype=new_buf.indices.dtype) - payload_data = np.empty(total_nnz, dtype=np.float64) - for j, i in enumerate(idx): - s, e = int(new_indptr[i]), int(new_indptr[i + 1]) - ps, pe = int(payload_indptr[j]), int(payload_indptr[j + 1]) - payload_indices[ps:pe] = new_buf.indices[s:e] - payload_data[ps:pe] = new_buf.data[s:e] - update.coef_indptr = payload_indptr - update.coef_indices = payload_indices - update.coef_data = payload_data - diff.n_coef_updates += total_nnz + coef_rows, coef_cols, coef_vals = _expand_coefs_coo( + new_buf, con_l2p, row_value_changed + ) + + rhs_idx = rhs_vals = rhs_signs_arr = None if rhs_changed.any(): idx = np.flatnonzero(rhs_changed) - update.rhs_row_indices = con_l2p[ + rhs_idx = con_l2p[ new_buf.active_labels[idx] ].astype(np.int32, copy=False) - update.rhs_values = new_buf.rhs[idx] - update.rhs_signs = new_buf.sign[idx] + rhs_vals = new_buf.rhs[idx].astype(np.float64, copy=False) + rhs_signs_arr = new_buf.sign[idx] + + sign_idx = sign_vals = None if sign_changed.any(): idx = np.flatnonzero(sign_changed) - update.sign_row_indices = con_l2p[ + sign_idx = con_l2p[ new_buf.active_labels[idx] ].astype(np.int32, copy=False) - update.sign_values = new_buf.sign[idx] - diff.cons[name] = update + sign_vals = new_buf.sign[idx] + + builder.push_con( + name, + coef_rows, coef_cols, coef_vals, + rhs_idx, rhs_vals, rhs_signs_arr, + sign_idx, sign_vals, + ) return None +def _expand_coefs_coo( + new_buf: ContainerConBuffers, + con_l2p: np.ndarray, + row_value_changed: np.ndarray, +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + idx = np.flatnonzero(row_value_changed) + row_positions = con_l2p[ + new_buf.active_labels[idx] + ].astype(np.int32, copy=False) + indptr = new_buf.indptr + nnz_per_changed = (indptr[idx + 1] - indptr[idx]).astype(np.int32) + total_nnz = int(nnz_per_changed.sum()) + rows = np.repeat(row_positions, nnz_per_changed) + cols = np.empty(total_nnz, dtype=np.int32) + vals = np.empty(total_nnz, dtype=np.float64) + cursor = 0 + for i in idx: + s, e = int(indptr[i]), int(indptr[i + 1]) + n = e - s + cols[cursor:cursor + n] = new_buf.indices[s:e] + vals[cursor:cursor + n] = new_buf.data[s:e] + cursor += n + return rows, cols, vals + + def _diff_objective( - diff: ModelDiff, + builder: _DiffBuilder, model: Model, base_obj_c: np.ndarray, base_obj_quad: bool, @@ -448,12 +668,14 @@ def _diff_objective( obj_c = _objective_linear_vector(model) if obj_c.shape != base_obj_c.shape: return RebuildReason.COORD_REINDEX + c_indices = c_values = None obj_diff_mask = obj_c != base_obj_c if obj_diff_mask.any(): - idx = np.flatnonzero(obj_diff_mask).astype(np.int32, copy=False) - diff.obj_c_indices = idx - diff.obj_c_values = obj_c[idx] + c_indices = np.flatnonzero(obj_diff_mask).astype(np.int32, copy=False) + c_values = obj_c[c_indices].astype(np.float64, copy=False) - if model.objective.sense != base_obj_sense: - diff.obj_sense = model.objective.sense + sense = ( + model.objective.sense if model.objective.sense != base_obj_sense else None + ) + builder.set_objective(c_indices, c_values, sense) return None diff --git a/linopy/solvers.py b/linopy/solvers.py index 59236572..7df055f5 100644 --- a/linopy/solvers.py +++ b/linopy/solvers.py @@ -1381,84 +1381,63 @@ class Highs(Solver[None]): def is_available(cls) -> bool: return _has_module("highspy") + _HIGHS_VTYPE_MAP: ClassVar[dict[VarKind, Any]] = {} + def apply_update( self, diff: ModelDiff, var_label_index: Any, con_label_index: Any, ) -> None: - for upd in diff.cons.values(): - if upd.sign_values is not None: - raise UnsupportedUpdate( - "HiGHS does not support in-place constraint sign change" - ) + if diff.con_sign_indices.size: + raise UnsupportedUpdate( + "HiGHS does not support in-place constraint sign change" + ) - variables = var_label_index._variables h = self.solver_model + type_map = self._HIGHS_VTYPE_MAP or self._init_highs_vtype_map() - type_map = { - VarKind.CONTINUOUS: highspy.HighsVarType.kContinuous, - VarKind.BINARY: highspy.HighsVarType.kInteger, - VarKind.INTEGER: highspy.HighsVarType.kInteger, - VarKind.SEMI_CONTINUOUS: highspy.HighsVarType.kSemiContinuous, - } + if diff.var_bounds_indices.size: + indices = diff.var_bounds_indices + h.changeColsBounds( + indices.size, indices, diff.var_bounds_lower, diff.var_bounds_upper + ) - for name, upd in diff.vars.items(): - var = variables[name] - if upd.type_change is VarKind.BINARY: - labels = var.labels.values.ravel() - mask = labels != -1 - container_positions = var_label_index.label_to_pos[labels[mask]].astype( - np.int32 - ) - lb = np.zeros(container_positions.size, dtype=np.float64) - ub = np.ones(container_positions.size, dtype=np.float64) - h.changeColsBounds(container_positions.size, container_positions, lb, ub) - elif upd.bounds_indices is not None: - indices = upd.bounds_indices - lower = np.asarray(upd.lower, dtype=np.float64) - upper = np.asarray(upd.upper, dtype=np.float64) - h.changeColsBounds(indices.size, indices, lower, upper) - - if upd.type_change is not None: - labels = var.labels.values.ravel() - mask = labels != -1 - container_positions = var_label_index.label_to_pos[labels[mask]].astype( - np.int32 - ) - integrality = np.full( - container_positions.size, - int(type_map[upd.type_change]), - dtype=np.uint8, - ) - h.changeColsIntegrality( - container_positions.size, container_positions, integrality + if diff.var_type_positions.size: + positions = diff.var_type_positions + kinds = diff.var_type_kinds + integrality = np.fromiter( + (int(type_map[k]) for k in kinds), + dtype=np.uint8, + count=positions.size, + ) + h.changeColsIntegrality(positions.size, positions, integrality) + binary_mask = kinds == VarKind.BINARY + if binary_mask.any(): + bin_positions = positions[binary_mask] + n = bin_positions.size + h.changeColsBounds( + n, + bin_positions, + np.zeros(n, dtype=np.float64), + np.ones(n, dtype=np.float64), ) - for name, upd in diff.cons.items(): - if upd.rhs_values is not None: - positions = upd.rhs_row_indices - rhs_values = np.asarray(upd.rhs_values, dtype=np.float64) - sign_for_rows = upd.rhs_signs - inf = np.inf - lower = np.where(sign_for_rows == short_LESS_EQUAL, -inf, rhs_values) - upper = np.where(sign_for_rows == short_GREATER_EQUAL, inf, rhs_values) - for pos, lo, up in zip(positions, lower, upper): - h.changeRowBounds(int(pos), float(lo), float(up)) - - if upd.coef_data is not None: - rows = upd.coef_row_indices - indptr = upd.coef_indptr - indices = upd.coef_indices - data = upd.coef_data - for i, r in enumerate(rows): - for j in range(int(indptr[i]), int(indptr[i + 1])): - h.changeCoeff(int(r), int(indices[j]), float(data[j])) + if diff.con_rhs_indices.size: + lower, upper = diff.con_rhs_as_bounds() + for pos, lo, up in zip(diff.con_rhs_indices, lower, upper): + h.changeRowBounds(int(pos), float(lo), float(up)) + + if diff.con_coef_vals.size: + rows = diff.con_coef_rows + cols = diff.con_coef_cols + vals = diff.con_coef_vals + for i in range(rows.size): + h.changeCoeff(int(rows[i]), int(cols[i]), float(vals[i])) if diff.obj_c_indices is not None: indices = diff.obj_c_indices - costs = np.asarray(diff.obj_c_values, dtype=np.float64) - h.changeColsCost(indices.size, indices, costs) + h.changeColsCost(indices.size, indices, diff.obj_c_values) if diff.obj_sense is not None: sense = ( @@ -1469,6 +1448,16 @@ def apply_update( h.changeObjectiveSense(sense) self.sense = diff.obj_sense + @classmethod + def _init_highs_vtype_map(cls) -> dict[VarKind, Any]: + cls._HIGHS_VTYPE_MAP = { + VarKind.CONTINUOUS: highspy.HighsVarType.kContinuous, + VarKind.BINARY: highspy.HighsVarType.kInteger, + VarKind.INTEGER: highspy.HighsVarType.kInteger, + VarKind.SEMI_CONTINUOUS: highspy.HighsVarType.kSemiContinuous, + } + return cls._HIGHS_VTYPE_MAP + def _build_direct( self, explicit_coordinate_names: bool = False, @@ -1915,59 +1904,45 @@ def apply_update( if len(gurobi_cons) != n_active_cons: raise UnsupportedUpdate("gurobi con count mismatch") - variables = var_label_index._variables - var_l2p = var_label_index.label_to_pos - - for name, upd in diff.vars.items(): - if upd.bounds_indices is not None: - indices = upd.bounds_indices - var_subset = [gurobi_vars[int(i)] for i in indices] - if upd.lower is not None: - gm.setAttr("LB", var_subset, upd.lower.tolist()) - if upd.upper is not None: - gm.setAttr("UB", var_subset, upd.upper.tolist()) - if upd.type_change is not None: - vtype = self._GUROBI_VTYPE_MAP[upd.type_change] - var = variables[name] - labels = var.labels.values.ravel() - mask = labels != -1 - container_positions = var_l2p[labels[mask]] - container_subset = [ - gurobi_vars[int(p)] for p in container_positions - ] - gm.setAttr("VType", container_subset, [vtype] * len(container_subset)) - - for name, upd in diff.cons.items(): - if upd.rhs_values is not None: - rows = upd.rhs_row_indices - con_subset = [gurobi_cons[int(r)] for r in rows] - gm.setAttr("RHS", con_subset, upd.rhs_values.tolist()) - if upd.sign_values is not None: - rows = upd.sign_row_indices - con_subset = [gurobi_cons[int(r)] for r in rows] - senses = [] - for s in upd.sign_values: - s_str = str(s) - if s_str not in self._GUROBI_SIGN_MAP: - raise UnsupportedUpdate(f"unknown sign {s_str!r}") - senses.append(self._GUROBI_SIGN_MAP[s_str]) - gm.setAttr("Sense", con_subset, senses) - if upd.coef_data is not None: - rows = upd.coef_row_indices - indptr = upd.coef_indptr - indices = upd.coef_indices - data = upd.coef_data - for i, r in enumerate(rows): - for j in range(int(indptr[i]), int(indptr[i + 1])): - gm.chgCoeff( - gurobi_cons[int(r)], - gurobi_vars[int(indices[j])], - float(data[j]), - ) + if diff.var_bounds_indices.size: + var_subset = [gurobi_vars[int(i)] for i in diff.var_bounds_indices] + gm.setAttr("LB", var_subset, diff.var_bounds_lower.tolist()) + gm.setAttr("UB", var_subset, diff.var_bounds_upper.tolist()) + + if diff.var_type_positions.size: + vtype_map = self._GUROBI_VTYPE_MAP + type_subset = [gurobi_vars[int(p)] for p in diff.var_type_positions] + vtypes = [vtype_map[k] for k in diff.var_type_kinds] + gm.setAttr("VType", type_subset, vtypes) + + if diff.con_rhs_indices.size: + con_subset = [gurobi_cons[int(r)] for r in diff.con_rhs_indices] + gm.setAttr("RHS", con_subset, diff.con_rhs_values.tolist()) + + if diff.con_sign_indices.size: + sign_map = self._GUROBI_SIGN_MAP + con_subset = [gurobi_cons[int(r)] for r in diff.con_sign_indices] + senses = [] + for s in diff.con_sign_values: + s_str = str(s) + if s_str not in sign_map: + raise UnsupportedUpdate(f"unknown sign {s_str!r}") + senses.append(sign_map[s_str]) + gm.setAttr("Sense", con_subset, senses) + + if diff.con_coef_vals.size: + rows = diff.con_coef_rows + cols = diff.con_coef_cols + vals = diff.con_coef_vals + for i in range(rows.size): + gm.chgCoeff( + gurobi_cons[int(rows[i])], + gurobi_vars[int(cols[i])], + float(vals[i]), + ) if diff.obj_c_indices is not None: - indices = diff.obj_c_indices - var_subset = [gurobi_vars[int(i)] for i in indices] + var_subset = [gurobi_vars[int(i)] for i in diff.obj_c_indices] gm.setAttr("Obj", var_subset, diff.obj_c_values.tolist()) if diff.obj_sense is not None: diff --git a/test/test_persistent_snapshot_diff.py b/test/test_persistent_snapshot_diff.py index 9f36685f..4be6dfe5 100644 --- a/test/test_persistent_snapshot_diff.py +++ b/test/test_persistent_snapshot_diff.py @@ -55,11 +55,10 @@ def test_bounds_only_mutation(baseline: Model) -> None: baseline.variables["x"].lower = 1 diff = ModelDiff.from_snapshot(snap, baseline) assert diff.rebuild_reason is RebuildReason.NONE - assert "x" in diff.vars - assert "y" not in diff.vars - upd = diff.vars["x"] - assert upd.lower is not None - np.testing.assert_array_equal(upd.lower, np.ones(3)) + assert "x" in diff.changed_variables + assert "y" not in diff.changed_variables + sl = diff.var_slices["x"].bounds + np.testing.assert_array_equal(diff.var_bounds_lower[sl], np.ones(3)) def test_rhs_only_mutation(baseline: Model) -> None: @@ -67,10 +66,10 @@ def test_rhs_only_mutation(baseline: Model) -> None: baseline.constraints["c1"].rhs = 9 diff = ModelDiff.from_snapshot(snap, baseline) assert diff.rebuild_reason is RebuildReason.NONE - assert "c1" in diff.cons - upd = diff.cons["c1"] - assert upd.rhs_values is not None - assert upd.coef_data is None + assert "c1" in diff.changed_constraints + sl = diff.con_slices["c1"] + assert sl.rhs.stop > sl.rhs.start + assert sl.coef.stop == sl.coef.start def test_objective_linear_change(baseline: Model) -> None: @@ -119,10 +118,10 @@ def test_coef_value_change_same_sparsity(baseline: Model) -> None: c.coeffs = c.coeffs * 3 diff = ModelDiff.from_snapshot(snap, baseline) assert diff.rebuild_reason is RebuildReason.NONE - assert "c1" in diff.cons - upd = diff.cons["c1"] - assert upd.coef_data is not None - np.testing.assert_array_equal(upd.coef_data, np.full(upd.coef_data.size, 6.0)) + assert "c1" in diff.changed_constraints + sl = diff.con_slices["c1"].coef + vals = diff.con_coef_vals[sl] + np.testing.assert_array_equal(vals, np.full(vals.size, 6.0)) def test_coef_sparsity_change(baseline: Model) -> None: @@ -137,7 +136,7 @@ def test_deep_copy_invariant(baseline: Model) -> None: snap = ModelSnapshot.capture(baseline) baseline.variables["x"].lower.values[...] = 99 diff = ModelDiff.from_snapshot(snap, baseline) - assert "x" in diff.vars + assert "x" in diff.changed_variables def test_same_model_false_ignores_dirty_flag(baseline: Model) -> None: @@ -146,10 +145,11 @@ def test_same_model_false_ignores_dirty_flag(baseline: Model) -> None: c.coeffs = c.coeffs * 5 c._coef_dirty = False diff_fast = ModelDiff.from_snapshot(snap, baseline, same_model=True) - assert "c1" not in diff_fast.cons or diff_fast.cons["c1"].coef_data is None + fast_coef = diff_fast.con_slices.get("c1") + assert fast_coef is None or fast_coef.coef.stop == fast_coef.coef.start diff_full = ModelDiff.from_snapshot(snap, baseline, same_model=False) - assert "c1" in diff_full.cons - assert diff_full.cons["c1"].coef_data is not None + full_coef = diff_full.con_slices["c1"].coef + assert full_coef.stop > full_coef.start def test_modeldiff_default_is_empty() -> None: @@ -171,9 +171,9 @@ def test_from_models_diffs_two_models() -> None: diff = ModelDiff.from_models(m1, m2) assert diff.rebuild_reason is RebuildReason.NONE - assert "c1" in diff.cons - assert diff.cons["c1"].rhs_values is not None - np.testing.assert_array_equal(diff.cons["c1"].rhs_values, np.full(3, 7.0)) + assert "c1" in diff.changed_constraints + sl = diff.con_slices["c1"].rhs + np.testing.assert_array_equal(diff.con_rhs_values[sl], np.full(3, 7.0)) def test_ignore_dims_detects_coord_change() -> None: diff --git a/test/test_persistent_solver_extras.py b/test/test_persistent_solver_extras.py index 4c642a06..f3ea6b97 100644 --- a/test/test_persistent_solver_extras.py +++ b/test/test_persistent_solver_extras.py @@ -452,5 +452,6 @@ def test_track_updates_false_cross_instance_update(solver_name: str) -> None: m2 = _base_model() m2.constraints["c1"].rhs = 8.0 diff = s.update(m2, apply=False) - assert diff.summary()["con_rhs"] == 1 + assert diff.summary()["con_rhs"] == 3 + assert "c1" in diff.changed_constraints assert s.snapshot is None From 4995e8b86dbc8edd097f2cae5fa432ea3ab2fc7a Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 21 May 2026 17:04:29 +0200 Subject: [PATCH 18/21] feat(persistent): apply_update for Xpress and Mosek Implement in-place model updates for Xpress (chgbounds/chgrhs/chgmcoef/ chgrowtype/chgobj/chgobjsense/chgcoltype) and Mosek (chgvarbound/ chgconbound/putaijlist/putclist/putvartypelist/putobjsense). Mosek rejects constraint sign change to trigger rebuild. Consolidate gurobi/highs apply_update tests into a single parametrized file that also covers xpress and mosek. --- linopy/solvers.py | 159 ++++++++++++++++++++ test/test_persistent_apply_update.py | 217 +++++++++++++++++++++++++++ test/test_persistent_gurobi.py | 149 ------------------ test/test_persistent_highs.py | 161 -------------------- 4 files changed, 376 insertions(+), 310 deletions(-) create mode 100644 test/test_persistent_apply_update.py delete mode 100644 test/test_persistent_gurobi.py delete mode 100644 test/test_persistent_highs.py diff --git a/linopy/solvers.py b/linopy/solvers.py index 7df055f5..c737d11b 100644 --- a/linopy/solvers.py +++ b/linopy/solvers.py @@ -2431,12 +2431,96 @@ class Xpress(Solver[None]): SolverFeature.SOS_CONSTRAINTS, } ) + supports_persistent_update: ClassVar[bool] = True + + _XPRESS_VTYPE_MAP: ClassVar[dict[VarKind, str]] = { + VarKind.CONTINUOUS: "C", + VarKind.BINARY: "B", + VarKind.INTEGER: "I", + VarKind.SEMI_CONTINUOUS: "S", + } + _XPRESS_ROWTYPE_MAP: ClassVar[dict[str, str]] = { + short_LESS_EQUAL: "L", + short_GREATER_EQUAL: "G", + EQUAL: "E", + } @classmethod @functools.cache def is_available(cls) -> bool: return _has_module("xpress") + def apply_update( + self, + diff: ModelDiff, + var_label_index: Any, + con_label_index: Any, + ) -> None: + p = self.solver_model + + if diff.var_bounds_indices.size: + idx = diff.var_bounds_indices + cols = np.concatenate([idx, idx]).astype(np.int64, copy=False) + btypes = ["L"] * idx.size + ["U"] * idx.size + lb = np.where(np.isneginf(diff.var_bounds_lower), -xpress.infinity, diff.var_bounds_lower) + ub = np.where(np.isposinf(diff.var_bounds_upper), xpress.infinity, diff.var_bounds_upper) + vals = np.concatenate([lb, ub]).astype(float, copy=False) + p.chgbounds(cols.tolist(), btypes, vals.tolist()) + + if diff.var_type_positions.size: + vtype_map = self._XPRESS_VTYPE_MAP + positions = diff.var_type_positions + coltypes = [vtype_map[k] for k in diff.var_type_kinds] + p.chgcoltype(positions.tolist(), coltypes) + binary_mask = diff.var_type_kinds == VarKind.BINARY + if binary_mask.any(): + bin_positions = positions[binary_mask].astype(np.int64, copy=False) + n = bin_positions.size + cols = np.concatenate([bin_positions, bin_positions]) + btypes = ["L"] * n + ["U"] * n + vals = np.concatenate([np.zeros(n), np.ones(n)]) + p.chgbounds(cols.tolist(), btypes, vals.tolist()) + + if diff.con_rhs_indices.size: + p.chgrhs( + diff.con_rhs_indices.astype(np.int64, copy=False).tolist(), + diff.con_rhs_values.astype(float, copy=False).tolist(), + ) + + if diff.con_sign_indices.size: + rowtype_map = self._XPRESS_ROWTYPE_MAP + rowtypes = [] + for s in diff.con_sign_values: + s_str = str(s) + if s_str not in rowtype_map: + raise UnsupportedUpdate(f"unknown sign {s_str!r}") + rowtypes.append(rowtype_map[s_str]) + p.chgrowtype( + diff.con_sign_indices.astype(np.int64, copy=False).tolist(), rowtypes + ) + + if diff.con_coef_vals.size: + p.chgmcoef( + diff.con_coef_rows.astype(np.int64, copy=False).tolist(), + diff.con_coef_cols.astype(np.int64, copy=False).tolist(), + diff.con_coef_vals.astype(float, copy=False).tolist(), + ) + + if diff.obj_c_indices is not None: + p.chgobj( + diff.obj_c_indices.astype(np.int64, copy=False).tolist(), + diff.obj_c_values.astype(float, copy=False).tolist(), + ) + + if diff.obj_sense is not None: + if diff.obj_sense == "max": + p.chgobjsense(xpress.maximize) + elif diff.obj_sense == "min": + p.chgobjsense(xpress.minimize) + else: + raise UnsupportedUpdate(f"unknown obj sense {diff.obj_sense!r}") + self.sense = diff.obj_sense + def _build_direct( self, explicit_coordinate_names: bool = False, @@ -3012,6 +3096,7 @@ class Mosek(Solver[None]): SolverFeature.SOLUTION_FILE_NOT_NEEDED, } ) + supports_persistent_update: ClassVar[bool] = True @classmethod @functools.cache @@ -3023,6 +3108,80 @@ def _license_probe(cls) -> None: t = mosek.Task() t.optimize() + def apply_update( + self, + diff: ModelDiff, + var_label_index: Any, + con_label_index: Any, + ) -> None: + if diff.con_sign_indices.size: + raise UnsupportedUpdate( + "MOSEK does not support in-place constraint sign change" + ) + + t = self.solver_model + + if diff.var_bounds_indices.size: + indices = diff.var_bounds_indices + lowers = diff.var_bounds_lower + uppers = diff.var_bounds_upper + for k in range(indices.size): + j = int(indices[k]) + lb = float(lowers[k]) + ub = float(uppers[k]) + t.chgvarbound(j, 1, int(np.isfinite(lb)), lb) + t.chgvarbound(j, 0, int(np.isfinite(ub)), ub) + + if diff.var_type_positions.size: + positions = diff.var_type_positions + kinds = diff.var_type_kinds + if (kinds == VarKind.SEMI_CONTINUOUS).any(): + raise UnsupportedUpdate( + "MOSEK does not support semi-continuous variables" + ) + integer_mask = (kinds == VarKind.BINARY) | (kinds == VarKind.INTEGER) + vartypes = np.where( + integer_mask, + mosek.variabletype.type_int, + mosek.variabletype.type_cont, + ).tolist() + t.putvartypelist(positions.astype(np.int32, copy=False).tolist(), vartypes) + binary_mask = kinds == VarKind.BINARY + if binary_mask.any(): + for j in positions[binary_mask]: + t.chgvarbound(int(j), 1, 1, 0.0) + t.chgvarbound(int(j), 0, 1, 1.0) + + if diff.con_rhs_indices.size: + lower, upper = diff.con_rhs_as_bounds() + for k, i in enumerate(diff.con_rhs_indices): + lo = float(lower[k]) + up = float(upper[k]) + t.chgconbound(int(i), 1, int(np.isfinite(lo)), lo) + t.chgconbound(int(i), 0, int(np.isfinite(up)), up) + + if diff.con_coef_vals.size: + t.putaijlist( + diff.con_coef_rows.astype(np.int32, copy=False).tolist(), + diff.con_coef_cols.astype(np.int32, copy=False).tolist(), + diff.con_coef_vals.astype(float, copy=False).tolist(), + ) + + if diff.obj_c_indices is not None: + t.putclist( + diff.obj_c_indices.astype(np.int32, copy=False).tolist(), + diff.obj_c_values.astype(float, copy=False).tolist(), + ) + + if diff.obj_sense is not None: + if diff.obj_sense == "max": + t.putobjsense(mosek.objsense.maximize) + elif diff.obj_sense == "min": + t.putobjsense(mosek.objsense.minimize) + else: + raise UnsupportedUpdate(f"unknown obj sense {diff.obj_sense!r}") + self.sense = diff.obj_sense + def _run_direct( self, solution_fn: Path | None = None, diff --git a/test/test_persistent_apply_update.py b/test/test_persistent_apply_update.py new file mode 100644 index 00000000..50012ec1 --- /dev/null +++ b/test/test_persistent_apply_update.py @@ -0,0 +1,217 @@ +from __future__ import annotations + +from typing import Any + +import numpy as np +import pytest + +from linopy import Model +from linopy.persistent import RebuildReason +from linopy.solvers import Gurobi, Highs, Mosek, Solver, Xpress + +_BACKENDS: dict[str, tuple[type[Solver], dict[str, Any]]] = { + "gurobi": (Gurobi, {"OutputFlag": 0}), + "highs": (Highs, {"output_flag": False}), + "xpress": (Xpress, {"OUTPUTLOG": 0}), + "mosek": (Mosek, {"MSK_IPAR_LOG": 0}), +} + +_SIGN_CHANGE_IN_PLACE: dict[str, bool] = { + "gurobi": True, + "highs": False, + "xpress": True, + "mosek": False, +} + + +def _have(name: str) -> bool: + cls = _BACKENDS[name][0] + if not cls.is_available(): + return False + try: + cls._license_probe() + except Exception: + return False + if name == "xpress": + try: + import xpress + + xpress.problem() + except Exception: + return False + return True + + +SOLVER_PARAMS = [ + pytest.param( + name, + marks=pytest.mark.skipif( + not _have(name), reason=f"{name} not installed" + ), + ) + for name in _BACKENDS +] + + +def _base_model() -> Model: + m = Model() + x = m.add_variables(0, 10, coords=[range(3)], name="x") + y = m.add_variables(0, 10, coords=[range(3)], name="y") + m.add_constraints(x + y >= 4, name="c1") + m.add_constraints(2 * x + y <= 20, name="c2") + m.add_objective(x.sum() + 2 * y.sum()) + return m + + +def _built(solver_name: str, model: Model) -> Solver: + cls, opts = _BACKENDS[solver_name] + s = cls(model=model, io_api="direct", track_updates=True) + s.options = opts + s._build() + return s + + +def _solve(solver: Solver, model: Model) -> float: + result = solver.solve(model, assign=True) + return float(result.solution.objective) + + +@pytest.mark.parametrize("solver_name", SOLVER_PARAMS) +def test_var_lb_in_place(solver_name: str) -> None: + m = _base_model() + s = _built(solver_name, m) + s.solve(assign=True) + base_obj = float(m.objective.value) + + m.variables["x"].lower.values[...] = 5.0 + obj = _solve(s, m) + assert s._in_place_updates == 1 + assert s._rebuilds == 0 + assert s._last_rebuild_reason is RebuildReason.NONE + assert obj > base_obj + + +@pytest.mark.parametrize("solver_name", SOLVER_PARAMS) +def test_var_ub_in_place(solver_name: str) -> None: + m = _base_model() + s = _built(solver_name, m) + s.solve(assign=True) + + m.variables["x"].upper.values[...] = 1.0 + _solve(s, m) + assert s._in_place_updates == 1 + assert s._rebuilds == 0 + + +@pytest.mark.parametrize("solver_name", SOLVER_PARAMS) +def test_rhs_only_in_place(solver_name: str) -> None: + m = _base_model() + s = _built(solver_name, m) + s.solve(assign=True) + base_obj = float(m.objective.value) + + c = m.constraints["c1"] + c.rhs = 8.0 + assert c._coef_dirty is False + obj = _solve(s, m) + assert s._in_place_updates == 1 + assert s._rebuilds == 0 + assert obj > base_obj + + +@pytest.mark.parametrize("solver_name", SOLVER_PARAMS) +def test_constraint_coef_change_in_place(solver_name: str) -> None: + m = _base_model() + s = _built(solver_name, m) + s.solve(assign=True) + base_obj = float(m.objective.value) + + c = m.constraints["c1"] + c.coeffs = c.coeffs * 2 + obj = _solve(s, m) + assert s._in_place_updates == 1 + assert s._rebuilds == 0 + assert not np.isclose(obj, base_obj) + + +@pytest.mark.parametrize("solver_name", SOLVER_PARAMS) +def test_objective_linear_change_in_place(solver_name: str) -> None: + m = _base_model() + s = _built(solver_name, m) + s.solve(assign=True) + base_obj = float(m.objective.value) + + x = m.variables["x"] + y = m.variables["y"] + m.objective.expression = 5 * x.sum() + 3 * y.sum() + obj = _solve(s, m) + assert s._in_place_updates == 1 + assert s._rebuilds == 0 + assert not np.isclose(obj, base_obj) + + +@pytest.mark.parametrize("solver_name", SOLVER_PARAMS) +def test_objective_sense_flip_in_place(solver_name: str) -> None: + m = _base_model() + s = _built(solver_name, m) + s.solve(assign=True) + min_obj = float(m.objective.value) + + m.objective.sense = "max" + max_obj = _solve(s, m) + assert s._in_place_updates == 1 + assert s._rebuilds == 0 + assert max_obj > min_obj + + +@pytest.mark.parametrize("solver_name", SOLVER_PARAMS) +def test_sparsity_change_triggers_rebuild(solver_name: str) -> None: + m = _base_model() + s = _built(solver_name, m) + s.solve(assign=True) + + x = m.variables["x"] + m.add_constraints(x <= 5, name="c3") + s.solve(m, assign=True) + assert s._rebuilds == 1 + assert s._last_rebuild_reason in { + RebuildReason.STRUCTURAL_LABELS, + RebuildReason.STRUCTURAL_CONTAINERS, + } + + +@pytest.mark.parametrize("solver_name", SOLVER_PARAMS) +def test_cross_model_in_place(solver_name: str) -> None: + m1 = _base_model() + s = _built(solver_name, m1) + s.solve(assign=True) + + m2 = _base_model() + m2.constraints["c1"].rhs = 8.0 + + s.solve(m2, assign=True) + assert s._in_place_updates == 1 + assert s._rebuilds == 0 + + cross_obj = float(m2.objective.value) + m3 = _base_model() + m3.constraints["c1"].rhs = 8.0 + s_fresh = _built(solver_name, m3) + s_fresh.solve(assign=True) + assert np.isclose(cross_obj, float(m3.objective.value)) + + +@pytest.mark.parametrize("solver_name", SOLVER_PARAMS) +def test_sign_flip(solver_name: str) -> None: + m = _base_model() + s = _built(solver_name, m) + s.solve(assign=True) + + m.constraints["c1"].sign = "<=" + s.solve(m, assign=True) + if _SIGN_CHANGE_IN_PLACE[solver_name]: + assert s._in_place_updates == 1 + assert s._rebuilds == 0 + else: + assert s._rebuilds == 1 + assert s._last_rebuild_reason is RebuildReason.BACKEND_REJECTED diff --git a/test/test_persistent_gurobi.py b/test/test_persistent_gurobi.py deleted file mode 100644 index f108bfd2..00000000 --- a/test/test_persistent_gurobi.py +++ /dev/null @@ -1,149 +0,0 @@ -from __future__ import annotations - -import numpy as np -import pytest - -from linopy import Model -from linopy.persistent import RebuildReason -from linopy.solvers import Gurobi - -pytest.importorskip("gurobipy") - - -def _base_model() -> Model: - m = Model() - x = m.add_variables(0, 10, coords=[range(3)], name="x") - y = m.add_variables(0, 10, coords=[range(3)], name="y") - m.add_constraints(x + y >= 4, name="c1") - m.add_constraints(2 * x + y <= 20, name="c2") - m.add_objective(x.sum() + 2 * y.sum()) - return m - - -def _built(model: Model) -> Gurobi: - s = Gurobi(model=model, io_api="direct", track_updates=True) - s.options = {"OutputFlag": 0} - s._build() - return s - - -def _solve_and_assign(solver: Gurobi, model: Model) -> float: - result = solver.solve(model, assign=True) - return float(result.solution.objective) - - -def test_var_lb_in_place() -> None: - m = _base_model() - s = _built(m) - s.solve(assign=True) - assert s._rebuilds == 0 - assert s._in_place_updates == 0 - base_obj = float(m.objective.value) - - m.variables["x"].lower.values[...] = 5.0 - obj = _solve_and_assign(s, m) - assert s._rebuilds == 0 - assert s._in_place_updates == 1 - assert s._last_rebuild_reason is RebuildReason.NONE - assert obj > base_obj - - -def test_var_ub_in_place() -> None: - m = _base_model() - s = _built(m) - s.solve(assign=True) - - m.variables["x"].upper.values[...] = 1.0 - _solve_and_assign(s, m) - assert s._in_place_updates == 1 - assert s._rebuilds == 0 - - -def test_rhs_only_in_place() -> None: - m = _base_model() - s = _built(m) - s.solve(assign=True) - base_obj = float(m.objective.value) - - c = m.constraints["c1"] - c.rhs = 8.0 - assert c._coef_dirty is False - obj = _solve_and_assign(s, m) - assert s._in_place_updates == 1 - assert s._rebuilds == 0 - assert obj > base_obj - - -def test_constraint_coef_change_in_place() -> None: - m = _base_model() - s = _built(m) - s.solve(assign=True) - base_obj = float(m.objective.value) - - c = m.constraints["c1"] - new_coeffs = c.coeffs * 2 - c.coeffs = new_coeffs - obj = _solve_and_assign(s, m) - assert s._in_place_updates == 1 - assert s._rebuilds == 0 - assert obj != base_obj - - -def test_objective_linear_change_in_place() -> None: - m = _base_model() - s = _built(m) - s.solve(assign=True) - base_obj = float(m.objective.value) - - x = m.variables["x"] - y = m.variables["y"] - m.objective.expression = 3 * x.sum() + 7 * y.sum() - obj = _solve_and_assign(s, m) - assert s._in_place_updates == 1 - assert s._rebuilds == 0 - assert obj != base_obj - - -def test_objective_sense_flip_in_place() -> None: - m = _base_model() - s = _built(m) - s.solve(assign=True) - min_obj = float(m.objective.value) - - m.objective.sense = "max" - max_obj = _solve_and_assign(s, m) - assert s._in_place_updates == 1 - assert s._rebuilds == 0 - assert max_obj > min_obj - - -def test_sparsity_change_triggers_rebuild() -> None: - m = _base_model() - s = _built(m) - s.solve(assign=True) - - x = m.variables["x"] - m.add_constraints(x <= 5, name="c3") - s.solve(m, assign=True) - assert s._rebuilds == 1 - assert s._last_rebuild_reason is RebuildReason.STRUCTURAL_CONTAINERS - - -def test_cross_model_in_place() -> None: - m1 = _base_model() - s = _built(m1) - s.solve(assign=True) - - m2 = _base_model() - m2.constraints["c1"].rhs = 8.0 - - s.solve(m2, assign=True) - assert s._in_place_updates == 1 - assert s._rebuilds == 0 - - fresh_obj = m2.objective.value - m3 = _base_model() - m3.constraints["c1"].rhs = 8.0 - s_fresh = _built(m3) - s_fresh.solve(assign=True) - assert np.isclose(float(fresh_obj), float(m3.objective.value)) diff --git a/test/test_persistent_highs.py b/test/test_persistent_highs.py deleted file mode 100644 index 77325ddc..00000000 --- a/test/test_persistent_highs.py +++ /dev/null @@ -1,161 +0,0 @@ -from __future__ import annotations - -import numpy as np -import pytest - -from linopy import Model -from linopy.persistent import RebuildReason -from linopy.solvers import Highs - -pytest.importorskip("highspy") - - -def _base_model() -> Model: - m = Model() - x = m.add_variables(0, 10, coords=[range(3)], name="x") - y = m.add_variables(0, 10, coords=[range(3)], name="y") - m.add_constraints(x + y >= 4, name="c1") - m.add_constraints(2 * x + y <= 20, name="c2") - m.add_objective(x.sum() + 2 * y.sum()) - return m - - -def _built(model: Model) -> Highs: - s = Highs(model=model, io_api="direct", track_updates=True) - s.options = {"output_flag": False} - s._build() - return s - - -def _solve_and_assign(solver: Highs, model: Model) -> float: - result = solver.solve(model, assign=True) - return float(result.solution.objective) - - -def test_var_lb_in_place() -> None: - m = _base_model() - s = _built(m) - s.solve(assign=True) - base_obj = float(m.objective.value) - - m.variables["x"].lower.values[...] = 6.0 - obj = _solve_and_assign(s, m) - assert s._in_place_updates == 1 - assert s._rebuilds == 0 - assert s._last_rebuild_reason is RebuildReason.NONE - assert obj > base_obj - - -def test_var_ub_in_place() -> None: - m = _base_model() - s = _built(m) - s.solve(assign=True) - - m.variables["x"].upper.values[...] = 1.0 - _solve_and_assign(s, m) - assert s._in_place_updates == 1 - assert s._rebuilds == 0 - - -def test_rhs_only_in_place() -> None: - m = _base_model() - s = _built(m) - s.solve(assign=True) - base_obj = float(m.objective.value) - - c = m.constraints["c1"] - c.rhs = 8.0 - assert c._coef_dirty is False - obj = _solve_and_assign(s, m) - assert s._in_place_updates == 1 - assert s._rebuilds == 0 - assert obj > base_obj - - -def test_constraint_coef_change_in_place() -> None: - m = _base_model() - s = _built(m) - s.solve(assign=True) - base_obj = float(m.objective.value) - - c = m.constraints["c1"] - c.coeffs = c.coeffs * 2 - obj = _solve_and_assign(s, m) - assert s._in_place_updates == 1 - assert s._rebuilds == 0 - assert not np.isclose(obj, base_obj) - - -def test_objective_linear_change_in_place() -> None: - m = _base_model() - s = _built(m) - s.solve(assign=True) - base_obj = float(m.objective.value) - - x = m.variables["x"] - y = m.variables["y"] - m.objective.expression = 5 * x.sum() + 3 * y.sum() - obj = _solve_and_assign(s, m) - assert s._in_place_updates == 1 - assert s._rebuilds == 0 - assert not np.isclose(obj, base_obj) - - -def test_objective_sense_flip_in_place() -> None: - m = _base_model() - s = _built(m) - s.solve(assign=True) - min_obj = float(m.objective.value) - - m.objective.sense = "max" - max_obj = _solve_and_assign(s, m) - assert s._in_place_updates == 1 - assert s._rebuilds == 0 - assert max_obj > min_obj - - -def test_sign_flip_falls_back_to_rebuild() -> None: - m = _base_model() - s = _built(m) - s.solve(assign=True) - - c = m.constraints["c1"] - c.sign = "<=" - s.solve(m, assign=True) - assert s._rebuilds == 1 - assert s._last_rebuild_reason is RebuildReason.BACKEND_REJECTED - - -def test_sparsity_change_triggers_rebuild() -> None: - m = _base_model() - s = _built(m) - s.solve(assign=True) - - x = m.variables["x"] - m.add_constraints(x <= 5, name="c3") - s.solve(m, assign=True) - assert s._rebuilds == 1 - assert s._last_rebuild_reason in { - RebuildReason.STRUCTURAL_LABELS, - RebuildReason.STRUCTURAL_CONTAINERS, - } - - -def test_cross_model_in_place() -> None: - m1 = _base_model() - s = _built(m1) - s.solve(assign=True) - - m2 = _base_model() - m2.constraints["c1"].rhs = 8.0 - - s.solve(m2, assign=True) - assert s._in_place_updates == 1 - assert s._rebuilds == 0 - - cross_obj = float(m2.objective.value) - m3 = _base_model() - m3.constraints["c1"].rhs = 8.0 - s_fresh = _built(m3) - s_fresh.solve(assign=True) - assert np.isclose(cross_obj, float(m3.objective.value)) From 5c0a369744e207c0f207d96a6e7bfd12004d3b61 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 21 May 2026 15:06:03 +0000 Subject: [PATCH 19/21] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- linopy/persistent/diff.py | 98 +++++++++++++++++----------- linopy/persistent/snapshot.py | 4 +- linopy/solvers.py | 12 +++- test/test_persistent_apply_update.py | 4 +- 4 files changed, 73 insertions(+), 45 deletions(-) diff --git a/linopy/persistent/diff.py b/linopy/persistent/diff.py index 0731abc7..ce96dae5 100644 --- a/linopy/persistent/diff.py +++ b/linopy/persistent/diff.py @@ -378,8 +378,14 @@ def from_snapshot( for name, var in model.variables.items(): base_coords = snapshot.var_coords[name] if check_coords else None reason = _diff_var_container( - builder, name, var, snapshot.var_buffers[name], - base_coords, var_l2p, ignored, check_coords, + builder, + name, + var, + snapshot.var_buffers[name], + base_coords, + var_l2p, + ignored, + check_coords, ) if reason is not None: diff.rebuild_reason = reason @@ -389,8 +395,15 @@ def from_snapshot( base_coords = snapshot.con_coords[name] if check_coords else None skip_coef_compare = same_model and not con._coef_dirty reason = _diff_con_container( - builder, name, con, snapshot.con_buffers[name], - base_coords, var_label_index, con_l2p, ignored, check_coords, + builder, + name, + con, + snapshot.con_buffers[name], + base_coords, + var_label_index, + con_l2p, + ignored, + check_coords, skip_coef_compare, ) if reason is not None: @@ -398,8 +411,11 @@ def from_snapshot( return diff reason = _diff_objective( - builder, model, - snapshot.obj_c, snapshot.obj_quad_present, snapshot.obj_sense, + builder, + model, + snapshot.obj_c, + snapshot.obj_quad_present, + snapshot.obj_sense, ) if reason is not None: diff.rebuild_reason = reason @@ -428,9 +444,8 @@ def from_models( var_names_a = tuple(model_a.variables) con_names_a = tuple(model_a.constraints) - if ( - var_names_a != tuple(model_b.variables) - or con_names_a != tuple(model_b.constraints) + if var_names_a != tuple(model_b.variables) or con_names_a != tuple( + model_b.constraints ): diff.rebuild_reason = RebuildReason.STRUCTURAL_CONTAINERS return diff @@ -455,8 +470,14 @@ def from_models( base_buf = _extract_var_buffers(var_a) base_coords = _coord_snapshot(var_a) if check_coords else None reason = _diff_var_container( - builder, name, var_b, base_buf, - base_coords, var_l2p, ignored, check_coords, + builder, + name, + var_b, + base_buf, + base_coords, + var_l2p, + ignored, + check_coords, ) if reason is not None: diff.rebuild_reason = reason @@ -467,8 +488,15 @@ def from_models( base_buf = _extract_con_buffers(con_a, var_idx_a) base_coords = _coord_snapshot(con_a) if check_coords else None reason = _diff_con_container( - builder, name, con_b, base_buf, - base_coords, var_idx_b, con_l2p, ignored, check_coords, + builder, + name, + con_b, + base_buf, + base_coords, + var_idx_b, + con_l2p, + ignored, + check_coords, skip_coef_compare=False, ) if reason is not None: @@ -476,7 +504,8 @@ def from_models( return diff reason = _diff_objective( - builder, model_b, + builder, + model_b, _objective_linear_vector(model_a), model_a.objective.is_quadratic, model_a.objective.sense, @@ -498,9 +527,7 @@ def _coords_equal( return all(np.array_equal(a[k], b[k]) for k in keys) -def _active_container_positions( - var: Variable, var_l2p: np.ndarray -) -> np.ndarray: +def _active_container_positions(var: Variable, var_l2p: np.ndarray) -> np.ndarray: labels = var.labels.values.ravel() active = labels[labels != -1] return var_l2p[active].astype(np.int32, copy=False) @@ -535,9 +562,9 @@ def _diff_var_container( bounds_idx = lower = upper = None if bound_mask.any(): local_idx = np.flatnonzero(bound_mask) - bounds_idx = var_l2p[ - new_buf.active_labels[local_idx] - ].astype(np.int32, copy=False) + bounds_idx = var_l2p[new_buf.active_labels[local_idx]].astype( + np.int32, copy=False + ) lower = new_buf.lower[local_idx].astype(np.float64, copy=False) upper = new_buf.upper[local_idx].astype(np.float64, copy=False) @@ -607,25 +634,26 @@ def _diff_con_container( rhs_idx = rhs_vals = rhs_signs_arr = None if rhs_changed.any(): idx = np.flatnonzero(rhs_changed) - rhs_idx = con_l2p[ - new_buf.active_labels[idx] - ].astype(np.int32, copy=False) + rhs_idx = con_l2p[new_buf.active_labels[idx]].astype(np.int32, copy=False) rhs_vals = new_buf.rhs[idx].astype(np.float64, copy=False) rhs_signs_arr = new_buf.sign[idx] sign_idx = sign_vals = None if sign_changed.any(): idx = np.flatnonzero(sign_changed) - sign_idx = con_l2p[ - new_buf.active_labels[idx] - ].astype(np.int32, copy=False) + sign_idx = con_l2p[new_buf.active_labels[idx]].astype(np.int32, copy=False) sign_vals = new_buf.sign[idx] builder.push_con( name, - coef_rows, coef_cols, coef_vals, - rhs_idx, rhs_vals, rhs_signs_arr, - sign_idx, sign_vals, + coef_rows, + coef_cols, + coef_vals, + rhs_idx, + rhs_vals, + rhs_signs_arr, + sign_idx, + sign_vals, ) return None @@ -636,9 +664,7 @@ def _expand_coefs_coo( row_value_changed: np.ndarray, ) -> tuple[np.ndarray, np.ndarray, np.ndarray]: idx = np.flatnonzero(row_value_changed) - row_positions = con_l2p[ - new_buf.active_labels[idx] - ].astype(np.int32, copy=False) + row_positions = con_l2p[new_buf.active_labels[idx]].astype(np.int32, copy=False) indptr = new_buf.indptr nnz_per_changed = (indptr[idx + 1] - indptr[idx]).astype(np.int32) total_nnz = int(nnz_per_changed.sum()) @@ -649,8 +675,8 @@ def _expand_coefs_coo( for i in idx: s, e = int(indptr[i]), int(indptr[i + 1]) n = e - s - cols[cursor:cursor + n] = new_buf.indices[s:e] - vals[cursor:cursor + n] = new_buf.data[s:e] + cols[cursor : cursor + n] = new_buf.indices[s:e] + vals[cursor : cursor + n] = new_buf.data[s:e] cursor += n return rows, cols, vals @@ -674,8 +700,6 @@ def _diff_objective( c_indices = np.flatnonzero(obj_diff_mask).astype(np.int32, copy=False) c_values = obj_c[c_indices].astype(np.float64, copy=False) - sense = ( - model.objective.sense if model.objective.sense != base_obj_sense else None - ) + sense = model.objective.sense if model.objective.sense != base_obj_sense else None builder.set_objective(c_indices, c_values, sense) return None diff --git a/linopy/persistent/snapshot.py b/linopy/persistent/snapshot.py index 8820bbab..bece987b 100644 --- a/linopy/persistent/snapshot.py +++ b/linopy/persistent/snapshot.py @@ -125,9 +125,7 @@ class ModelSnapshot: con_buffers: dict[str, ContainerConBuffers] = field(default_factory=dict) var_coords: dict[str, dict[str, np.ndarray]] = field(default_factory=dict) con_coords: dict[str, dict[str, np.ndarray]] = field(default_factory=dict) - obj_c: np.ndarray = field( - default_factory=lambda: np.zeros(0, dtype=np.float64) - ) + obj_c: np.ndarray = field(default_factory=lambda: np.zeros(0, dtype=np.float64)) obj_quad_present: bool = False obj_sense: str = "min" diff --git a/linopy/solvers.py b/linopy/solvers.py index c737d11b..26bd37d8 100644 --- a/linopy/solvers.py +++ b/linopy/solvers.py @@ -2462,8 +2462,16 @@ def apply_update( idx = diff.var_bounds_indices cols = np.concatenate([idx, idx]).astype(np.int64, copy=False) btypes = ["L"] * idx.size + ["U"] * idx.size - lb = np.where(np.isneginf(diff.var_bounds_lower), -xpress.infinity, diff.var_bounds_lower) - ub = np.where(np.isposinf(diff.var_bounds_upper), xpress.infinity, diff.var_bounds_upper) + lb = np.where( + np.isneginf(diff.var_bounds_lower), + -xpress.infinity, + diff.var_bounds_lower, + ) + ub = np.where( + np.isposinf(diff.var_bounds_upper), + xpress.infinity, + diff.var_bounds_upper, + ) vals = np.concatenate([lb, ub]).astype(float, copy=False) p.chgbounds(cols.tolist(), btypes, vals.tolist()) diff --git a/test/test_persistent_apply_update.py b/test/test_persistent_apply_update.py index 50012ec1..ba108560 100644 --- a/test/test_persistent_apply_update.py +++ b/test/test_persistent_apply_update.py @@ -45,9 +45,7 @@ def _have(name: str) -> bool: SOLVER_PARAMS = [ pytest.param( name, - marks=pytest.mark.skipif( - not _have(name), reason=f"{name} not installed" - ), + marks=pytest.mark.skipif(not _have(name), reason=f"{name} not installed"), ) for name in _BACKENDS ] From 9fd88dead92d59ff3adbaaefcbac82faab6e1d68 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 22 May 2026 10:39:01 +0200 Subject: [PATCH 20/21] fix(persistent): serialize concurrent solves; satisfy mypy * hold solver lock through _run_direct so two threads calling solve(model) on the same Solver no longer race on the native handle (HiGHS returned 0.0 from the second concurrent solve). * narrow Optional ndarrays in persistent.diff.push_var / push_con and in HiGHS/Gurobi/Xpress/Mosek apply_update objective paths. * widen Constraint.rhs setter to ExpressionLike | VariableLike | ConstantLike to match the as_expression call in the body. * widen Constraints.__getitem__(str) return type to Constraint (the dominant case) so tests can set .rhs/.coeffs/.sign without ignores. * add docs for in-place solver updates. --- doc/release_notes.rst | 6 +++ linopy/constraints.py | 8 ++-- linopy/persistent/diff.py | 36 ++++++++++------ linopy/persistent/snapshot.py | 4 +- linopy/solvers.py | 48 +++++++++++---------- test/test_constraint.py | 4 +- test/test_persistent_apply_update.py | 21 ++++++--- test/test_persistent_solver_extras.py | 28 +++++++----- test/test_persistent_solver_orchestrator.py | 4 +- 9 files changed, 98 insertions(+), 61 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index e5b7033f..edd4ed07 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -40,6 +40,12 @@ Most users should keep calling ``model.solve(...)``. If you want more control, y * Opt in globally via ``Model(freeze_constraints=True)`` or per-call via ``model.add_constraints(..., freeze=True)``. * Lossless conversion both ways with ``Constraint.freeze()`` / ``CSRConstraint.mutable()``. +*In-place solver updates (persistent re-solve)* + +* A built solver can now be re-solved against a mutated ``Model`` without a full rebuild. Construct with ``Solver.from_name(..., track_updates=True)`` and re-call ``solver.solve(model)`` after edits — the diff against the previous build is applied in place when the backend supports it, falling back to a rebuild otherwise. Supported on HiGHS, Gurobi, Xpress, and Mosek (``io_api="direct"``). +* Pass ``disallow_rebuild=True`` to ``solve(model, ...)`` to guarantee an in-place update or raise ``RebuildRequiredError``. Inspect ``solver._last_rebuild_reason`` (a ``RebuildReason``) to understand why a rebuild was triggered. +* New ``linopy.persistent`` module exposes ``ModelSnapshot``, ``ModelDiff``, and ``RebuildReason`` for users who want to introspect or build the diff themselves. + **Performance** * ~10× faster direct solver communication (``io_api="direct"``), thanks to the new CSR-based matrix construction. Conversion helpers like ``to_highspy`` benefit too. diff --git a/linopy/constraints.py b/linopy/constraints.py index 1b51f48d..6f11b137 100644 --- a/linopy/constraints.py +++ b/linopy/constraints.py @@ -1153,7 +1153,7 @@ def rhs(self) -> DataArray: return self.data.rhs @rhs.setter - def rhs(self, value: ExpressionLike) -> None: + def rhs(self, value: ExpressionLike | VariableLike | ConstantLike) -> None: value = expressions.as_expression( value, self.model, coords=self.coords, dims=self.coord_dims ) @@ -1512,14 +1512,14 @@ def __repr__(self) -> str: return r @overload - def __getitem__(self, names: str) -> ConstraintBase: ... + def __getitem__(self, names: str) -> Constraint: ... @overload def __getitem__(self, names: list[str]) -> Constraints: ... - def __getitem__(self, names: str | list[str]) -> ConstraintBase | Constraints: + def __getitem__(self, names: str | list[str]) -> Constraint | Constraints: if isinstance(names, str): - return self.data[names] + return self.data[names] # type: ignore[return-value] return Constraints({name: self.data[name] for name in names}, self.model) def __getattr__(self, name: str) -> ConstraintBase: diff --git a/linopy/persistent/diff.py b/linopy/persistent/diff.py index ce96dae5..56133bdb 100644 --- a/linopy/persistent/diff.py +++ b/linopy/persistent/diff.py @@ -8,6 +8,7 @@ import numpy as np from linopy.constants import short_GREATER_EQUAL, short_LESS_EQUAL +from linopy.constraints import Constraint from linopy.persistent.snapshot import ( ContainerConBuffers, ContainerVarBuffers, @@ -101,6 +102,7 @@ def push_var( ) -> None: b_start = self._vb_cur if bounds_idx is not None: + assert lower is not None and upper is not None self.var_bounds_idx.append(bounds_idx) self.var_bounds_lo.append(lower) self.var_bounds_up.append(upper) @@ -131,18 +133,21 @@ def push_con( ) -> None: c_start = self._cc_cur if coef_rows is not None: + assert coef_cols is not None and coef_vals is not None self.con_coef_rows.append(coef_rows) self.con_coef_cols.append(coef_cols) self.con_coef_vals.append(coef_vals) self._cc_cur += coef_rows.size r_start = self._cr_cur if rhs_idx is not None: + assert rhs_vals is not None and rhs_signs is not None self.con_rhs_idx.append(rhs_idx) self.con_rhs_vals.append(rhs_vals) self.con_rhs_signs.append(rhs_signs) self._cr_cur += rhs_idx.size s_start = self._cs_cur if sign_idx is not None: + assert sign_vals is not None self.con_sign_idx.append(sign_idx) self.con_sign_vals.append(sign_vals) self._cs_cur += sign_idx.size @@ -393,7 +398,8 @@ def from_snapshot( for name, con in model.constraints.items(): base_coords = snapshot.con_coords[name] if check_coords else None - skip_coef_compare = same_model and not con._coef_dirty + coef_dirty = isinstance(con, Constraint) and con._coef_dirty + skip_coef_compare = same_model and not coef_dirty reason = _diff_con_container( builder, name, @@ -467,14 +473,14 @@ def from_models( for name, var_b in model_b.variables.items(): var_a = model_a.variables[name] - base_buf = _extract_var_buffers(var_a) - base_coords = _coord_snapshot(var_a) if check_coords else None + var_base_buf = _extract_var_buffers(var_a) + var_base_coords = _coord_snapshot(var_a) if check_coords else None reason = _diff_var_container( builder, name, var_b, - base_buf, - base_coords, + var_base_buf, + var_base_coords, var_l2p, ignored, check_coords, @@ -485,14 +491,14 @@ def from_models( for name, con_b in model_b.constraints.items(): con_a = model_a.constraints[name] - base_buf = _extract_con_buffers(con_a, var_idx_a) - base_coords = _coord_snapshot(con_a) if check_coords else None + con_base_buf = _extract_con_buffers(con_a, var_idx_a) + con_base_coords = _coord_snapshot(con_a) if check_coords else None reason = _diff_con_container( builder, name, con_b, - base_buf, - base_coords, + con_base_buf, + con_base_coords, var_idx_b, con_l2p, ignored, @@ -548,8 +554,10 @@ def _diff_var_container( return RebuildReason.COORD_REINDEX if not np.array_equal(new_buf.active_labels, base_buf.active_labels): return RebuildReason.STRUCTURAL_LABELS - if check_coords and not _coords_equal(base_coords, _coord_snapshot(var), ignored): - return RebuildReason.COORD_REINDEX + if check_coords: + assert base_coords is not None + if not _coords_equal(base_coords, _coord_snapshot(var), ignored): + return RebuildReason.COORD_REINDEX lower_diff = new_buf.lower != base_buf.lower upper_diff = new_buf.upper != base_buf.upper @@ -595,8 +603,10 @@ def _diff_con_container( return RebuildReason.COORD_REINDEX if not np.array_equal(new_buf.active_labels, base_buf.active_labels): return RebuildReason.STRUCTURAL_LABELS - if check_coords and not _coords_equal(base_coords, _coord_snapshot(con), ignored): - return RebuildReason.COORD_REINDEX + if check_coords: + assert base_coords is not None + if not _coords_equal(base_coords, _coord_snapshot(con), ignored): + return RebuildReason.COORD_REINDEX if not np.array_equal(new_buf.indptr, base_buf.indptr): return RebuildReason.SPARSITY if not np.array_equal(new_buf.indices, base_buf.indices): diff --git a/linopy/persistent/snapshot.py b/linopy/persistent/snapshot.py index bece987b..55072673 100644 --- a/linopy/persistent/snapshot.py +++ b/linopy/persistent/snapshot.py @@ -7,6 +7,7 @@ import numpy as np from linopy import expressions +from linopy.constraints import Constraint if TYPE_CHECKING: from linopy.constraints import ConstraintBase @@ -156,7 +157,8 @@ def capture(cls, model: Model) -> ModelSnapshot: } for con in model.constraints.data.values(): - con._coef_dirty = False + if isinstance(con, Constraint): + con._coef_dirty = False return cls( structural_key=structural_key, diff --git a/linopy/solvers.py b/linopy/solvers.py index 402d780e..59cd6c6f 100644 --- a/linopy/solvers.py +++ b/linopy/solvers.py @@ -698,10 +698,11 @@ def solve( :class:`RebuildRequiredError` instead. The initial build on the first ``solve(model, ...)`` is still allowed. """ - if model is not None: - if self.io_api != "direct": - raise ValueError("solve(model=...) requires io_api='direct'") - with self._lock: + if model is not None and self.io_api != "direct": + raise ValueError("solve(model=...) requires io_api='direct'") + + with self._lock: + if model is not None: if self.solver_model is None: self.model = model self._build() @@ -720,26 +721,26 @@ def solve( ignore_dims=ignore_dims, disallow_rebuild=disallow_rebuild, ) - target = model - else: - target = self.model # type: ignore[assignment] + target = model + else: + target = self.model # type: ignore[assignment] - if self.model is not None and self.model.objective.expression.empty: - raise ValueError( - "No objective has been set on the model. Use `m.add_objective(...)` " - "first (e.g. `m.add_objective(0 * x)` for a pure feasibility problem)." - ) - if self.io_api == "direct" or self.solver_model is not None: - result = self._run_direct(**run_kwargs) - elif self._problem_fn is not None: - result = self._run_file(**run_kwargs) - else: - raise RuntimeError( - "Solver has not been built; call Solver.from_name(...) or _build() first." - ) + if self.model is not None and self.model.objective.expression.empty: + raise ValueError( + "No objective has been set on the model. Use `m.add_objective(...)` " + "first (e.g. `m.add_objective(0 * x)` for a pure feasibility problem)." + ) + if self.io_api == "direct" or self.solver_model is not None: + result = self._run_direct(**run_kwargs) + elif self._problem_fn is not None: + result = self._run_file(**run_kwargs) + else: + raise RuntimeError( + "Solver has not been built; call Solver.from_name(...) or _build() first." + ) - if assign and target is not None: - target.assign_result(result, solver=self) + if assign and target is not None: + target.assign_result(result, solver=self) return result def update( @@ -1942,6 +1943,7 @@ def apply_update( ) if diff.obj_c_indices is not None: + assert diff.obj_c_values is not None var_subset = [gurobi_vars[int(i)] for i in diff.obj_c_indices] gm.setAttr("Obj", var_subset, diff.obj_c_values.tolist()) @@ -2515,6 +2517,7 @@ def apply_update( ) if diff.obj_c_indices is not None: + assert diff.obj_c_values is not None p.chgobj( diff.obj_c_indices.astype(np.int64, copy=False).tolist(), diff.obj_c_values.astype(float, copy=False).tolist(), @@ -3237,6 +3240,7 @@ def apply_update( ) if diff.obj_c_indices is not None: + assert diff.obj_c_values is not None t.putclist( diff.obj_c_indices.astype(np.int32, copy=False).tolist(), diff.obj_c_values.astype(float, copy=False).tolist(), diff --git a/test/test_constraint.py b/test/test_constraint.py index a1b33d66..690da8f6 100644 --- a/test/test_constraint.py +++ b/test/test_constraint.py @@ -421,7 +421,7 @@ def test_constraint_sign_setter_invalid( def test_constraint_rhs_setter(mc: linopy.constraints.Constraint) -> None: sizes = mc.sizes - mc.rhs = 2 # type: ignore + mc.rhs = 2 assert (mc.rhs == 2).all() assert mc.sizes == sizes @@ -429,7 +429,7 @@ def test_constraint_rhs_setter(mc: linopy.constraints.Constraint) -> None: def test_constraint_rhs_setter_with_variable( mc: linopy.constraints.Constraint, x: linopy.Variable ) -> None: - mc.rhs = x # type: ignore + mc.rhs = x assert (mc.rhs == 0).all() assert (mc.coeffs.isel({mc.term_dim: -1}) == -1).all() assert mc.lhs.nterm == 2 diff --git a/test/test_persistent_apply_update.py b/test/test_persistent_apply_update.py index ba108560..8f3d44d7 100644 --- a/test/test_persistent_apply_update.py +++ b/test/test_persistent_apply_update.py @@ -71,15 +71,22 @@ def _built(solver_name: str, model: Model) -> Solver: def _solve(solver: Solver, model: Model) -> float: result = solver.solve(model, assign=True) + assert result.solution is not None return float(result.solution.objective) +def _obj(model: Model) -> float: + value = model.objective.value + assert value is not None + return float(value) + + @pytest.mark.parametrize("solver_name", SOLVER_PARAMS) def test_var_lb_in_place(solver_name: str) -> None: m = _base_model() s = _built(solver_name, m) s.solve(assign=True) - base_obj = float(m.objective.value) + base_obj = _obj(m) m.variables["x"].lower.values[...] = 5.0 obj = _solve(s, m) @@ -106,7 +113,7 @@ def test_rhs_only_in_place(solver_name: str) -> None: m = _base_model() s = _built(solver_name, m) s.solve(assign=True) - base_obj = float(m.objective.value) + base_obj = _obj(m) c = m.constraints["c1"] c.rhs = 8.0 @@ -122,7 +129,7 @@ def test_constraint_coef_change_in_place(solver_name: str) -> None: m = _base_model() s = _built(solver_name, m) s.solve(assign=True) - base_obj = float(m.objective.value) + base_obj = _obj(m) c = m.constraints["c1"] c.coeffs = c.coeffs * 2 @@ -137,7 +144,7 @@ def test_objective_linear_change_in_place(solver_name: str) -> None: m = _base_model() s = _built(solver_name, m) s.solve(assign=True) - base_obj = float(m.objective.value) + base_obj = _obj(m) x = m.variables["x"] y = m.variables["y"] @@ -153,7 +160,7 @@ def test_objective_sense_flip_in_place(solver_name: str) -> None: m = _base_model() s = _built(solver_name, m) s.solve(assign=True) - min_obj = float(m.objective.value) + min_obj = _obj(m) m.objective.sense = "max" max_obj = _solve(s, m) @@ -191,12 +198,12 @@ def test_cross_model_in_place(solver_name: str) -> None: assert s._in_place_updates == 1 assert s._rebuilds == 0 - cross_obj = float(m2.objective.value) + cross_obj = _obj(m2) m3 = _base_model() m3.constraints["c1"].rhs = 8.0 s_fresh = _built(solver_name, m3) s_fresh.solve(assign=True) - assert np.isclose(cross_obj, float(m3.objective.value)) + assert np.isclose(cross_obj, _obj(m3)) @pytest.mark.parametrize("solver_name", SOLVER_PARAMS) diff --git a/test/test_persistent_solver_extras.py b/test/test_persistent_solver_extras.py index f3ea6b97..f9e6f0a0 100644 --- a/test/test_persistent_solver_extras.py +++ b/test/test_persistent_solver_extras.py @@ -58,17 +58,23 @@ def _built(solver_name: str, model: Model) -> Solver: return s +def _obj(model: Model) -> float: + value = model.objective.value + assert value is not None + return float(value) + + @pytest.mark.parametrize("solver_name", SOLVER_PARAMS) def test_noop_resolve_increments_in_place(solver_name: str) -> None: m = _base_model() s = _built(solver_name, m) s.solve(assign=True) - first_obj = float(m.objective.value) + first_obj = _obj(m) s.solve(m, assign=True) assert s._in_place_updates == 1 assert s._rebuilds == 0 - assert np.isclose(float(m.objective.value), first_obj) + assert np.isclose(_obj(m), first_obj) @pytest.mark.parametrize("solver_name", SOLVER_PARAMS) @@ -82,7 +88,7 @@ def test_two_consecutive_solves_no_stale_state(solver_name: str) -> None: s.solve(m, assign=True) assert s.status is not first_status assert s.solution is not None - assert np.isclose(float(s.solution.objective), float(m.objective.value)) + assert np.isclose(float(s.solution.objective), _obj(m)) @pytest.mark.parametrize("solver_name", SOLVER_PARAMS) @@ -95,7 +101,7 @@ def test_cross_model_scenario_sweep(solver_name: str) -> None: s = _built(solver_name, m1) s.solve(assign=True) - obj1 = float(m1.objective.value) + obj1 = _obj(m1) sol1 = m1.solution s.solve(m2, assign=True) @@ -117,7 +123,7 @@ def test_cross_model_scenario_sweep(solver_name: str) -> None: fresh.variables["x"].lower.values[...] = 2.0 s_fresh = _built(solver_name, fresh) s_fresh.solve(assign=True) - assert np.isclose(float(mk.objective.value), float(fresh.objective.value)) + assert np.isclose(_obj(mk), _obj(fresh)) s_fresh.close() @@ -184,7 +190,7 @@ def test_dirty_flag_ignored_across_models(solver_name: str) -> None: cf.coeffs = cf.coeffs * 3 s_fresh = _built(solver_name, fresh) s_fresh.solve(assign=True) - assert np.isclose(float(m2.objective.value), float(fresh.objective.value)) + assert np.isclose(_obj(m2), _obj(fresh)) s_fresh.close() @@ -216,7 +222,7 @@ def test_model_pickle_round_trip_no_native_handle(solver_name: str) -> None: assert s2.solver_model is not None s2.solve(assign=True) assert s2._rebuilds == 0 - assert np.isclose(float(m.objective.value), float(m2.objective.value)) + assert np.isclose(_obj(m), _obj(m2)) s2.close() @@ -257,7 +263,7 @@ def test_concurrent_solves_serialize(solver_name: str) -> None: m = _base_model() s = _built(solver_name, m) s.solve(assign=True) - expected = float(m.objective.value) + expected = _obj(m) barrier = threading.Barrier(2) results: list[float] = [] @@ -267,6 +273,7 @@ def _run() -> None: try: barrier.wait() res = s.solve(m, assign=True) + assert res.solution is not None results.append(float(res.solution.objective)) except BaseException as e: errors.append(e) @@ -338,7 +345,7 @@ def test_scenario_sweep_in_place( _apply_scenario(fresh, scenario) s_fresh = _built(solver_name, fresh) s_fresh.solve(assign=True) - assert np.isclose(float(target.objective.value), float(fresh.objective.value)) + assert np.isclose(_obj(target), _obj(fresh)) s_fresh.close() @@ -428,7 +435,7 @@ def test_track_updates_false_cross_instance_resolve(solver_name: str) -> None: s.options = opts s._build() s.solve(assign=True) - base_obj = float(m1.objective.value) + base_obj = _obj(m1) m2 = _base_model() m2.constraints["c1"].rhs = 8.0 @@ -437,6 +444,7 @@ def test_track_updates_false_cross_instance_resolve(solver_name: str) -> None: assert s._rebuilds == 0 assert s.snapshot is None assert s.model is m2 + assert result.solution is not None assert float(result.solution.objective) > base_obj diff --git a/test/test_persistent_solver_orchestrator.py b/test/test_persistent_solver_orchestrator.py index 4fcdb58f..d622cdf8 100644 --- a/test/test_persistent_solver_orchestrator.py +++ b/test/test_persistent_solver_orchestrator.py @@ -24,11 +24,11 @@ class FakeSolver(Solver[None]): supports_persistent_update = False @classmethod - def is_available(cls) -> bool: + def is_available(cls) -> bool: # type: ignore[override] return True @property - def solver_name(self): # type: ignore[override] + def solver_name(self) -> Any: class _N: value = "fake" From 089cf2e0930dd88c736d1313a6ddad3644ff9122 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 22 May 2026 11:27:32 +0200 Subject: [PATCH 21/21] harden coords comparison --- linopy/persistent/diff.py | 23 ++++++++++++----------- linopy/solvers.py | 12 ++++++------ test/test_persistent_snapshot_diff.py | 3 +-- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/linopy/persistent/diff.py b/linopy/persistent/diff.py index 56133bdb..46a866f2 100644 --- a/linopy/persistent/diff.py +++ b/linopy/persistent/diff.py @@ -344,18 +344,18 @@ def from_snapshot( snapshot: ModelSnapshot, model: Model, same_model: bool = True, - ignore_dims: Iterable[str] | None = None, + ignore_dims: Iterable[str] = (), ) -> ModelDiff: """ Diff ``model`` against a captured ``snapshot``. - Coordinate values are not compared by default. Pass ``ignore_dims`` - (e.g. ``ignore_dims=()`` or ``ignore_dims={"snapshot"}``) to opt into - per-container coord-equality on every dim *not* in the set — a - mismatch triggers ``RebuildReason.COORD_REINDEX``. + Coordinate values are compared on every dim *not* in ``ignore_dims``; + a mismatch triggers ``RebuildReason.COORD_REINDEX``. Pass + ``ignore_dims={"snapshot"}`` for rolling-horizon use cases where the + snapshot coord legitimately shifts between solves. """ - check_coords = ignore_dims is not None - ignored = frozenset(ignore_dims) if ignore_dims is not None else frozenset() + ignored = frozenset(ignore_dims) + check_coords = True diff = cls() var_names = tuple(model.variables) @@ -435,17 +435,18 @@ def from_models( cls, model_a: Model, model_b: Model, - ignore_dims: Iterable[str] | None = None, + ignore_dims: Iterable[str] = (), ) -> ModelDiff: """ Diff two linopy models directly, without capturing a snapshot. ``model_a`` is the baseline, ``model_b`` is the target. The coefficient comparison runs unconditionally — no ``_coef_dirty`` - shortcut applies between independently-built models. + shortcut applies between independently-built models. Coordinates + are compared on every dim not in ``ignore_dims``. """ - check_coords = ignore_dims is not None - ignored = frozenset(ignore_dims) if ignore_dims is not None else frozenset() + ignored = frozenset(ignore_dims) + check_coords = True diff = cls() var_names_a = tuple(model_a.variables) diff --git a/linopy/solvers.py b/linopy/solvers.py index 59cd6c6f..9f3434f6 100644 --- a/linopy/solvers.py +++ b/linopy/solvers.py @@ -676,7 +676,7 @@ def solve( self, model: Model | None = None, assign: bool = False, - ignore_dims: Iterable[str] | None = None, + ignore_dims: Iterable[str] = (), disallow_rebuild: bool = False, **run_kwargs: Any, ) -> Result: @@ -688,9 +688,9 @@ def solve( With ``assign=True`` the Result is written back to the target Model via :meth:`Model.assign_result`. - Pass ``ignore_dims`` (e.g. ``{"snapshot"}``) to opt into per-container - coordinate-equality checking on every dim *not* in the set. Default - (``None``) skips the coord check entirely. + Coordinate alignment is checked on every dim by default. Pass + ``ignore_dims`` to exclude dims whose coord values legitimately shift + between solves. Pass ``disallow_rebuild=True`` to guarantee that an existing solver model is updated in place — any condition that would force a rebuild @@ -747,7 +747,7 @@ def update( self, model: Model, apply: bool = True, - ignore_dims: Iterable[str] | None = None, + ignore_dims: Iterable[str] = (), ) -> ModelDiff: if self.io_api != "direct": raise ValueError("update requires io_api='direct'") @@ -768,7 +768,7 @@ def _update_locked( self, model: Model, apply: bool, - ignore_dims: Iterable[str] | None = None, + ignore_dims: Iterable[str] = (), disallow_rebuild: bool = False, ) -> ModelDiff: if apply and not type(self).supports_persistent_update: diff --git a/test/test_persistent_snapshot_diff.py b/test/test_persistent_snapshot_diff.py index 4be6dfe5..ab48d8e8 100644 --- a/test/test_persistent_snapshot_diff.py +++ b/test/test_persistent_snapshot_diff.py @@ -188,8 +188,7 @@ def test_ignore_dims_detects_coord_change() -> None: m2.add_constraints(m2.variables["x"] >= 0, name="c1") m2.add_objective(m2.variables["x"].sum()) - assert ModelDiff.from_snapshot(snap, m2).rebuild_reason is RebuildReason.NONE - assert ModelDiff.from_snapshot(snap, m2, ignore_dims=()).rebuild_reason is ( + assert ModelDiff.from_snapshot(snap, m2).rebuild_reason is ( RebuildReason.COORD_REINDEX ) assert ModelDiff.from_snapshot(snap, m2, ignore_dims={"t"}).rebuild_reason is (