fix: load clustered NetCDF files written before flixopt 7.0

flixopt/io.py

@@ -1846,13 +1846,72 @@ def _restore_clustering(
         # such files remain loadable.
         clustering_structure.pop('_original_data_refs', None)
         clustering_structure.pop('_metrics_refs', None)
+        clustering_structure = cls._migrate_legacy_clustering_result(clustering_structure, flow_system)
         clustering = fs_cls._resolve_reference_structure(clustering_structure, {})
         flow_system.clustering = clustering
 
         # Restore cluster_weight from clustering's cluster_occurrences
         if hasattr(clustering, 'cluster_occurrences'):
             flow_system.cluster_weight = clustering.cluster_occurrences.rename('cluster_weight')
 
+    LEGACY_SLICE_DIM_RENAMES = {'period': '_period', 'cluster': '_cluster'}
+    LEGACY_KEY_SEPARATOR = '|'
+
+    @classmethod
+    def _migrate_legacy_clustering_result(
+        cls,
+        clustering_structure: dict[str, Any],
+        flow_system: FlowSystem,
+    ) -> dict[str, Any]:
+        """Convert a pre-7.0 ``clustering.results`` blob to the current schema.
+
+        Files written before flixopt 7.0 stored the clustering as
+        ``{'dim_names': [...], 'results': {key: tsam_blob}}``, keyed by the slice
+        coordinates joined into a string (``'2030|low'``, or ``'__single__'`` when
+        undivided). ``Clustering`` now expects a ``clustering_result`` matching
+        ``tsam_xarray.ClusteringResult.from_dict``, which takes a list of
+        ``{'key': [...], 'clustering': tsam_blob}`` entries and indexes them by the
+        coordinate values themselves -- so ``'2030'`` has to become the integer 2030
+        or every lookup silently misses. The values are recovered by matching against
+        the coordinates already restored on ``flow_system``, rather than by guessing a
+        type, so labels that merely look numeric survive unchanged.
+
+        The per-slice tsam blobs are identical between the two layouts; only the
+        structure around them is rewritten.
+        """
+        if 'clustering_result' in clustering_structure or 'results' not in clustering_structure:
+            return clustering_structure
+
+        legacy = clustering_structure.pop('results')
+        dims = list(legacy.get('dim_names') or [])
+
+        lookups = []
+        for dim in dims:
+            index = getattr(flow_system, f'{dim}s', None)
+            values = [] if index is None else list(index)
+            # .item() unwraps numpy scalars so the keys stay plain Python values
+            lookups.append({str(value): value.item() if hasattr(value, 'item') else value for value in values})
+
+        clusterings = []
+        for key, blob in legacy.get('results', {}).items():
+            # Split only when several dims share the key, so single-dim labels
+            # containing the separator stay intact.
+            parts = key.split(cls.LEGACY_KEY_SEPARATOR) if len(dims) > 1 else [key]
+            clusterings.append(
+                {
+                    'key': [lookup.get(part, part) for lookup, part in zip(lookups, parts, strict=False)],
+                    'clustering': blob,
+                }
+            )
+
+        clustering_structure['clustering_result'] = {
+            'time_dim': 'time',
+            'cluster_dim': ['variable'],
+            'slice_dims': [cls.LEGACY_SLICE_DIM_RENAMES.get(dim, dim) for dim in dims],
+            'clusterings': clusterings,
+        }
+        return clustering_structure
+
     @staticmethod
     def _restore_metadata(
         flow_system: FlowSystem,

tests/test_clustering/test_clustering_io.py

@@ -521,50 +521,51 @@ def test_clustering_preserves_component_labels(self, simple_system_8_days, solve
         assert 'source' in fs_expanded.components
 
 
-class TestMultiDimensionalClusteringIO:
-    """Test IO for clustering with both periods and scenarios (multi-dimensional)."""
+@pytest.fixture
+def system_with_periods_and_scenarios():
+    """Create a flow system with both periods and scenarios, with different demand patterns."""
+    n_days = 3
+    hours = 24 * n_days
+    timesteps = pd.date_range('2024-01-01', periods=hours, freq='h')
+    periods = pd.Index([2024, 2025], name='period')
+    scenarios = pd.Index(['high', 'low'], name='scenario')
+
+    # Create DIFFERENT demand patterns per period/scenario to get different cluster assignments
+    # Pattern structure: (base_mean, amplitude) for each day
+    patterns = {
+        (2024, 'high'): [(100, 40), (100, 40), (50, 20)],  # Days 0&1 similar
+        (2024, 'low'): [(50, 20), (100, 40), (100, 40)],  # Days 1&2 similar
+        (2025, 'high'): [(100, 40), (50, 20), (100, 40)],  # Days 0&2 similar
+        (2025, 'low'): [(50, 20), (50, 20), (100, 40)],  # Days 0&1 similar
+    }
+
+    demand_values = np.zeros((hours, len(periods), len(scenarios)))
+    for pi, period in enumerate(periods):
+        for si, scenario in enumerate(scenarios):
+            base = np.zeros(hours)
+            for d, (mean, amp) in enumerate(patterns[(period, scenario)]):
+                start = d * 24
+                base[start : start + 24] = mean + amp * np.sin(np.linspace(0, 2 * np.pi, 24))
+            demand_values[:, pi, si] = base
+
+    demand = xr.DataArray(
+        demand_values,
+        dims=['time', 'period', 'scenario'],
+        coords={'time': timesteps, 'period': periods, 'scenario': scenarios},
+    )
 
-    @pytest.fixture
-    def system_with_periods_and_scenarios(self):
-        """Create a flow system with both periods and scenarios, with different demand patterns."""
-        n_days = 3
-        hours = 24 * n_days
-        timesteps = pd.date_range('2024-01-01', periods=hours, freq='h')
-        periods = pd.Index([2024, 2025], name='period')
-        scenarios = pd.Index(['high', 'low'], name='scenario')
-
-        # Create DIFFERENT demand patterns per period/scenario to get different cluster assignments
-        # Pattern structure: (base_mean, amplitude) for each day
-        patterns = {
-            (2024, 'high'): [(100, 40), (100, 40), (50, 20)],  # Days 0&1 similar
-            (2024, 'low'): [(50, 20), (100, 40), (100, 40)],  # Days 1&2 similar
-            (2025, 'high'): [(100, 40), (50, 20), (100, 40)],  # Days 0&2 similar
-            (2025, 'low'): [(50, 20), (50, 20), (100, 40)],  # Days 0&1 similar
-        }
-
-        demand_values = np.zeros((hours, len(periods), len(scenarios)))
-        for pi, period in enumerate(periods):
-            for si, scenario in enumerate(scenarios):
-                base = np.zeros(hours)
-                for d, (mean, amp) in enumerate(patterns[(period, scenario)]):
-                    start = d * 24
-                    base[start : start + 24] = mean + amp * np.sin(np.linspace(0, 2 * np.pi, 24))
-                demand_values[:, pi, si] = base
-
-        demand = xr.DataArray(
-            demand_values,
-            dims=['time', 'period', 'scenario'],
-            coords={'time': timesteps, 'period': periods, 'scenario': scenarios},
-        )
+    fs = fx.FlowSystem(timesteps, periods=periods, scenarios=scenarios)
+    fs.add_elements(
+        fx.Bus('heat'),
+        fx.Effect('costs', unit='EUR', description='costs', is_objective=True, is_standard=True),
+        fx.Sink('demand', inputs=[fx.Flow('in', bus='heat', fixed_relative_profile=demand, size=1)]),
+        fx.Source('source', outputs=[fx.Flow('out', bus='heat', size=200, effects_per_flow_hour={'costs': 0.05})]),
+    )
+    return fs
 
-        fs = fx.FlowSystem(timesteps, periods=periods, scenarios=scenarios)
-        fs.add_elements(
-            fx.Bus('heat'),
-            fx.Effect('costs', unit='EUR', description='costs', is_objective=True, is_standard=True),
-            fx.Sink('demand', inputs=[fx.Flow('in', bus='heat', fixed_relative_profile=demand, size=1)]),
-            fx.Source('source', outputs=[fx.Flow('out', bus='heat', size=200, effects_per_flow_hour={'costs': 0.05})]),
-        )
-        return fs
+
+class TestMultiDimensionalClusteringIO:
+    """Test IO for clustering with both periods and scenarios (multi-dimensional)."""
 
     def test_cluster_assignments_has_correct_dimensions(self, system_with_periods_and_scenarios):
         """cluster_assignments should have dimensions for original_cluster, period, and scenario."""
@@ -761,3 +762,114 @@ def test_load_legacy_clustered_netcdf(self, simple_system_8_days, tmp_path):
 
         assert isinstance(fs_restored.clustering, Clustering)
         assert fs_restored.clustering.n_clusters == 2
+
+
+class TestPreV7ClusteringSchema:
+    """Loading clustered files written before flixopt 7.0, which serialized the
+    clustering under a ``results`` key with string-joined slice keys instead of the
+    ``clustering_result`` schema that tsam_xarray's ClusteringResult now expects.
+
+    The per-slice tsam blobs are unchanged between the two layouts, so these tests
+    rewrite a current dataset into the legacy layout rather than shipping a binary
+    fixture. The layout was verified against files generated by flixopt 6.2.1.
+    """
+
+    def _to_legacy_schema(self, ds: xr.Dataset) -> xr.Dataset:
+        """Rewrite a clustered dataset's clustering attrs into the pre-7.0 layout."""
+        import json
+
+        clustering = json.loads(ds.attrs['clustering'])
+        result = clustering.pop('clustering_result')
+
+        unrename = {'_period': 'period', '_cluster': 'cluster'}
+        dim_names = [unrename.get(dim, dim) for dim in result['slice_dims']]
+
+        legacy_results = {}
+        for entry in result['clusterings']:
+            key = '|'.join(str(part) for part in entry['key']) if entry['key'] else '__single__'
+            legacy_results[key] = entry['clustering']
+
+        clustering['results'] = {'dim_names': dim_names, 'results': legacy_results}
+        # Pre-7.0 files always carried these; they are dropped on load.
+        clustering['_original_data_refs'] = []
+        clustering['_metrics_refs'] = None
+        ds.attrs['clustering'] = json.dumps(clustering, ensure_ascii=False)
+        return ds
+
+    def _assignments(self, flow_system) -> xr.DataArray:
+        return flow_system.clustering.cluster_assignments
+
+    def test_legacy_schema_is_rejected_without_migration(self, simple_system_8_days):
+        """Sanity check: the legacy key really is incompatible with Clustering.__init__."""
+        from flixopt.clustering import Clustering
+
+        fs_clustered = simple_system_8_days.transform.cluster(n_clusters=2, cluster_duration='1D')
+        ds = self._to_legacy_schema(fs_clustered.to_dataset(include_solution=False))
+
+        import json
+
+        legacy = json.loads(ds.attrs['clustering'])
+        assert 'results' in legacy and 'clustering_result' not in legacy
+        with pytest.raises(TypeError):
+            Clustering(results=legacy['results'])
+
+    def test_load_legacy_schema_preserves_assignments(self, simple_system_8_days):
+        """A pre-7.0 clustered dataset loads with its cluster assignments intact."""
+        from flixopt.clustering import Clustering
+
+        fs_clustered = simple_system_8_days.transform.cluster(n_clusters=2, cluster_duration='1D')
+        expected = self._assignments(fs_clustered).values.copy()
+        ds = self._to_legacy_schema(fs_clustered.to_dataset(include_solution=False))
+
+        fs_restored = fx.FlowSystem.from_dataset(ds)
+
+        assert isinstance(fs_restored.clustering, Clustering)
+        assert fs_restored.clustering.n_clusters == 2
+        np.testing.assert_array_equal(self._assignments(fs_restored).values, expected)
+
+    def test_load_legacy_schema_netcdf_roundtrip(self, simple_system_8_days, tmp_path):
+        """Same, but through a real NetCDF file rather than an in-memory dataset."""
+        from flixopt.io import save_dataset_to_netcdf
+
+        fs_clustered = simple_system_8_days.transform.cluster(n_clusters=2, cluster_duration='1D')
+        expected = self._assignments(fs_clustered).values.copy()
+        ds = self._to_legacy_schema(fs_clustered.to_dataset(include_solution=False))
+
+        nc_path = tmp_path / 'legacy_schema.nc'
+        save_dataset_to_netcdf(ds, nc_path)
+        fs_restored = fx.FlowSystem.from_netcdf(nc_path)
+
+        np.testing.assert_array_equal(self._assignments(fs_restored).values, expected)
+
+    def test_legacy_schema_expands_to_full_timesteps(self, simple_system_8_days):
+        """A loaded pre-7.0 file can still be expanded back to the original timesteps."""
+        fs_clustered = simple_system_8_days.transform.cluster(n_clusters=2, cluster_duration='1D')
+        ds = self._to_legacy_schema(fs_clustered.to_dataset(include_solution=False))
+
+        fs_expanded = fx.FlowSystem.from_dataset(ds).transform.expand()
+
+        assert len(fs_expanded.timesteps) == 8 * 24
+
+    def test_legacy_multi_dim_keys_map_to_the_right_slice(self, system_with_periods_and_scenarios):
+        """Legacy keys are strings ('2024|high'); each must land on its own slice.
+
+        Sensitivity: if the string keys were parsed without recovering the original
+        coordinate dtype, every lookup would miss and the assignments would silently
+        collapse onto one slice or swap between periods.
+        """
+        fs_clustered = system_with_periods_and_scenarios.transform.cluster(n_clusters=2, cluster_duration='1D')
+        expected = self._assignments(fs_clustered)
+        ds = self._to_legacy_schema(fs_clustered.to_dataset(include_solution=False))
+
+        fs_restored = fx.FlowSystem.from_dataset(ds)
+        restored = self._assignments(fs_restored)
+
+        assert set(restored.dims) == set(expected.dims)
+        for period in fs_restored.periods:
+            for scenario in fs_restored.scenarios:
+                sel = {'period': period, 'scenario': scenario}
+                np.testing.assert_array_equal(
+                    restored.sel(sel).values,
+                    expected.sel(sel).values,
+                    err_msg=f'assignments differ for {sel}',
+                )