From bae11138afd593b16c23d8abaa2b682e1e3861bc Mon Sep 17 00:00:00 2001 From: cwasicki <126617870+cwasicki@users.noreply.github.com> Date: Tue, 18 Aug 2026 14:58:13 +0200 Subject: [PATCH 1/2] Reset release notes Signed-off-by: cwasicki <126617870+cwasicki@users.noreply.github.com> --- RELEASE_NOTES.md | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index fc8def2..d466455 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -6,18 +6,7 @@ ## Upgrading -- Microgrid config files should nest their entries under `assets.microgrids`: - - ```toml - assets.microgrids.23.meta.name = "..." - ``` - - Bare microgrid IDs at the top level still load but log a deprecation warning, - and support for them will be removed. The namespace keeps generated inventory - data apart from operator settings under `app.*`, so a config file can carry - both without its keys colliding. - - A file mixing both layouts is rejected. + ## New Features From c1c74936e2c5e84357ae8f3a71a7647ecf101396 Mon Sep 17 00:00:00 2001 From: cwasicki <126617870+cwasicki@users.noreply.github.com> Date: Tue, 18 Aug 2026 12:57:55 +0200 Subject: [PATCH 2/2] feat(config): add `AssetsConfig` for the whole config document Gives the `assets` namespace a type instead of string literals threaded through dict navigation, so the entities still to come are added as fields rather than as more lookup code. The document-level loader moves onto it, since what a config file holds is an assets document, of which microgrids are one entity. Entries are checked against the ID they are filed under in `__post_init__`, so the check also covers readers that load the class directly rather than through `load_from_file`. Unknown entity tables are skipped with a warning, so an older reader keeps working against files that already carry newer entities. Signed-off-by: cwasicki <126617870+cwasicki@users.noreply.github.com> --- RELEASE_NOTES.md | 17 +++- src/frequenz/gridpool/__init__.py | 2 + src/frequenz/gridpool/cli/__main__.py | 6 +- src/frequenz/gridpool/config/__init__.py | 2 + src/frequenz/gridpool/config/assets.py | 112 ++++++++++++++++++++++ src/frequenz/gridpool/config/load.py | 3 +- src/frequenz/gridpool/config/microgrid.py | 65 ------------- tests/test_config.py | 34 ++++++- 8 files changed, 166 insertions(+), 75 deletions(-) create mode 100644 src/frequenz/gridpool/config/assets.py diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index d466455..592d704 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -6,11 +6,24 @@ ## Upgrading - +- `MicrogridConfig.load_from_file` is replaced by `AssetsConfig.load_from_file`, + which returns the whole document rather than just its microgrids: + + ```python + configs = AssetsConfig.load_from_file(path).microgrids + ``` + + `load_configs_from_files` and `load_configs` are unchanged. ## New Features - +- `AssetsConfig` gives the `assets` namespace a type, so the entities still to + come are added as fields rather than as more dict lookups. Entries are checked + against the ID they are filed under wherever the class is loaded, not only via + `load_from_file`. + + Entity tables a version does not know are ignored with a warning, so a reader + keeps working against files that already carry newer entities. ## Bug Fixes diff --git a/src/frequenz/gridpool/__init__.py b/src/frequenz/gridpool/__init__.py index fa68cdb..15830f4 100644 --- a/src/frequenz/gridpool/__init__.py +++ b/src/frequenz/gridpool/__init__.py @@ -15,12 +15,14 @@ merge_config_maps, merge_microgrid_configs, ) +from .config.assets import AssetsConfig __all__ = [ "ComponentGraphConfig", "ComponentGraphGenerator", "FormulaOverrides", "Metadata", + "AssetsConfig", "MicrogridConfig", "load_configs", "load_configs_from_api", diff --git a/src/frequenz/gridpool/cli/__main__.py b/src/frequenz/gridpool/cli/__main__.py index 159cc67..85ab652 100644 --- a/src/frequenz/gridpool/cli/__main__.py +++ b/src/frequenz/gridpool/cli/__main__.py @@ -12,9 +12,9 @@ from frequenz.client.common.microgrid import MicrogridId from frequenz.gridpool import ( + AssetsConfig, ComponentGraphConfig, ComponentGraphGenerator, - MicrogridConfig, load_configs, ) from frequenz.gridpool.cli._dump_config import dump_map @@ -207,7 +207,9 @@ async def generate_config( ids = list(dict.fromkeys(microgrid_ids)) or None if inplace and ids is None: assert default_file is not None - ids = sorted(int(mid) for mid in MicrogridConfig.load_from_file(default_file)) + ids = sorted( + int(mid) for mid in AssetsConfig.load_from_file(default_file).microgrids + ) async with AssetsApiClient(url, auth_key=key, sign_secret=secret) as client: if inplace: diff --git a/src/frequenz/gridpool/config/__init__.py b/src/frequenz/gridpool/config/__init__.py index 7be40f1..b0d399a 100644 --- a/src/frequenz/gridpool/config/__init__.py +++ b/src/frequenz/gridpool/config/__init__.py @@ -5,6 +5,7 @@ from frequenz.microgrid_component_graph import ComponentGraphConfig, FormulaOverrides +from .assets import AssetsConfig from .load import ( load_configs, load_configs_from_api, @@ -31,6 +32,7 @@ "ComponentTypeConfig", "FormulaOverrides", "Metadata", + "AssetsConfig", "MicrogridConfig", "PVConfig", "WindConfig", diff --git a/src/frequenz/gridpool/config/assets.py b/src/frequenz/gridpool/config/assets.py new file mode 100644 index 0000000..4785f38 --- /dev/null +++ b/src/frequenz/gridpool/config/assets.py @@ -0,0 +1,112 @@ +# License: MIT +# Copyright © 2026 Frequenz Energy-as-a-Service GmbH + +"""Data model for the `assets` config namespace.""" + +import logging +import tomllib +from dataclasses import field +from pathlib import Path +from typing import Any, ClassVar, Self, Type + +import marshmallow +from marshmallow import Schema +from marshmallow_dataclass import dataclass + +from .microgrid import MicrogridConfig + +_logger = logging.getLogger(__name__) + + +@dataclass +class AssetsConfig: + """Entities described by a config document, keyed by their ID.""" + + microgrids: dict[str, MicrogridConfig] = field(default_factory=dict) + """Microgrids, keyed by microgrid ID.""" + + class Meta: + """Ignore entity tables this version does not know about. + + A reader must keep working against files that already carry entities + added after it, so unknown tables are skipped rather than rejected. + `_warn_unknown_entities` reports them, so a mistyped table is still + visible instead of silently loading as empty. + """ + + unknown = marshmallow.EXCLUDE + + Schema: ClassVar[Type[Schema]] = Schema + + def __post_init__(self) -> None: + """Check that each entry is filed under its own ID. + + Raises: + ValueError: If a key is not a numeric microgrid ID, or does not + match its entry's `meta.microgrid_id`. + """ + for mid, cfg in self.microgrids.items(): + if not mid.isdigit(): + raise ValueError(f"Microgrid ID key must be numeric, got {mid}") + if int(cfg.meta.microgrid_id) != int(mid): + raise ValueError( + f"Microgrid ID mismatch: key {mid} != {cfg.meta.microgrid_id}" + ) + + @classmethod + def _warn_unknown_entities(cls, assets: dict[str, Any], source: Path) -> None: + """Warn about entity tables that this version drops on load.""" + if unknown := sorted(set(assets) - set(cls.Schema().fields)): + _logger.warning( + "%s: ignoring unknown entity tables under `assets`: %s", + source, + ", ".join(unknown), + ) + + @classmethod + def load_from_file(cls, config_path: Path) -> Self: + """Load and validate a config document from a TOML file. + + Entries live under `assets`. A document without an `assets` table is + read in the deprecated layout, where the microgrid entries sit at the + top level. + + Args: + config_path: The path to the TOML configuration file. + + Returns: + The loaded configuration. + + Raises: + TypeError: If `assets` is not a table. + ValueError: If both layouts are present, which means a + half-migrated file rather than a merge. + """ + with config_path.open("rb") as f: + data: dict[str, Any] = tomllib.load(f) + + if "assets" not in data: + _logger.warning( + "%s: top-level microgrid IDs are deprecated, " + "nest the entries under `assets.microgrids` instead.", + config_path, + ) + data = {"assets": {"microgrids": data}} + + assets = data["assets"] + if not isinstance(assets, dict): + raise TypeError( + f"{config_path}: `assets` must be a table, got {type(assets)}" + ) + + if unprefixed := sorted(k for k in data if k != "assets"): + raise ValueError( + f"{config_path}: keys {unprefixed} sit outside `assets` while the " + "file already has an `assets` table; move them under " + "`assets.microgrids`." + ) + + cls._warn_unknown_entities(assets, config_path) + loaded = cls.Schema().load(assets) + assert isinstance(loaded, cls) + return loaded diff --git a/src/frequenz/gridpool/config/load.py b/src/frequenz/gridpool/config/load.py index 5aaf170..2dacf64 100644 --- a/src/frequenz/gridpool/config/load.py +++ b/src/frequenz/gridpool/config/load.py @@ -24,6 +24,7 @@ pv_inverter_ids, pv_meter_ids, ) +from .assets import AssetsConfig from .microgrid import ( ComponentTypeConfig, Metadata, @@ -168,7 +169,7 @@ def load_configs_from_files( _logger.warning("Config path %s is not a file, skipping.", config_path) continue - mcfgs = MicrogridConfig.load_from_file(config_path) + mcfgs = AssetsConfig.load_from_file(config_path).microgrids microgrid_configs.update({str(key): value for key, value in mcfgs.items()}) return microgrid_configs diff --git a/src/frequenz/gridpool/config/microgrid.py b/src/frequenz/gridpool/config/microgrid.py index b59c356..be0c6d5 100644 --- a/src/frequenz/gridpool/config/microgrid.py +++ b/src/frequenz/gridpool/config/microgrid.py @@ -5,11 +5,9 @@ import logging import re -import tomllib from copy import deepcopy from dataclasses import field from datetime import datetime -from pathlib import Path from typing import Any, ClassVar, Literal, Self, Type, cast, get_args from marshmallow import Schema @@ -337,69 +335,6 @@ def _load_table_entries(cls, data: dict[str, Any]) -> dict[str, Self]: return mgrids - @classmethod - def _microgrid_table(cls, data: dict[str, Any], source: str) -> dict[str, Any]: - """Pick the microgrid entries out of a parsed config document. - - Entries live under `assets.microgrids`. A document without an `assets` - table is read in the deprecated layout, where the entries sit at the - top level. - - Args: - data: The parsed TOML document. - source: Name of the document, used in messages. - - Returns: - The table mapping microgrid IDs to their entries. - - Raises: - TypeError: If `assets` is not a table. - ValueError: If both layouts are present, which means a - half-migrated file rather than a merge. - """ - if "assets" not in data: - _logger.warning( - "%s: top-level microgrid IDs are deprecated, " - "nest the entries under `assets.microgrids` instead.", - source, - ) - return data - - assets = data["assets"] - if not isinstance(assets, dict): - raise TypeError(f"{source}: `assets` must be a table, got {type(assets)}") - - if unprefixed := sorted(k for k in data if k != "assets"): - raise ValueError( - f"{source}: keys {unprefixed} sit outside `assets` while the file " - "already has an `assets` table; move them under `assets.microgrids`." - ) - - microgrids = assets.get("microgrids", {}) - if not isinstance(microgrids, dict): - raise TypeError( - f"{source}: `assets.microgrids` must be a table, got {type(microgrids)}" - ) - return microgrids - - @classmethod - def load_from_file(cls, config_path: Path) -> dict[str, Self]: - """ - Load and validate configuration settings from a TOML file. - - Args: - config_path: the path to the TOML configuration file. - - Returns: - A dict mapping microgrid IDs to MicrogridConfig instances. - """ - with config_path.open("rb") as f: - data = tomllib.load(f) - - assert isinstance(data, dict) - - return cls._load_table_entries(cls._microgrid_table(data, str(config_path))) - def merge_microgrid_configs( base: MicrogridConfig, diff --git a/tests/test_config.py b/tests/test_config.py index 8102a19..6629cea 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -16,6 +16,7 @@ load_configs, load_configs_from_files, ) +from frequenz.gridpool.config.assets import AssetsConfig VALID_CONFIG: dict[str, dict[str, Any]] = { "1": { @@ -189,9 +190,9 @@ def _write(tmp_path: Path, name: str, text: str) -> Path: def test_load_prefixed(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: """Entries under `assets.microgrids` load without a deprecation warning.""" - configs = MicrogridConfig.load_from_file( + configs = AssetsConfig.load_from_file( _write(tmp_path, "prefixed.toml", _PREFIXED_TOML) - ) + ).microgrids assert configs["1"].meta.name == "Test Grid" assert configs["1"].component_type_ids("pv") == [101, 102] @@ -203,7 +204,7 @@ def test_load_legacy_warns(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> path = _write(tmp_path, "legacy.toml", _LEGACY_TOML) with caplog.at_level(logging.WARNING): - configs = MicrogridConfig.load_from_file(path) + configs = AssetsConfig.load_from_file(path).microgrids assert configs["1"].meta.name == "Test Grid" assert "deprecated" in caplog.text @@ -219,14 +220,14 @@ def test_load_mixed_layouts_rejected(tmp_path: Path) -> None: ) with pytest.raises(ValueError, match="outside `assets`"): - MicrogridConfig.load_from_file(path) + AssetsConfig.load_from_file(path) def test_load_assets_without_microgrids(tmp_path: Path) -> None: """An `assets` table without microgrids yields no configs and no warning.""" path = _write(tmp_path, "other.toml", 'assets.gridpool.7.name = "GP"\n') - assert not MicrogridConfig.load_from_file(path) + assert not AssetsConfig.load_from_file(path).microgrids async def test_merge_prefixed_base_with_legacy_override(tmp_path: Path) -> None: @@ -240,3 +241,26 @@ async def test_merge_prefixed_base_with_legacy_override(tmp_path: Path) -> None: assert configs["1"].meta.name == "Renamed" assert configs["1"].component_type_ids("pv") == [101, 102] + + +def test_assets_config_rejects_mismatched_id() -> None: + """An entry filed under the wrong ID is rejected wherever it is loaded.""" + with pytest.raises(ValueError, match="Microgrid ID mismatch"): + AssetsConfig.Schema().load( + {"microgrids": {"23": {"meta": {"microgrid_id": 99}}}} + ) + + +def test_assets_config_warns_on_unknown_entities( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """Entity tables this version does not know are skipped, but reported.""" + path = _write( + tmp_path, "future.toml", _PREFIXED_TOML + 'assets.gridpool.7.name = "GP"\n' + ) + + with caplog.at_level(logging.WARNING): + config = AssetsConfig.load_from_file(path) + + assert sorted(config.microgrids) == ["1"] + assert "gridpool" in caplog.text