-
Notifications
You must be signed in to change notification settings - Fork 2.2k
Python: Isolate checkpoint state from live workflow state across restoration and storage boundaries #7697
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Python: Isolate checkpoint state from live workflow state across restoration and storage boundaries #7697
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,6 @@ | ||
| # Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| import copy | ||
| from typing import Any | ||
|
|
||
|
|
||
|
|
@@ -104,18 +105,31 @@ def discard(self) -> None: | |
| self._pending.clear() | ||
|
|
||
| def export_state(self) -> dict[str, Any]: | ||
| """Export a serialized copy of the committed state. | ||
| """Export an isolated copy of the committed state. | ||
|
|
||
| Note: Does not include pending changes. | ||
|
|
||
| The returned dict (and any mutable containers reachable through its | ||
| values) is a deep copy, so later mutations to the live ``State`` — | ||
| including in-place mutation of values handed out by :meth:`get` — | ||
| will not be reflected in the exported snapshot. This matches the | ||
| isolation that :class:`RunnerContext` already applies to ``messages`` | ||
| in the checkpoint-construction path and is what makes a snapshot | ||
| safe to share with another workflow instance. | ||
| """ | ||
| return dict(self._committed) | ||
| return copy.deepcopy(self._committed) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Checkpoint state supports custom pickle-serializable values through |
||
|
|
||
| def import_state(self, state: dict[str, Any]) -> None: | ||
| """Import state from a serialized dictionary. | ||
|
|
||
| Merges into committed state. Does not affect pending changes. | ||
|
|
||
| The incoming ``state`` dict is deep-copied before merge, so later | ||
| in-place mutations of the caller's dict (or any mutable container | ||
| reachable through its values) will not leak into the committed | ||
| state. This is the import-side counterpart to :meth:`export_state`. | ||
| """ | ||
| self._committed.update(state) | ||
| self._committed.update(copy.deepcopy(state)) | ||
|
|
||
|
|
||
| class _DeleteSentinelType: | ||
|
|
||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Prefer adding the new tests to the existing |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,198 @@ | ||
| # Copyright (c) Microsoft. All rights reserved. | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Please replace this file with The suite is parametrized over memory and file backends so the ownership contract is asserted for every in-tree storage, not only Same file is in #7712 # Copyright (c) Microsoft. All rights reserved.
"""Conformance tests for the CheckpointStorage ownership contract.
A checkpoint handed to the caller is owned by the caller, and a checkpoint
handed to ``save()`` is snapshotted at call time. Backends that serialize
(file, Cosmos) get this for free because decoding allocates a fresh object
graph; backends that hold live objects must copy explicitly.
"""
from __future__ import annotations
from pathlib import Path
from typing import Any, cast
import pytest
from agent_framework import (
CheckpointStorage,
FileCheckpointStorage,
InMemoryCheckpointStorage,
WorkflowCheckpoint,
WorkflowCheckpointException,
WorkflowEvent,
)
from agent_framework._workflows._runner_context import WorkflowMessage
@pytest.fixture(params=["memory", "file"])
def conformance_storage(request: pytest.FixtureRequest, tmp_path: Path) -> CheckpointStorage:
"""Yield each in-tree checkpoint storage backend for the shared contract tests."""
if request.param == "memory":
return InMemoryCheckpointStorage()
return FileCheckpointStorage(tmp_path / "conformance")
def _conformance_checkpoint(workflow_name: str = "conformance-workflow") -> WorkflowCheckpoint:
"""Build a checkpoint whose object graph holds nested mutable containers."""
return WorkflowCheckpoint(
workflow_name=workflow_name,
graph_signature_hash="conformance-hash",
state={
"shared": {"counter": 0, "history": ["initial"]},
"_executor_state": {"executor1": {"visits": ["first"]}},
},
messages={
"executor1": [
WorkflowMessage(
data={"text": "hello", "tags": ["initial"]},
source_id="src",
target_id="tgt",
)
]
},
pending_request_info_events={
"req1": WorkflowEvent.request_info(
request_id="req1",
source_executor_id="executor1",
request_data={"payload": ["initial"]},
response_type=str,
),
},
metadata={"tags": ["initial"]},
)
def _mutate(checkpoint: WorkflowCheckpoint) -> None:
"""Mutate every nested container the ownership contract covers."""
checkpoint.state["shared"]["counter"] = 999
checkpoint.state["shared"]["history"].append("mutated")
checkpoint.state["_executor_state"]["executor1"]["visits"].append("mutated")
cast(dict[str, Any], checkpoint.messages["executor1"][0].data)["tags"].append("mutated")
cast(dict[str, Any], checkpoint.pending_request_info_events["req1"].data)["payload"].append("mutated")
checkpoint.metadata["tags"].append("mutated")
def _assert_pristine(checkpoint: WorkflowCheckpoint) -> None:
"""Assert nested containers still hold their original values."""
assert checkpoint.state["shared"]["counter"] == 0
assert checkpoint.state["shared"]["history"] == ["initial"]
assert checkpoint.state["_executor_state"]["executor1"]["visits"] == ["first"]
assert cast(dict[str, Any], checkpoint.messages["executor1"][0].data)["tags"] == ["initial"]
assert cast(dict[str, Any], checkpoint.pending_request_info_events["req1"].data)["payload"] == ["initial"]
assert checkpoint.metadata["tags"] == ["initial"]
async def test_conformance_load_returns_caller_owned_copy(conformance_storage: CheckpointStorage) -> None:
"""Mutating a loaded checkpoint must not alter what the backend has stored."""
checkpoint = _conformance_checkpoint()
await conformance_storage.save(checkpoint)
loaded = await conformance_storage.load(checkpoint.checkpoint_id)
_mutate(loaded)
reloaded = await conformance_storage.load(checkpoint.checkpoint_id)
_assert_pristine(reloaded)
async def test_conformance_repeated_loads_are_independent(conformance_storage: CheckpointStorage) -> None:
"""Two loads of one checkpoint must not share mutable state."""
checkpoint = _conformance_checkpoint()
await conformance_storage.save(checkpoint)
first = await conformance_storage.load(checkpoint.checkpoint_id)
second = await conformance_storage.load(checkpoint.checkpoint_id)
assert first is not second
_mutate(first)
_assert_pristine(second)
async def test_conformance_get_latest_returns_caller_owned_copy(conformance_storage: CheckpointStorage) -> None:
"""Mutating the result of get_latest must not alter stored state."""
checkpoint = _conformance_checkpoint()
await conformance_storage.save(checkpoint)
latest = await conformance_storage.get_latest(workflow_name=checkpoint.workflow_name)
assert latest is not None
_mutate(latest)
reloaded = await conformance_storage.get_latest(workflow_name=checkpoint.workflow_name)
assert reloaded is not None
_assert_pristine(reloaded)
async def test_conformance_list_checkpoints_returns_caller_owned_copies(
conformance_storage: CheckpointStorage,
) -> None:
"""Mutating a checkpoint from list_checkpoints must not alter stored state."""
checkpoint = _conformance_checkpoint()
await conformance_storage.save(checkpoint)
listed = await conformance_storage.list_checkpoints(workflow_name=checkpoint.workflow_name)
assert len(listed) == 1
_mutate(listed[0])
relisted = await conformance_storage.list_checkpoints(workflow_name=checkpoint.workflow_name)
assert len(relisted) == 1
_assert_pristine(relisted[0])
async def test_conformance_save_snapshots_state_at_call_time(conformance_storage: CheckpointStorage) -> None:
"""Mutating the caller's object after save must not alter the stored checkpoint."""
checkpoint = _conformance_checkpoint()
await conformance_storage.save(checkpoint)
_mutate(checkpoint)
loaded = await conformance_storage.load(checkpoint.checkpoint_id)
_assert_pristine(loaded)
async def test_conformance_list_checkpoints_filters_by_workflow_name(
conformance_storage: CheckpointStorage,
) -> None:
"""list_checkpoints must continue to filter by workflow_name after isolation."""
first = _conformance_checkpoint(workflow_name="workflow-a")
second = _conformance_checkpoint(workflow_name="workflow-b")
await conformance_storage.save(first)
await conformance_storage.save(second)
listed_a = await conformance_storage.list_checkpoints(workflow_name="workflow-a")
listed_b = await conformance_storage.list_checkpoints(workflow_name="workflow-b")
assert {cp.checkpoint_id for cp in listed_a} == {first.checkpoint_id}
assert {cp.checkpoint_id for cp in listed_b} == {second.checkpoint_id}
async def test_conformance_get_latest_returns_none_when_empty(conformance_storage: CheckpointStorage) -> None:
"""get_latest must return None for an unknown workflow, not raise."""
result = await conformance_storage.get_latest(workflow_name="missing-workflow")
assert result is None
async def test_conformance_load_missing_id_raises(conformance_storage: CheckpointStorage) -> None:
"""load must raise WorkflowCheckpointException for an unknown checkpoint id."""
with pytest.raises(WorkflowCheckpointException):
await conformance_storage.load("does-not-exist")
async def test_conformance_save_returns_id(conformance_storage: CheckpointStorage) -> None:
"""save must return the checkpoint_id."""
checkpoint = _conformance_checkpoint()
result = await conformance_storage.save(checkpoint)
assert result == checkpoint.checkpoint_id |
||
|
|
||
| """Regression tests for https://github.com/microsoft/agent-framework/issues/7683. | ||
|
|
||
| The InMemoryCheckpointStorage must return deep copies from | ||
| ``load()``, ``list_checkpoints()``, and ``get_latest()`` so that callers | ||
| cannot mutate the storage backend's internal checkpoint objects. This | ||
| matches the defensive copy already applied in ``save()``. | ||
|
|
||
| These tests import directly from ``_workflows._checkpoint`` and | ||
| ``agent_framework.exceptions`` to avoid pulling the top-level | ||
| ``agent_framework`` package, which has heavy runtime dependencies | ||
| (opentelemetry, etc.) that are not relevant to the isolation contract | ||
| under test. | ||
|
|
||
| The tests are written as synchronous functions that drive an asyncio | ||
| event loop internally. This keeps the test file runnable both locally | ||
| (without ``pytest-asyncio``) and in CI (where the project's standard | ||
| ``asyncio_mode = "auto"`` setting would also be honoured). | ||
| """ | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| import asyncio | ||
| from datetime import datetime, timezone | ||
|
|
||
| from agent_framework._workflows._checkpoint import ( | ||
| InMemoryCheckpointStorage, | ||
| WorkflowCheckpoint, | ||
| ) | ||
| from agent_framework.exceptions import WorkflowCheckpointException | ||
|
|
||
|
|
||
| def _make_checkpoint( | ||
| *, | ||
| checkpoint_id: str, | ||
| workflow_name: str = "demo", | ||
| state: dict | None = None, | ||
| messages: dict | None = None, | ||
| timestamp: str | None = None, | ||
| ) -> WorkflowCheckpoint: | ||
| return WorkflowCheckpoint( | ||
| workflow_name=workflow_name, | ||
| graph_signature_hash="hash-1", | ||
| checkpoint_id=checkpoint_id, | ||
| previous_checkpoint_id=None, | ||
| timestamp=timestamp or datetime.now(timezone.utc).isoformat(), | ||
| messages=messages if messages is not None else {}, | ||
| state=state if state is not None else {"history": ["step-1"]}, | ||
| pending_request_info_events={}, | ||
| iteration_count=1, | ||
| ) | ||
|
|
||
|
|
||
| def test_load_returns_isolated_checkpoint() -> None: | ||
| """Mutating a loaded checkpoint must not affect the storage backend.""" | ||
|
|
||
| async def _run() -> None: | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We can make the tests themselves async. |
||
| storage = InMemoryCheckpointStorage() | ||
| await storage.save(_make_checkpoint(checkpoint_id="cp-1")) | ||
|
|
||
| loaded = await storage.load("cp-1") | ||
| loaded.state["history"].append("step-2") # mutate the returned snapshot | ||
|
|
||
| reloaded = await storage.load("cp-1") | ||
| assert reloaded.state == {"history": ["step-1"]} | ||
|
|
||
| asyncio.run(_run()) | ||
|
|
||
|
|
||
| def test_load_returns_isolated_messages() -> None: | ||
| """The ``messages`` field is also nested mutable state and must be isolated.""" | ||
|
|
||
| async def _run() -> None: | ||
| storage = InMemoryCheckpointStorage() | ||
| checkpoint = _make_checkpoint( | ||
| checkpoint_id="cp-1", | ||
| state={"history": ["step-1"]}, | ||
| messages={"src": ["msg-1"]}, | ||
| ) | ||
| await storage.save(checkpoint) | ||
|
|
||
| loaded = await storage.load("cp-1") | ||
| loaded.messages["src"].append("msg-2") | ||
|
|
||
| reloaded = await storage.load("cp-1") | ||
| assert reloaded.messages == {"src": ["msg-1"]} | ||
|
|
||
| asyncio.run(_run()) | ||
|
|
||
|
|
||
| def test_list_checkpoints_returns_isolated_snapshots() -> None: | ||
| """Mutating checkpoints returned from list_checkpoints must not affect the backend.""" | ||
|
|
||
| async def _run() -> None: | ||
| storage = InMemoryCheckpointStorage() | ||
| await storage.save(_make_checkpoint(checkpoint_id="cp-1")) | ||
| await storage.save(_make_checkpoint(checkpoint_id="cp-2")) | ||
|
|
||
| listed = await storage.list_checkpoints(workflow_name="demo") | ||
| assert len(listed) == 2 | ||
|
|
||
| for cp in listed: | ||
| cp.state["history"].append("step-2") # mutate each returned snapshot | ||
|
|
||
| # Reload: storage must be untouched. | ||
| cp1 = await storage.load("cp-1") | ||
| cp2 = await storage.load("cp-2") | ||
| assert cp1.state == {"history": ["step-1"]} | ||
| assert cp2.state == {"history": ["step-1"]} | ||
|
|
||
| asyncio.run(_run()) | ||
|
|
||
|
|
||
| def test_list_checkpoints_filters_by_workflow_name() -> None: | ||
| """list_checkpoints must continue to filter by workflow_name after isolation.""" | ||
|
|
||
| async def _run() -> None: | ||
| storage = InMemoryCheckpointStorage() | ||
| await storage.save(_make_checkpoint(checkpoint_id="cp-1", workflow_name="a")) | ||
| await storage.save(_make_checkpoint(checkpoint_id="cp-2", workflow_name="b")) | ||
|
|
||
| listed_a = await storage.list_checkpoints(workflow_name="a") | ||
| listed_b = await storage.list_checkpoints(workflow_name="b") | ||
|
|
||
| assert {cp.checkpoint_id for cp in listed_a} == {"cp-1"} | ||
| assert {cp.checkpoint_id for cp in listed_b} == {"cp-2"} | ||
|
|
||
| asyncio.run(_run()) | ||
|
|
||
|
|
||
| def test_get_latest_returns_isolated_snapshot() -> None: | ||
| """Mutating the result of get_latest must not affect any stored checkpoint.""" | ||
|
|
||
| async def _run() -> None: | ||
| storage = InMemoryCheckpointStorage() | ||
| cp1 = _make_checkpoint( | ||
| checkpoint_id="cp-1", | ||
| timestamp="2026-01-01T00:00:00+00:00", | ||
| ) | ||
| cp2 = _make_checkpoint( | ||
| checkpoint_id="cp-2", | ||
| timestamp="2026-02-01T00:00:00+00:00", | ||
| ) | ||
| await storage.save(cp1) | ||
| await storage.save(cp2) | ||
|
|
||
| latest = await storage.get_latest(workflow_name="demo") | ||
| assert latest is not None | ||
| assert latest.checkpoint_id == "cp-2" | ||
|
|
||
| latest.state["history"].append("step-2") # mutate the latest snapshot | ||
|
|
||
| # Both stored checkpoints must be unchanged. | ||
| reloaded_latest = await storage.get_latest(workflow_name="demo") | ||
| assert reloaded_latest is not None | ||
| assert reloaded_latest.state == {"history": ["step-1"]} | ||
| # And the older one wasn't touched either. | ||
| reloaded_cp1 = await storage.load("cp-1") | ||
| assert reloaded_cp1.state == {"history": ["step-1"]} | ||
|
|
||
| asyncio.run(_run()) | ||
|
|
||
|
|
||
| def test_get_latest_returns_none_when_empty() -> None: | ||
| """get_latest must return None for an unknown workflow, not raise.""" | ||
|
|
||
| async def _run() -> None: | ||
| storage = InMemoryCheckpointStorage() | ||
| result = await storage.get_latest(workflow_name="demo") | ||
| assert result is None | ||
|
|
||
| asyncio.run(_run()) | ||
|
|
||
|
|
||
| def test_load_missing_id_raises() -> None: | ||
| """load must raise WorkflowCheckpointException for an unknown checkpoint id.""" | ||
|
|
||
| async def _run() -> None: | ||
| storage = InMemoryCheckpointStorage() | ||
| try: | ||
| await storage.load("does-not-exist") | ||
| except WorkflowCheckpointException: | ||
| return | ||
| raise AssertionError("Expected WorkflowCheckpointException for missing checkpoint id") | ||
|
|
||
| asyncio.run(_run()) | ||
|
|
||
|
|
||
| def test_save_returns_id() -> None: | ||
| """save must return the checkpoint_id (and the contract must hold post-isolation).""" | ||
|
|
||
| async def _run() -> None: | ||
| storage = InMemoryCheckpointStorage() | ||
| result = await storage.save(_make_checkpoint(checkpoint_id="cp-1")) | ||
| assert result == "cp-1" | ||
|
|
||
| asyncio.run(_run()) | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
ruff format(line-length 120) collapses this back to a single line, which is likely the failing code quality check.