Recreate stale sandbox containers after image rebuild

Co-Authored-By: GPT-5 Codex <codex@openai.com>

Author: dcramer

src/ash/sandbox/executor.py

@@ -269,6 +269,7 @@ def _clear_managed_container_state(self) -> None:
     async def _resolve_managed_container(self, managed_name: str | None) -> str | None:
         if not managed_name:
             return None
+        expected_image_id = await self._manager.get_image_id(self._config.image)
         state = self._read_managed_container_state()
         refs: list[str] = []
         if state:
@@ -284,11 +285,34 @@ async def _resolve_managed_container(self, managed_name: str | None) -> str | No
             if container is None:
                 continue
             attrs = container.attrs if isinstance(container.attrs, dict) else {}
+            container_image_id = str(attrs.get("Image") or "")
+            if (
+                expected_image_id
+                and container_image_id
+                and container_image_id != expected_image_id
+            ):
+                logger.info(
+                    "sandbox_container_image_id_mismatch_pruned",
+                    extra={
+                        "container.id": container.id[:12],
+                        "container.image_id": container_image_id,
+                        "sandbox.image": self._config.image,
+                        "sandbox.image_id": expected_image_id,
+                    },
+                )
+                await self._manager.remove_container(container.id, force=True)
+                continue
             config = attrs.get("Config", {})
             config_image = (
                 str(config.get("Image") or "") if isinstance(config, dict) else ""
             )
-            if config_image != self._config.image:
+            # Fallback for older container metadata: tag/name should match config image.
+            # TODO(2026-03-10): Remove this fallback after stale managed containers
+            # have rolled out and image-id matching has been in production for a bit.
+            if config_image and config_image not in {
+                self._config.image,
+                expected_image_id,
+            }:
                 logger.info(
                     "sandbox_container_image_mismatch_pruned",
                     extra={

src/ash/sandbox/manager.py

@@ -152,6 +152,16 @@ async def ensure_image(self, dockerfile_path: Path | None = None) -> bool:
             )
             return False
 
+    async def get_image_id(self, image_ref: str) -> str | None:
+        """Return canonical image id (sha256:...) for an image reference."""
+        client = await self._ensure_client()
+        try:
+            image = await asyncio.to_thread(client.images.get, image_ref)
+        except ImageNotFound:
+            return None
+        image_id = str(getattr(image, "id", "") or "")
+        return image_id or None
+
     async def _build_image(self, dockerfile_path: Path) -> None:
         client = await self._ensure_client()
         await asyncio.to_thread(

tests/test_sandbox_executor.py

@@ -11,12 +11,17 @@ def __init__(
         *,
         container_id: str,
         image: str,
+        image_id: str = "sha256:latest",
         status: str = "running",
         mounts: list[dict[str, str]] | None = None,
     ) -> None:
         self.id = container_id
         self.status = status
-        self.attrs = {"Config": {"Image": image}, "Mounts": mounts or []}
+        self.attrs = {
+            "Image": image_id,
+            "Config": {"Image": image},
+            "Mounts": mounts or [],
+        }
 
 
 class _FakeManager:
@@ -25,6 +30,7 @@ def __init__(self) -> None:
         self.removed: list[str] = []
         self.start_should_fail = False
         self.exec_should_fail = False
+        self.expected_image_id: str | None = "sha256:latest"
         self._containers: dict[str, _FakeContainer] = {}
         self._by_name: dict[str, str] = {}
 
@@ -35,7 +41,9 @@ async def create_container(self, name=None, environment=None) -> str:
         container_id = f"c{len(self.created) + 1}"
         self.created.append(container_id)
         container = _FakeContainer(
-            container_id=container_id, image="ash-sandbox:latest"
+            container_id=container_id,
+            image="ash-sandbox:latest",
+            image_id=self.expected_image_id or "sha256:latest",
         )
         self._containers[container_id] = container
         if name:
@@ -89,6 +97,10 @@ async def remove_container(self, container_id: str, force: bool = True) -> None:
             if cid == container_id:
                 self._by_name.pop(name, None)
 
+    async def get_image_id(self, image_ref: str) -> str | None:
+        _ = image_ref
+        return self.expected_image_id
+
 
 def _patch_runtime_paths(monkeypatch, tmp_path: Path) -> None:
     monkeypatch.setattr("ash.sandbox.executor.get_run_path", lambda: tmp_path / "run")
@@ -181,7 +193,36 @@ async def test_reuse_container_prunes_image_mismatch_then_recreates(
     manager = _FakeManager()
     shared_name = "ash-sandbox-deadbeef"
     manager._containers["stale"] = _FakeContainer(
-        container_id="stale", image="old-image-id", status="running"
+        container_id="stale",
+        image="old-image-id",
+        image_id="sha256:stale",
+        status="running",
+    )
+    manager._by_name[shared_name] = "stale"
+
+    executor = SandboxExecutor()
+    executor._manager = manager
+    executor._initialized = True
+    monkeypatch.setattr(executor, "_managed_container_name", lambda: shared_name)
+
+    result = await executor.execute("echo hi", reuse_container=True)
+    assert result.success is True
+    assert "stale" in manager.removed
+    assert manager.created == ["c1"]
+
+
+async def test_reuse_container_prunes_stale_latest_image_id_then_recreates(
+    tmp_path: Path, monkeypatch
+) -> None:
+    _patch_runtime_paths(monkeypatch, tmp_path)
+    manager = _FakeManager()
+    manager.expected_image_id = "sha256:newlatest"
+    shared_name = "ash-sandbox-deadbeef"
+    manager._containers["stale"] = _FakeContainer(
+        container_id="stale",
+        image="ash-sandbox:latest",
+        image_id="sha256:oldlatest",
+        status="running",
     )
     manager._by_name[shared_name] = "stale"