Feat: release stopped server capacity

docs/PLAN.md

@@ -236,11 +236,14 @@ it is feature-complete for IPv4 TCP/UDP.)*
 - **Retry/backoff:** today a failed reconcile sets `status=error` and just relies
   on the 2s tick; replace with `attempts` + `next_attempt_at` (exponential
   backoff) and have `ListReconcilable` respect it.
-- **Host-failure reschedule:** the host reaper marks a host `down` but **nothing
-  reschedules its servers** — `host_id` is only cleared on delete. Add: on
-  sustained host-down, clear `host_id`/`vm_id` and re-place, **with fencing**
-  (generation token / ensure the old VM is gone) to avoid split-brain. P5's
-  store-mediated reschedule is the data half; this is the control half.
+- **Host-failure reschedule:** a *stop* now releases a server's host (clears
+  `host_id`, returns the reservation) so its next *start* re-places it on any
+  host — voluntary reschedule works. But the host reaper marks a dead host
+  `down` while **nothing reschedules the servers still desired-running on it**:
+  that involuntary path is the gap. Add: on sustained host-down, clear
+  `host_id`/`vm_id` and re-place, **with fencing** (generation token / ensure the
+  old VM is gone) to avoid split-brain. P5's store-mediated reschedule is the
+  data half; this is the control half.
 - **Draining:** `model.HostDraining` exists but is never entered or honored — wire
   a drain that blocks new placement and migrates servers off.
 - Optional leader election (advisory lock/lease) for multiple control-plane

internal/agent/agent_test.go

@@ -47,9 +47,39 @@ func TestFakeRuntimeLifecycle(t *testing.T) {
 	assertState(t, rt, vm.ID, StateMissing)
 }
 
+// TestFakeRuntimeEvict verifies Evict tears a VM down like Deprovision for the
+// fake (which holds no durable world), freeing its capacity and host port.
+func TestFakeRuntimeEvict(t *testing.T) {
+	ctx := context.Background()
+	rt := NewFakeRuntime("10.0.0.7", WithCapacity(4, 8192))
+
+	vm, err := rt.Provision(ctx, VMSpec{ServerID: "a", CPUs: 4, MemoryMB: 8192})
+	if err != nil {
+		t.Fatalf("provision: %v", err)
+	}
+	if err := rt.Evict(ctx, vm.ID); err != nil {
+		t.Fatalf("evict: %v", err)
+	}
+	assertState(t, rt, vm.ID, StateMissing)
+
+	// Capacity and the host port are freed: a fresh VM fits and reuses the port.
+	next, err := rt.Provision(ctx, VMSpec{ServerID: "b", CPUs: 4, MemoryMB: 8192})
+	if err != nil {
+		t.Fatalf("provision after evict should fit: %v", err)
+	}
+	if next.Port != vm.Port {
+		t.Errorf("port after evict = %d, want freed port %d reused", next.Port, vm.Port)
+	}
+	// Evicting an unknown VM is a no-op.
+	if err := rt.Evict(ctx, "ghost"); err != nil {
+		t.Errorf("evict unknown = %v, want nil (idempotent)", err)
+	}
+}
+
 // TestFakeRuntimeRefusesOvercommit verifies a capacity-bounded host agent will
 // not boot a VM that would exceed its total: two 4/8192 servers cannot both run
-// on a single 4/8192 host. Deprovisioning the first frees the slot for another.
+// on a single 4/8192 host. Stopping the first frees its slot for another, and
+// restarting it back over that taken slot is itself refused.
 func TestFakeRuntimeRefusesOvercommit(t *testing.T) {
 	ctx := context.Background()
 	rt := NewFakeRuntime("10.0.0.7", WithCapacity(4, 8192))
@@ -61,19 +91,24 @@ func TestFakeRuntimeRefusesOvercommit(t *testing.T) {
 	if _, err := rt.Provision(ctx, VMSpec{ServerID: "b", CPUs: 4, MemoryMB: 8192}); !errors.Is(err, ErrInsufficientCapacity) {
 		t.Fatalf("second provision err = %v, want ErrInsufficientCapacity", err)
 	}
-	// A stopped VM keeps its slot, so it still cannot fit a second.
+	// A stopped VM frees its slot, so a second now fits.
 	if err := rt.Stop(ctx, first.ID); err != nil {
 		t.Fatalf("stop: %v", err)
 	}
-	if _, err := rt.Provision(ctx, VMSpec{ServerID: "b", CPUs: 4, MemoryMB: 8192}); !errors.Is(err, ErrInsufficientCapacity) {
-		t.Fatalf("provision over a stopped VM err = %v, want ErrInsufficientCapacity", err)
+	second, err := rt.Provision(ctx, VMSpec{ServerID: "b", CPUs: 4, MemoryMB: 8192})
+	if err != nil {
+		t.Fatalf("provision over a stopped VM should fit: %v", err)
+	}
+	// Restarting the first back over the now-taken slot is refused.
+	if _, err := rt.Start(ctx, first.ID); !errors.Is(err, ErrInsufficientCapacity) {
+		t.Fatalf("restart over a taken slot err = %v, want ErrInsufficientCapacity", err)
 	}
-	// Deprovisioning frees the capacity.
-	if err := rt.Deprovision(ctx, first.ID); err != nil {
+	// Freeing the second's slot lets the first restart.
+	if err := rt.Deprovision(ctx, second.ID); err != nil {
 		t.Fatalf("deprovision: %v", err)
 	}
-	if _, err := rt.Provision(ctx, VMSpec{ServerID: "b", CPUs: 4, MemoryMB: 8192}); err != nil {
-		t.Fatalf("provision after deprovision should fit: %v", err)
+	if _, err := rt.Start(ctx, first.ID); err != nil {
+		t.Fatalf("restart after freeing capacity should fit: %v", err)
 	}
 }
 

internal/agent/firecracker/runtime.go

@@ -298,15 +298,19 @@ func (r *Runtime) Provision(ctx context.Context, spec agent.VMSpec) (*agent.VM,
 }
 
 // checkCapacity reports whether a VM needing cpus/memMB fits the host's
-// remaining capacity, returning agent.ErrInsufficientCapacity if not. A live VM
-// holds its slot whether running or stopped, so every tracked VM counts. A zero
-// configured total leaves that dimension unconstrained.
+// remaining capacity, returning agent.ErrInsufficientCapacity if not. Only a
+// running VM holds its slot — a stopped VM's process is gone, freeing its
+// cpu/memory for other work, at the cost that restarting it may then fail. A
+// zero configured total leaves that dimension unconstrained.
 func (r *Runtime) checkCapacity(cpus, memMB int) error {
 	r.mu.Lock()
 	defer r.mu.Unlock()
 
 	var usedCPUs, usedMem int
 	for _, m := range r.vms {
+		if !m.running() {
+			continue
+		}
 		usedCPUs += m.vcpus
 		usedMem += m.memoryMB
 	}
@@ -331,6 +335,12 @@ func (r *Runtime) Start(ctx context.Context, vmID string) (*agent.VM, error) {
 	if m.running() {
 		return r.vmView(m), nil
 	}
+	// A stopped VM gave up its capacity slot, so restarting it must clear the
+	// host backstop again — the cpu/memory it needs may have been handed to
+	// another VM while it was down.
+	if err := r.checkCapacity(m.vcpus, m.memoryMB); err != nil {
+		return nil, err
+	}
 	if err := m.boot(ctx); err != nil {
 		return nil, fmt.Errorf("firecracker: restart vm: %w", err)
 	}
@@ -360,8 +370,25 @@ func (r *Runtime) Stop(ctx context.Context, vmID string) error {
 	return nil
 }
 
-// Deprovision force-stops a VM and removes its working directory (idempotent).
+// Evict force-stops a VM and removes its host-local footprint — working dir and
+// local world disk — but keeps the durable world snapshot, so the server can be
+// rescheduled onto another host and restore its world from the store there. It
+// backs releasing a stopped server from its host (idempotent).
+func (r *Runtime) Evict(_ context.Context, vmID string) error {
+	return r.teardown(vmID, false)
+}
+
+// Deprovision force-stops a VM and removes it entirely, including its durable
+// world snapshot — it is a server delete (idempotent).
 func (r *Runtime) Deprovision(_ context.Context, vmID string) error {
+	return r.teardown(vmID, true)
+}
+
+// teardown force-stops a VM and removes its host-local footprint. When
+// deleteWorld is set it also deletes the durable world snapshot (a server
+// delete); otherwise the durable copy is preserved so the world can be restored
+// after a reschedule. An unknown VM is a no-op.
+func (r *Runtime) teardown(vmID string, deleteWorld bool) error {
 	r.mu.Lock()
 	m, ok := r.vms[vmID]
 	if ok {
@@ -383,18 +410,17 @@ func (r *Runtime) Deprovision(_ context.Context, vmID string) error {
 		}
 		_ = deleteTAP(m.tapName)
 	}
-	// Destroy the world disk along with the VM. This is the point P5b will
-	// snapshot-then-upload before removing, so a deprovision becomes a safe
-	// teardown rather than data loss; for now destroy means destroy. Removing
-	// the keyed parent dir (DataDir/<key>) takes the disk with it.
+	// Remove the host-local world disk. Removing the keyed parent dir
+	// (DataDir/<key>) takes the disk with it. The durable copy in the store is
+	// the system of record across hosts; we delete it only on a true server
+	// delete, keeping it for a reschedule.
 	if m.worldDisk != "" {
 		if err := os.RemoveAll(filepath.Dir(m.worldDisk)); err != nil {
 			return fmt.Errorf("firecracker: remove world disk: %w", err)
 		}
-		// Delete the durable copy too: deprovision is a server delete, so the
-		// world is meant to be gone. Best-effort — an orphaned blob is harmless
-		// (a later GC can sweep it) and must not block teardown.
-		if r.store != nil {
+		if deleteWorld && r.store != nil {
+			// Best-effort — an orphaned blob is harmless (a later GC can sweep
+			// it) and must not block teardown.
 			_ = r.store.Delete(context.Background(), m.worldKey)
 		}
 	}

internal/agent/firecracker/worldsnapshot_test.go

@@ -137,3 +137,69 @@ func TestPrepareWorldDiskFreshWhenStoreEmpty(t *testing.T) {
 		t.Fatal("expected fresh-format fallback to invoke mkfs and fail")
 	}
 }
+
+// TestEvictPreservesDurableWorld verifies the teardown split: Evict removes a
+// VM's host-local footprint but keeps the durable world snapshot (so the server
+// can restore it after a reschedule), while Deprovision deletes the durable copy
+// too (a true server delete).
+func TestEvictPreservesDurableWorld(t *testing.T) {
+	ctx := context.Background()
+	store, err := storage.NewDirStore(t.TempDir())
+	if err != nil {
+		t.Fatal(err)
+	}
+	rt := newTestRuntime(t)
+	rt.store = store
+
+	// seed builds a durable snapshot for key and a tracked machine with a local
+	// world disk, returning the machine.
+	seed := func(id, key string) *machine {
+		if err := store.Put(ctx, key, bytes.NewReader([]byte("world-"+key))); err != nil {
+			t.Fatalf("seed durable %s: %v", key, err)
+		}
+		base := t.TempDir()
+		vmDir := filepath.Join(base, id)
+		worldDir := filepath.Join(base, "worlds", key)
+		for _, d := range []string{vmDir, worldDir} {
+			if err := os.MkdirAll(d, 0o750); err != nil {
+				t.Fatal(err)
+			}
+		}
+		disk := filepath.Join(worldDir, "world.ext4")
+		if err := os.WriteFile(disk, []byte("local-"+key), 0o640); err != nil {
+			t.Fatal(err)
+		}
+		m := &machine{id: id, serverID: key, dir: vmDir, worldDisk: disk, worldKey: key}
+		rt.vms[id] = m
+		return m
+	}
+
+	gone := func(path string) bool {
+		_, err := os.Stat(path)
+		return os.IsNotExist(err)
+	}
+
+	// Evict: durable world survives, host-local footprint is gone.
+	mE := seed("vm-evict", "srv-evict")
+	if err := rt.Evict(ctx, mE.id); err != nil {
+		t.Fatalf("evict: %v", err)
+	}
+	if ok, _ := store.Exists(ctx, "srv-evict"); !ok {
+		t.Error("evict deleted the durable world; want preserved")
+	}
+	if !gone(mE.dir) || !gone(filepath.Dir(mE.worldDisk)) {
+		t.Error("evict left host-local footprint behind")
+	}
+	if _, ok := rt.vms[mE.id]; ok {
+		t.Error("evict left the machine tracked")
+	}
+
+	// Deprovision: durable world is deleted too.
+	mD := seed("vm-deprov", "srv-deprov")
+	if err := rt.Deprovision(ctx, mD.id); err != nil {
+		t.Fatalf("deprovision: %v", err)
+	}
+	if ok, _ := store.Exists(ctx, "srv-deprov"); ok {
+		t.Error("deprovision kept the durable world; want deleted")
+	}
+}

internal/agent/link.go

@@ -21,6 +21,7 @@ const (
 	OpStart       = "start"
 	OpStop        = "stop"
 	OpSnapshot    = "snapshot"
+	OpEvict       = "evict"
 	OpDeprovision = "deprovision"
 	OpStatus      = "status"
 )
@@ -168,6 +169,8 @@ func execOp(ctx context.Context, rt Runtime, cmd *pb.Command) (payload []byte, e
 		return nil, errString(rt.Stop(ctx, vmRef(cmd)))
 	case OpSnapshot:
 		return nil, errString(rt.Snapshot(ctx, vmRef(cmd)))
+	case OpEvict:
+		return nil, errString(rt.Evict(ctx, vmRef(cmd)))
 	case OpDeprovision:
 		return nil, errString(rt.Deprovision(ctx, vmRef(cmd)))
 	case OpStatus:

internal/agent/runtime.go

@@ -87,7 +87,15 @@ type Runtime interface {
 	Start(ctx context.Context, vmID string) (*VM, error)
 	// Stop halts a VM without destroying it. Idempotent (missing VM is success).
 	Stop(ctx context.Context, vmID string) error
-	// Deprovision destroys a VM. Idempotent (missing VM is success).
+	// Evict destroys a VM and its host-local footprint (working dir, local world
+	// disk) while preserving the durable world snapshot, so the server can be
+	// rescheduled onto another host and restore its world there. It is the
+	// teardown half of releasing a stopped server from its host; Deprovision is
+	// the stronger form that also deletes the durable world. Idempotent (missing
+	// VM is success).
+	Evict(ctx context.Context, vmID string) error
+	// Deprovision destroys a VM, including its durable world. Idempotent (missing
+	// VM is success).
 	Deprovision(ctx context.Context, vmID string) error
 	// Status reports a VM's observed state, returning StateMissing for an
 	// unknown id rather than an error.
@@ -117,9 +125,10 @@ type FakeRuntime struct {
 	usedPorts map[int]struct{} // host ports currently bound by a live VM
 }
 
-// fakeVM is a simulated VM plus the host resources it holds. A provisioned VM
-// occupies its cpu/memory until it is deprovisioned, even while stopped —
-// mirroring a real host, where a halted VM keeps its slot.
+// fakeVM is a simulated VM plus the host resources it holds. A running VM
+// occupies its cpu/memory; stopping it frees that slot back to the host —
+// mirroring a real host that reclaims a halted VM's resources — so only running
+// VMs count against capacity.
 type fakeVM struct {
 	vm    *VM
 	cpus  int
@@ -187,11 +196,16 @@ func (r *FakeRuntime) allocatePortLocked() int {
 }
 
 // fitsLocked reports whether a VM needing cpus/memMB fits the host's remaining
-// capacity, returning ErrInsufficientCapacity if not. A zero total leaves that
+// capacity, returning ErrInsufficientCapacity if not. Only running VMs hold a
+// slot — a stopped VM frees its cpu/memory so the host can pack other work into
+// it, at the cost that restarting it may then fail. A zero total leaves that
 // dimension unconstrained. Caller holds r.mu.
 func (r *FakeRuntime) fitsLocked(cpus, memMB int) error {
 	var usedCPUs, usedMem int
 	for _, fv := range r.vms {
+		if fv.vm.State != StateRunning {
+			continue
+		}
 		usedCPUs += fv.cpus
 		usedMem += fv.memMB
 	}
@@ -204,7 +218,10 @@ func (r *FakeRuntime) fitsLocked(cpus, memMB int) error {
 	return nil
 }
 
-// Start boots an existing VM back to running.
+// Start boots an existing VM back to running, refusing to overcommit. Because a
+// stopped VM gave up its slot, the cpu/memory it needs may have been handed to
+// another VM while it was down, so a restart can fail with
+// ErrInsufficientCapacity. Restarting an already-running VM is a no-op.
 func (r *FakeRuntime) Start(_ context.Context, vmID string) (*VM, error) {
 	r.mu.Lock()
 	defer r.mu.Unlock()
@@ -213,7 +230,12 @@ func (r *FakeRuntime) Start(_ context.Context, vmID string) (*VM, error) {
 	if !ok {
 		return nil, ErrVMNotFound
 	}
-	fv.vm.State = StateRunning
+	if fv.vm.State != StateRunning {
+		if err := r.fitsLocked(fv.cpus, fv.memMB); err != nil {
+			return nil, err
+		}
+		fv.vm.State = StateRunning
+	}
 	return clone(fv.vm), nil
 }
 
@@ -228,17 +250,35 @@ func (r *FakeRuntime) Stop(_ context.Context, vmID string) error {
 	return nil
 }
 
+// Evict destroys a VM, freeing its capacity and host port. The fake keeps no
+// durable world, so it is indistinguishable from Deprovision here; the
+// distinction (preserving the durable snapshot) only matters for the real
+// runtime. Unknown VM is a no-op.
+func (r *FakeRuntime) Evict(_ context.Context, vmID string) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	r.removeLocked(vmID)
+	return nil
+}
+
 // Deprovision destroys a VM, freeing its capacity and host port. Unknown VM is
 // a no-op.
 func (r *FakeRuntime) Deprovision(_ context.Context, vmID string) error {
 	r.mu.Lock()
 	defer r.mu.Unlock()
 
+	r.removeLocked(vmID)
+	return nil
+}
+
+// removeLocked drops a VM from the inventory and releases its host port. Caller
+// holds r.mu; an unknown VM is a no-op.
+func (r *FakeRuntime) removeLocked(vmID string) {
 	if fv, ok := r.vms[vmID]; ok {
 		delete(r.usedPorts, fv.vm.Port)
 		delete(r.vms, vmID)
 	}
-	return nil
 }
 
 // Status reports a VM's state, or a missing VM for an unknown id.

internal/agentlink/hub.go

@@ -222,7 +222,14 @@ func (h *Hub) Snapshot(ctx context.Context, hostID, vmID string) error {
 	return err
 }
 
-// Deprovision asks the host's agent to destroy a VM.
+// Evict asks the host's agent to tear down a VM while preserving its durable
+// world, releasing the host so the server can be rescheduled elsewhere.
+func (h *Hub) Evict(ctx context.Context, hostID, vmID string) error {
+	_, err := h.call(ctx, hostID, agent.OpEvict, agent.VMRef{VMID: vmID})
+	return err
+}
+
+// Deprovision asks the host's agent to destroy a VM, including its durable world.
 func (h *Hub) Deprovision(ctx context.Context, hostID, vmID string) error {
 	_, err := h.call(ctx, hostID, agent.OpDeprovision, agent.VMRef{VMID: vmID})
 	return err