vm: add reconcile-stale-create verb over a shared ownerless-create reclaim

cmd/core/utils.go

@@ -24,12 +24,17 @@ import (
 )
 
 func FindHypervisor(ctx context.Context, conf *config.Config, ref string) (hypervisor.Hypervisor, error) {
+	owner, _, err := FindVM(ctx, conf, ref)
+	return owner, err
+}
+
+// FindVM resolves ref to its owning hypervisor and the resolved VM.
+func FindVM(ctx context.Context, conf *config.Config, ref string) (hypervisor.Hypervisor, *types.VM, error) {
 	hypers, err := InitAllHypervisors(ctx, conf)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
-	owner, _, err := resolveVMOwner(ctx, hypers, ref)
-	return owner, err
+	return resolveVMOwner(ctx, hypers, ref)
 }
 
 func ListAllVMs(ctx context.Context, hypers []hypervisor.Hypervisor) ([]*types.VM, error) {

cmd/storebench/main.go

@@ -26,6 +26,20 @@ const opGet = "get"
 
 var benchPayload = json.RawMessage(`{"name":"bench","state":"running","config":{"cpu":2,"memory":1073741824}}`)
 
+type benchConfig struct{ dir string }
+
+func (c benchConfig) BinaryName() string                  { return "bench" }
+func (c benchConfig) RootDirPath() string                 { return c.dir }
+func (c benchConfig) PIDFileName() string                 { return "pid" }
+func (c benchConfig) TerminateGracePeriod() time.Duration { return time.Second }
+func (c benchConfig) SocketWaitTimeout() time.Duration    { return time.Second }
+func (c benchConfig) EffectivePoolSize() int              { return 4 }
+func (c benchConfig) EnsureDirs() error                   { return nil }
+func (c benchConfig) RunDir() string                      { return c.dir }
+func (c benchConfig) LogDir() string                      { return c.dir }
+func (c benchConfig) VMRunDir(id string) string           { return filepath.Join(c.dir, id) }
+func (c benchConfig) VMLogDir(id string) string           { return filepath.Join(c.dir, id) }
+
 func main() {
 	if len(os.Args) < 3 {
 		fmt.Fprintln(os.Stderr, "usage: storebench update|get <engine> <n> <ops> [dir]")
@@ -277,17 +291,3 @@ func argDir(i int) string {
 func uniquePayload(i int) json.RawMessage {
 	return json.RawMessage(fmt.Sprintf(`{"name":"bench","state":"running","seq":%d}`, i))
 }
-
-type benchConfig struct{ dir string }
-
-func (c benchConfig) BinaryName() string                  { return "bench" }
-func (c benchConfig) RootDirPath() string                 { return c.dir }
-func (c benchConfig) PIDFileName() string                 { return "pid" }
-func (c benchConfig) TerminateGracePeriod() time.Duration { return time.Second }
-func (c benchConfig) SocketWaitTimeout() time.Duration    { return time.Second }
-func (c benchConfig) EffectivePoolSize() int              { return 4 }
-func (c benchConfig) EnsureDirs() error                   { return nil }
-func (c benchConfig) RunDir() string                      { return c.dir }
-func (c benchConfig) LogDir() string                      { return c.dir }
-func (c benchConfig) VMRunDir(id string) string           { return filepath.Join(c.dir, id) }
-func (c benchConfig) VMLogDir(id string) string           { return filepath.Join(c.dir, id) }

cmd/vm/commands.go

@@ -122,6 +122,14 @@ func Command(h Handler) *cobra.Command {
 	rmCmd.Flags().Bool("force", false, "force delete running VMs (immediate SIGTERM/SIGKILL, no graceful window)")
 	cliutil.AddOutputFlag(rmCmd)
 
+	reconcileStaleCreateCmd := &cobra.Command{
+		Use:   "reconcile-stale-create VM",
+		Short: "Reclaim an ownerless creating placeholder (refuses while a create or clone is in flight)",
+		Args:  cobra.ExactArgs(1),
+		RunE:  h.ReconcileStaleCreate,
+	}
+	cliutil.AddOutputFlag(reconcileStaleCreateCmd)
+
 	restoreCmd := &cobra.Command{
 		Use:   "restore [flags] VM [SNAPSHOT]",
 		Short: "Restore a running or stopped VM to a previous snapshot (or a directory via --from-dir)",
@@ -184,6 +192,7 @@ func Command(h Handler) *cobra.Command {
 		reseedCmd,
 		logsCmd,
 		rmCmd,
+		reconcileStaleCreateCmd,
 		restoreCmd,
 		hibernateCmd,
 		debugCmd,

cmd/vm/reconcile.go

@@ -0,0 +1,50 @@
+package vm
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/projecteru2/core/log"
+	"github.com/spf13/cobra"
+
+	"github.com/cocoonstack/cocoon/cmd/cliutil"
+	cmdcore "github.com/cocoonstack/cocoon/cmd/core"
+	"github.com/cocoonstack/cocoon/hypervisor"
+)
+
+type staleCreateResult struct {
+	ID      string                        `json:"id,omitempty"`
+	Outcome hypervisor.StaleCreateOutcome `json:"outcome"`
+}
+
+func (h Handler) ReconcileStaleCreate(cmd *cobra.Command, args []string) error {
+	ctx, conf, err := h.Init(cmd)
+	if err != nil {
+		return err
+	}
+	hyper, vm, err := cmdcore.FindVM(ctx, conf, args[0])
+	if errors.Is(err, hypervisor.ErrNotFound) {
+		return outputStaleCreate(ctx, cmd, args[0], staleCreateResult{Outcome: hypervisor.StaleCreateNotFound})
+	}
+	if err != nil {
+		return err
+	}
+	sup, ok := hyper.(hypervisor.Supervisable)
+	if !ok {
+		return fmt.Errorf("backend %s cannot reconcile stale creates", hyper.Type())
+	}
+	outcome, err := sup.ReconcileStaleCreate(ctx, vm.ID)
+	if err != nil {
+		return fmt.Errorf("reconcile stale create: %w", err)
+	}
+	return outputStaleCreate(ctx, cmd, args[0], staleCreateResult{ID: vm.ID, Outcome: outcome})
+}
+
+func outputStaleCreate(ctx context.Context, cmd *cobra.Command, ref string, res staleCreateResult) error {
+	if done, jsonErr := cliutil.MaybeOutputJSON(cmd, res); done {
+		return jsonErr
+	}
+	log.WithFunc("cmd.vm.reconcileStaleCreate").Infof(ctx, "%s: %s", ref, res.Outcome)
+	return nil
+}

daemon/reconcile.go

@@ -138,23 +138,17 @@ func (d *Daemon) convergeDead(ctx context.Context, b Supervisor, key watchKey, r
 	return nil
 }
 
-// collectOwnerless reclaims a creating placeholder whose owner died; a free ops lock is the proof, since create and clone hold it from prereserve through the final record commit.
+// collectOwnerless routes a creating placeholder to the backend's shared stale-create reclaim; busy and raced outcomes are healthy skips.
 func (d *Daemon) collectOwnerless(ctx context.Context, b Supervisor, rec *hypervisor.VMRecord) error {
-	unlock, ok, err := d.tryLock(ctx, b, rec.ID)
-	if !ok {
-		return err
-	}
-	defer unlock()
-	fresh, err := b.PeekRecord(ctx, rec.ID)
-	if err != nil || fresh == nil || fresh.State != types.VMStateCreating {
-		return err
-	}
 	logger := log.WithFunc("daemon.collectOwnerless")
-	if err := b.CollectStaleCreate(ctx, rec.ID, fresh); err != nil {
+	outcome, err := b.ReconcileStaleCreate(ctx, rec.ID)
+	if err != nil {
 		logger.Errorf(ctx, err, "collect ownerless create %s", rec.ID)
 		return err
 	}
-	logger.Warnf(ctx, "collected ownerless creating VM %s", rec.ID)
+	if outcome == hypervisor.StaleCreateCollected {
+		logger.Warnf(ctx, "collected ownerless creating VM %s", rec.ID)
+	}
 	return nil
 }
 

daemon/reconcile_test.go

@@ -263,11 +263,6 @@ type fakeSupervisor struct {
 	resumed   []string
 }
 
-func (f *fakeSupervisor) put(rec *hypervisor.VMRecord) *fakeSupervisor {
-	f.records[rec.ID] = rec
-	return f
-}
-
 func (f *fakeSupervisor) Type() string { return "fake-hv" }
 
 func (f *fakeSupervisor) ScanSupervision(context.Context) (hypervisor.SupervisionScan, error) {
@@ -303,11 +298,9 @@ func (f *fakeSupervisor) ObserveVMM(_ context.Context, rec *hypervisor.VMRecord)
 func (f *fakeSupervisor) TryLockVMOps(_ context.Context, vmID string) (func(), bool, error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
-	if f.lockErr != nil {
-		return nil, false, f.lockErr
-	}
-	if _, held := f.busy[vmID]; held {
-		return nil, false, nil
+	busy, err := f.lockState(vmID)
+	if err != nil || busy {
+		return nil, false, err
 	}
 	return func() {}, true, nil
 }
@@ -359,12 +352,40 @@ func (f *fakeSupervisor) RecoverTombstone(_ context.Context, vmID string) (bool,
 	return true, nil
 }
 
-func (f *fakeSupervisor) CollectStaleCreate(_ context.Context, vmID string, _ *hypervisor.VMRecord) error {
+func (f *fakeSupervisor) ReconcileStaleCreate(_ context.Context, vmID string) (hypervisor.StaleCreateOutcome, error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()
+	busy, err := f.lockState(vmID)
+	if err != nil {
+		return "", err
+	}
+	if busy {
+		return hypervisor.StaleCreateBusy, nil
+	}
+	rec := f.records[vmID]
+	if rec == nil {
+		return hypervisor.StaleCreateNotFound, nil
+	}
+	if rec.State != types.VMStateCreating {
+		return hypervisor.StaleCreateNotCreating, nil
+	}
 	f.collected = append(f.collected, vmID)
 	delete(f.records, vmID)
-	return nil
+	return hypervisor.StaleCreateCollected, nil
+}
+
+// lockState reads the scripted lock condition; the caller holds f.mu.
+func (f *fakeSupervisor) lockState(vmID string) (busy bool, err error) {
+	if f.lockErr != nil {
+		return false, f.lockErr
+	}
+	_, held := f.busy[vmID]
+	return held, nil
+}
+
+func (f *fakeSupervisor) put(rec *hypervisor.VMRecord) *fakeSupervisor {
+	f.records[rec.ID] = rec
+	return f
 }
 
 func newFake() *fakeSupervisor {

docs/cli.md

@@ -25,6 +25,7 @@ cocoon
 │   ├── reseed [--machine-id] VM   Force a CRNG reseed inside the guest (fresh entropy over vsock)
 │   ├── logs [-f] [--tail N] VM    Print the per-VM hypervisor log file
 │   ├── rm [flags] VM [VM...]      Delete VM(s) (--force kills running VMs immediately)
+│   ├── reconcile-stale-create VM  Reclaim an ownerless creating placeholder (JSON outcome)
 │   ├── restore [flags] VM SNAP   Restore a VM (running or stopped) to a snapshot
 │   ├── hibernate [flags] VM       Atomically snapshot a running VM and stop it
 │   ├── status [VM...]             Watch VM status in real time
@@ -205,6 +206,19 @@ cocoon vm restore my-vm --from-dir /unrelated/lineage --force
 
 The dir is read-only across the call, so multiple clones of the same dir (golden image use case) are safe. Pass `--pull` if the base image's blobs may not be present locally — `EnsureImage` reads `image_blob_ids` from the envelope and pulls as needed.
 
+### Reconcile Stale Create
+
+`cocoon vm reconcile-stale-create VM` reclaims a `creating` placeholder whose owning create or clone died mid-flight, freeing the record and its name. A free VM ops lock is the proof of ownerlessness — create and clone hold it from prereserve through the final record commit — so the verb never races an in-flight operation and needs no age heuristics (contrast `vm rm --force`, which queues behind a live clone and then deletes the freshly created VM). With `--output json` it prints `{"id": "...", "outcome": "..."}`:
+
+| Outcome | Meaning |
+|------|-------------|
+| `collected` | Placeholder reclaimed; record and name are free |
+| `busy` | An in-flight operation owns the VM; nothing was touched — not worth retrying blindly |
+| `not-creating` | The record has left the creating state; nothing was touched |
+| `not-found` | No record under that ref (already collected, or never existed) |
+
+All four outcomes exit 0; a non-zero exit is a real failure (store I/O, an orphan VMM that would not die). At a startup reconcile, `busy` means an external create or clone legitimately owns the record: leave it un-indexed and revisit on the next pass — it either becomes a live VM or becomes collectable. The [daemon](daemon.md)'s ownerless-create reconcile and [GC](gc.md)'s `stale-creating` sweep run the same reclaim; this verb is for embedders that must clear skeletons synchronously (e.g. a startup reconcile) without waiting for either.
+
 ### Reseed Flags
 
 Applies to `cocoon vm reseed` (forces fresh guest entropy over vsock; runs automatically best-effort after clone/restore, invoke manually to retry a failed auto-reseed):

gc/orchestrator.go

@@ -20,11 +20,6 @@ type Orchestrator struct {
 // New returns an Orchestrator with no registered modules.
 func New() *Orchestrator { return &Orchestrator{} }
 
-// Register is package-level because Go methods can't have type params.
-func Register[S any](o *Orchestrator, m Module[S]) {
-	o.modules = append(o.modules, m)
-}
-
 // Run executes one GC cycle: recover tombstones by phase, then snapshot →
 // resolve → collect; modules revalidate every destructive decision under
 // their own entity locks (§5 loose-snapshot rule).
@@ -71,6 +66,11 @@ func (o *Orchestrator) Run(ctx context.Context) error {
 	return errors.Join(errs...)
 }
 
+// Register is package-level because Go methods can't have type params.
+func Register[S any](o *Orchestrator, m Module[S]) {
+	o.modules = append(o.modules, m)
+}
+
 // formatSummary renders counts as `m1=N m2=M`, sorted.
 func formatSummary(s map[string]int) string {
 	if len(s) == 0 {

hypervisor/backend.go

@@ -62,6 +62,8 @@ type BackendConfig interface {
 	VMLogDir(id string) string
 }
 
+var _ Supervisable = (*Backend)(nil)
+
 // Backend provides shared store operations for hypervisor backends.
 type Backend struct {
 	Typ      string

hypervisor/clone.go

@@ -43,6 +43,25 @@ func (b *Backend) CloneFromStream(
 	})
 }
 
+// FinalizeClone persists the record and emits the clone open-interval pair.
+func (b *Backend) FinalizeClone(ctx context.Context, vmID string, info *types.VM, bootCfg *types.BootConfig, blobIDs map[string]struct{}, sourceSnapshotID string) error {
+	if err := b.UpdateRecord(ctx, vmID, func(r *VMRecord) error {
+		r.VM = *info
+		// The subpackage building info cannot reach markTransition; without this the clone commits at generation zero.
+		markTransition(r, info.State, types.TransitionClone, timeNow())
+		r.BootConfig = bootCfg
+		r.FirstBooted = true
+		if blobIDs != nil {
+			r.ImageBlobIDs = blobIDs
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+	b.emitOpenInterval(ctx, info, metering.ReasonClone, sourceSnapshotID, timeNow())
+	return nil
+}
+
 func (b *Backend) cloneBase(
 	ctx context.Context, vmID string, vmCfg *types.VMConfig,
 	net types.NetSetup, snapshotConfig *types.SnapshotConfig,
@@ -62,22 +81,3 @@ func (b *Backend) cloneBase(
 	}
 	return afterExtract(ctx, vmID, vmCfg, net, runDir, logDir, now, snapshotConfig.ID)
 }
-
-// FinalizeClone persists the record and emits the clone open-interval pair.
-func (b *Backend) FinalizeClone(ctx context.Context, vmID string, info *types.VM, bootCfg *types.BootConfig, blobIDs map[string]struct{}, sourceSnapshotID string) error {
-	if err := b.UpdateRecord(ctx, vmID, func(r *VMRecord) error {
-		r.VM = *info
-		// The subpackage building info cannot reach markTransition; without this the clone commits at generation zero.
-		markTransition(r, info.State, types.TransitionClone, timeNow())
-		r.BootConfig = bootCfg
-		r.FirstBooted = true
-		if blobIDs != nil {
-			r.ImageBlobIDs = blobIDs
-		}
-		return nil
-	}); err != nil {
-		return err
-	}
-	b.emitOpenInterval(ctx, info, metering.ReasonClone, sourceSnapshotID, timeNow())
-	return nil
-}

hypervisor/cloudhypervisor/cloudhypervisor.go

@@ -5,6 +5,10 @@ import (
 	"fmt"
 
 	"github.com/cocoonstack/cocoon/config"
+	"github.com/cocoonstack/cocoon/extend/disk"
+	"github.com/cocoonstack/cocoon/extend/fs"
+	"github.com/cocoonstack/cocoon/extend/netresize"
+	"github.com/cocoonstack/cocoon/extend/vfio"
 	"github.com/cocoonstack/cocoon/hypervisor"
 	"github.com/cocoonstack/cocoon/meta"
 	"github.com/cocoonstack/cocoon/metering"
@@ -15,6 +19,13 @@ const typ = "cloud-hypervisor"
 var (
 	_ hypervisor.Hypervisor = (*CloudHypervisor)(nil)
 	_ hypervisor.Direct     = (*CloudHypervisor)(nil)
+	_ disk.Attacher         = (*CloudHypervisor)(nil)
+	_ disk.Lister           = (*CloudHypervisor)(nil)
+	_ fs.Attacher           = (*CloudHypervisor)(nil)
+	_ fs.Lister             = (*CloudHypervisor)(nil)
+	_ vfio.Attacher         = (*CloudHypervisor)(nil)
+	_ vfio.Lister           = (*CloudHypervisor)(nil)
+	_ netresize.Resizer     = (*CloudHypervisor)(nil)
 )
 
 // CloudHypervisor implements hypervisor.Hypervisor.

hypervisor/cloudhypervisor/extend.go

@@ -13,7 +13,6 @@ import (
 
 	"github.com/cocoonstack/cocoon/extend/disk"
 	"github.com/cocoonstack/cocoon/extend/fs"
-	"github.com/cocoonstack/cocoon/extend/netresize"
 	"github.com/cocoonstack/cocoon/extend/vfio"
 	"github.com/cocoonstack/cocoon/hypervisor"
 	"github.com/cocoonstack/cocoon/types"
@@ -22,16 +21,6 @@ import (
 
 const chStatePaused = "Paused"
 
-var (
-	_ disk.Attacher     = (*CloudHypervisor)(nil)
-	_ disk.Lister       = (*CloudHypervisor)(nil)
-	_ fs.Attacher       = (*CloudHypervisor)(nil)
-	_ fs.Lister         = (*CloudHypervisor)(nil)
-	_ vfio.Attacher     = (*CloudHypervisor)(nil)
-	_ vfio.Lister       = (*CloudHypervisor)(nil)
-	_ netresize.Resizer = (*CloudHypervisor)(nil)
-)
-
 func (ch *CloudHypervisor) DiskAttach(ctx context.Context, vmRef string, spec disk.Spec) (string, error) {
 	if err := spec.Normalize(); err != nil {
 		return "", err