diff --git a/hypervisor/cloudhypervisor/extend.go b/hypervisor/cloudhypervisor/extend.go index d0932bc2..baada1f0 100644 --- a/hypervisor/cloudhypervisor/extend.go +++ b/hypervisor/cloudhypervisor/extend.go @@ -11,6 +11,8 @@ import ( "slices" "strings" + "github.com/projecteru2/core/log" + "github.com/cocoonstack/cocoon/extend/disk" "github.com/cocoonstack/cocoon/extend/fs" "github.com/cocoonstack/cocoon/extend/vfio" @@ -238,7 +240,7 @@ func (ch *CloudHypervisor) attachWith( return "", err } defer unlock() - if err = ensureNotPaused(info); err != nil { + if info, err = convergeOrphanedPause(ctx, hc, rec.ID, info); err != nil { return "", err } if checkErr := preCheck(info); checkErr != nil { @@ -271,12 +273,12 @@ func (ch *CloudHypervisor) detachWith( ctx context.Context, vmRef string, findID func(*chVMInfoResponse) (string, error), ) error { - hc, _, info, unlock, err := ch.lockedDeviceOp(ctx, vmRef) + hc, rec, info, unlock, err := ch.lockedDeviceOp(ctx, vmRef) if err != nil { return err } defer unlock() - if err = ensureNotPaused(info); err != nil { + if info, err = convergeOrphanedPause(ctx, hc, rec.ID, info); err != nil { return err } deviceID, err := findID(info) @@ -313,12 +315,18 @@ func (ch *CloudHypervisor) runningVMClientWithRecord(ctx context.Context, vmRef return utils.NewSocketHTTPClient(sockPath), vmID, rec, nil } -// ensureNotPaused refuses device-set mutations while a capture window is open — mutating mid-capture would desync config and memory. -func ensureNotPaused(info *chVMInfoResponse) error { - if info.State == chStatePaused { - return fmt.Errorf("vm is paused (snapshot or hibernate in flight); retry after it completes") - } - return nil +// convergeOrphanedPause resumes a VM left paused by a capture whose process died, returning a refreshed vm.info for device classification. +// The pause is provably ownerless: callers hold the VM ops lock every capture window holds end to end, and the Running-record gate in runningVMClientWithRecord keeps restore/clone pause windows (record Stopped/creating) out of reach. +func convergeOrphanedPause(ctx context.Context, hc *http.Client, vmID string, info *chVMInfoResponse) (*chVMInfoResponse, error) { + if info.State != chStatePaused { + return info, nil + } + log.WithFunc("cloudhypervisor.convergeOrphanedPause"). + Warnf(ctx, "vm %s is paused with no capture in flight (interrupted snapshot or hibernate), resuming", vmID) + if err := resumeVM(ctx, hc); err != nil { + return nil, fmt.Errorf("resume orphaned pause: %w", err) + } + return getVMInfo(ctx, hc) } // listWith returns nil (not error) for stopped VMs so inspect can omit the field. diff --git a/hypervisor/cloudhypervisor/netresize.go b/hypervisor/cloudhypervisor/netresize.go index 52ef1f3c..f062a272 100644 --- a/hypervisor/cloudhypervisor/netresize.go +++ b/hypervisor/cloudhypervisor/netresize.go @@ -43,7 +43,7 @@ func (ch *CloudHypervisor) NetResize(ctx context.Context, vmRef string, spec net if err != nil { return netresize.Result{}, err } - if err = ensureNotPaused(info); err != nil { + if info, err = convergeOrphanedPause(ctx, hc, vmID, info); err != nil { return netresize.Result{}, err } current := len(rec.NetworkConfigs) diff --git a/hypervisor/cloudhypervisor/netresize_test.go b/hypervisor/cloudhypervisor/netresize_test.go index fa40b5cc..59536344 100644 --- a/hypervisor/cloudhypervisor/netresize_test.go +++ b/hypervisor/cloudhypervisor/netresize_test.go @@ -155,6 +155,52 @@ func TestNICPersisted(t *testing.T) { } } +// TestConvergeOrphanedPause pins the interrupted-capture wedge: a save killed +// inside its pause window leaves CH Paused with nobody left to resume it. +func TestConvergeOrphanedPause(t *testing.T) { + var ( + mu sync.Mutex + resumes int + state = chStatePaused + ) + mux := http.NewServeMux() + mux.HandleFunc("/api/v1/vm.resume", func(w http.ResponseWriter, _ *http.Request) { + mu.Lock() + resumes++ + state = "Running" + mu.Unlock() + w.WriteHeader(http.StatusNoContent) + }) + mux.HandleFunc("/api/v1/vm.info", func(w http.ResponseWriter, _ *http.Request) { + mu.Lock() + defer mu.Unlock() + _ = json.NewEncoder(w).Encode(chVMInfoResponse{State: state}) + }) + hc := newStubHTTPClient(t, mux) + + info, err := getVMInfo(t.Context(), hc) + if err != nil { + t.Fatalf("vm.info: %v", err) + } + fresh, err := convergeOrphanedPause(t.Context(), hc, "vm1", info) + if err != nil { + t.Fatalf("convergeOrphanedPause: %v", err) + } + if resumes != 1 { + t.Fatalf("resumes = %d, want the orphaned pause resumed once", resumes) + } + // The returned info must be re-read: callers classify devices off it. + if fresh.State != "Running" { + t.Errorf("returned state = %q, want the refreshed Running", fresh.State) + } + if _, err = convergeOrphanedPause(t.Context(), hc, "vm1", fresh); err != nil { + t.Fatalf("convergeOrphanedPause on a running VM: %v", err) + } + if resumes != 1 { + t.Errorf("resumes = %d, want a running VM left untouched", resumes) + } +} + type stubPlumbing struct { removed []int } @@ -209,6 +255,18 @@ func newTestCH(t *testing.T) *CloudHypervisor { return &CloudHypervisor{Backend: backend, conf: cfg} } +func newStubHTTPClient(t *testing.T, mux *http.ServeMux) *http.Client { + t.Helper() + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + addr := strings.TrimPrefix(srv.URL, "http://") + return &http.Client{Transport: &http.Transport{ + DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { + return net.Dial("tcp", addr) + }, + }} +} + // newCHStubClient serves vm.info and vm.remove-device over an httptest server; // removed() snapshots the eject calls. stickyIDs stay in the device tree after // removal, simulating a guest that never acks B0EJ. @@ -247,14 +305,7 @@ func newCHStubClient(t *testing.T, nets []chNet, stickyIDs ...string) (*http.Cli } _ = json.NewEncoder(w).Encode(chVMInfoResponse{Config: chVMInfoConfig{Nets: live}, DeviceTree: tree}) }) - srv := httptest.NewServer(mux) - t.Cleanup(srv.Close) - addr := strings.TrimPrefix(srv.URL, "http://") - hc := &http.Client{Transport: &http.Transport{ - DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { - return net.Dial("tcp", addr) - }, - }} + hc := newStubHTTPClient(t, mux) return hc, func() []string { mu.Lock() defer mu.Unlock()