cloudhypervisor: converge a VM left paused by a killed capture

hypervisor/cloudhypervisor/extend.go

@@ -11,6 +11,8 @@ import (
 	"slices"
 	"strings"
 
+	"github.com/projecteru2/core/log"
+
 	"github.com/cocoonstack/cocoon/extend/disk"
 	"github.com/cocoonstack/cocoon/extend/fs"
 	"github.com/cocoonstack/cocoon/extend/vfio"
@@ -238,7 +240,7 @@ func (ch *CloudHypervisor) attachWith(
 		return "", err
 	}
 	defer unlock()
-	if err = ensureNotPaused(info); err != nil {
+	if info, err = convergeOrphanedPause(ctx, hc, rec.ID, info); err != nil {
 		return "", err
 	}
 	if checkErr := preCheck(info); checkErr != nil {
@@ -271,12 +273,12 @@ func (ch *CloudHypervisor) detachWith(
 	ctx context.Context, vmRef string,
 	findID func(*chVMInfoResponse) (string, error),
 ) error {
-	hc, _, info, unlock, err := ch.lockedDeviceOp(ctx, vmRef)
+	hc, rec, info, unlock, err := ch.lockedDeviceOp(ctx, vmRef)
 	if err != nil {
 		return err
 	}
 	defer unlock()
-	if err = ensureNotPaused(info); err != nil {
+	if info, err = convergeOrphanedPause(ctx, hc, rec.ID, info); err != nil {
 		return err
 	}
 	deviceID, err := findID(info)
@@ -313,12 +315,18 @@ func (ch *CloudHypervisor) runningVMClientWithRecord(ctx context.Context, vmRef
 	return utils.NewSocketHTTPClient(sockPath), vmID, rec, nil
 }
 
-// ensureNotPaused refuses device-set mutations while a capture window is open — mutating mid-capture would desync config and memory.
-func ensureNotPaused(info *chVMInfoResponse) error {
-	if info.State == chStatePaused {
-		return fmt.Errorf("vm is paused (snapshot or hibernate in flight); retry after it completes")
-	}
-	return nil
+// convergeOrphanedPause resumes a VM left paused by a capture whose process died, returning a refreshed vm.info for device classification.
+// The pause is provably ownerless: callers hold the VM ops lock every capture window holds end to end, and the Running-record gate in runningVMClientWithRecord keeps restore/clone pause windows (record Stopped/creating) out of reach.
+func convergeOrphanedPause(ctx context.Context, hc *http.Client, vmID string, info *chVMInfoResponse) (*chVMInfoResponse, error) {
+	if info.State != chStatePaused {
+		return info, nil
+	}
+	log.WithFunc("cloudhypervisor.convergeOrphanedPause").
+		Warnf(ctx, "vm %s is paused with no capture in flight (interrupted snapshot or hibernate), resuming", vmID)
+	if err := resumeVM(ctx, hc); err != nil {
+		return nil, fmt.Errorf("resume orphaned pause: %w", err)
+	}
+	return getVMInfo(ctx, hc)
 }
 
 // listWith returns nil (not error) for stopped VMs so inspect can omit the field.

hypervisor/cloudhypervisor/netresize.go

@@ -43,7 +43,7 @@ func (ch *CloudHypervisor) NetResize(ctx context.Context, vmRef string, spec net
 	if err != nil {
 		return netresize.Result{}, err
 	}
-	if err = ensureNotPaused(info); err != nil {
+	if info, err = convergeOrphanedPause(ctx, hc, vmID, info); err != nil {
 		return netresize.Result{}, err
 	}
 	current := len(rec.NetworkConfigs)

hypervisor/cloudhypervisor/netresize_test.go

@@ -155,6 +155,52 @@ func TestNICPersisted(t *testing.T) {
 	}
 }
 
+// TestConvergeOrphanedPause pins the interrupted-capture wedge: a save killed
+// inside its pause window leaves CH Paused with nobody left to resume it.
+func TestConvergeOrphanedPause(t *testing.T) {
+	var (
+		mu      sync.Mutex
+		resumes int
+		state   = chStatePaused
+	)
+	mux := http.NewServeMux()
+	mux.HandleFunc("/api/v1/vm.resume", func(w http.ResponseWriter, _ *http.Request) {
+		mu.Lock()
+		resumes++
+		state = "Running"
+		mu.Unlock()
+		w.WriteHeader(http.StatusNoContent)
+	})
+	mux.HandleFunc("/api/v1/vm.info", func(w http.ResponseWriter, _ *http.Request) {
+		mu.Lock()
+		defer mu.Unlock()
+		_ = json.NewEncoder(w).Encode(chVMInfoResponse{State: state})
+	})
+	hc := newStubHTTPClient(t, mux)
+
+	info, err := getVMInfo(t.Context(), hc)
+	if err != nil {
+		t.Fatalf("vm.info: %v", err)
+	}
+	fresh, err := convergeOrphanedPause(t.Context(), hc, "vm1", info)
+	if err != nil {
+		t.Fatalf("convergeOrphanedPause: %v", err)
+	}
+	if resumes != 1 {
+		t.Fatalf("resumes = %d, want the orphaned pause resumed once", resumes)
+	}
+	// The returned info must be re-read: callers classify devices off it.
+	if fresh.State != "Running" {
+		t.Errorf("returned state = %q, want the refreshed Running", fresh.State)
+	}
+	if _, err = convergeOrphanedPause(t.Context(), hc, "vm1", fresh); err != nil {
+		t.Fatalf("convergeOrphanedPause on a running VM: %v", err)
+	}
+	if resumes != 1 {
+		t.Errorf("resumes = %d, want a running VM left untouched", resumes)
+	}
+}
+
 type stubPlumbing struct {
 	removed []int
 }
@@ -209,6 +255,18 @@ func newTestCH(t *testing.T) *CloudHypervisor {
 	return &CloudHypervisor{Backend: backend, conf: cfg}
 }
 
+func newStubHTTPClient(t *testing.T, mux *http.ServeMux) *http.Client {
+	t.Helper()
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+	addr := strings.TrimPrefix(srv.URL, "http://")
+	return &http.Client{Transport: &http.Transport{
+		DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+			return net.Dial("tcp", addr)
+		},
+	}}
+}
+
 // newCHStubClient serves vm.info and vm.remove-device over an httptest server;
 // removed() snapshots the eject calls. stickyIDs stay in the device tree after
 // removal, simulating a guest that never acks B0EJ.
@@ -247,14 +305,7 @@ func newCHStubClient(t *testing.T, nets []chNet, stickyIDs ...string) (*http.Cli
 		}
 		_ = json.NewEncoder(w).Encode(chVMInfoResponse{Config: chVMInfoConfig{Nets: live}, DeviceTree: tree})
 	})
-	srv := httptest.NewServer(mux)
-	t.Cleanup(srv.Close)
-	addr := strings.TrimPrefix(srv.URL, "http://")
-	hc := &http.Client{Transport: &http.Transport{
-		DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
-			return net.Dial("tcp", addr)
-		},
-	}}
+	hc := newStubHTTPClient(t, mux)
 	return hc, func() []string {
 		mu.Lock()
 		defer mu.Unlock()