Refactor supervisor management to start after container creation and pass overlay root via environment variable

Author: aledbf

cmd/vminitd/main.go

@@ -18,7 +18,6 @@ import (
 
 	"github.com/spin-stack/spinbox/internal/guest/vminit/config"
 	"github.com/spin-stack/spinbox/internal/guest/vminit/service"
-	"github.com/spin-stack/spinbox/internal/guest/vminit/supervisor"
 	"github.com/spin-stack/spinbox/internal/guest/vminit/system"
 	"github.com/spin-stack/spinbox/internal/guest/vminit/systools"
 
@@ -108,18 +107,9 @@ func run(ctx context.Context, cfg *config.ServiceConfig) error {
 
 	log.G(ctx).WithField("t", time.Since(t1)).Debug("initialized vminitd")
 
-	// Start supervisor agent in background with automatic restart on crash.
-	// The supervisor binary is injected via extras disk and will be available
-	// at /var/lib/spin-stack/bin/spin-supervisor after VM boot.
-	// RunWithMonitoring blocks until context is cancelled, handling all restarts.
-	go func() {
-		// Wait a bit for extras disk to be mounted and files extracted
-		time.Sleep(2 * time.Second)
-
-		if err := supervisor.RunWithMonitoring(ctx); err != nil {
-			log.G(ctx).WithError(err).Error("supervisor monitor exited with error")
-		}
-	}()
+	// Note: The supervisor agent is now started after container creation
+	// (in runc/container.go) with the overlay root path passed via env var.
+	// This ensures the supervisor has access to the container's rootfs.
 
 	// Limit GOMAXPROCS for VM environment to prevent scheduler overhead
 	// Cap at maxGOMAXPROCS to improve cache locality, but respect available CPUs

internal/guest/vminit/runc/container.go

@@ -21,6 +21,7 @@ import (
 	spinbox "github.com/spin-stack/spinbox/api/types/spinbox/v1"
 	"github.com/spin-stack/spinbox/internal/guest/vminit/process"
 	"github.com/spin-stack/spinbox/internal/guest/vminit/stream"
+	"github.com/spin-stack/spinbox/internal/guest/vminit/supervisor"
 	"github.com/spin-stack/spinbox/internal/host/mountutil"
 )
 
@@ -132,6 +133,11 @@ func NewContainer(ctx context.Context, platform stdio.Platform, r *task.CreateTa
 			return nil, err
 		}
 		log.G(ctx).WithField("rootfs", rootfs).Info("rootfs components mounted")
+
+		// Start the supervisor agent now that the rootfs is available.
+		// Pass the rootfs path via environment variable so the supervisor
+		// knows where to find files generated by the container.
+		go supervisor.StartWithOverlayRoot(ctx, rootfs)
 	}
 
 	// Relax OCI spec restrictions - VM provides the security boundary

internal/guest/vminit/supervisor/monitor.go

@@ -34,6 +34,7 @@ const (
 // Monitor manages the supervisor process lifecycle with automatic restart.
 type Monitor struct {
 	binaryPath string
+	extraEnv   []string // Additional environment variables to pass to supervisor
 
 	mu          sync.Mutex
 	cmd         *exec.Cmd
@@ -52,6 +53,12 @@ func NewMonitor(binaryPath string) *Monitor {
 	}
 }
 
+// SetExtraEnv sets additional environment variables to pass to the supervisor process.
+// These are appended to the current process environment on each start.
+func (m *Monitor) SetExtraEnv(env []string) {
+	m.extraEnv = env
+}
+
 // Run starts the supervisor and monitors it for crashes.
 // It blocks until the context is cancelled or max restarts is exceeded.
 func (m *Monitor) Run(ctx context.Context) error {
@@ -144,7 +151,7 @@ func (m *Monitor) start(ctx context.Context) error {
 	cmd := exec.Command(m.binaryPath) //nolint:gosec
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
-	cmd.Env = os.Environ()
+	cmd.Env = append(os.Environ(), m.extraEnv...)
 	cmd.SysProcAttr = &syscall.SysProcAttr{
 		Setsid: true, // Create new session
 	}

internal/guest/vminit/supervisor/supervisor.go

@@ -30,6 +30,10 @@ const (
 
 	// LogFile is where supervisor logs are written.
 	LogFile = "/var/log/spin-supervisor.log"
+
+	// OverlayRootEnvVar is the environment variable name used to pass
+	// the overlay filesystem root path to the supervisor.
+	OverlayRootEnvVar = "SPINBOX_OVERLAY_ROOT"
 )
 
 // findBinary searches for the supervisor binary.
@@ -82,6 +86,29 @@ func findBinary(ctx context.Context) string {
 //
 // Returns nil immediately if supervisor binary is not found.
 func RunWithMonitoring(ctx context.Context) error {
+	return runWithEnv(ctx, nil)
+}
+
+// StartWithOverlayRoot starts the supervisor agent with the overlay root path
+// passed via environment variable. This should be called after the container's
+// rootfs is mounted so the supervisor can access files in the overlay.
+//
+// The overlayRoot parameter is the path to the container's mounted rootfs,
+// which will be passed to the supervisor via SPINBOX_OVERLAY_ROOT env var.
+//
+// This function runs in a goroutine and does not block.
+func StartWithOverlayRoot(ctx context.Context, overlayRoot string) {
+	env := []string{OverlayRootEnvVar + "=" + overlayRoot}
+
+	log.G(ctx).WithField("overlay_root", overlayRoot).Info("starting supervisor with overlay root")
+
+	if err := runWithEnv(ctx, env); err != nil {
+		log.G(ctx).WithError(err).Error("supervisor monitor exited with error")
+	}
+}
+
+// runWithEnv starts the supervisor with optional additional environment variables.
+func runWithEnv(ctx context.Context, extraEnv []string) error {
 	binaryPath := findBinary(ctx)
 	if binaryPath == "" {
 		log.G(ctx).Info("supervisor binary not found, skipping")
@@ -90,5 +117,6 @@ func RunWithMonitoring(ctx context.Context) error {
 
 	log.G(ctx).WithField("path", binaryPath).Info("starting supervisor with monitoring")
 	monitor := NewMonitor(binaryPath)
+	monitor.SetExtraEnv(extraEnv)
 	return monitor.Run(ctx)
 }