feat(cloud-hypervisor): expose a configurable REST API control socket

.github/contributors.yaml

@@ -107,3 +107,6 @@ users:
   Chennamma-Hotkar:
     name: Chennamma Hotkar
     email: channuhotkar@gmail.com
+  Anamika1608:
+    name: Anamika Aggarwal
+    email: anamikaagg18@gmail.com

.github/linters/urunc-dict.txt

@@ -84,6 +84,7 @@ Sharedfs
 Syscalls
 TUNSETGROUP
 TUNSETOWNER
+TUNSETPERSIST
 TUNTAP
 Timestamping
 Tmpfs

docs/configuration.md

@@ -99,6 +99,7 @@ values.
 
 - [QEMU/KVM](./hypervisor-support#qemu) - `qemu`
 - [Firecracker](./hypervisor-support#firecracker) - `firecracker`
+- [Cloud Hypervisor](./hypervisor-support#cloud-hypervisor) - `cloud-hypervisor`
 - [Solo5-hvt](./hypervisor-support#solo5-hvt) - `hvt` - Solo5 hvt (KVM-based tender)
 - [Solo5-spt](./hypervisor-support#solo5-spt) - `spt` - Solo5 spt (Seccomp-based tender)
 
@@ -112,6 +113,8 @@ Each monitor subsection supports the following options:
 | `default_vcpus` | integer | `1` | Default number of virtual CPUs |
 | `path` | string | (empty) | Optional custom path to the monitor binary. If not specified, urunc will search for the binary in PATH |
 | `data_path` | string | (empty) | Optional custom path for the monitor's data file directory |
+| `socket_path` | string | (empty) | Optional path for the monitor's control socket. If not specified, urunc uses a per-container default under `/tmp`. When a custom path is set, urunc creates its parent directory; setting it to an invalid location (a file already exists on the path) makes the monitor fail to start. Currently used by Cloud Hypervisor. |
+| `boot_mode` | string | (empty) | Optional boot mode. When set to `api`, urunc starts the monitor as a supervised child and triggers the guest's start through the monitor's control socket; when unset, the monitor boots from its command line as before. Currently only used by Cloud Hypervisor. |
 
 Since Qemu is the only currently supported monitor which requires extra data to
 boot a VM, `urunc` will first check `/usr/local/share` and then `/usr/share` for
@@ -130,6 +133,11 @@ data_path = "/usr/local/share/"
 default_memory_mb = 512
 default_vcpus = 2
 path = "/opt/firecracker/firecracker"
+
+[monitors.cloud-hypervisor]
+default_memory_mb = 256
+default_vcpus = 1
+boot_mode = "api"
 ```
 
 ### Extra binaries Configuration

pkg/network/network.go

@@ -101,16 +101,35 @@ func createTapDevice(name string, mtu int, ownerUID, ownerGID uint32) (netlink.L
 		return nil, fmt.Errorf("failed to create tap device: %w", err)
 	}
 
+	// LinkAdd opened the tap device's fd and set TUNSETPERSIST, so the
+	// interface survives independently of any open fd. Nothing after the
+	// owner/group ioctl calls uses the fd (the remaining setup goes through
+	// netlink and the monitor attaches to the device by name with its own
+	// open), so close it right away instead of relying on a later exec to
+	// close it (O_CLOEXEC). A single-queue tap device only allows one
+	// attached fd at a time, so holding it open blocks the monitor from
+	// attaching if the caller does not exec away.
 	for _, tapFd := range tapLink.Fds {
 		err = unix.IoctlSetInt(int(tapFd.Fd()), unix.TUNSETOWNER, int(ownerUID))
 		if err != nil {
+			if closeErr := tapFd.Close(); closeErr != nil {
+				netlog.Warnf("failed to close tap %s fd after owner ioctl error: %v", name, closeErr)
+			}
 			return nil, fmt.Errorf("failed to set tap %s owner to uid %d: %w", name, ownerUID, err)
 		}
 
 		err = unix.IoctlSetInt(int(tapFd.Fd()), unix.TUNSETGROUP, int(ownerGID))
 		if err != nil {
+			if closeErr := tapFd.Close(); closeErr != nil {
+				netlog.Warnf("failed to close tap %s fd after group ioctl error: %v", name, closeErr)
+			}
 			return nil, fmt.Errorf("failed to set tap %s group to gid %d: %w", name, ownerGID, err)
 		}
+
+		err = tapFd.Close()
+		if err != nil {
+			return nil, fmt.Errorf("failed to close the fd of tap %s: %w", name, err)
+		}
 	}
 
 	err = netlink.LinkSetMTU(tapLink, mtu)

pkg/unikontainers/hypervisors/cloud_hypervisor.go

@@ -15,8 +15,15 @@
 package hypervisors
 
 import (
+	"context"
+	"errors"
 	"fmt"
+	"os"
+	"os/exec"
+	"os/signal"
 	"strings"
+	"syscall"
+	"time"
 
 	"github.com/urunc-dev/urunc/pkg/unikontainers/types"
 	"golang.org/x/sys/unix"
@@ -70,6 +77,10 @@ func (ch *CloudHypervisor) BuildExecCmd(args types.ExecArgs, ukernel types.Unike
 	// Start building the command
 	exArgs := []string{ch.binaryPath}
 
+	// Expose the REST API over a control socket so the runtime can talk to
+	// Cloud Hypervisor after boot (e.g. for graceful shutdown).
+	exArgs = append(exArgs, "--api-socket", "path="+ResolveSocketPath(args))
+
 	// Memory configuration
 	if args.Sharedfs.Type == "virtiofs" {
 		exArgs = append(exArgs, "--memory", fmt.Sprintf("size=%sM,shared=on", chMem))
@@ -159,3 +170,230 @@ func (ch *CloudHypervisor) BuildExecCmd(args types.ExecArgs, ukernel types.Unike
 func (ch *CloudHypervisor) PreExec(_ types.ExecArgs) error {
 	return nil
 }
+
+// The following types mirror the subset of Cloud Hypervisor's REST VmConfig
+// that urunc's command-line boot already uses (verified against the installed
+// cloud-hypervisor v53 API).
+
+type CHPayload struct {
+	Kernel    string `json:"kernel,omitempty"`
+	Cmdline   string `json:"cmdline,omitempty"`
+	Initramfs string `json:"initramfs,omitempty"`
+}
+
+type CHMemory struct {
+	Size   uint64 `json:"size"`
+	Shared bool   `json:"shared,omitempty"`
+}
+
+type CHCpus struct {
+	BootVcpus uint `json:"boot_vcpus"`
+	MaxVcpus  uint `json:"max_vcpus"`
+}
+
+type CHNet struct {
+	Tap string `json:"tap,omitempty"`
+	Mac string `json:"mac,omitempty"`
+	Mtu int    `json:"mtu,omitempty"`
+}
+
+type CHDisk struct {
+	Path string `json:"path"`
+	ID   string `json:"id,omitempty"`
+}
+
+type CHFs struct {
+	Tag    string `json:"tag"`
+	Socket string `json:"socket"`
+}
+
+type CHVsock struct {
+	Cid    int    `json:"cid"`
+	Socket string `json:"socket"`
+}
+
+type CHConsole struct {
+	Mode string `json:"mode"`
+}
+
+type CHVMConfig struct {
+	Payload CHPayload  `json:"payload"`
+	Memory  *CHMemory  `json:"memory,omitempty"`
+	Cpus    *CHCpus    `json:"cpus,omitempty"`
+	Net     []CHNet    `json:"net,omitempty"`
+	Disks   []CHDisk   `json:"disks,omitempty"`
+	Fs      []CHFs     `json:"fs,omitempty"`
+	Vsock   *CHVsock   `json:"vsock,omitempty"`
+	Serial  *CHConsole `json:"serial,omitempty"`
+	Console *CHConsole `json:"console,omitempty"`
+}
+
+// buildCHVMConfig maps the same data BuildExecCmd puts on the command line
+// into the REST VmConfig. Unikernel-specific raw CLI argument strings cannot
+// be mapped to JSON, so they yield an error instead of being dropped.
+func buildCHVMConfig(args types.ExecArgs, ukernel types.Unikernel) (*CHVMConfig, error) {
+	mem := args.MemSizeB
+	if mem < 1<<20 {
+		mem = DefaultMemory << 20
+	}
+	cfg := &CHVMConfig{
+		Payload: CHPayload{Kernel: args.UnikernelPath, Cmdline: args.Command},
+		Memory:  &CHMemory{Size: mem, Shared: args.Sharedfs.Type == "virtiofs"},
+		Serial:  &CHConsole{Mode: "Tty"},
+		Console: &CHConsole{Mode: "Off"},
+	}
+	if args.VCPUs > 0 {
+		cfg.Cpus = &CHCpus{BootVcpus: args.VCPUs, MaxVcpus: args.VCPUs}
+	}
+
+	extraMonArgs := ukernel.MonitorCli()
+	if extraMonArgs.OtherArgs != "" {
+		return nil, fmt.Errorf("boot_mode=api does not support unikernel-specific monitor arguments (%q)", extraMonArgs.OtherArgs)
+	}
+	initrdPath := args.InitrdPath
+	if initrdPath == "" {
+		initrdPath = extraMonArgs.ExtraInitrd
+	}
+	cfg.Payload.Initramfs = initrdPath
+
+	if args.Net.TapDev != "" {
+		if netCli := ukernel.MonitorNetCli(args.Net.TapDev, args.Net.MAC); netCli != "" {
+			return nil, fmt.Errorf("boot_mode=api does not support unikernel-specific network arguments (%q)", netCli)
+		}
+		cfg.Net = append(cfg.Net, CHNet{Tap: args.Net.TapDev, Mac: args.Net.MAC, Mtu: args.Net.MTU})
+	}
+
+	for _, blockArg := range ukernel.MonitorBlockCli() {
+		if blockArg.ExactArgs != "" {
+			return nil, fmt.Errorf("boot_mode=api does not support unikernel-specific block arguments (%q)", blockArg.ExactArgs)
+		}
+		if blockArg.Path != "" {
+			cfg.Disks = append(cfg.Disks, CHDisk{Path: blockArg.Path, ID: blockArg.ID})
+		}
+	}
+
+	if args.Sharedfs.Type == "virtiofs" {
+		cfg.Fs = append(cfg.Fs, CHFs{Tag: "fs0", Socket: "/tmp/vhostqemu"})
+	}
+	if args.VAccelType == "vsock" {
+		cfg.Vsock = &CHVsock{Cid: args.VSockDevID, Socket: args.VSockDevPath + "/vaccel.sock"}
+	}
+	return cfg, nil
+}
+
+// CHSession is a Cloud Hypervisor child process driven over its REST API
+// socket. Create it with SpawnSocketVMM, send the full configuration with
+// ConfigureVM, boot the guest with BootVM once the caller's start handshake
+// allows it, then hand the calling process over with Supervise.
+type CHSession struct {
+	cmd    *exec.Cmd
+	client *chAPIClient
+}
+
+// SpawnSocketVMM starts Cloud Hypervisor as a supervised child with only its
+// API socket enabled. It is called after changeRoot, so the child inherits
+// the pivoted root and its socket lives inside the monitor rootfs. The
+// caller is still privileged here and only drops its own privileges
+// afterwards, so when uid/gid are non-zero the child is started directly
+// under that credential.
+func (ch *CloudHypervisor) SpawnSocketVMM(args types.ExecArgs, uid, gid uint32) (*CHSession, error) {
+	socketPath := ResolveSocketPath(args)
+	// Cloud Hypervisor binds this path itself; remove any stale leftover.
+	if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) {
+		return nil, fmt.Errorf("failed to remove stale socket %q: %w", socketPath, err)
+	}
+	execCmd := []string{ch.binaryPath, "--api-socket", "path=" + socketPath}
+	if args.Seccomp {
+		execCmd = append(execCmd, "--seccomp", "true")
+	} else {
+		execCmd = append(execCmd, "--seccomp", "false")
+	}
+
+	cmd := exec.Command(execCmd[0], execCmd[1:]...) //nolint: gosec
+	cmd.Env = args.Environment
+	cmd.Stdin = os.Stdin
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if uid != 0 || gid != 0 {
+		cmd.SysProcAttr = &syscall.SysProcAttr{
+			Credential: &syscall.Credential{Uid: uid, Gid: gid},
+		}
+	}
+	vmmLog.WithField("command", execCmd).Debug("starting cloud-hypervisor as a supervised child")
+	if err := cmd.Start(); err != nil {
+		return nil, fmt.Errorf("failed to start cloud-hypervisor: %w", err)
+	}
+
+	client := newCHAPIClient(socketPath)
+	if err := client.connect(5 * time.Second); err != nil {
+		s := &CHSession{cmd: cmd, client: client}
+		s.Kill()
+		return nil, fmt.Errorf("cloud-hypervisor API socket never became ready: %w", err)
+	}
+	return &CHSession{cmd: cmd, client: client}, nil
+}
+
+// ConfigureVM sends the complete VM configuration in one request.
+func (s *CHSession) ConfigureVM(ctx context.Context, args types.ExecArgs, ukernel types.Unikernel) error {
+	cfg, err := buildCHVMConfig(args, ukernel)
+	if err != nil {
+		return err
+	}
+	vmmLog.Debug("api boot: sending vm.create")
+	return s.client.createVM(ctx, cfg)
+}
+
+// BootVM boots the configured VM.
+func (s *CHSession) BootVM(ctx context.Context) error {
+	vmmLog.Debug("api boot: sending vm.boot")
+	return s.client.bootVM(ctx)
+}
+
+// Kill terminates the child and reaps it. For error paths before Supervise.
+func (s *CHSession) Kill() {
+	_ = s.cmd.Process.Kill()
+	_, _ = s.cmd.Process.Wait()
+}
+
+// Supervise hands the calling process over to the child for the rest of its
+// life: it forwards SIGTERM/SIGINT and, once the child exits, exits this
+// process with the child's exit code, mirroring the semantics syscall.Exec
+// would have had. The caller must not exit before the child, since it is
+// the container's init process.
+//
+// On success this function does not return: it calls os.Exit with the
+// child's exit status once the child exits.
+func (s *CHSession) Supervise() error {
+	// Forward the signals containerd would send to stop the container.
+	// SIGKILL cannot be caught, so it is not listed here: if it arrives,
+	// this process dies immediately and the child is left running, a known
+	// gap for this bounded experiment, not yet handled.
+	sigCh := make(chan os.Signal, 1)
+	signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT)
+	go func() {
+		sig, ok := <-sigCh
+		if !ok {
+			return
+		}
+		if sg, ok := sig.(syscall.Signal); ok {
+			_ = s.cmd.Process.Signal(sg)
+		}
+	}()
+
+	waitErr := s.cmd.Wait()
+	signal.Stop(sigCh)
+	close(sigCh)
+
+	exitCode := 0
+	if waitErr != nil {
+		var exitErr *exec.ExitError
+		if errors.As(waitErr, &exitErr) {
+			exitCode = exitErr.ExitCode()
+		} else {
+			vmmLog.WithError(waitErr).Error("cloud-hypervisor exited with an unexpected error")
+			exitCode = 1
+		}
+	}
+	os.Exit(exitCode)
+	return nil // unreachable
+}