From a23cece1d8abd0c92e40630e548f513d085ed609 Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Sat, 25 Jul 2026 20:47:58 +0530 Subject: [PATCH 1/9] feat(cloud-hypervisor): add a configurable control socket path Signed-off-by: Anamika Aggarwal --- pkg/unikontainers/hypervisors/utils.go | 16 ++++++++++++++++ pkg/unikontainers/hypervisors/utils_test.go | 21 +++++++++++++++++++++ pkg/unikontainers/hypervisors/vmm.go | 11 +++++++++++ pkg/unikontainers/types/types.go | 8 +++++--- pkg/unikontainers/unikontainers.go | 9 +++++++++ pkg/unikontainers/urunc_config.go | 3 +++ 6 files changed, 65 insertions(+), 3 deletions(-) diff --git a/pkg/unikontainers/hypervisors/utils.go b/pkg/unikontainers/hypervisors/utils.go index 1bee33104..3c4cb5555 100644 --- a/pkg/unikontainers/hypervisors/utils.go +++ b/pkg/unikontainers/hypervisors/utils.go @@ -17,10 +17,12 @@ package hypervisors import ( "errors" "fmt" + "path/filepath" "runtime" "strconv" "time" + "github.com/urunc-dev/urunc/pkg/unikontainers/types" "golang.org/x/sys/unix" ) @@ -93,3 +95,17 @@ func killProcess(pid int) error { return nil } + +// DefaultSocketDir is the directory used for a monitor's control socket when +// no socket_path is configured. It always exists inside the monitor rootfs. +const DefaultSocketDir = "/tmp" + +// ResolveSocketPath returns the path for a monitor's control socket: the +// configured SocketPath when set, otherwise a per-container default under +// DefaultSocketDir. +func ResolveSocketPath(args types.ExecArgs) string { + if args.SocketPath != "" { + return args.SocketPath + } + return filepath.Join(DefaultSocketDir, args.ContainerID+".sock") +} diff --git a/pkg/unikontainers/hypervisors/utils_test.go b/pkg/unikontainers/hypervisors/utils_test.go index ab4f73738..6ba858ff2 100644 --- a/pkg/unikontainers/hypervisors/utils_test.go +++ b/pkg/unikontainers/hypervisors/utils_test.go @@ -18,6 +18,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/urunc-dev/urunc/pkg/unikontainers/types" ) func TestBytesToMiB(t *testing.T) { @@ -71,3 +72,23 @@ func TestBytesToMB(t *testing.T) { }) } } + +func TestResolveSocketPath(t *testing.T) { + t.Parallel() + if got := ResolveSocketPath(types.ExecArgs{SocketPath: "/run/urunc/x.sock"}); got != "/run/urunc/x.sock" { + t.Fatalf("configured: got %q", got) + } + if got := ResolveSocketPath(types.ExecArgs{ContainerID: "abc"}); got != "/tmp/abc.sock" { + t.Fatalf("default: got %q", got) + } +} + +func TestUsesControlSocket(t *testing.T) { + t.Parallel() + if !UsesControlSocket(CloudHypervisorVmm) { + t.Fatal("cloud-hypervisor should use a control socket") + } + if UsesControlSocket(HvtVmm) { + t.Fatal("hvt should not") + } +} diff --git a/pkg/unikontainers/hypervisors/vmm.go b/pkg/unikontainers/hypervisors/vmm.go index c8600957b..2ac4972e1 100644 --- a/pkg/unikontainers/hypervisors/vmm.go +++ b/pkg/unikontainers/hypervisors/vmm.go @@ -27,6 +27,17 @@ const DefaultMemory uint64 = 256 // The default memory for every hypervisor: 256 type VmmType string +// UsesControlSocket reports whether a monitor exposes a control socket whose +// path (socket_path) urunc must make reachable before the monitor launches. +func UsesControlSocket(vmmType VmmType) bool { + switch vmmType { + case CloudHypervisorVmm: + return true + default: + return false + } +} + var ErrVMMNotInstalled = errors.New("vmm not found") var vmmLog = logrus.WithField("subsystem", "monitors") diff --git a/pkg/unikontainers/types/types.go b/pkg/unikontainers/types/types.go index c6388e2cc..89a3e9ebf 100644 --- a/pkg/unikontainers/types/types.go +++ b/pkg/unikontainers/types/types.go @@ -106,6 +106,7 @@ type ExecArgs struct { VAccelType string // Specifies the vAccel acceleration type(e.g. vsock). When empty, vAccel is disabled VSockDevPath string // The host directory where the fc unix socket is created VSockDevID int // The guest-cid + SocketPath string // The path of the monitor's control socket (empty means the monitor's default) Net NetDevParams Sharedfs SharedfsParams } @@ -133,7 +134,8 @@ type ExtraBinConfig struct { type MonitorConfig struct { DefaultMemoryMB uint `toml:"default_memory_mb"` DefaultVCPUs uint `toml:"default_vcpus"` - BinaryPath string `toml:"path,omitempty"` // Optional path to the hypervisor binary - DataPath string `toml:"data_path,omitempty"` // Optional path to the hypervisor data files (e.g. qemu bios stuff) - Vhost bool `toml:"vhost,omitempty"` // Optional: enable vhost for network performance optimization + BinaryPath string `toml:"path,omitempty"` // Optional path to the hypervisor binary + DataPath string `toml:"data_path,omitempty"` // Optional path to the hypervisor data files (e.g. qemu bios stuff) + Vhost bool `toml:"vhost,omitempty"` // Optional: enable vhost for network performance optimization + SocketPath string `toml:"socket_path,omitempty"` // Optional path for the monitor's control socket (falls back to a per-container default) } diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index 84172aca4..d8561d951 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -402,6 +402,7 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { defaultVCPUs = 1 } defaultMemSizeMB := u.UruncCfg.Monitors[vmmType].DefaultMemoryMB + socketPath := u.UruncCfg.Monitors[vmmType].SocketPath // ExecArgs vmmArgs := types.ExecArgs{ @@ -412,6 +413,7 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { MemSizeB: uint64(defaultMemSizeMB * 1024 * 1024), VCPUs: uint(defaultVCPUs), Environment: os.Environ(), + SocketPath: socketPath, } // ExecArgs @@ -648,6 +650,13 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return err } + if hypervisors.UsesControlSocket(hypervisors.VmmType(vmmType)) { + sockDir := filepath.Dir(hypervisors.ResolveSocketPath(vmmArgs)) + if err = os.MkdirAll(sockDir, 0o755); err != nil { + return fmt.Errorf("failed to create control socket directory %q: %w", sockDir, err) + } + } + // uid/gid // Setup uid, gid and additional groups for the monitor process err = setupUser(u.Spec.Process.User) diff --git a/pkg/unikontainers/urunc_config.go b/pkg/unikontainers/urunc_config.go index 22573f43c..1476eb3ee 100644 --- a/pkg/unikontainers/urunc_config.go +++ b/pkg/unikontainers/urunc_config.go @@ -145,6 +145,7 @@ func (p *UruncConfig) Map() map[string]string { cfgMap[prefix+"binary_path"] = hvCfg.BinaryPath cfgMap[prefix+"data_path"] = hvCfg.DataPath cfgMap[prefix+"vhost"] = strconv.FormatBool(hvCfg.Vhost) + cfgMap[prefix+"socket_path"] = hvCfg.SocketPath } for eb, ebCfg := range p.ExtraBins { prefix := "urunc_config.extra_binaries." + eb + "." @@ -198,6 +199,8 @@ func UruncConfigFromMap(cfgMap map[string]string) *UruncConfig { } else { hvCfg.Vhost = boolVal } + case "socket_path": + hvCfg.SocketPath = val } cfg.Monitors[hv] = hvCfg } From 1b6ace7ac980b6c18b785b5cfa97b984d7e4beec Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Sat, 25 Jul 2026 20:48:14 +0530 Subject: [PATCH 2/9] feat(cloud-hypervisor): expose the REST API control socket Signed-off-by: Anamika Aggarwal --- pkg/unikontainers/hypervisors/cloud_hypervisor.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/unikontainers/hypervisors/cloud_hypervisor.go b/pkg/unikontainers/hypervisors/cloud_hypervisor.go index 606a3c02e..c334b752b 100644 --- a/pkg/unikontainers/hypervisors/cloud_hypervisor.go +++ b/pkg/unikontainers/hypervisors/cloud_hypervisor.go @@ -70,6 +70,10 @@ func (ch *CloudHypervisor) BuildExecCmd(args types.ExecArgs, ukernel types.Unike // Start building the command exArgs := []string{ch.binaryPath} + // Expose the REST API over a control socket so the runtime can talk to + // Cloud Hypervisor after boot (e.g. for graceful shutdown). + exArgs = append(exArgs, "--api-socket", "path="+ResolveSocketPath(args)) + // Memory configuration if args.Sharedfs.Type == "virtiofs" { exArgs = append(exArgs, "--memory", fmt.Sprintf("size=%sM,shared=on", chMem)) From b1045ade3e7be3c6a19ae9fba58343bde672308f Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Sat, 25 Jul 2026 20:48:42 +0530 Subject: [PATCH 3/9] docs(configuration): document socket_path for cloud-hypervisor Signed-off-by: Anamika Aggarwal --- docs/configuration.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/configuration.md b/docs/configuration.md index a2daa4ba4..59d3ddcb6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -112,6 +112,7 @@ Each monitor subsection supports the following options: | `default_vcpus` | integer | `1` | Default number of virtual CPUs | | `path` | string | (empty) | Optional custom path to the monitor binary. If not specified, urunc will search for the binary in PATH | | `data_path` | string | (empty) | Optional custom path for the monitor's data file directory | +| `socket_path` | string | (empty) | Optional path for the monitor's control socket. If not specified, urunc uses a per-container default under `/tmp`. When a custom path is set, urunc creates its parent directory; setting it to an invalid location (a file already exists on the path) makes the monitor fail to start. Currently used by Cloud Hypervisor. | Since Qemu is the only currently supported monitor which requires extra data to boot a VM, `urunc` will first check `/usr/local/share` and then `/usr/share` for From 22ed1151b5a48bb78a44fe234f4f41b615e6d961 Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Tue, 28 Jul 2026 07:26:35 +0530 Subject: [PATCH 4/9] feat(cloud-hypervisor): add a boot_mode monitor option Add an opt-in boot_mode field to the monitor configuration. It is inert in this commit; the following commit makes cloud-hypervisor honor boot_mode=api. Signed-off-by: Anamika Aggarwal --- pkg/unikontainers/types/types.go | 2 ++ pkg/unikontainers/unikontainers.go | 2 ++ pkg/unikontainers/urunc_config.go | 3 +++ 3 files changed, 7 insertions(+) diff --git a/pkg/unikontainers/types/types.go b/pkg/unikontainers/types/types.go index 89a3e9ebf..21e7b024b 100644 --- a/pkg/unikontainers/types/types.go +++ b/pkg/unikontainers/types/types.go @@ -107,6 +107,7 @@ type ExecArgs struct { VSockDevPath string // The host directory where the fc unix socket is created VSockDevID int // The guest-cid SocketPath string // The path of the monitor's control socket (empty means the monitor's default) + BootMode string // Optional boot mode for the monitor. "api" drives the guest's start over the monitor's control socket; any other value boots the monitor from its command line as before. Net NetDevParams Sharedfs SharedfsParams } @@ -138,4 +139,5 @@ type MonitorConfig struct { DataPath string `toml:"data_path,omitempty"` // Optional path to the hypervisor data files (e.g. qemu bios stuff) Vhost bool `toml:"vhost,omitempty"` // Optional: enable vhost for network performance optimization SocketPath string `toml:"socket_path,omitempty"` // Optional path for the monitor's control socket (falls back to a per-container default) + BootMode string `toml:"boot_mode,omitempty"` } diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index d8561d951..97b8f9f13 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -403,6 +403,7 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { } defaultMemSizeMB := u.UruncCfg.Monitors[vmmType].DefaultMemoryMB socketPath := u.UruncCfg.Monitors[vmmType].SocketPath + bootMode := u.UruncCfg.Monitors[vmmType].BootMode // ExecArgs vmmArgs := types.ExecArgs{ @@ -414,6 +415,7 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { VCPUs: uint(defaultVCPUs), Environment: os.Environ(), SocketPath: socketPath, + BootMode: bootMode, } // ExecArgs diff --git a/pkg/unikontainers/urunc_config.go b/pkg/unikontainers/urunc_config.go index 1476eb3ee..277a82299 100644 --- a/pkg/unikontainers/urunc_config.go +++ b/pkg/unikontainers/urunc_config.go @@ -146,6 +146,7 @@ func (p *UruncConfig) Map() map[string]string { cfgMap[prefix+"data_path"] = hvCfg.DataPath cfgMap[prefix+"vhost"] = strconv.FormatBool(hvCfg.Vhost) cfgMap[prefix+"socket_path"] = hvCfg.SocketPath + cfgMap[prefix+"boot_mode"] = hvCfg.BootMode } for eb, ebCfg := range p.ExtraBins { prefix := "urunc_config.extra_binaries." + eb + "." @@ -201,6 +202,8 @@ func UruncConfigFromMap(cfgMap map[string]string) *UruncConfig { } case "socket_path": hvCfg.SocketPath = val + case "boot_mode": + hvCfg.BootMode = val } cfg.Monitors[hv] = hvCfg } From 132d02d5160763cd891cb309cef884dff6d3144b Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Tue, 28 Jul 2026 07:29:58 +0530 Subject: [PATCH 5/9] chore(cloud-hypervisor): align the boot_mode field comment Signed-off-by: Anamika Aggarwal --- pkg/unikontainers/types/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/unikontainers/types/types.go b/pkg/unikontainers/types/types.go index 21e7b024b..dd2013289 100644 --- a/pkg/unikontainers/types/types.go +++ b/pkg/unikontainers/types/types.go @@ -139,5 +139,5 @@ type MonitorConfig struct { DataPath string `toml:"data_path,omitempty"` // Optional path to the hypervisor data files (e.g. qemu bios stuff) Vhost bool `toml:"vhost,omitempty"` // Optional: enable vhost for network performance optimization SocketPath string `toml:"socket_path,omitempty"` // Optional path for the monitor's control socket (falls back to a per-container default) - BootMode string `toml:"boot_mode,omitempty"` + BootMode string `toml:"boot_mode,omitempty"` // Optional boot mode for the monitor. "api" drives the guest's start over the monitor's control socket; any other value boots the monitor from its command line as before. } From 638f92628ef09dc597b2a9713aa4cec529b21577 Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Tue, 28 Jul 2026 07:38:13 +0530 Subject: [PATCH 6/9] feat(cloud-hypervisor): drive the boot over the REST API With boot_mode=api, Cloud Hypervisor is spawned after changeRoot as a supervised child with only its API socket enabled, so the monitor and the socket are confined inside the monitor rootfs. urunc sends the whole VM configuration with vm.create and boots the guest with vm.boot only after the start-success handshake, preserving OCI start ordering, then supervises the child for its lifetime. Any other boot_mode value keeps the exec-based boot unchanged. Unikernel-specific raw CLI argument overrides cannot be expressed in the REST configuration and yield a clear error in this mode. Add an HTTP-over-Unix API client and unit tests driving it against a fake REST server, asserting the full vm.create payload and ordering. Signed-off-by: Anamika Aggarwal --- .../hypervisors/cloud_hypervisor.go | 234 ++++++++++++++++++ .../hypervisors/cloud_hypervisor_client.go | 131 ++++++++++ .../cloud_hypervisor_session_test.go | 188 ++++++++++++++ pkg/unikontainers/unikontainers.go | 68 ++++- 4 files changed, 616 insertions(+), 5 deletions(-) create mode 100644 pkg/unikontainers/hypervisors/cloud_hypervisor_client.go create mode 100644 pkg/unikontainers/hypervisors/cloud_hypervisor_session_test.go diff --git a/pkg/unikontainers/hypervisors/cloud_hypervisor.go b/pkg/unikontainers/hypervisors/cloud_hypervisor.go index c334b752b..d9b9a7909 100644 --- a/pkg/unikontainers/hypervisors/cloud_hypervisor.go +++ b/pkg/unikontainers/hypervisors/cloud_hypervisor.go @@ -15,8 +15,15 @@ package hypervisors import ( + "context" + "errors" "fmt" + "os" + "os/exec" + "os/signal" "strings" + "syscall" + "time" "github.com/urunc-dev/urunc/pkg/unikontainers/types" "golang.org/x/sys/unix" @@ -163,3 +170,230 @@ func (ch *CloudHypervisor) BuildExecCmd(args types.ExecArgs, ukernel types.Unike func (ch *CloudHypervisor) PreExec(_ types.ExecArgs) error { return nil } + +// The following types mirror the subset of Cloud Hypervisor's REST VmConfig +// that urunc's command-line boot already uses (verified against the installed +// cloud-hypervisor v53 API). + +type CHPayload struct { + Kernel string `json:"kernel,omitempty"` + Cmdline string `json:"cmdline,omitempty"` + Initramfs string `json:"initramfs,omitempty"` +} + +type CHMemory struct { + Size uint64 `json:"size"` + Shared bool `json:"shared,omitempty"` +} + +type CHCpus struct { + BootVcpus uint `json:"boot_vcpus"` + MaxVcpus uint `json:"max_vcpus"` +} + +type CHNet struct { + Tap string `json:"tap,omitempty"` + Mac string `json:"mac,omitempty"` + Mtu int `json:"mtu,omitempty"` +} + +type CHDisk struct { + Path string `json:"path"` + ID string `json:"id,omitempty"` +} + +type CHFs struct { + Tag string `json:"tag"` + Socket string `json:"socket"` +} + +type CHVsock struct { + Cid int `json:"cid"` + Socket string `json:"socket"` +} + +type CHConsole struct { + Mode string `json:"mode"` +} + +type CHVMConfig struct { + Payload CHPayload `json:"payload"` + Memory *CHMemory `json:"memory,omitempty"` + Cpus *CHCpus `json:"cpus,omitempty"` + Net []CHNet `json:"net,omitempty"` + Disks []CHDisk `json:"disks,omitempty"` + Fs []CHFs `json:"fs,omitempty"` + Vsock *CHVsock `json:"vsock,omitempty"` + Serial *CHConsole `json:"serial,omitempty"` + Console *CHConsole `json:"console,omitempty"` +} + +// buildCHVMConfig maps the same data BuildExecCmd puts on the command line +// into the REST VmConfig. Unikernel-specific raw CLI argument strings cannot +// be mapped to JSON, so they yield an error instead of being dropped. +func buildCHVMConfig(args types.ExecArgs, ukernel types.Unikernel) (*CHVMConfig, error) { + mem := args.MemSizeB + if mem < 1<<20 { + mem = DefaultMemory << 20 + } + cfg := &CHVMConfig{ + Payload: CHPayload{Kernel: args.UnikernelPath, Cmdline: args.Command}, + Memory: &CHMemory{Size: mem, Shared: args.Sharedfs.Type == "virtiofs"}, + Serial: &CHConsole{Mode: "Tty"}, + Console: &CHConsole{Mode: "Off"}, + } + if args.VCPUs > 0 { + cfg.Cpus = &CHCpus{BootVcpus: args.VCPUs, MaxVcpus: args.VCPUs} + } + + extraMonArgs := ukernel.MonitorCli() + if extraMonArgs.OtherArgs != "" { + return nil, fmt.Errorf("boot_mode=api does not support unikernel-specific monitor arguments (%q)", extraMonArgs.OtherArgs) + } + initrdPath := args.InitrdPath + if initrdPath == "" { + initrdPath = extraMonArgs.ExtraInitrd + } + cfg.Payload.Initramfs = initrdPath + + if args.Net.TapDev != "" { + if netCli := ukernel.MonitorNetCli(args.Net.TapDev, args.Net.MAC); netCli != "" { + return nil, fmt.Errorf("boot_mode=api does not support unikernel-specific network arguments (%q)", netCli) + } + cfg.Net = append(cfg.Net, CHNet{Tap: args.Net.TapDev, Mac: args.Net.MAC, Mtu: args.Net.MTU}) + } + + for _, blockArg := range ukernel.MonitorBlockCli() { + if blockArg.ExactArgs != "" { + return nil, fmt.Errorf("boot_mode=api does not support unikernel-specific block arguments (%q)", blockArg.ExactArgs) + } + if blockArg.Path != "" { + cfg.Disks = append(cfg.Disks, CHDisk{Path: blockArg.Path, ID: blockArg.ID}) + } + } + + if args.Sharedfs.Type == "virtiofs" { + cfg.Fs = append(cfg.Fs, CHFs{Tag: "fs0", Socket: "/tmp/vhostqemu"}) + } + if args.VAccelType == "vsock" { + cfg.Vsock = &CHVsock{Cid: args.VSockDevID, Socket: args.VSockDevPath + "/vaccel.sock"} + } + return cfg, nil +} + +// CHSession is a Cloud Hypervisor child process driven over its REST API +// socket. Create it with SpawnSocketVMM, send the full configuration with +// ConfigureVM, boot the guest with BootVM once the caller's start handshake +// allows it, then hand the calling process over with Supervise. +type CHSession struct { + cmd *exec.Cmd + client *chAPIClient +} + +// SpawnSocketVMM starts Cloud Hypervisor as a supervised child with only its +// API socket enabled. It is called after changeRoot, so the child inherits +// the pivoted root and its socket lives inside the monitor rootfs. The +// caller is still privileged here and only drops its own privileges +// afterwards, so when uid/gid are non-zero the child is started directly +// under that credential. +func (ch *CloudHypervisor) SpawnSocketVMM(args types.ExecArgs, uid, gid uint32) (*CHSession, error) { + socketPath := ResolveSocketPath(args) + // Cloud Hypervisor binds this path itself; remove any stale leftover. + if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) { + return nil, fmt.Errorf("failed to remove stale socket %q: %w", socketPath, err) + } + execCmd := []string{ch.binaryPath, "--api-socket", "path=" + socketPath} + if args.Seccomp { + execCmd = append(execCmd, "--seccomp", "true") + } else { + execCmd = append(execCmd, "--seccomp", "false") + } + + cmd := exec.Command(execCmd[0], execCmd[1:]...) //nolint: gosec + cmd.Env = args.Environment + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if uid != 0 || gid != 0 { + cmd.SysProcAttr = &syscall.SysProcAttr{ + Credential: &syscall.Credential{Uid: uid, Gid: gid}, + } + } + vmmLog.WithField("command", execCmd).Debug("starting cloud-hypervisor as a supervised child") + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("failed to start cloud-hypervisor: %w", err) + } + + client := newCHAPIClient(socketPath) + if err := client.connect(5 * time.Second); err != nil { + s := &CHSession{cmd: cmd, client: client} + s.Kill() + return nil, fmt.Errorf("cloud-hypervisor API socket never became ready: %w", err) + } + return &CHSession{cmd: cmd, client: client}, nil +} + +// ConfigureVM sends the complete VM configuration in one request. +func (s *CHSession) ConfigureVM(ctx context.Context, args types.ExecArgs, ukernel types.Unikernel) error { + cfg, err := buildCHVMConfig(args, ukernel) + if err != nil { + return err + } + vmmLog.Debug("api boot: sending vm.create") + return s.client.createVM(ctx, cfg) +} + +// BootVM boots the configured VM. +func (s *CHSession) BootVM(ctx context.Context) error { + vmmLog.Debug("api boot: sending vm.boot") + return s.client.bootVM(ctx) +} + +// Kill terminates the child and reaps it. For error paths before Supervise. +func (s *CHSession) Kill() { + _ = s.cmd.Process.Kill() + _, _ = s.cmd.Process.Wait() +} + +// Supervise hands the calling process over to the child for the rest of its +// life: it forwards SIGTERM/SIGINT and, once the child exits, exits this +// process with the child's exit code, mirroring the semantics syscall.Exec +// would have had. The caller must not exit before the child, since it is +// the container's init process. +// +// On success this function does not return: it calls os.Exit with the +// child's exit status once the child exits. +func (s *CHSession) Supervise() error { + // Forward the signals containerd would send to stop the container. + // SIGKILL cannot be caught, so it is not listed here: if it arrives, + // this process dies immediately and the child is left running, a known + // gap for this bounded experiment, not yet handled. + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT) + go func() { + sig, ok := <-sigCh + if !ok { + return + } + if sg, ok := sig.(syscall.Signal); ok { + _ = s.cmd.Process.Signal(sg) + } + }() + + waitErr := s.cmd.Wait() + signal.Stop(sigCh) + close(sigCh) + + exitCode := 0 + if waitErr != nil { + var exitErr *exec.ExitError + if errors.As(waitErr, &exitErr) { + exitCode = exitErr.ExitCode() + } else { + vmmLog.WithError(waitErr).Error("cloud-hypervisor exited with an unexpected error") + exitCode = 1 + } + } + os.Exit(exitCode) + return nil // unreachable +} diff --git a/pkg/unikontainers/hypervisors/cloud_hypervisor_client.go b/pkg/unikontainers/hypervisors/cloud_hypervisor_client.go new file mode 100644 index 000000000..927a6777a --- /dev/null +++ b/pkg/unikontainers/hypervisors/cloud_hypervisor_client.go @@ -0,0 +1,131 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hypervisors + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "sync" + "time" +) + +// chAPIClient drives a running Cloud Hypervisor process over its HTTP-over-Unix +// REST API socket. urunc uses it to create the whole VM configuration in one +// request and boot the guest when the start handshake allows it. +// +// The client establishes a single connection in connect() and keeps it alive +// for all requests, so every request reuses the connection whose readiness +// connect() already waited for, instead of re-dialing. +type chAPIClient struct { + socketPath string + httpClient *http.Client + + dialMu sync.Mutex + heldConn net.Conn +} + +func newCHAPIClient(socketPath string) *chAPIClient { + c := &chAPIClient{socketPath: socketPath} + transport := &http.Transport{ + DialContext: c.dialContext, + IdleConnTimeout: 0, + MaxIdleConns: 1, + MaxIdleConnsPerHost: 1, + } + c.httpClient = &http.Client{Transport: transport} + return c +} + +// dialContext hands the transport the connection pre-established by connect(), +// and falls back to dialing the socket path directly if that connection was +// already consumed. +func (c *chAPIClient) dialContext(ctx context.Context, _, _ string) (net.Conn, error) { + c.dialMu.Lock() + defer c.dialMu.Unlock() + if c.heldConn != nil { + conn := c.heldConn + c.heldConn = nil + return conn, nil + } + var d net.Dialer + return d.DialContext(ctx, "unix", c.socketPath) +} + +// connect blocks until the API socket exists and accepts connections, or the +// timeout elapses. Cloud Hypervisor creates the socket shortly after it +// starts, so the dial is retried tightly. +func (c *chAPIClient) connect(timeout time.Duration) error { + deadline := time.Now().Add(timeout) + var lastErr error + for time.Now().Before(deadline) { + conn, err := net.DialTimeout("unix", c.socketPath, 50*time.Millisecond) + if err == nil { + c.dialMu.Lock() + c.heldConn = conn + c.dialMu.Unlock() + return nil + } + lastErr = err + time.Sleep(1 * time.Millisecond) + } + if lastErr == nil { + lastErr = context.DeadlineExceeded + } + return fmt.Errorf("cloud-hypervisor API socket %q not ready within %s: %w", c.socketPath, timeout, lastErr) +} + +// createVM sends the full VM configuration. +func (c *chAPIClient) createVM(ctx context.Context, cfg *CHVMConfig) error { + return c.put(ctx, "/api/v1/vm.create", cfg) +} + +// bootVM boots the created VM. +func (c *chAPIClient) bootVM(ctx context.Context) error { + return c.put(ctx, "/api/v1/vm.boot", nil) +} + +// put sends body as an HTTP PUT to the given API path over the Unix socket, +// returning an error for any non-2xx response. A nil body sends an empty +// request. +func (c *chAPIClient) put(ctx context.Context, path string, body any) error { + var payload []byte + if body != nil { + var err error + payload, err = json.Marshal(body) + if err != nil { + return fmt.Errorf("marshal %s request: %w", path, err) + } + } + req, err := http.NewRequestWithContext(ctx, http.MethodPut, "http://localhost"+path, bytes.NewReader(payload)) + if err != nil { + return fmt.Errorf("build %s request: %w", path, err) + } + req.Header.Set("Content-Type", "application/json") + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("send %s request: %w", path, err) + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + respBody, _ := io.ReadAll(resp.Body) + return fmt.Errorf("%s returned HTTP %d: %s", path, resp.StatusCode, string(respBody)) + } + return nil +} diff --git a/pkg/unikontainers/hypervisors/cloud_hypervisor_session_test.go b/pkg/unikontainers/hypervisors/cloud_hypervisor_session_test.go new file mode 100644 index 000000000..21aec3b7a --- /dev/null +++ b/pkg/unikontainers/hypervisors/cloud_hypervisor_session_test.go @@ -0,0 +1,188 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hypervisors + +import ( + "context" + "encoding/json" + "net" + "net/http" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/urunc-dev/urunc/pkg/unikontainers/types" +) + +type chRecordedRequest struct { + path string + body map[string]any +} + +type fakeCHAPI struct { + mu sync.Mutex + requests []chRecordedRequest +} + +func (f *fakeCHAPI) recorded() []chRecordedRequest { + f.mu.Lock() + defer f.mu.Unlock() + out := make([]chRecordedRequest, len(f.requests)) + copy(out, f.requests) + return out +} + +func startFakeCHAPI(t *testing.T) (*fakeCHAPI, string) { + t.Helper() + sockPath := filepath.Join(t.TempDir(), "ch.sock") + ln, err := net.Listen("unix", sockPath) + if err != nil { + t.Fatalf("failed to listen on %s: %v", sockPath, err) + } + api := &fakeCHAPI{} + srv := &http.Server{ + ReadHeaderTimeout: time.Second, + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body map[string]any + _ = json.NewDecoder(r.Body).Decode(&body) + api.mu.Lock() + api.requests = append(api.requests, chRecordedRequest{path: r.URL.Path, body: body}) + api.mu.Unlock() + w.WriteHeader(http.StatusNoContent) + }), + } + go func() { _ = srv.Serve(ln) }() + t.Cleanup(func() { _ = srv.Close() }) + return api, sockPath +} + +func newTestCHSession(t *testing.T, sockPath string) *CHSession { + t.Helper() + client := newCHAPIClient(sockPath) + if err := client.connect(2 * time.Second); err != nil { + t.Fatalf("connect failed: %v", err) + } + return &CHSession{client: client} +} + +// TestCHConfigureVMSendsFullConfig verifies vm.create carries the complete +// machine configuration mapped from ExecArgs: payload paths verbatim (no +// rootfs prefixing), memory in bytes, both vcpu fields, net, disks, and the +// fixed console/serial modes. +func TestCHConfigureVMSendsFullConfig(t *testing.T) { + api, sockPath := startFakeCHAPI(t) + session := newTestCHSession(t, sockPath) + + args := types.ExecArgs{ + ContainerID: "test-container", + Command: "app -arg", + UnikernelPath: "/unikernel/app.bin", + InitrdPath: "/unikernel/initrd", + MemSizeB: 256 * 1024 * 1024, + VCPUs: 2, + } + args.Net.TapDev = "tap0" + args.Net.MAC = "aa:bb:cc:dd:ee:ff" + args.Net.MTU = 1500 + ukernel := &fakeUnikernel{ + blockCli: []types.MonitorBlockArgs{{ID: "rootfs", Path: "/dev/mapper/test-snap-1"}}, + } + + if err := session.ConfigureVM(context.Background(), args, ukernel); err != nil { + t.Fatalf("ConfigureVM failed: %v", err) + } + + reqs := api.recorded() + if len(reqs) != 1 || reqs[0].path != "/api/v1/vm.create" { + t.Fatalf("expected one PUT to /api/v1/vm.create, got %+v", reqs) + } + body := reqs[0].body + + payload, _ := body["payload"].(map[string]any) + if payload["kernel"] != "/unikernel/app.bin" || payload["cmdline"] != "app -arg" || payload["initramfs"] != "/unikernel/initrd" { + t.Errorf("unexpected payload: %v", payload) + } + memory, _ := body["memory"].(map[string]any) + if memory["size"] != float64(256*1024*1024) { + t.Errorf("memory.size = %v, want bytes not MB", memory["size"]) + } + cpus, _ := body["cpus"].(map[string]any) + if cpus["boot_vcpus"] != float64(2) || cpus["max_vcpus"] != float64(2) { + t.Errorf("unexpected cpus: %v", cpus) + } + nets, _ := body["net"].([]any) + if len(nets) != 1 { + t.Fatalf("expected one net device, got %v", body["net"]) + } + net0, _ := nets[0].(map[string]any) + if net0["tap"] != "tap0" || net0["mac"] != "aa:bb:cc:dd:ee:ff" || net0["mtu"] != float64(1500) { + t.Errorf("unexpected net[0]: %v", net0) + } + disks, _ := body["disks"].([]any) + if len(disks) != 1 { + t.Fatalf("expected one disk, got %v", body["disks"]) + } + disk0, _ := disks[0].(map[string]any) + if disk0["path"] != "/dev/mapper/test-snap-1" { + t.Errorf("disk path = %v, want unprefixed", disk0["path"]) + } + serial, _ := body["serial"].(map[string]any) + console, _ := body["console"].(map[string]any) + if serial["mode"] != "Tty" || console["mode"] != "Off" { + t.Errorf("serial=%v console=%v", serial, console) + } +} + +// TestCHBootAfterCreate verifies the request ordering create then boot. +func TestCHBootAfterCreate(t *testing.T) { + api, sockPath := startFakeCHAPI(t) + session := newTestCHSession(t, sockPath) + ctx := context.Background() + + args := types.ExecArgs{ContainerID: "test-container", Command: "app", UnikernelPath: "/unikernel/app.bin", MemSizeB: 256 * 1024 * 1024, VCPUs: 1} + if err := session.ConfigureVM(ctx, args, &fakeUnikernel{}); err != nil { + t.Fatalf("ConfigureVM failed: %v", err) + } + if err := session.BootVM(ctx); err != nil { + t.Fatalf("BootVM failed: %v", err) + } + want := []string{"/api/v1/vm.create", "/api/v1/vm.boot"} + reqs := api.recorded() + if len(reqs) != len(want) { + t.Fatalf("expected %v, got %+v", want, reqs) + } + for i := range want { + if reqs[i].path != want[i] { + t.Errorf("request %d: got %s, want %s", i, reqs[i].path, want[i]) + } + } +} + +// TestCHConfigureVMRejectsRawCliOverrides verifies the documented limitation: +// unikernel-provided raw CLI argument strings cannot be mapped to the REST +// config and must fail loudly rather than be silently dropped. +func TestCHConfigureVMRejectsRawCliOverrides(t *testing.T) { + _, sockPath := startFakeCHAPI(t) + session := newTestCHSession(t, sockPath) + + args := types.ExecArgs{ContainerID: "test-container", Command: "app", UnikernelPath: "/unikernel/app.bin", MemSizeB: 256 * 1024 * 1024, VCPUs: 1} + ukernel := &fakeUnikernel{ + blockCli: []types.MonitorBlockArgs{{ID: "vol0", ExactArgs: "--disk path=/x"}}, + } + if err := session.ConfigureVM(context.Background(), args, ukernel); err == nil { + t.Fatal("ConfigureVM accepted a raw CLI override it cannot map") + } +} diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index 97b8f9f13..1649aa24e 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -16,6 +16,7 @@ package unikontainers import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -641,6 +642,27 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { // ExecArgs vmmArgs.Command = unikernelCmd + // api boot mode: Cloud Hypervisor is spawned bare right after changeRoot + // below, so the monitor and its API socket are confined inside the + // monitor rootfs. The whole VM configuration is sent over the socket, + // and the guest only boots when vm.boot is sent after the start-success + // handshake, preserving OCI start ordering. + isAPIBoot := bootMode == "api" && vmmType == string(hypervisors.CloudHypervisorVmm) + if isAPIBoot && vmmArgs.Sharedfs.Type == "virtiofs" { + return fmt.Errorf("boot_mode=api does not support the virtiofs shared filesystem yet") + } + var chSession *hypervisors.CHSession + chHandedOff := false + defer func() { + // Any error return after the spawn must not leave the VMM child + // behind. Supervise never returns (os.Exit), so this only fires on + // error paths. + if chSession != nil && !chHandedOff { + chSession.Kill() + } + }() + ctx := context.Background() + // pivot _, err = findNS(u.Spec.Linux.Namespaces, specs.MountNamespace) // We just want to check if a mount namespace was define din the list @@ -659,6 +681,25 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { } } + // urunc is still privileged at this point; the child is started directly + // under the container user's credentials, and urunc drops its own + // privileges right after (setupUser below). + if isAPIBoot { + ch, ok := vmm.(*hypervisors.CloudHypervisor) + if !ok { + return fmt.Errorf("boot_mode=api is only supported for the cloud-hypervisor monitor") + } + chSession, err = ch.SpawnSocketVMM(vmmArgs, procAttrs.UID, procAttrs.GID) + if err != nil { + uniklog.Errorf("failed to spawn cloud-hypervisor: %v", err) + return err + } + if err = chSession.ConfigureVM(ctx, vmmArgs, unikernel); err != nil { + uniklog.Errorf("failed to configure the vm over the socket: %v", err) + return err + } + } + // uid/gid // Setup uid, gid and additional groups for the monitor process err = setupUser(u.Spec.Process.User) @@ -682,15 +723,20 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return err } - uniklog.Debug("calling vmm execve") + uniklog.Debug("preparing to start the vmm") metrics.Capture(m.TS18) // Build the VMM command once and verify it can be constructed successfully. // This ensures we don't report the container as started if command building fails. - execCmd, err := vmm.BuildExecCmd(vmmArgs, unikernel) - if err != nil { - uniklog.WithError(err).Error("failed to build VMM command") - return err + // For the api boot mode the VMM is already running and fully configured + // (the equivalent validation), so there is no command to build. + var execCmd []string + if !isAPIBoot { + execCmd, err = vmm.BuildExecCmd(vmmArgs, unikernel) + if err != nil { + uniklog.WithError(err).Error("failed to build VMM command") + return err + } } // Notify urunc start that the monitor is ready to execute. @@ -711,6 +757,18 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return err } + if isAPIBoot { + // The VMM is up and configured; boot the guest and hand this process + // over to supervising the child. This process must not exit before + // the child, since it is the container's init process. + if err = chSession.BootVM(ctx); err != nil { + uniklog.Errorf("failed to boot the guest: %v", err) + return err + } + chHandedOff = true + return chSession.Supervise() + } + // Execute the VMM using the command we built earlier. uniklog.WithField("command", execCmd).Debug("Ready to execve VMM") return syscall.Exec(vmm.Path(), execCmd, vmmArgs.Environment) //nolint: gosec From d7c212f78325facf1ab0ef0d4565fb2957da1824 Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Wed, 22 Jul 2026 16:05:05 +0530 Subject: [PATCH 7/9] fix(network): close the tap device fd after creation createTapDevice relies on a later syscall.Exec into the monitor to close the tap device fd that netlink.LinkAdd opened (O_CLOEXEC). In any flow where urunc keeps running instead of exec-ing away, the fd stays open and, since the tap device is single-queue, blocks the monitor from attaching to it ("Resource busy"). Close the fd explicitly after the owner and group are set (handling the close error), and add the author to contributors. Signed-off-by: Anamika Aggarwal --- .github/contributors.yaml | 3 +++ .github/linters/urunc-dict.txt | 1 + pkg/network/network.go | 19 +++++++++++++++++++ 3 files changed, 23 insertions(+) diff --git a/.github/contributors.yaml b/.github/contributors.yaml index 411f9f42a..7047f49a7 100644 --- a/.github/contributors.yaml +++ b/.github/contributors.yaml @@ -107,3 +107,6 @@ users: Chennamma-Hotkar: name: Chennamma Hotkar email: channuhotkar@gmail.com + Anamika1608: + name: Anamika Aggarwal + email: anamikaagg18@gmail.com diff --git a/.github/linters/urunc-dict.txt b/.github/linters/urunc-dict.txt index 07a4855b5..d7d2189a2 100644 --- a/.github/linters/urunc-dict.txt +++ b/.github/linters/urunc-dict.txt @@ -84,6 +84,7 @@ Sharedfs Syscalls TUNSETGROUP TUNSETOWNER +TUNSETPERSIST TUNTAP Timestamping Tmpfs diff --git a/pkg/network/network.go b/pkg/network/network.go index 375af431b..cbc73c707 100644 --- a/pkg/network/network.go +++ b/pkg/network/network.go @@ -101,16 +101,35 @@ func createTapDevice(name string, mtu int, ownerUID, ownerGID uint32) (netlink.L return nil, fmt.Errorf("failed to create tap device: %w", err) } + // LinkAdd opened the tap device's fd and set TUNSETPERSIST, so the + // interface survives independently of any open fd. Nothing after the + // owner/group ioctl calls uses the fd (the remaining setup goes through + // netlink and the monitor attaches to the device by name with its own + // open), so close it right away instead of relying on a later exec to + // close it (O_CLOEXEC). A single-queue tap device only allows one + // attached fd at a time, so holding it open blocks the monitor from + // attaching if the caller does not exec away. for _, tapFd := range tapLink.Fds { err = unix.IoctlSetInt(int(tapFd.Fd()), unix.TUNSETOWNER, int(ownerUID)) if err != nil { + if closeErr := tapFd.Close(); closeErr != nil { + netlog.Warnf("failed to close tap %s fd after owner ioctl error: %v", name, closeErr) + } return nil, fmt.Errorf("failed to set tap %s owner to uid %d: %w", name, ownerUID, err) } err = unix.IoctlSetInt(int(tapFd.Fd()), unix.TUNSETGROUP, int(ownerGID)) if err != nil { + if closeErr := tapFd.Close(); closeErr != nil { + netlog.Warnf("failed to close tap %s fd after group ioctl error: %v", name, closeErr) + } return nil, fmt.Errorf("failed to set tap %s group to gid %d: %w", name, ownerGID, err) } + + err = tapFd.Close() + if err != nil { + return nil, fmt.Errorf("failed to close the fd of tap %s: %w", name, err) + } } err = netlink.LinkSetMTU(tapLink, mtu) From 8614578c646e9ee3b00c886426d322c77b5250f5 Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Tue, 28 Jul 2026 07:58:22 +0530 Subject: [PATCH 8/9] docs(configuration): document the boot_mode option Signed-off-by: Anamika Aggarwal --- docs/configuration.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 59d3ddcb6..ee0dff5e6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -113,6 +113,7 @@ Each monitor subsection supports the following options: | `path` | string | (empty) | Optional custom path to the monitor binary. If not specified, urunc will search for the binary in PATH | | `data_path` | string | (empty) | Optional custom path for the monitor's data file directory | | `socket_path` | string | (empty) | Optional path for the monitor's control socket. If not specified, urunc uses a per-container default under `/tmp`. When a custom path is set, urunc creates its parent directory; setting it to an invalid location (a file already exists on the path) makes the monitor fail to start. Currently used by Cloud Hypervisor. | +| `boot_mode` | string | (empty) | Optional boot mode. When set to `api`, urunc starts the monitor as a supervised child and triggers the guest's start through the monitor's control socket; when unset, the monitor boots from its command line as before. Currently only used by Cloud Hypervisor. | Since Qemu is the only currently supported monitor which requires extra data to boot a VM, `urunc` will first check `/usr/local/share` and then `/usr/share` for @@ -131,6 +132,11 @@ data_path = "/usr/local/share/" default_memory_mb = 512 default_vcpus = 2 path = "/opt/firecracker/firecracker" + +[monitors.cloud-hypervisor] +default_memory_mb = 256 +default_vcpus = 1 +boot_mode = "api" ``` ### Extra binaries Configuration From ddfc0e337c5027ddc770385fb23760cd28c42c1c Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Tue, 28 Jul 2026 08:11:34 +0530 Subject: [PATCH 9/9] docs(configuration): list cloud-hypervisor as a supported monitor Signed-off-by: Anamika Aggarwal --- docs/configuration.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/configuration.md b/docs/configuration.md index ee0dff5e6..cb4700987 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -99,6 +99,7 @@ values. - [QEMU/KVM](./hypervisor-support#qemu) - `qemu` - [Firecracker](./hypervisor-support#firecracker) - `firecracker` +- [Cloud Hypervisor](./hypervisor-support#cloud-hypervisor) - `cloud-hypervisor` - [Solo5-hvt](./hypervisor-support#solo5-hvt) - `hvt` - Solo5 hvt (KVM-based tender) - [Solo5-spt](./hypervisor-support#solo5-spt) - `spt` - Solo5 spt (Seccomp-based tender)