From 5937270eb2191600d0931928a88ff00667d2aba3 Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Sat, 25 Jul 2026 20:37:06 +0530 Subject: [PATCH 1/8] feat(qemu): add a configurable control socket path and setup Signed-off-by: Anamika Aggarwal --- pkg/unikontainers/hypervisors/utils.go | 16 ++++++++++++++++ pkg/unikontainers/hypervisors/utils_test.go | 21 +++++++++++++++++++++ pkg/unikontainers/hypervisors/vmm.go | 11 +++++++++++ pkg/unikontainers/types/types.go | 8 +++++--- pkg/unikontainers/unikontainers.go | 9 +++++++++ pkg/unikontainers/urunc_config.go | 3 +++ 6 files changed, 65 insertions(+), 3 deletions(-) diff --git a/pkg/unikontainers/hypervisors/utils.go b/pkg/unikontainers/hypervisors/utils.go index 1bee33104..d06d0ce82 100644 --- a/pkg/unikontainers/hypervisors/utils.go +++ b/pkg/unikontainers/hypervisors/utils.go @@ -17,10 +17,12 @@ package hypervisors import ( "errors" "fmt" + "path/filepath" "runtime" "strconv" "time" + "github.com/urunc-dev/urunc/pkg/unikontainers/types" "golang.org/x/sys/unix" ) @@ -66,6 +68,20 @@ func BytesToStringMB(argMem uint64) string { return stringMem } +// DefaultSocketDir is the directory used for a monitor's control socket when +// no socket_path is configured. It always exists inside the monitor rootfs. +const DefaultSocketDir = "/tmp" + +// ResolveSocketPath returns the path for a monitor's control socket: the +// configured SocketPath when set, otherwise a per-container default under +// DefaultSocketDir. +func ResolveSocketPath(args types.ExecArgs) string { + if args.SocketPath != "" { + return args.SocketPath + } + return filepath.Join(DefaultSocketDir, args.ContainerID+".sock") +} + func killProcess(pid int) error { const timeout = 2 * time.Second err := unix.Kill(pid, unix.SIGKILL) diff --git a/pkg/unikontainers/hypervisors/utils_test.go b/pkg/unikontainers/hypervisors/utils_test.go index ab4f73738..d8ac54b15 100644 --- a/pkg/unikontainers/hypervisors/utils_test.go +++ b/pkg/unikontainers/hypervisors/utils_test.go @@ -18,6 +18,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/urunc-dev/urunc/pkg/unikontainers/types" ) func TestBytesToMiB(t *testing.T) { @@ -71,3 +72,23 @@ func TestBytesToMB(t *testing.T) { }) } } + +func TestResolveSocketPath(t *testing.T) { + t.Parallel() + if got := ResolveSocketPath(types.ExecArgs{SocketPath: "/run/urunc/x.sock"}); got != "/run/urunc/x.sock" { + t.Fatalf("configured: got %q", got) + } + if got := ResolveSocketPath(types.ExecArgs{ContainerID: "abc"}); got != "/tmp/abc.sock" { + t.Fatalf("default: got %q", got) + } +} + +func TestUsesControlSocket(t *testing.T) { + t.Parallel() + if !UsesControlSocket(QemuVmm) { + t.Fatal("qemu should use a control socket") + } + if UsesControlSocket(HvtVmm) { + t.Fatal("hvt should not") + } +} diff --git a/pkg/unikontainers/hypervisors/vmm.go b/pkg/unikontainers/hypervisors/vmm.go index c8600957b..e52c0a83b 100644 --- a/pkg/unikontainers/hypervisors/vmm.go +++ b/pkg/unikontainers/hypervisors/vmm.go @@ -27,6 +27,17 @@ const DefaultMemory uint64 = 256 // The default memory for every hypervisor: 256 type VmmType string +// UsesControlSocket reports whether a monitor exposes a control socket whose +// path (socket_path) urunc must make reachable before the monitor launches. +func UsesControlSocket(vmmType VmmType) bool { + switch vmmType { + case QemuVmm: + return true + default: + return false + } +} + var ErrVMMNotInstalled = errors.New("vmm not found") var vmmLog = logrus.WithField("subsystem", "monitors") diff --git a/pkg/unikontainers/types/types.go b/pkg/unikontainers/types/types.go index c6388e2cc..ff3d06928 100644 --- a/pkg/unikontainers/types/types.go +++ b/pkg/unikontainers/types/types.go @@ -108,6 +108,7 @@ type ExecArgs struct { VSockDevID int // The guest-cid Net NetDevParams Sharedfs SharedfsParams + SocketPath string // The path of the monitor's control socket (empty means the monitor's default) } type MonitorCliArgs struct { @@ -133,7 +134,8 @@ type ExtraBinConfig struct { type MonitorConfig struct { DefaultMemoryMB uint `toml:"default_memory_mb"` DefaultVCPUs uint `toml:"default_vcpus"` - BinaryPath string `toml:"path,omitempty"` // Optional path to the hypervisor binary - DataPath string `toml:"data_path,omitempty"` // Optional path to the hypervisor data files (e.g. qemu bios stuff) - Vhost bool `toml:"vhost,omitempty"` // Optional: enable vhost for network performance optimization + BinaryPath string `toml:"path,omitempty"` // Optional path to the hypervisor binary + DataPath string `toml:"data_path,omitempty"` // Optional path to the hypervisor data files (e.g. qemu bios stuff) + Vhost bool `toml:"vhost,omitempty"` // Optional: enable vhost for network performance optimization + SocketPath string `toml:"socket_path,omitempty"` // Optional path for the monitor's control socket (falls back to a per-container default) } diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index 84172aca4..d8561d951 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -402,6 +402,7 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { defaultVCPUs = 1 } defaultMemSizeMB := u.UruncCfg.Monitors[vmmType].DefaultMemoryMB + socketPath := u.UruncCfg.Monitors[vmmType].SocketPath // ExecArgs vmmArgs := types.ExecArgs{ @@ -412,6 +413,7 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { MemSizeB: uint64(defaultMemSizeMB * 1024 * 1024), VCPUs: uint(defaultVCPUs), Environment: os.Environ(), + SocketPath: socketPath, } // ExecArgs @@ -648,6 +650,13 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return err } + if hypervisors.UsesControlSocket(hypervisors.VmmType(vmmType)) { + sockDir := filepath.Dir(hypervisors.ResolveSocketPath(vmmArgs)) + if err = os.MkdirAll(sockDir, 0o755); err != nil { + return fmt.Errorf("failed to create control socket directory %q: %w", sockDir, err) + } + } + // uid/gid // Setup uid, gid and additional groups for the monitor process err = setupUser(u.Spec.Process.User) diff --git a/pkg/unikontainers/urunc_config.go b/pkg/unikontainers/urunc_config.go index 22573f43c..1476eb3ee 100644 --- a/pkg/unikontainers/urunc_config.go +++ b/pkg/unikontainers/urunc_config.go @@ -145,6 +145,7 @@ func (p *UruncConfig) Map() map[string]string { cfgMap[prefix+"binary_path"] = hvCfg.BinaryPath cfgMap[prefix+"data_path"] = hvCfg.DataPath cfgMap[prefix+"vhost"] = strconv.FormatBool(hvCfg.Vhost) + cfgMap[prefix+"socket_path"] = hvCfg.SocketPath } for eb, ebCfg := range p.ExtraBins { prefix := "urunc_config.extra_binaries." + eb + "." @@ -198,6 +199,8 @@ func UruncConfigFromMap(cfgMap map[string]string) *UruncConfig { } else { hvCfg.Vhost = boolVal } + case "socket_path": + hvCfg.SocketPath = val } cfg.Monitors[hv] = hvCfg } From 2a6eaee813e9a3ccf639ac062760b8947fc5401f Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Sat, 25 Jul 2026 20:37:58 +0530 Subject: [PATCH 2/8] feat(qemu): expose a QMP control socket Signed-off-by: Anamika Aggarwal --- pkg/unikontainers/hypervisors/qemu.go | 4 ++++ pkg/unikontainers/hypervisors/qemu_test.go | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/pkg/unikontainers/hypervisors/qemu.go b/pkg/unikontainers/hypervisors/qemu.go index 1ac77f870..269077bd8 100644 --- a/pkg/unikontainers/hypervisors/qemu.go +++ b/pkg/unikontainers/hypervisors/qemu.go @@ -67,6 +67,10 @@ func (q *Qemu) BuildExecCmd(args types.ExecArgs, ukernel types.Unikernel) ([]str cmdString += " -cpu host" // Choose CPU cmdString += " -enable-kvm" // Enable KVM to use CPU virt extensions cmdString += " -display none -vga none -serial stdio -monitor null" // Disable graphic output + // Expose a QMP control socket so the runtime can talk to QEMU after boot + // (e.g. for graceful shutdown). server,nowait lets QEMU boot without + // waiting for a client to connect. + cmdString += " -qmp unix:" + ResolveSocketPath(args) + ",server,nowait" if args.VCPUs > 0 { cmdString += fmt.Sprintf(" -smp %d", args.VCPUs) diff --git a/pkg/unikontainers/hypervisors/qemu_test.go b/pkg/unikontainers/hypervisors/qemu_test.go index fff45244c..a861b57b9 100644 --- a/pkg/unikontainers/hypervisors/qemu_test.go +++ b/pkg/unikontainers/hypervisors/qemu_test.go @@ -83,6 +83,7 @@ func TestQemuBuildExecCmd(t *testing.T) { "-m 256M", "-kernel " + testKernelPath, "-nic none", + "-qmp unix:/tmp/.sock,server,nowait", }, mustNotContain: []string{ "-smp", @@ -94,6 +95,18 @@ func TestQemuBuildExecCmd(t *testing.T) { "vhost-vsock-pci", }, }, + { + name: "configured SocketPath renders -qmp on that path", + args: types.ExecArgs{UnikernelPath: testKernelPath, Command: testCommand, SocketPath: "/run/urunc/q.sock"}, + unikernel: &fakeUnikernel{}, + mustContain: []string{"-qmp unix:/run/urunc/q.sock,server,nowait"}, + }, + { + name: "default SocketPath uses the container-id path", + args: types.ExecArgs{UnikernelPath: testKernelPath, Command: testCommand, ContainerID: "abc123"}, + unikernel: &fakeUnikernel{}, + mustContain: []string{"-qmp unix:/tmp/abc123.sock,server,nowait"}, + }, { name: "custom MemSizeB renders -m in MB", args: types.ExecArgs{ From 02c0ac808cea7888ad686d85a18ddd304241dabd Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Sat, 25 Jul 2026 20:38:22 +0530 Subject: [PATCH 3/8] docs(configuration): document the socket_path option for qemu Signed-off-by: Anamika Aggarwal --- docs/configuration.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/configuration.md b/docs/configuration.md index a2daa4ba4..b0031d7a9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -112,6 +112,7 @@ Each monitor subsection supports the following options: | `default_vcpus` | integer | `1` | Default number of virtual CPUs | | `path` | string | (empty) | Optional custom path to the monitor binary. If not specified, urunc will search for the binary in PATH | | `data_path` | string | (empty) | Optional custom path for the monitor's data file directory | +| `socket_path` | string | (empty) | Optional path for the monitor's control socket. If not specified, urunc uses a per-container default under `/tmp`. When a custom path is set, urunc creates its parent directory; setting it to an invalid location (a file already exists on the path) makes the monitor fail to start. Currently used by Qemu. | Since Qemu is the only currently supported monitor which requires extra data to boot a VM, `urunc` will first check `/usr/local/share` and then `/usr/share` for From 4bd29dc342210197026f21dbf11d813d71fd13ee Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Tue, 28 Jul 2026 06:35:47 +0530 Subject: [PATCH 4/8] feat(qemu): add a boot_mode monitor option Add an opt-in boot_mode field to the monitor configuration. It is inert in this commit; the following commit makes qemu honor boot_mode=api. Signed-off-by: Anamika Aggarwal --- pkg/unikontainers/types/types.go | 2 ++ pkg/unikontainers/unikontainers.go | 2 ++ pkg/unikontainers/urunc_config.go | 3 +++ 3 files changed, 7 insertions(+) diff --git a/pkg/unikontainers/types/types.go b/pkg/unikontainers/types/types.go index ff3d06928..69ca17dea 100644 --- a/pkg/unikontainers/types/types.go +++ b/pkg/unikontainers/types/types.go @@ -109,6 +109,7 @@ type ExecArgs struct { Net NetDevParams Sharedfs SharedfsParams SocketPath string // The path of the monitor's control socket (empty means the monitor's default) + BootMode string // Optional boot mode for the monitor. "api" drives the guest's start over the monitor's control socket; any other value boots the monitor from its command line as before. } type MonitorCliArgs struct { @@ -138,4 +139,5 @@ type MonitorConfig struct { DataPath string `toml:"data_path,omitempty"` // Optional path to the hypervisor data files (e.g. qemu bios stuff) Vhost bool `toml:"vhost,omitempty"` // Optional: enable vhost for network performance optimization SocketPath string `toml:"socket_path,omitempty"` // Optional path for the monitor's control socket (falls back to a per-container default) + BootMode string `toml:"boot_mode,omitempty"` // Optional boot mode for the monitor. "api" drives the guest's start over the monitor's control socket; any other value boots the monitor from its command line as before. } diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index d8561d951..97b8f9f13 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -403,6 +403,7 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { } defaultMemSizeMB := u.UruncCfg.Monitors[vmmType].DefaultMemoryMB socketPath := u.UruncCfg.Monitors[vmmType].SocketPath + bootMode := u.UruncCfg.Monitors[vmmType].BootMode // ExecArgs vmmArgs := types.ExecArgs{ @@ -414,6 +415,7 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { VCPUs: uint(defaultVCPUs), Environment: os.Environ(), SocketPath: socketPath, + BootMode: bootMode, } // ExecArgs diff --git a/pkg/unikontainers/urunc_config.go b/pkg/unikontainers/urunc_config.go index 1476eb3ee..277a82299 100644 --- a/pkg/unikontainers/urunc_config.go +++ b/pkg/unikontainers/urunc_config.go @@ -146,6 +146,7 @@ func (p *UruncConfig) Map() map[string]string { cfgMap[prefix+"data_path"] = hvCfg.DataPath cfgMap[prefix+"vhost"] = strconv.FormatBool(hvCfg.Vhost) cfgMap[prefix+"socket_path"] = hvCfg.SocketPath + cfgMap[prefix+"boot_mode"] = hvCfg.BootMode } for eb, ebCfg := range p.ExtraBins { prefix := "urunc_config.extra_binaries." + eb + "." @@ -201,6 +202,8 @@ func UruncConfigFromMap(cfgMap map[string]string) *UruncConfig { } case "socket_path": hvCfg.SocketPath = val + case "boot_mode": + hvCfg.BootMode = val } cfg.Monitors[hv] = hvCfg } From 90923984dfe0d5e90ce28c6d132a6fa7f501dc55 Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Tue, 28 Jul 2026 06:43:00 +0530 Subject: [PATCH 5/8] feat(qemu): start the guest over QMP in api boot mode With boot_mode=api, QEMU is spawned after changeRoot as a supervised child with its CPUs frozen (-S), so the monitor and its QMP socket are confined inside the monitor rootfs. urunc performs the QMP handshake and sends cont only after the start-success handshake, preserving OCI start ordering, then supervises the child for its lifetime. The machine configuration itself stays on the command line: QMP cannot configure a machine, so for QEMU the api mode drives only the guest's start. Any other boot_mode value keeps the exec-based boot unchanged. Add a QMP client implementing the handshake and single commands, and unit tests driving it against a fake QMP server on a Unix socket. Signed-off-by: Anamika Aggarwal --- pkg/unikontainers/hypervisors/qemu.go | 112 +++++++++++++++ .../hypervisors/qemu_qmp_client.go | 113 +++++++++++++++ .../hypervisors/qemu_session_test.go | 133 ++++++++++++++++++ pkg/unikontainers/unikontainers.go | 56 +++++++- 4 files changed, 410 insertions(+), 4 deletions(-) create mode 100644 pkg/unikontainers/hypervisors/qemu_qmp_client.go create mode 100644 pkg/unikontainers/hypervisors/qemu_session_test.go diff --git a/pkg/unikontainers/hypervisors/qemu.go b/pkg/unikontainers/hypervisors/qemu.go index 269077bd8..b35729997 100644 --- a/pkg/unikontainers/hypervisors/qemu.go +++ b/pkg/unikontainers/hypervisors/qemu.go @@ -15,9 +15,15 @@ package hypervisors import ( + "errors" "fmt" + "os" + "os/exec" + "os/signal" "runtime" "strings" + "syscall" + "time" "github.com/urunc-dev/urunc/pkg/unikontainers/types" "golang.org/x/sys/unix" @@ -156,6 +162,112 @@ func (q *Qemu) PreExec(_ types.ExecArgs) error { return nil } +// QemuSession is a QEMU child process started with its CPUs frozen (-S) and +// controlled over its QMP socket. Create it with SpawnPausedVMM, start the +// guest with Resume once the caller's start handshake allows it, then hand +// the calling process over with Supervise. +type QemuSession struct { + cmd *exec.Cmd + client *qmpClient +} + +// SpawnPausedVMM starts QEMU as a supervised child with the guest frozen (-S) +// and performs the QMP handshake over the control socket. It is called after +// changeRoot, so the child inherits the pivoted root and its QMP socket lives +// inside the monitor rootfs. The caller is still privileged here and only +// drops its own privileges afterwards, so when uid/gid are non-zero the child +// is started directly under that credential. +func (q *Qemu) SpawnPausedVMM(args types.ExecArgs, ukernel types.Unikernel, uid, gid uint32) (*QemuSession, error) { + execCmd, err := q.BuildExecCmd(args, ukernel) + if err != nil { + return nil, err + } + execCmd = append(execCmd, "-S") + + socketPath := ResolveSocketPath(args) + // QEMU binds the QMP socket itself; a stale file left from an earlier run + // would make its bind fail, so remove any leftover first. + if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) { + return nil, fmt.Errorf("failed to remove stale socket %q: %w", socketPath, err) + } + + cmd := exec.Command(execCmd[0], execCmd[1:]...) //nolint: gosec + cmd.Env = args.Environment + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if uid != 0 || gid != 0 { + cmd.SysProcAttr = &syscall.SysProcAttr{ + Credential: &syscall.Credential{Uid: uid, Gid: gid}, + } + } + vmmLog.WithField("command", execCmd).Debug("starting QEMU as a paused supervised child") + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("failed to start qemu: %w", err) + } + + client, err := connectQMP(socketPath, 5*time.Second) + if err != nil { + s := &QemuSession{cmd: cmd} + s.Kill() + return nil, err + } + return &QemuSession{cmd: cmd, client: client}, nil +} + +// Resume unfreezes the guest CPUs; the guest boots from this moment. +func (s *QemuSession) Resume() error { + vmmLog.Debug("api boot: sending QMP cont") + return s.client.execute("cont") +} + +// Kill terminates the child and reaps it. For error paths before Supervise. +func (s *QemuSession) Kill() { + if s.client != nil { + s.client.close() + } + _ = s.cmd.Process.Kill() + _, _ = s.cmd.Process.Wait() +} + +// Supervise hands the calling process over to the child for the rest of its +// life: it forwards SIGTERM/SIGINT and, once the child exits, exits this +// process with the child's exit code, mirroring the semantics syscall.Exec +// would have had. The caller must not exit before the child, since it is +// the container's init process. +// +// On success this function does not return: it calls os.Exit with the +// child's exit status once the child exits. +func (s *QemuSession) Supervise() error { + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT) + go func() { + sig, ok := <-sigCh + if !ok { + return + } + if sg, ok := sig.(syscall.Signal); ok { + _ = s.cmd.Process.Signal(sg) + } + }() + + waitErr := s.cmd.Wait() + signal.Stop(sigCh) + close(sigCh) + + exitCode := 0 + if waitErr != nil { + var exitErr *exec.ExitError + if errors.As(waitErr, &exitErr) { + exitCode = exitErr.ExitCode() + } else { + vmmLog.WithError(waitErr).Error("qemu exited with an unexpected error") + exitCode = 1 + } + } + os.Exit(exitCode) + return nil // unreachable +} + func getVirtioNetArg() string { devType := "virtio-net-pci" if runtime.GOARCH == "arm64" { diff --git a/pkg/unikontainers/hypervisors/qemu_qmp_client.go b/pkg/unikontainers/hypervisors/qemu_qmp_client.go new file mode 100644 index 000000000..d27a7a73c --- /dev/null +++ b/pkg/unikontainers/hypervisors/qemu_qmp_client.go @@ -0,0 +1,113 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hypervisors + +import ( + "bufio" + "encoding/json" + "fmt" + "net" + "time" +) + +// qmpClient drives a running QEMU process over its QMP control socket +// (line-delimited JSON over a Unix socket). It implements only what urunc +// needs: the connection handshake with capability negotiation, and single +// commands such as cont for resuming a guest that was started frozen (-S). +type qmpClient struct { + conn net.Conn + rd *bufio.Reader +} + +// connectQMP blocks until the QMP socket accepts a connection, then performs +// the mandatory handshake: read the server greeting, negotiate +// qmp_capabilities. QEMU binds the socket shortly after it starts, so the +// dial is retried tightly until the timeout elapses. +func connectQMP(socketPath string, timeout time.Duration) (*qmpClient, error) { + deadline := time.Now().Add(timeout) + var conn net.Conn + var lastErr error + for time.Now().Before(deadline) { + conn, lastErr = net.DialTimeout("unix", socketPath, 50*time.Millisecond) + if lastErr == nil { + break + } + time.Sleep(1 * time.Millisecond) + } + if conn == nil { + return nil, fmt.Errorf("qmp socket %q not ready within %s: %w", socketPath, timeout, lastErr) + } + c := &qmpClient{conn: conn, rd: bufio.NewReader(conn)} + // The server speaks first: nothing may be sent before its greeting is read. + greeting, err := c.readMessage() + if err != nil { + c.close() + return nil, fmt.Errorf("failed to read the QMP greeting: %w", err) + } + if _, ok := greeting["QMP"]; !ok { + c.close() + return nil, fmt.Errorf("unexpected QMP greeting: %v", greeting) + } + if err := c.execute("qmp_capabilities"); err != nil { + c.close() + return nil, err + } + return c, nil +} + +// execute sends one argument-less QMP command and waits for its result. +func (c *qmpClient) execute(command string) error { + req, err := json.Marshal(map[string]string{"execute": command}) + if err != nil { + return fmt.Errorf("failed to marshal QMP %s: %w", command, err) + } + if _, err := c.conn.Write(append(req, '\n')); err != nil { + return fmt.Errorf("failed to send QMP %s: %w", command, err) + } + resp, err := c.readMessage() + if err != nil { + return fmt.Errorf("failed to read the QMP response to %s: %w", command, err) + } + if errObj, ok := resp["error"]; ok { + return fmt.Errorf("QMP %s failed: %v", command, errObj) + } + if _, ok := resp["return"]; !ok { + return fmt.Errorf("unexpected QMP response to %s: %v", command, resp) + } + return nil +} + +// readMessage returns the next QMP message that is not an asynchronous event. +// QEMU may interleave event lines (e.g. RESUME) with command responses. +func (c *qmpClient) readMessage() (map[string]any, error) { + for { + line, err := c.rd.ReadBytes('\n') + if err != nil { + return nil, err + } + var msg map[string]any + if err := json.Unmarshal(line, &msg); err != nil { + return nil, fmt.Errorf("invalid QMP message %q: %w", line, err) + } + if _, isEvent := msg["event"]; isEvent { + continue + } + return msg, nil + } +} + +func (c *qmpClient) close() { + _ = c.conn.Close() +} diff --git a/pkg/unikontainers/hypervisors/qemu_session_test.go b/pkg/unikontainers/hypervisors/qemu_session_test.go new file mode 100644 index 000000000..29e292cb7 --- /dev/null +++ b/pkg/unikontainers/hypervisors/qemu_session_test.go @@ -0,0 +1,133 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hypervisors + +import ( + "bufio" + "encoding/json" + "net" + "path/filepath" + "sync" + "testing" + "time" +) + +// fakeQMPServer speaks just enough of the QMP wire protocol for these tests: +// it sends the greeting on accept, answers qmp_capabilities and cont with +// {"return":{}}, and records every command in order. Before answering cont it +// emits an asynchronous RESUME event line, which clients must skip. +type fakeQMPServer struct { + mu sync.Mutex + commands []string + failCont bool +} + +func (f *fakeQMPServer) recorded() []string { + f.mu.Lock() + defer f.mu.Unlock() + out := make([]string, len(f.commands)) + copy(out, f.commands) + return out +} + +func startFakeQMPServer(t *testing.T, failCont bool) (*fakeQMPServer, string) { + t.Helper() + sockPath := filepath.Join(t.TempDir(), "qmp.sock") + ln, err := net.Listen("unix", sockPath) + if err != nil { + t.Fatalf("failed to listen on %s: %v", sockPath, err) + } + t.Cleanup(func() { ln.Close() }) + + srv := &fakeQMPServer{failCont: failCont} + go func() { + conn, err := ln.Accept() + if err != nil { + return + } + defer conn.Close() + _, _ = conn.Write([]byte(`{"QMP":{"version":{},"capabilities":[]}}` + "\n")) + rd := bufio.NewReader(conn) + for { + line, err := rd.ReadBytes('\n') + if err != nil { + return + } + var msg map[string]any + if json.Unmarshal(line, &msg) != nil { + return + } + cmd, _ := msg["execute"].(string) + srv.mu.Lock() + srv.commands = append(srv.commands, cmd) + srv.mu.Unlock() + if cmd == "cont" { + _, _ = conn.Write([]byte(`{"event":"RESUME","timestamp":{"seconds":0,"microseconds":0}}` + "\n")) + if srv.failCont { + _, _ = conn.Write([]byte(`{"error":{"class":"GenericError","desc":"cont refused"}}` + "\n")) + continue + } + } + _, _ = conn.Write([]byte(`{"return":{}}` + "\n")) + } + }() + return srv, sockPath +} + +// TestQMPConnectNegotiatesAndResumes verifies the client reads the greeting, +// negotiates capabilities before anything else, and that Resume sends cont and +// tolerates the interleaved asynchronous event line. +func TestQMPConnectNegotiatesAndResumes(t *testing.T) { + srv, sockPath := startFakeQMPServer(t, false) + + client, err := connectQMP(sockPath, 2*time.Second) + if err != nil { + t.Fatalf("connectQMP failed: %v", err) + } + defer client.close() + + session := &QemuSession{client: client} + if err := session.Resume(); err != nil { + t.Fatalf("Resume failed: %v", err) + } + + want := []string{"qmp_capabilities", "cont"} + got := srv.recorded() + if len(got) != len(want) { + t.Fatalf("expected commands %v, got %v", want, got) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("command %d: got %s, want %s", i, got[i], want[i]) + } + } +} + +// TestQMPResumeSurfacesErrors verifies a QMP error response becomes a Go error. +func TestQMPResumeSurfacesErrors(t *testing.T) { + _, sockPath := startFakeQMPServer(t, true) + + client, err := connectQMP(sockPath, 2*time.Second) + if err != nil { + t.Fatalf("connectQMP failed: %v", err) + } + defer client.close() + + session := &QemuSession{client: client} + err = session.Resume() + if err == nil { + t.Fatal("Resume succeeded against a server that refused cont") + } +} diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index 97b8f9f13..8d220b9cb 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -641,6 +641,22 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { // ExecArgs vmmArgs.Command = unikernelCmd + // api boot mode: QEMU is spawned frozen (-S) as a supervised child right + // after changeRoot below, so the monitor and its QMP socket are confined + // inside the monitor rootfs. The guest only starts when the QMP cont is + // sent after the start-success handshake, preserving OCI start ordering. + isAPIBoot := bootMode == "api" && vmmType == string(hypervisors.QemuVmm) + var qSession *hypervisors.QemuSession + qHandedOff := false + defer func() { + // Any error return after the spawn must not leave the VMM child + // behind. Supervise never returns (os.Exit), so this only fires on + // error paths. + if qSession != nil && !qHandedOff { + qSession.Kill() + } + }() + // pivot _, err = findNS(u.Spec.Linux.Namespaces, specs.MountNamespace) // We just want to check if a mount namespace was define din the list @@ -659,6 +675,21 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { } } + // urunc is still privileged at this point; the child is started directly + // under the container user's credentials, and urunc drops its own + // privileges right after (setupUser below). + if isAPIBoot { + q, ok := vmm.(*hypervisors.Qemu) + if !ok { + return fmt.Errorf("boot_mode=api is only supported for the qemu monitor") + } + qSession, err = q.SpawnPausedVMM(vmmArgs, unikernel, procAttrs.UID, procAttrs.GID) + if err != nil { + uniklog.Errorf("failed to spawn qemu: %v", err) + return err + } + } + // uid/gid // Setup uid, gid and additional groups for the monitor process err = setupUser(u.Spec.Process.User) @@ -687,10 +718,15 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { // Build the VMM command once and verify it can be constructed successfully. // This ensures we don't report the container as started if command building fails. - execCmd, err := vmm.BuildExecCmd(vmmArgs, unikernel) - if err != nil { - uniklog.WithError(err).Error("failed to build VMM command") - return err + // For the api boot mode the VMM is already running, frozen and fully + // configured (the equivalent validation), so there is no command to build. + var execCmd []string + if !isAPIBoot { + execCmd, err = vmm.BuildExecCmd(vmmArgs, unikernel) + if err != nil { + uniklog.WithError(err).Error("failed to build VMM command") + return err + } } // Notify urunc start that the monitor is ready to execute. @@ -711,6 +747,18 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return err } + if isAPIBoot { + // The VMM is up and configured; unfreeze the guest and hand this + // process over to supervising the child. This process must not exit + // before the child, since it is the container's init process. + if err = qSession.Resume(); err != nil { + uniklog.Errorf("failed to resume the guest: %v", err) + return err + } + qHandedOff = true + return qSession.Supervise() + } + // Execute the VMM using the command we built earlier. uniklog.WithField("command", execCmd).Debug("Ready to execve VMM") return syscall.Exec(vmm.Path(), execCmd, vmmArgs.Environment) //nolint: gosec From 8d133ddf493183107d339ac6940aa6a0c9b3cc7f Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Wed, 22 Jul 2026 16:05:05 +0530 Subject: [PATCH 6/8] fix(network): close the tap device fd after creation createTapDevice relies on a later syscall.Exec into the monitor to close the tap device fd that netlink.LinkAdd opened (O_CLOEXEC). In any flow where urunc keeps running instead of exec-ing away, the fd stays open and, since the tap device is single-queue, blocks the monitor from attaching to it ("Resource busy"). Close the fd explicitly after the owner and group are set (handling the close error), and add the author to contributors. Signed-off-by: Anamika Aggarwal --- .github/contributors.yaml | 3 +++ .github/linters/urunc-dict.txt | 1 + pkg/network/network.go | 19 +++++++++++++++++++ 3 files changed, 23 insertions(+) diff --git a/.github/contributors.yaml b/.github/contributors.yaml index 411f9f42a..7047f49a7 100644 --- a/.github/contributors.yaml +++ b/.github/contributors.yaml @@ -107,3 +107,6 @@ users: Chennamma-Hotkar: name: Chennamma Hotkar email: channuhotkar@gmail.com + Anamika1608: + name: Anamika Aggarwal + email: anamikaagg18@gmail.com diff --git a/.github/linters/urunc-dict.txt b/.github/linters/urunc-dict.txt index 07a4855b5..d7d2189a2 100644 --- a/.github/linters/urunc-dict.txt +++ b/.github/linters/urunc-dict.txt @@ -84,6 +84,7 @@ Sharedfs Syscalls TUNSETGROUP TUNSETOWNER +TUNSETPERSIST TUNTAP Timestamping Tmpfs diff --git a/pkg/network/network.go b/pkg/network/network.go index 375af431b..cbc73c707 100644 --- a/pkg/network/network.go +++ b/pkg/network/network.go @@ -101,16 +101,35 @@ func createTapDevice(name string, mtu int, ownerUID, ownerGID uint32) (netlink.L return nil, fmt.Errorf("failed to create tap device: %w", err) } + // LinkAdd opened the tap device's fd and set TUNSETPERSIST, so the + // interface survives independently of any open fd. Nothing after the + // owner/group ioctl calls uses the fd (the remaining setup goes through + // netlink and the monitor attaches to the device by name with its own + // open), so close it right away instead of relying on a later exec to + // close it (O_CLOEXEC). A single-queue tap device only allows one + // attached fd at a time, so holding it open blocks the monitor from + // attaching if the caller does not exec away. for _, tapFd := range tapLink.Fds { err = unix.IoctlSetInt(int(tapFd.Fd()), unix.TUNSETOWNER, int(ownerUID)) if err != nil { + if closeErr := tapFd.Close(); closeErr != nil { + netlog.Warnf("failed to close tap %s fd after owner ioctl error: %v", name, closeErr) + } return nil, fmt.Errorf("failed to set tap %s owner to uid %d: %w", name, ownerUID, err) } err = unix.IoctlSetInt(int(tapFd.Fd()), unix.TUNSETGROUP, int(ownerGID)) if err != nil { + if closeErr := tapFd.Close(); closeErr != nil { + netlog.Warnf("failed to close tap %s fd after group ioctl error: %v", name, closeErr) + } return nil, fmt.Errorf("failed to set tap %s group to gid %d: %w", name, ownerGID, err) } + + err = tapFd.Close() + if err != nil { + return nil, fmt.Errorf("failed to close the fd of tap %s: %w", name, err) + } } err = netlink.LinkSetMTU(tapLink, mtu) From 848dbbdbad93c7bf578b91a50d4197a29268226e Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Tue, 28 Jul 2026 06:54:45 +0530 Subject: [PATCH 7/8] fix(qemu): address api boot review findings Reject boot_mode=api with virtiofs sharedfs up front, since QMP cannot be sequenced ahead of virtiofsd's own startup. Pass real stdin through to the paused QEMU child, matching the exec path's -serial stdio. Bound every QMP read/write with a deadline instead of only the initial dial, and restore the SIGKILL-cannot-be-caught note on Supervise. Signed-off-by: Anamika Aggarwal --- pkg/unikontainers/hypervisors/qemu.go | 5 +++++ pkg/unikontainers/hypervisors/qemu_qmp_client.go | 10 ++++++++++ pkg/unikontainers/unikontainers.go | 5 ++++- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/pkg/unikontainers/hypervisors/qemu.go b/pkg/unikontainers/hypervisors/qemu.go index b35729997..7e01ab7dc 100644 --- a/pkg/unikontainers/hypervisors/qemu.go +++ b/pkg/unikontainers/hypervisors/qemu.go @@ -193,6 +193,7 @@ func (q *Qemu) SpawnPausedVMM(args types.ExecArgs, ukernel types.Unikernel, uid, cmd := exec.Command(execCmd[0], execCmd[1:]...) //nolint: gosec cmd.Env = args.Environment + cmd.Stdin = os.Stdin cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if uid != 0 || gid != 0 { @@ -238,6 +239,10 @@ func (s *QemuSession) Kill() { // On success this function does not return: it calls os.Exit with the // child's exit status once the child exits. func (s *QemuSession) Supervise() error { + // Forward the signals containerd would send to stop the container. + // SIGKILL cannot be caught, so it is not listed here: if it arrives, + // this process dies immediately and the child is left running, a known + // gap for this bounded experiment, not yet handled. sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT) go func() { diff --git a/pkg/unikontainers/hypervisors/qemu_qmp_client.go b/pkg/unikontainers/hypervisors/qemu_qmp_client.go index d27a7a73c..4fd4647d2 100644 --- a/pkg/unikontainers/hypervisors/qemu_qmp_client.go +++ b/pkg/unikontainers/hypervisors/qemu_qmp_client.go @@ -47,9 +47,15 @@ func connectQMP(socketPath string, timeout time.Duration) (*qmpClient, error) { time.Sleep(1 * time.Millisecond) } if conn == nil { + if lastErr == nil { + lastErr = fmt.Errorf("dial timed out after %s", timeout) + } return nil, fmt.Errorf("qmp socket %q not ready within %s: %w", socketPath, timeout, lastErr) } c := &qmpClient{conn: conn, rd: bufio.NewReader(conn)} + // Bound the handshake itself: a dial succeeding does not guarantee QEMU + // keeps talking, so reads/writes below must not block forever. + _ = c.conn.SetDeadline(deadline) // The server speaks first: nothing may be sent before its greeting is read. greeting, err := c.readMessage() if err != nil { @@ -64,11 +70,15 @@ func connectQMP(socketPath string, timeout time.Duration) (*qmpClient, error) { c.close() return nil, err } + _ = c.conn.SetDeadline(time.Time{}) return c, nil } // execute sends one argument-less QMP command and waits for its result. func (c *qmpClient) execute(command string) error { + _ = c.conn.SetDeadline(time.Now().Add(5 * time.Second)) + defer func() { _ = c.conn.SetDeadline(time.Time{}) }() + req, err := json.Marshal(map[string]string{"execute": command}) if err != nil { return fmt.Errorf("failed to marshal QMP %s: %w", command, err) diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index 8d220b9cb..0430ade01 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -646,6 +646,9 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { // inside the monitor rootfs. The guest only starts when the QMP cont is // sent after the start-success handshake, preserving OCI start ordering. isAPIBoot := bootMode == "api" && vmmType == string(hypervisors.QemuVmm) + if isAPIBoot && vmmArgs.Sharedfs.Type == "virtiofs" { + return fmt.Errorf("boot_mode=api does not support the virtiofs shared filesystem yet") + } var qSession *hypervisors.QemuSession qHandedOff := false defer func() { @@ -713,7 +716,7 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return err } - uniklog.Debug("calling vmm execve") + uniklog.Debug("preparing to start the vmm") metrics.Capture(m.TS18) // Build the VMM command once and verify it can be constructed successfully. From 2beebe43d4199b1d80b5309641da3539d835f426 Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Tue, 28 Jul 2026 07:10:12 +0530 Subject: [PATCH 8/8] docs(configuration): document the boot_mode option Signed-off-by: Anamika Aggarwal --- docs/configuration.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index b0031d7a9..bfcb41a8a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -113,6 +113,7 @@ Each monitor subsection supports the following options: | `path` | string | (empty) | Optional custom path to the monitor binary. If not specified, urunc will search for the binary in PATH | | `data_path` | string | (empty) | Optional custom path for the monitor's data file directory | | `socket_path` | string | (empty) | Optional path for the monitor's control socket. If not specified, urunc uses a per-container default under `/tmp`. When a custom path is set, urunc creates its parent directory; setting it to an invalid location (a file already exists on the path) makes the monitor fail to start. Currently used by Qemu. | +| `boot_mode` | string | (empty) | Optional boot mode. When set to `api`, urunc starts the monitor as a supervised child and triggers the guest's start through the monitor's control socket; when unset, the monitor boots from its command line as before. Currently only used by Qemu. | Since Qemu is the only currently supported monitor which requires extra data to boot a VM, `urunc` will first check `/usr/local/share` and then `/usr/share` for @@ -126,6 +127,7 @@ default_memory_mb = 1024 default_vcpus = 4 path = "/usr/local/bin/qemu-system-x86_64" data_path = "/usr/local/share/" +boot_mode = "api" [monitors.firecracker] default_memory_mb = 512