diff --git a/.github/contributors.yaml b/.github/contributors.yaml index 411f9f42a..7047f49a7 100644 --- a/.github/contributors.yaml +++ b/.github/contributors.yaml @@ -107,3 +107,6 @@ users: Chennamma-Hotkar: name: Chennamma Hotkar email: channuhotkar@gmail.com + Anamika1608: + name: Anamika Aggarwal + email: anamikaagg18@gmail.com diff --git a/.github/linters/urunc-dict.txt b/.github/linters/urunc-dict.txt index 07a4855b5..d7d2189a2 100644 --- a/.github/linters/urunc-dict.txt +++ b/.github/linters/urunc-dict.txt @@ -84,6 +84,7 @@ Sharedfs Syscalls TUNSETGROUP TUNSETOWNER +TUNSETPERSIST TUNTAP Timestamping Tmpfs diff --git a/docs/configuration.md b/docs/configuration.md index a2daa4ba4..bfcb41a8a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -112,6 +112,8 @@ Each monitor subsection supports the following options: | `default_vcpus` | integer | `1` | Default number of virtual CPUs | | `path` | string | (empty) | Optional custom path to the monitor binary. If not specified, urunc will search for the binary in PATH | | `data_path` | string | (empty) | Optional custom path for the monitor's data file directory | +| `socket_path` | string | (empty) | Optional path for the monitor's control socket. If not specified, urunc uses a per-container default under `/tmp`. When a custom path is set, urunc creates its parent directory; setting it to an invalid location (a file already exists on the path) makes the monitor fail to start. Currently used by Qemu. | +| `boot_mode` | string | (empty) | Optional boot mode. When set to `api`, urunc starts the monitor as a supervised child and triggers the guest's start through the monitor's control socket; when unset, the monitor boots from its command line as before. Currently only used by Qemu. | Since Qemu is the only currently supported monitor which requires extra data to boot a VM, `urunc` will first check `/usr/local/share` and then `/usr/share` for @@ -125,6 +127,7 @@ default_memory_mb = 1024 default_vcpus = 4 path = "/usr/local/bin/qemu-system-x86_64" data_path = "/usr/local/share/" +boot_mode = "api" [monitors.firecracker] default_memory_mb = 512 diff --git a/pkg/network/network.go b/pkg/network/network.go index 375af431b..cbc73c707 100644 --- a/pkg/network/network.go +++ b/pkg/network/network.go @@ -101,16 +101,35 @@ func createTapDevice(name string, mtu int, ownerUID, ownerGID uint32) (netlink.L return nil, fmt.Errorf("failed to create tap device: %w", err) } + // LinkAdd opened the tap device's fd and set TUNSETPERSIST, so the + // interface survives independently of any open fd. Nothing after the + // owner/group ioctl calls uses the fd (the remaining setup goes through + // netlink and the monitor attaches to the device by name with its own + // open), so close it right away instead of relying on a later exec to + // close it (O_CLOEXEC). A single-queue tap device only allows one + // attached fd at a time, so holding it open blocks the monitor from + // attaching if the caller does not exec away. for _, tapFd := range tapLink.Fds { err = unix.IoctlSetInt(int(tapFd.Fd()), unix.TUNSETOWNER, int(ownerUID)) if err != nil { + if closeErr := tapFd.Close(); closeErr != nil { + netlog.Warnf("failed to close tap %s fd after owner ioctl error: %v", name, closeErr) + } return nil, fmt.Errorf("failed to set tap %s owner to uid %d: %w", name, ownerUID, err) } err = unix.IoctlSetInt(int(tapFd.Fd()), unix.TUNSETGROUP, int(ownerGID)) if err != nil { + if closeErr := tapFd.Close(); closeErr != nil { + netlog.Warnf("failed to close tap %s fd after group ioctl error: %v", name, closeErr) + } return nil, fmt.Errorf("failed to set tap %s group to gid %d: %w", name, ownerGID, err) } + + err = tapFd.Close() + if err != nil { + return nil, fmt.Errorf("failed to close the fd of tap %s: %w", name, err) + } } err = netlink.LinkSetMTU(tapLink, mtu) diff --git a/pkg/unikontainers/hypervisors/qemu.go b/pkg/unikontainers/hypervisors/qemu.go index 1ac77f870..7e01ab7dc 100644 --- a/pkg/unikontainers/hypervisors/qemu.go +++ b/pkg/unikontainers/hypervisors/qemu.go @@ -15,9 +15,15 @@ package hypervisors import ( + "errors" "fmt" + "os" + "os/exec" + "os/signal" "runtime" "strings" + "syscall" + "time" "github.com/urunc-dev/urunc/pkg/unikontainers/types" "golang.org/x/sys/unix" @@ -67,6 +73,10 @@ func (q *Qemu) BuildExecCmd(args types.ExecArgs, ukernel types.Unikernel) ([]str cmdString += " -cpu host" // Choose CPU cmdString += " -enable-kvm" // Enable KVM to use CPU virt extensions cmdString += " -display none -vga none -serial stdio -monitor null" // Disable graphic output + // Expose a QMP control socket so the runtime can talk to QEMU after boot + // (e.g. for graceful shutdown). server,nowait lets QEMU boot without + // waiting for a client to connect. + cmdString += " -qmp unix:" + ResolveSocketPath(args) + ",server,nowait" if args.VCPUs > 0 { cmdString += fmt.Sprintf(" -smp %d", args.VCPUs) @@ -152,6 +162,117 @@ func (q *Qemu) PreExec(_ types.ExecArgs) error { return nil } +// QemuSession is a QEMU child process started with its CPUs frozen (-S) and +// controlled over its QMP socket. Create it with SpawnPausedVMM, start the +// guest with Resume once the caller's start handshake allows it, then hand +// the calling process over with Supervise. +type QemuSession struct { + cmd *exec.Cmd + client *qmpClient +} + +// SpawnPausedVMM starts QEMU as a supervised child with the guest frozen (-S) +// and performs the QMP handshake over the control socket. It is called after +// changeRoot, so the child inherits the pivoted root and its QMP socket lives +// inside the monitor rootfs. The caller is still privileged here and only +// drops its own privileges afterwards, so when uid/gid are non-zero the child +// is started directly under that credential. +func (q *Qemu) SpawnPausedVMM(args types.ExecArgs, ukernel types.Unikernel, uid, gid uint32) (*QemuSession, error) { + execCmd, err := q.BuildExecCmd(args, ukernel) + if err != nil { + return nil, err + } + execCmd = append(execCmd, "-S") + + socketPath := ResolveSocketPath(args) + // QEMU binds the QMP socket itself; a stale file left from an earlier run + // would make its bind fail, so remove any leftover first. + if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) { + return nil, fmt.Errorf("failed to remove stale socket %q: %w", socketPath, err) + } + + cmd := exec.Command(execCmd[0], execCmd[1:]...) //nolint: gosec + cmd.Env = args.Environment + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if uid != 0 || gid != 0 { + cmd.SysProcAttr = &syscall.SysProcAttr{ + Credential: &syscall.Credential{Uid: uid, Gid: gid}, + } + } + vmmLog.WithField("command", execCmd).Debug("starting QEMU as a paused supervised child") + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("failed to start qemu: %w", err) + } + + client, err := connectQMP(socketPath, 5*time.Second) + if err != nil { + s := &QemuSession{cmd: cmd} + s.Kill() + return nil, err + } + return &QemuSession{cmd: cmd, client: client}, nil +} + +// Resume unfreezes the guest CPUs; the guest boots from this moment. +func (s *QemuSession) Resume() error { + vmmLog.Debug("api boot: sending QMP cont") + return s.client.execute("cont") +} + +// Kill terminates the child and reaps it. For error paths before Supervise. +func (s *QemuSession) Kill() { + if s.client != nil { + s.client.close() + } + _ = s.cmd.Process.Kill() + _, _ = s.cmd.Process.Wait() +} + +// Supervise hands the calling process over to the child for the rest of its +// life: it forwards SIGTERM/SIGINT and, once the child exits, exits this +// process with the child's exit code, mirroring the semantics syscall.Exec +// would have had. The caller must not exit before the child, since it is +// the container's init process. +// +// On success this function does not return: it calls os.Exit with the +// child's exit status once the child exits. +func (s *QemuSession) Supervise() error { + // Forward the signals containerd would send to stop the container. + // SIGKILL cannot be caught, so it is not listed here: if it arrives, + // this process dies immediately and the child is left running, a known + // gap for this bounded experiment, not yet handled. + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT) + go func() { + sig, ok := <-sigCh + if !ok { + return + } + if sg, ok := sig.(syscall.Signal); ok { + _ = s.cmd.Process.Signal(sg) + } + }() + + waitErr := s.cmd.Wait() + signal.Stop(sigCh) + close(sigCh) + + exitCode := 0 + if waitErr != nil { + var exitErr *exec.ExitError + if errors.As(waitErr, &exitErr) { + exitCode = exitErr.ExitCode() + } else { + vmmLog.WithError(waitErr).Error("qemu exited with an unexpected error") + exitCode = 1 + } + } + os.Exit(exitCode) + return nil // unreachable +} + func getVirtioNetArg() string { devType := "virtio-net-pci" if runtime.GOARCH == "arm64" { diff --git a/pkg/unikontainers/hypervisors/qemu_qmp_client.go b/pkg/unikontainers/hypervisors/qemu_qmp_client.go new file mode 100644 index 000000000..4fd4647d2 --- /dev/null +++ b/pkg/unikontainers/hypervisors/qemu_qmp_client.go @@ -0,0 +1,123 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hypervisors + +import ( + "bufio" + "encoding/json" + "fmt" + "net" + "time" +) + +// qmpClient drives a running QEMU process over its QMP control socket +// (line-delimited JSON over a Unix socket). It implements only what urunc +// needs: the connection handshake with capability negotiation, and single +// commands such as cont for resuming a guest that was started frozen (-S). +type qmpClient struct { + conn net.Conn + rd *bufio.Reader +} + +// connectQMP blocks until the QMP socket accepts a connection, then performs +// the mandatory handshake: read the server greeting, negotiate +// qmp_capabilities. QEMU binds the socket shortly after it starts, so the +// dial is retried tightly until the timeout elapses. +func connectQMP(socketPath string, timeout time.Duration) (*qmpClient, error) { + deadline := time.Now().Add(timeout) + var conn net.Conn + var lastErr error + for time.Now().Before(deadline) { + conn, lastErr = net.DialTimeout("unix", socketPath, 50*time.Millisecond) + if lastErr == nil { + break + } + time.Sleep(1 * time.Millisecond) + } + if conn == nil { + if lastErr == nil { + lastErr = fmt.Errorf("dial timed out after %s", timeout) + } + return nil, fmt.Errorf("qmp socket %q not ready within %s: %w", socketPath, timeout, lastErr) + } + c := &qmpClient{conn: conn, rd: bufio.NewReader(conn)} + // Bound the handshake itself: a dial succeeding does not guarantee QEMU + // keeps talking, so reads/writes below must not block forever. + _ = c.conn.SetDeadline(deadline) + // The server speaks first: nothing may be sent before its greeting is read. + greeting, err := c.readMessage() + if err != nil { + c.close() + return nil, fmt.Errorf("failed to read the QMP greeting: %w", err) + } + if _, ok := greeting["QMP"]; !ok { + c.close() + return nil, fmt.Errorf("unexpected QMP greeting: %v", greeting) + } + if err := c.execute("qmp_capabilities"); err != nil { + c.close() + return nil, err + } + _ = c.conn.SetDeadline(time.Time{}) + return c, nil +} + +// execute sends one argument-less QMP command and waits for its result. +func (c *qmpClient) execute(command string) error { + _ = c.conn.SetDeadline(time.Now().Add(5 * time.Second)) + defer func() { _ = c.conn.SetDeadline(time.Time{}) }() + + req, err := json.Marshal(map[string]string{"execute": command}) + if err != nil { + return fmt.Errorf("failed to marshal QMP %s: %w", command, err) + } + if _, err := c.conn.Write(append(req, '\n')); err != nil { + return fmt.Errorf("failed to send QMP %s: %w", command, err) + } + resp, err := c.readMessage() + if err != nil { + return fmt.Errorf("failed to read the QMP response to %s: %w", command, err) + } + if errObj, ok := resp["error"]; ok { + return fmt.Errorf("QMP %s failed: %v", command, errObj) + } + if _, ok := resp["return"]; !ok { + return fmt.Errorf("unexpected QMP response to %s: %v", command, resp) + } + return nil +} + +// readMessage returns the next QMP message that is not an asynchronous event. +// QEMU may interleave event lines (e.g. RESUME) with command responses. +func (c *qmpClient) readMessage() (map[string]any, error) { + for { + line, err := c.rd.ReadBytes('\n') + if err != nil { + return nil, err + } + var msg map[string]any + if err := json.Unmarshal(line, &msg); err != nil { + return nil, fmt.Errorf("invalid QMP message %q: %w", line, err) + } + if _, isEvent := msg["event"]; isEvent { + continue + } + return msg, nil + } +} + +func (c *qmpClient) close() { + _ = c.conn.Close() +} diff --git a/pkg/unikontainers/hypervisors/qemu_session_test.go b/pkg/unikontainers/hypervisors/qemu_session_test.go new file mode 100644 index 000000000..29e292cb7 --- /dev/null +++ b/pkg/unikontainers/hypervisors/qemu_session_test.go @@ -0,0 +1,133 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hypervisors + +import ( + "bufio" + "encoding/json" + "net" + "path/filepath" + "sync" + "testing" + "time" +) + +// fakeQMPServer speaks just enough of the QMP wire protocol for these tests: +// it sends the greeting on accept, answers qmp_capabilities and cont with +// {"return":{}}, and records every command in order. Before answering cont it +// emits an asynchronous RESUME event line, which clients must skip. +type fakeQMPServer struct { + mu sync.Mutex + commands []string + failCont bool +} + +func (f *fakeQMPServer) recorded() []string { + f.mu.Lock() + defer f.mu.Unlock() + out := make([]string, len(f.commands)) + copy(out, f.commands) + return out +} + +func startFakeQMPServer(t *testing.T, failCont bool) (*fakeQMPServer, string) { + t.Helper() + sockPath := filepath.Join(t.TempDir(), "qmp.sock") + ln, err := net.Listen("unix", sockPath) + if err != nil { + t.Fatalf("failed to listen on %s: %v", sockPath, err) + } + t.Cleanup(func() { ln.Close() }) + + srv := &fakeQMPServer{failCont: failCont} + go func() { + conn, err := ln.Accept() + if err != nil { + return + } + defer conn.Close() + _, _ = conn.Write([]byte(`{"QMP":{"version":{},"capabilities":[]}}` + "\n")) + rd := bufio.NewReader(conn) + for { + line, err := rd.ReadBytes('\n') + if err != nil { + return + } + var msg map[string]any + if json.Unmarshal(line, &msg) != nil { + return + } + cmd, _ := msg["execute"].(string) + srv.mu.Lock() + srv.commands = append(srv.commands, cmd) + srv.mu.Unlock() + if cmd == "cont" { + _, _ = conn.Write([]byte(`{"event":"RESUME","timestamp":{"seconds":0,"microseconds":0}}` + "\n")) + if srv.failCont { + _, _ = conn.Write([]byte(`{"error":{"class":"GenericError","desc":"cont refused"}}` + "\n")) + continue + } + } + _, _ = conn.Write([]byte(`{"return":{}}` + "\n")) + } + }() + return srv, sockPath +} + +// TestQMPConnectNegotiatesAndResumes verifies the client reads the greeting, +// negotiates capabilities before anything else, and that Resume sends cont and +// tolerates the interleaved asynchronous event line. +func TestQMPConnectNegotiatesAndResumes(t *testing.T) { + srv, sockPath := startFakeQMPServer(t, false) + + client, err := connectQMP(sockPath, 2*time.Second) + if err != nil { + t.Fatalf("connectQMP failed: %v", err) + } + defer client.close() + + session := &QemuSession{client: client} + if err := session.Resume(); err != nil { + t.Fatalf("Resume failed: %v", err) + } + + want := []string{"qmp_capabilities", "cont"} + got := srv.recorded() + if len(got) != len(want) { + t.Fatalf("expected commands %v, got %v", want, got) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("command %d: got %s, want %s", i, got[i], want[i]) + } + } +} + +// TestQMPResumeSurfacesErrors verifies a QMP error response becomes a Go error. +func TestQMPResumeSurfacesErrors(t *testing.T) { + _, sockPath := startFakeQMPServer(t, true) + + client, err := connectQMP(sockPath, 2*time.Second) + if err != nil { + t.Fatalf("connectQMP failed: %v", err) + } + defer client.close() + + session := &QemuSession{client: client} + err = session.Resume() + if err == nil { + t.Fatal("Resume succeeded against a server that refused cont") + } +} diff --git a/pkg/unikontainers/hypervisors/qemu_test.go b/pkg/unikontainers/hypervisors/qemu_test.go index fff45244c..a861b57b9 100644 --- a/pkg/unikontainers/hypervisors/qemu_test.go +++ b/pkg/unikontainers/hypervisors/qemu_test.go @@ -83,6 +83,7 @@ func TestQemuBuildExecCmd(t *testing.T) { "-m 256M", "-kernel " + testKernelPath, "-nic none", + "-qmp unix:/tmp/.sock,server,nowait", }, mustNotContain: []string{ "-smp", @@ -94,6 +95,18 @@ func TestQemuBuildExecCmd(t *testing.T) { "vhost-vsock-pci", }, }, + { + name: "configured SocketPath renders -qmp on that path", + args: types.ExecArgs{UnikernelPath: testKernelPath, Command: testCommand, SocketPath: "/run/urunc/q.sock"}, + unikernel: &fakeUnikernel{}, + mustContain: []string{"-qmp unix:/run/urunc/q.sock,server,nowait"}, + }, + { + name: "default SocketPath uses the container-id path", + args: types.ExecArgs{UnikernelPath: testKernelPath, Command: testCommand, ContainerID: "abc123"}, + unikernel: &fakeUnikernel{}, + mustContain: []string{"-qmp unix:/tmp/abc123.sock,server,nowait"}, + }, { name: "custom MemSizeB renders -m in MB", args: types.ExecArgs{ diff --git a/pkg/unikontainers/hypervisors/utils.go b/pkg/unikontainers/hypervisors/utils.go index 1bee33104..d06d0ce82 100644 --- a/pkg/unikontainers/hypervisors/utils.go +++ b/pkg/unikontainers/hypervisors/utils.go @@ -17,10 +17,12 @@ package hypervisors import ( "errors" "fmt" + "path/filepath" "runtime" "strconv" "time" + "github.com/urunc-dev/urunc/pkg/unikontainers/types" "golang.org/x/sys/unix" ) @@ -66,6 +68,20 @@ func BytesToStringMB(argMem uint64) string { return stringMem } +// DefaultSocketDir is the directory used for a monitor's control socket when +// no socket_path is configured. It always exists inside the monitor rootfs. +const DefaultSocketDir = "/tmp" + +// ResolveSocketPath returns the path for a monitor's control socket: the +// configured SocketPath when set, otherwise a per-container default under +// DefaultSocketDir. +func ResolveSocketPath(args types.ExecArgs) string { + if args.SocketPath != "" { + return args.SocketPath + } + return filepath.Join(DefaultSocketDir, args.ContainerID+".sock") +} + func killProcess(pid int) error { const timeout = 2 * time.Second err := unix.Kill(pid, unix.SIGKILL) diff --git a/pkg/unikontainers/hypervisors/utils_test.go b/pkg/unikontainers/hypervisors/utils_test.go index ab4f73738..d8ac54b15 100644 --- a/pkg/unikontainers/hypervisors/utils_test.go +++ b/pkg/unikontainers/hypervisors/utils_test.go @@ -18,6 +18,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/urunc-dev/urunc/pkg/unikontainers/types" ) func TestBytesToMiB(t *testing.T) { @@ -71,3 +72,23 @@ func TestBytesToMB(t *testing.T) { }) } } + +func TestResolveSocketPath(t *testing.T) { + t.Parallel() + if got := ResolveSocketPath(types.ExecArgs{SocketPath: "/run/urunc/x.sock"}); got != "/run/urunc/x.sock" { + t.Fatalf("configured: got %q", got) + } + if got := ResolveSocketPath(types.ExecArgs{ContainerID: "abc"}); got != "/tmp/abc.sock" { + t.Fatalf("default: got %q", got) + } +} + +func TestUsesControlSocket(t *testing.T) { + t.Parallel() + if !UsesControlSocket(QemuVmm) { + t.Fatal("qemu should use a control socket") + } + if UsesControlSocket(HvtVmm) { + t.Fatal("hvt should not") + } +} diff --git a/pkg/unikontainers/hypervisors/vmm.go b/pkg/unikontainers/hypervisors/vmm.go index c8600957b..e52c0a83b 100644 --- a/pkg/unikontainers/hypervisors/vmm.go +++ b/pkg/unikontainers/hypervisors/vmm.go @@ -27,6 +27,17 @@ const DefaultMemory uint64 = 256 // The default memory for every hypervisor: 256 type VmmType string +// UsesControlSocket reports whether a monitor exposes a control socket whose +// path (socket_path) urunc must make reachable before the monitor launches. +func UsesControlSocket(vmmType VmmType) bool { + switch vmmType { + case QemuVmm: + return true + default: + return false + } +} + var ErrVMMNotInstalled = errors.New("vmm not found") var vmmLog = logrus.WithField("subsystem", "monitors") diff --git a/pkg/unikontainers/types/types.go b/pkg/unikontainers/types/types.go index c6388e2cc..69ca17dea 100644 --- a/pkg/unikontainers/types/types.go +++ b/pkg/unikontainers/types/types.go @@ -108,6 +108,8 @@ type ExecArgs struct { VSockDevID int // The guest-cid Net NetDevParams Sharedfs SharedfsParams + SocketPath string // The path of the monitor's control socket (empty means the monitor's default) + BootMode string // Optional boot mode for the monitor. "api" drives the guest's start over the monitor's control socket; any other value boots the monitor from its command line as before. } type MonitorCliArgs struct { @@ -133,7 +135,9 @@ type ExtraBinConfig struct { type MonitorConfig struct { DefaultMemoryMB uint `toml:"default_memory_mb"` DefaultVCPUs uint `toml:"default_vcpus"` - BinaryPath string `toml:"path,omitempty"` // Optional path to the hypervisor binary - DataPath string `toml:"data_path,omitempty"` // Optional path to the hypervisor data files (e.g. qemu bios stuff) - Vhost bool `toml:"vhost,omitempty"` // Optional: enable vhost for network performance optimization + BinaryPath string `toml:"path,omitempty"` // Optional path to the hypervisor binary + DataPath string `toml:"data_path,omitempty"` // Optional path to the hypervisor data files (e.g. qemu bios stuff) + Vhost bool `toml:"vhost,omitempty"` // Optional: enable vhost for network performance optimization + SocketPath string `toml:"socket_path,omitempty"` // Optional path for the monitor's control socket (falls back to a per-container default) + BootMode string `toml:"boot_mode,omitempty"` // Optional boot mode for the monitor. "api" drives the guest's start over the monitor's control socket; any other value boots the monitor from its command line as before. } diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index 84172aca4..0430ade01 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -402,6 +402,8 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { defaultVCPUs = 1 } defaultMemSizeMB := u.UruncCfg.Monitors[vmmType].DefaultMemoryMB + socketPath := u.UruncCfg.Monitors[vmmType].SocketPath + bootMode := u.UruncCfg.Monitors[vmmType].BootMode // ExecArgs vmmArgs := types.ExecArgs{ @@ -412,6 +414,8 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { MemSizeB: uint64(defaultMemSizeMB * 1024 * 1024), VCPUs: uint(defaultVCPUs), Environment: os.Environ(), + SocketPath: socketPath, + BootMode: bootMode, } // ExecArgs @@ -637,6 +641,25 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { // ExecArgs vmmArgs.Command = unikernelCmd + // api boot mode: QEMU is spawned frozen (-S) as a supervised child right + // after changeRoot below, so the monitor and its QMP socket are confined + // inside the monitor rootfs. The guest only starts when the QMP cont is + // sent after the start-success handshake, preserving OCI start ordering. + isAPIBoot := bootMode == "api" && vmmType == string(hypervisors.QemuVmm) + if isAPIBoot && vmmArgs.Sharedfs.Type == "virtiofs" { + return fmt.Errorf("boot_mode=api does not support the virtiofs shared filesystem yet") + } + var qSession *hypervisors.QemuSession + qHandedOff := false + defer func() { + // Any error return after the spawn must not leave the VMM child + // behind. Supervise never returns (os.Exit), so this only fires on + // error paths. + if qSession != nil && !qHandedOff { + qSession.Kill() + } + }() + // pivot _, err = findNS(u.Spec.Linux.Namespaces, specs.MountNamespace) // We just want to check if a mount namespace was define din the list @@ -648,6 +671,28 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return err } + if hypervisors.UsesControlSocket(hypervisors.VmmType(vmmType)) { + sockDir := filepath.Dir(hypervisors.ResolveSocketPath(vmmArgs)) + if err = os.MkdirAll(sockDir, 0o755); err != nil { + return fmt.Errorf("failed to create control socket directory %q: %w", sockDir, err) + } + } + + // urunc is still privileged at this point; the child is started directly + // under the container user's credentials, and urunc drops its own + // privileges right after (setupUser below). + if isAPIBoot { + q, ok := vmm.(*hypervisors.Qemu) + if !ok { + return fmt.Errorf("boot_mode=api is only supported for the qemu monitor") + } + qSession, err = q.SpawnPausedVMM(vmmArgs, unikernel, procAttrs.UID, procAttrs.GID) + if err != nil { + uniklog.Errorf("failed to spawn qemu: %v", err) + return err + } + } + // uid/gid // Setup uid, gid and additional groups for the monitor process err = setupUser(u.Spec.Process.User) @@ -671,15 +716,20 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return err } - uniklog.Debug("calling vmm execve") + uniklog.Debug("preparing to start the vmm") metrics.Capture(m.TS18) // Build the VMM command once and verify it can be constructed successfully. // This ensures we don't report the container as started if command building fails. - execCmd, err := vmm.BuildExecCmd(vmmArgs, unikernel) - if err != nil { - uniklog.WithError(err).Error("failed to build VMM command") - return err + // For the api boot mode the VMM is already running, frozen and fully + // configured (the equivalent validation), so there is no command to build. + var execCmd []string + if !isAPIBoot { + execCmd, err = vmm.BuildExecCmd(vmmArgs, unikernel) + if err != nil { + uniklog.WithError(err).Error("failed to build VMM command") + return err + } } // Notify urunc start that the monitor is ready to execute. @@ -700,6 +750,18 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return err } + if isAPIBoot { + // The VMM is up and configured; unfreeze the guest and hand this + // process over to supervising the child. This process must not exit + // before the child, since it is the container's init process. + if err = qSession.Resume(); err != nil { + uniklog.Errorf("failed to resume the guest: %v", err) + return err + } + qHandedOff = true + return qSession.Supervise() + } + // Execute the VMM using the command we built earlier. uniklog.WithField("command", execCmd).Debug("Ready to execve VMM") return syscall.Exec(vmm.Path(), execCmd, vmmArgs.Environment) //nolint: gosec diff --git a/pkg/unikontainers/urunc_config.go b/pkg/unikontainers/urunc_config.go index 22573f43c..277a82299 100644 --- a/pkg/unikontainers/urunc_config.go +++ b/pkg/unikontainers/urunc_config.go @@ -145,6 +145,8 @@ func (p *UruncConfig) Map() map[string]string { cfgMap[prefix+"binary_path"] = hvCfg.BinaryPath cfgMap[prefix+"data_path"] = hvCfg.DataPath cfgMap[prefix+"vhost"] = strconv.FormatBool(hvCfg.Vhost) + cfgMap[prefix+"socket_path"] = hvCfg.SocketPath + cfgMap[prefix+"boot_mode"] = hvCfg.BootMode } for eb, ebCfg := range p.ExtraBins { prefix := "urunc_config.extra_binaries." + eb + "." @@ -198,6 +200,10 @@ func UruncConfigFromMap(cfgMap map[string]string) *UruncConfig { } else { hvCfg.Vhost = boolVal } + case "socket_path": + hvCfg.SocketPath = val + case "boot_mode": + hvCfg.BootMode = val } cfg.Monitors[hv] = hvCfg }