diff --git a/.github/contributors.yaml b/.github/contributors.yaml index 411f9f42a..7047f49a7 100644 --- a/.github/contributors.yaml +++ b/.github/contributors.yaml @@ -107,3 +107,6 @@ users: Chennamma-Hotkar: name: Chennamma Hotkar email: channuhotkar@gmail.com + Anamika1608: + name: Anamika Aggarwal + email: anamikaagg18@gmail.com diff --git a/.github/linters/urunc-dict.txt b/.github/linters/urunc-dict.txt index 07a4855b5..d7d2189a2 100644 --- a/.github/linters/urunc-dict.txt +++ b/.github/linters/urunc-dict.txt @@ -84,6 +84,7 @@ Sharedfs Syscalls TUNSETGROUP TUNSETOWNER +TUNSETPERSIST TUNTAP Timestamping Tmpfs diff --git a/docs/configuration.md b/docs/configuration.md index a2daa4ba4..cb4700987 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -99,6 +99,7 @@ values. - [QEMU/KVM](./hypervisor-support#qemu) - `qemu` - [Firecracker](./hypervisor-support#firecracker) - `firecracker` +- [Cloud Hypervisor](./hypervisor-support#cloud-hypervisor) - `cloud-hypervisor` - [Solo5-hvt](./hypervisor-support#solo5-hvt) - `hvt` - Solo5 hvt (KVM-based tender) - [Solo5-spt](./hypervisor-support#solo5-spt) - `spt` - Solo5 spt (Seccomp-based tender) @@ -112,6 +113,8 @@ Each monitor subsection supports the following options: | `default_vcpus` | integer | `1` | Default number of virtual CPUs | | `path` | string | (empty) | Optional custom path to the monitor binary. If not specified, urunc will search for the binary in PATH | | `data_path` | string | (empty) | Optional custom path for the monitor's data file directory | +| `socket_path` | string | (empty) | Optional path for the monitor's control socket. If not specified, urunc uses a per-container default under `/tmp`. When a custom path is set, urunc creates its parent directory; setting it to an invalid location (a file already exists on the path) makes the monitor fail to start. Currently used by Cloud Hypervisor. | +| `boot_mode` | string | (empty) | Optional boot mode. When set to `api`, urunc starts the monitor as a supervised child and triggers the guest's start through the monitor's control socket; when unset, the monitor boots from its command line as before. Currently only used by Cloud Hypervisor. | Since Qemu is the only currently supported monitor which requires extra data to boot a VM, `urunc` will first check `/usr/local/share` and then `/usr/share` for @@ -130,6 +133,11 @@ data_path = "/usr/local/share/" default_memory_mb = 512 default_vcpus = 2 path = "/opt/firecracker/firecracker" + +[monitors.cloud-hypervisor] +default_memory_mb = 256 +default_vcpus = 1 +boot_mode = "api" ``` ### Extra binaries Configuration diff --git a/pkg/network/network.go b/pkg/network/network.go index 375af431b..cbc73c707 100644 --- a/pkg/network/network.go +++ b/pkg/network/network.go @@ -101,16 +101,35 @@ func createTapDevice(name string, mtu int, ownerUID, ownerGID uint32) (netlink.L return nil, fmt.Errorf("failed to create tap device: %w", err) } + // LinkAdd opened the tap device's fd and set TUNSETPERSIST, so the + // interface survives independently of any open fd. Nothing after the + // owner/group ioctl calls uses the fd (the remaining setup goes through + // netlink and the monitor attaches to the device by name with its own + // open), so close it right away instead of relying on a later exec to + // close it (O_CLOEXEC). A single-queue tap device only allows one + // attached fd at a time, so holding it open blocks the monitor from + // attaching if the caller does not exec away. for _, tapFd := range tapLink.Fds { err = unix.IoctlSetInt(int(tapFd.Fd()), unix.TUNSETOWNER, int(ownerUID)) if err != nil { + if closeErr := tapFd.Close(); closeErr != nil { + netlog.Warnf("failed to close tap %s fd after owner ioctl error: %v", name, closeErr) + } return nil, fmt.Errorf("failed to set tap %s owner to uid %d: %w", name, ownerUID, err) } err = unix.IoctlSetInt(int(tapFd.Fd()), unix.TUNSETGROUP, int(ownerGID)) if err != nil { + if closeErr := tapFd.Close(); closeErr != nil { + netlog.Warnf("failed to close tap %s fd after group ioctl error: %v", name, closeErr) + } return nil, fmt.Errorf("failed to set tap %s group to gid %d: %w", name, ownerGID, err) } + + err = tapFd.Close() + if err != nil { + return nil, fmt.Errorf("failed to close the fd of tap %s: %w", name, err) + } } err = netlink.LinkSetMTU(tapLink, mtu) diff --git a/pkg/unikontainers/hypervisors/cloud_hypervisor.go b/pkg/unikontainers/hypervisors/cloud_hypervisor.go index 606a3c02e..d9b9a7909 100644 --- a/pkg/unikontainers/hypervisors/cloud_hypervisor.go +++ b/pkg/unikontainers/hypervisors/cloud_hypervisor.go @@ -15,8 +15,15 @@ package hypervisors import ( + "context" + "errors" "fmt" + "os" + "os/exec" + "os/signal" "strings" + "syscall" + "time" "github.com/urunc-dev/urunc/pkg/unikontainers/types" "golang.org/x/sys/unix" @@ -70,6 +77,10 @@ func (ch *CloudHypervisor) BuildExecCmd(args types.ExecArgs, ukernel types.Unike // Start building the command exArgs := []string{ch.binaryPath} + // Expose the REST API over a control socket so the runtime can talk to + // Cloud Hypervisor after boot (e.g. for graceful shutdown). + exArgs = append(exArgs, "--api-socket", "path="+ResolveSocketPath(args)) + // Memory configuration if args.Sharedfs.Type == "virtiofs" { exArgs = append(exArgs, "--memory", fmt.Sprintf("size=%sM,shared=on", chMem)) @@ -159,3 +170,230 @@ func (ch *CloudHypervisor) BuildExecCmd(args types.ExecArgs, ukernel types.Unike func (ch *CloudHypervisor) PreExec(_ types.ExecArgs) error { return nil } + +// The following types mirror the subset of Cloud Hypervisor's REST VmConfig +// that urunc's command-line boot already uses (verified against the installed +// cloud-hypervisor v53 API). + +type CHPayload struct { + Kernel string `json:"kernel,omitempty"` + Cmdline string `json:"cmdline,omitempty"` + Initramfs string `json:"initramfs,omitempty"` +} + +type CHMemory struct { + Size uint64 `json:"size"` + Shared bool `json:"shared,omitempty"` +} + +type CHCpus struct { + BootVcpus uint `json:"boot_vcpus"` + MaxVcpus uint `json:"max_vcpus"` +} + +type CHNet struct { + Tap string `json:"tap,omitempty"` + Mac string `json:"mac,omitempty"` + Mtu int `json:"mtu,omitempty"` +} + +type CHDisk struct { + Path string `json:"path"` + ID string `json:"id,omitempty"` +} + +type CHFs struct { + Tag string `json:"tag"` + Socket string `json:"socket"` +} + +type CHVsock struct { + Cid int `json:"cid"` + Socket string `json:"socket"` +} + +type CHConsole struct { + Mode string `json:"mode"` +} + +type CHVMConfig struct { + Payload CHPayload `json:"payload"` + Memory *CHMemory `json:"memory,omitempty"` + Cpus *CHCpus `json:"cpus,omitempty"` + Net []CHNet `json:"net,omitempty"` + Disks []CHDisk `json:"disks,omitempty"` + Fs []CHFs `json:"fs,omitempty"` + Vsock *CHVsock `json:"vsock,omitempty"` + Serial *CHConsole `json:"serial,omitempty"` + Console *CHConsole `json:"console,omitempty"` +} + +// buildCHVMConfig maps the same data BuildExecCmd puts on the command line +// into the REST VmConfig. Unikernel-specific raw CLI argument strings cannot +// be mapped to JSON, so they yield an error instead of being dropped. +func buildCHVMConfig(args types.ExecArgs, ukernel types.Unikernel) (*CHVMConfig, error) { + mem := args.MemSizeB + if mem < 1<<20 { + mem = DefaultMemory << 20 + } + cfg := &CHVMConfig{ + Payload: CHPayload{Kernel: args.UnikernelPath, Cmdline: args.Command}, + Memory: &CHMemory{Size: mem, Shared: args.Sharedfs.Type == "virtiofs"}, + Serial: &CHConsole{Mode: "Tty"}, + Console: &CHConsole{Mode: "Off"}, + } + if args.VCPUs > 0 { + cfg.Cpus = &CHCpus{BootVcpus: args.VCPUs, MaxVcpus: args.VCPUs} + } + + extraMonArgs := ukernel.MonitorCli() + if extraMonArgs.OtherArgs != "" { + return nil, fmt.Errorf("boot_mode=api does not support unikernel-specific monitor arguments (%q)", extraMonArgs.OtherArgs) + } + initrdPath := args.InitrdPath + if initrdPath == "" { + initrdPath = extraMonArgs.ExtraInitrd + } + cfg.Payload.Initramfs = initrdPath + + if args.Net.TapDev != "" { + if netCli := ukernel.MonitorNetCli(args.Net.TapDev, args.Net.MAC); netCli != "" { + return nil, fmt.Errorf("boot_mode=api does not support unikernel-specific network arguments (%q)", netCli) + } + cfg.Net = append(cfg.Net, CHNet{Tap: args.Net.TapDev, Mac: args.Net.MAC, Mtu: args.Net.MTU}) + } + + for _, blockArg := range ukernel.MonitorBlockCli() { + if blockArg.ExactArgs != "" { + return nil, fmt.Errorf("boot_mode=api does not support unikernel-specific block arguments (%q)", blockArg.ExactArgs) + } + if blockArg.Path != "" { + cfg.Disks = append(cfg.Disks, CHDisk{Path: blockArg.Path, ID: blockArg.ID}) + } + } + + if args.Sharedfs.Type == "virtiofs" { + cfg.Fs = append(cfg.Fs, CHFs{Tag: "fs0", Socket: "/tmp/vhostqemu"}) + } + if args.VAccelType == "vsock" { + cfg.Vsock = &CHVsock{Cid: args.VSockDevID, Socket: args.VSockDevPath + "/vaccel.sock"} + } + return cfg, nil +} + +// CHSession is a Cloud Hypervisor child process driven over its REST API +// socket. Create it with SpawnSocketVMM, send the full configuration with +// ConfigureVM, boot the guest with BootVM once the caller's start handshake +// allows it, then hand the calling process over with Supervise. +type CHSession struct { + cmd *exec.Cmd + client *chAPIClient +} + +// SpawnSocketVMM starts Cloud Hypervisor as a supervised child with only its +// API socket enabled. It is called after changeRoot, so the child inherits +// the pivoted root and its socket lives inside the monitor rootfs. The +// caller is still privileged here and only drops its own privileges +// afterwards, so when uid/gid are non-zero the child is started directly +// under that credential. +func (ch *CloudHypervisor) SpawnSocketVMM(args types.ExecArgs, uid, gid uint32) (*CHSession, error) { + socketPath := ResolveSocketPath(args) + // Cloud Hypervisor binds this path itself; remove any stale leftover. + if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) { + return nil, fmt.Errorf("failed to remove stale socket %q: %w", socketPath, err) + } + execCmd := []string{ch.binaryPath, "--api-socket", "path=" + socketPath} + if args.Seccomp { + execCmd = append(execCmd, "--seccomp", "true") + } else { + execCmd = append(execCmd, "--seccomp", "false") + } + + cmd := exec.Command(execCmd[0], execCmd[1:]...) //nolint: gosec + cmd.Env = args.Environment + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if uid != 0 || gid != 0 { + cmd.SysProcAttr = &syscall.SysProcAttr{ + Credential: &syscall.Credential{Uid: uid, Gid: gid}, + } + } + vmmLog.WithField("command", execCmd).Debug("starting cloud-hypervisor as a supervised child") + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("failed to start cloud-hypervisor: %w", err) + } + + client := newCHAPIClient(socketPath) + if err := client.connect(5 * time.Second); err != nil { + s := &CHSession{cmd: cmd, client: client} + s.Kill() + return nil, fmt.Errorf("cloud-hypervisor API socket never became ready: %w", err) + } + return &CHSession{cmd: cmd, client: client}, nil +} + +// ConfigureVM sends the complete VM configuration in one request. +func (s *CHSession) ConfigureVM(ctx context.Context, args types.ExecArgs, ukernel types.Unikernel) error { + cfg, err := buildCHVMConfig(args, ukernel) + if err != nil { + return err + } + vmmLog.Debug("api boot: sending vm.create") + return s.client.createVM(ctx, cfg) +} + +// BootVM boots the configured VM. +func (s *CHSession) BootVM(ctx context.Context) error { + vmmLog.Debug("api boot: sending vm.boot") + return s.client.bootVM(ctx) +} + +// Kill terminates the child and reaps it. For error paths before Supervise. +func (s *CHSession) Kill() { + _ = s.cmd.Process.Kill() + _, _ = s.cmd.Process.Wait() +} + +// Supervise hands the calling process over to the child for the rest of its +// life: it forwards SIGTERM/SIGINT and, once the child exits, exits this +// process with the child's exit code, mirroring the semantics syscall.Exec +// would have had. The caller must not exit before the child, since it is +// the container's init process. +// +// On success this function does not return: it calls os.Exit with the +// child's exit status once the child exits. +func (s *CHSession) Supervise() error { + // Forward the signals containerd would send to stop the container. + // SIGKILL cannot be caught, so it is not listed here: if it arrives, + // this process dies immediately and the child is left running, a known + // gap for this bounded experiment, not yet handled. + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT) + go func() { + sig, ok := <-sigCh + if !ok { + return + } + if sg, ok := sig.(syscall.Signal); ok { + _ = s.cmd.Process.Signal(sg) + } + }() + + waitErr := s.cmd.Wait() + signal.Stop(sigCh) + close(sigCh) + + exitCode := 0 + if waitErr != nil { + var exitErr *exec.ExitError + if errors.As(waitErr, &exitErr) { + exitCode = exitErr.ExitCode() + } else { + vmmLog.WithError(waitErr).Error("cloud-hypervisor exited with an unexpected error") + exitCode = 1 + } + } + os.Exit(exitCode) + return nil // unreachable +} diff --git a/pkg/unikontainers/hypervisors/cloud_hypervisor_client.go b/pkg/unikontainers/hypervisors/cloud_hypervisor_client.go new file mode 100644 index 000000000..927a6777a --- /dev/null +++ b/pkg/unikontainers/hypervisors/cloud_hypervisor_client.go @@ -0,0 +1,131 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hypervisors + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "sync" + "time" +) + +// chAPIClient drives a running Cloud Hypervisor process over its HTTP-over-Unix +// REST API socket. urunc uses it to create the whole VM configuration in one +// request and boot the guest when the start handshake allows it. +// +// The client establishes a single connection in connect() and keeps it alive +// for all requests, so every request reuses the connection whose readiness +// connect() already waited for, instead of re-dialing. +type chAPIClient struct { + socketPath string + httpClient *http.Client + + dialMu sync.Mutex + heldConn net.Conn +} + +func newCHAPIClient(socketPath string) *chAPIClient { + c := &chAPIClient{socketPath: socketPath} + transport := &http.Transport{ + DialContext: c.dialContext, + IdleConnTimeout: 0, + MaxIdleConns: 1, + MaxIdleConnsPerHost: 1, + } + c.httpClient = &http.Client{Transport: transport} + return c +} + +// dialContext hands the transport the connection pre-established by connect(), +// and falls back to dialing the socket path directly if that connection was +// already consumed. +func (c *chAPIClient) dialContext(ctx context.Context, _, _ string) (net.Conn, error) { + c.dialMu.Lock() + defer c.dialMu.Unlock() + if c.heldConn != nil { + conn := c.heldConn + c.heldConn = nil + return conn, nil + } + var d net.Dialer + return d.DialContext(ctx, "unix", c.socketPath) +} + +// connect blocks until the API socket exists and accepts connections, or the +// timeout elapses. Cloud Hypervisor creates the socket shortly after it +// starts, so the dial is retried tightly. +func (c *chAPIClient) connect(timeout time.Duration) error { + deadline := time.Now().Add(timeout) + var lastErr error + for time.Now().Before(deadline) { + conn, err := net.DialTimeout("unix", c.socketPath, 50*time.Millisecond) + if err == nil { + c.dialMu.Lock() + c.heldConn = conn + c.dialMu.Unlock() + return nil + } + lastErr = err + time.Sleep(1 * time.Millisecond) + } + if lastErr == nil { + lastErr = context.DeadlineExceeded + } + return fmt.Errorf("cloud-hypervisor API socket %q not ready within %s: %w", c.socketPath, timeout, lastErr) +} + +// createVM sends the full VM configuration. +func (c *chAPIClient) createVM(ctx context.Context, cfg *CHVMConfig) error { + return c.put(ctx, "/api/v1/vm.create", cfg) +} + +// bootVM boots the created VM. +func (c *chAPIClient) bootVM(ctx context.Context) error { + return c.put(ctx, "/api/v1/vm.boot", nil) +} + +// put sends body as an HTTP PUT to the given API path over the Unix socket, +// returning an error for any non-2xx response. A nil body sends an empty +// request. +func (c *chAPIClient) put(ctx context.Context, path string, body any) error { + var payload []byte + if body != nil { + var err error + payload, err = json.Marshal(body) + if err != nil { + return fmt.Errorf("marshal %s request: %w", path, err) + } + } + req, err := http.NewRequestWithContext(ctx, http.MethodPut, "http://localhost"+path, bytes.NewReader(payload)) + if err != nil { + return fmt.Errorf("build %s request: %w", path, err) + } + req.Header.Set("Content-Type", "application/json") + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("send %s request: %w", path, err) + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + respBody, _ := io.ReadAll(resp.Body) + return fmt.Errorf("%s returned HTTP %d: %s", path, resp.StatusCode, string(respBody)) + } + return nil +} diff --git a/pkg/unikontainers/hypervisors/cloud_hypervisor_session_test.go b/pkg/unikontainers/hypervisors/cloud_hypervisor_session_test.go new file mode 100644 index 000000000..21aec3b7a --- /dev/null +++ b/pkg/unikontainers/hypervisors/cloud_hypervisor_session_test.go @@ -0,0 +1,188 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hypervisors + +import ( + "context" + "encoding/json" + "net" + "net/http" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/urunc-dev/urunc/pkg/unikontainers/types" +) + +type chRecordedRequest struct { + path string + body map[string]any +} + +type fakeCHAPI struct { + mu sync.Mutex + requests []chRecordedRequest +} + +func (f *fakeCHAPI) recorded() []chRecordedRequest { + f.mu.Lock() + defer f.mu.Unlock() + out := make([]chRecordedRequest, len(f.requests)) + copy(out, f.requests) + return out +} + +func startFakeCHAPI(t *testing.T) (*fakeCHAPI, string) { + t.Helper() + sockPath := filepath.Join(t.TempDir(), "ch.sock") + ln, err := net.Listen("unix", sockPath) + if err != nil { + t.Fatalf("failed to listen on %s: %v", sockPath, err) + } + api := &fakeCHAPI{} + srv := &http.Server{ + ReadHeaderTimeout: time.Second, + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body map[string]any + _ = json.NewDecoder(r.Body).Decode(&body) + api.mu.Lock() + api.requests = append(api.requests, chRecordedRequest{path: r.URL.Path, body: body}) + api.mu.Unlock() + w.WriteHeader(http.StatusNoContent) + }), + } + go func() { _ = srv.Serve(ln) }() + t.Cleanup(func() { _ = srv.Close() }) + return api, sockPath +} + +func newTestCHSession(t *testing.T, sockPath string) *CHSession { + t.Helper() + client := newCHAPIClient(sockPath) + if err := client.connect(2 * time.Second); err != nil { + t.Fatalf("connect failed: %v", err) + } + return &CHSession{client: client} +} + +// TestCHConfigureVMSendsFullConfig verifies vm.create carries the complete +// machine configuration mapped from ExecArgs: payload paths verbatim (no +// rootfs prefixing), memory in bytes, both vcpu fields, net, disks, and the +// fixed console/serial modes. +func TestCHConfigureVMSendsFullConfig(t *testing.T) { + api, sockPath := startFakeCHAPI(t) + session := newTestCHSession(t, sockPath) + + args := types.ExecArgs{ + ContainerID: "test-container", + Command: "app -arg", + UnikernelPath: "/unikernel/app.bin", + InitrdPath: "/unikernel/initrd", + MemSizeB: 256 * 1024 * 1024, + VCPUs: 2, + } + args.Net.TapDev = "tap0" + args.Net.MAC = "aa:bb:cc:dd:ee:ff" + args.Net.MTU = 1500 + ukernel := &fakeUnikernel{ + blockCli: []types.MonitorBlockArgs{{ID: "rootfs", Path: "/dev/mapper/test-snap-1"}}, + } + + if err := session.ConfigureVM(context.Background(), args, ukernel); err != nil { + t.Fatalf("ConfigureVM failed: %v", err) + } + + reqs := api.recorded() + if len(reqs) != 1 || reqs[0].path != "/api/v1/vm.create" { + t.Fatalf("expected one PUT to /api/v1/vm.create, got %+v", reqs) + } + body := reqs[0].body + + payload, _ := body["payload"].(map[string]any) + if payload["kernel"] != "/unikernel/app.bin" || payload["cmdline"] != "app -arg" || payload["initramfs"] != "/unikernel/initrd" { + t.Errorf("unexpected payload: %v", payload) + } + memory, _ := body["memory"].(map[string]any) + if memory["size"] != float64(256*1024*1024) { + t.Errorf("memory.size = %v, want bytes not MB", memory["size"]) + } + cpus, _ := body["cpus"].(map[string]any) + if cpus["boot_vcpus"] != float64(2) || cpus["max_vcpus"] != float64(2) { + t.Errorf("unexpected cpus: %v", cpus) + } + nets, _ := body["net"].([]any) + if len(nets) != 1 { + t.Fatalf("expected one net device, got %v", body["net"]) + } + net0, _ := nets[0].(map[string]any) + if net0["tap"] != "tap0" || net0["mac"] != "aa:bb:cc:dd:ee:ff" || net0["mtu"] != float64(1500) { + t.Errorf("unexpected net[0]: %v", net0) + } + disks, _ := body["disks"].([]any) + if len(disks) != 1 { + t.Fatalf("expected one disk, got %v", body["disks"]) + } + disk0, _ := disks[0].(map[string]any) + if disk0["path"] != "/dev/mapper/test-snap-1" { + t.Errorf("disk path = %v, want unprefixed", disk0["path"]) + } + serial, _ := body["serial"].(map[string]any) + console, _ := body["console"].(map[string]any) + if serial["mode"] != "Tty" || console["mode"] != "Off" { + t.Errorf("serial=%v console=%v", serial, console) + } +} + +// TestCHBootAfterCreate verifies the request ordering create then boot. +func TestCHBootAfterCreate(t *testing.T) { + api, sockPath := startFakeCHAPI(t) + session := newTestCHSession(t, sockPath) + ctx := context.Background() + + args := types.ExecArgs{ContainerID: "test-container", Command: "app", UnikernelPath: "/unikernel/app.bin", MemSizeB: 256 * 1024 * 1024, VCPUs: 1} + if err := session.ConfigureVM(ctx, args, &fakeUnikernel{}); err != nil { + t.Fatalf("ConfigureVM failed: %v", err) + } + if err := session.BootVM(ctx); err != nil { + t.Fatalf("BootVM failed: %v", err) + } + want := []string{"/api/v1/vm.create", "/api/v1/vm.boot"} + reqs := api.recorded() + if len(reqs) != len(want) { + t.Fatalf("expected %v, got %+v", want, reqs) + } + for i := range want { + if reqs[i].path != want[i] { + t.Errorf("request %d: got %s, want %s", i, reqs[i].path, want[i]) + } + } +} + +// TestCHConfigureVMRejectsRawCliOverrides verifies the documented limitation: +// unikernel-provided raw CLI argument strings cannot be mapped to the REST +// config and must fail loudly rather than be silently dropped. +func TestCHConfigureVMRejectsRawCliOverrides(t *testing.T) { + _, sockPath := startFakeCHAPI(t) + session := newTestCHSession(t, sockPath) + + args := types.ExecArgs{ContainerID: "test-container", Command: "app", UnikernelPath: "/unikernel/app.bin", MemSizeB: 256 * 1024 * 1024, VCPUs: 1} + ukernel := &fakeUnikernel{ + blockCli: []types.MonitorBlockArgs{{ID: "vol0", ExactArgs: "--disk path=/x"}}, + } + if err := session.ConfigureVM(context.Background(), args, ukernel); err == nil { + t.Fatal("ConfigureVM accepted a raw CLI override it cannot map") + } +} diff --git a/pkg/unikontainers/hypervisors/utils.go b/pkg/unikontainers/hypervisors/utils.go index 1bee33104..3c4cb5555 100644 --- a/pkg/unikontainers/hypervisors/utils.go +++ b/pkg/unikontainers/hypervisors/utils.go @@ -17,10 +17,12 @@ package hypervisors import ( "errors" "fmt" + "path/filepath" "runtime" "strconv" "time" + "github.com/urunc-dev/urunc/pkg/unikontainers/types" "golang.org/x/sys/unix" ) @@ -93,3 +95,17 @@ func killProcess(pid int) error { return nil } + +// DefaultSocketDir is the directory used for a monitor's control socket when +// no socket_path is configured. It always exists inside the monitor rootfs. +const DefaultSocketDir = "/tmp" + +// ResolveSocketPath returns the path for a monitor's control socket: the +// configured SocketPath when set, otherwise a per-container default under +// DefaultSocketDir. +func ResolveSocketPath(args types.ExecArgs) string { + if args.SocketPath != "" { + return args.SocketPath + } + return filepath.Join(DefaultSocketDir, args.ContainerID+".sock") +} diff --git a/pkg/unikontainers/hypervisors/utils_test.go b/pkg/unikontainers/hypervisors/utils_test.go index ab4f73738..6ba858ff2 100644 --- a/pkg/unikontainers/hypervisors/utils_test.go +++ b/pkg/unikontainers/hypervisors/utils_test.go @@ -18,6 +18,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/urunc-dev/urunc/pkg/unikontainers/types" ) func TestBytesToMiB(t *testing.T) { @@ -71,3 +72,23 @@ func TestBytesToMB(t *testing.T) { }) } } + +func TestResolveSocketPath(t *testing.T) { + t.Parallel() + if got := ResolveSocketPath(types.ExecArgs{SocketPath: "/run/urunc/x.sock"}); got != "/run/urunc/x.sock" { + t.Fatalf("configured: got %q", got) + } + if got := ResolveSocketPath(types.ExecArgs{ContainerID: "abc"}); got != "/tmp/abc.sock" { + t.Fatalf("default: got %q", got) + } +} + +func TestUsesControlSocket(t *testing.T) { + t.Parallel() + if !UsesControlSocket(CloudHypervisorVmm) { + t.Fatal("cloud-hypervisor should use a control socket") + } + if UsesControlSocket(HvtVmm) { + t.Fatal("hvt should not") + } +} diff --git a/pkg/unikontainers/hypervisors/vmm.go b/pkg/unikontainers/hypervisors/vmm.go index c8600957b..2ac4972e1 100644 --- a/pkg/unikontainers/hypervisors/vmm.go +++ b/pkg/unikontainers/hypervisors/vmm.go @@ -27,6 +27,17 @@ const DefaultMemory uint64 = 256 // The default memory for every hypervisor: 256 type VmmType string +// UsesControlSocket reports whether a monitor exposes a control socket whose +// path (socket_path) urunc must make reachable before the monitor launches. +func UsesControlSocket(vmmType VmmType) bool { + switch vmmType { + case CloudHypervisorVmm: + return true + default: + return false + } +} + var ErrVMMNotInstalled = errors.New("vmm not found") var vmmLog = logrus.WithField("subsystem", "monitors") diff --git a/pkg/unikontainers/types/types.go b/pkg/unikontainers/types/types.go index c6388e2cc..dd2013289 100644 --- a/pkg/unikontainers/types/types.go +++ b/pkg/unikontainers/types/types.go @@ -106,6 +106,8 @@ type ExecArgs struct { VAccelType string // Specifies the vAccel acceleration type(e.g. vsock). When empty, vAccel is disabled VSockDevPath string // The host directory where the fc unix socket is created VSockDevID int // The guest-cid + SocketPath string // The path of the monitor's control socket (empty means the monitor's default) + BootMode string // Optional boot mode for the monitor. "api" drives the guest's start over the monitor's control socket; any other value boots the monitor from its command line as before. Net NetDevParams Sharedfs SharedfsParams } @@ -133,7 +135,9 @@ type ExtraBinConfig struct { type MonitorConfig struct { DefaultMemoryMB uint `toml:"default_memory_mb"` DefaultVCPUs uint `toml:"default_vcpus"` - BinaryPath string `toml:"path,omitempty"` // Optional path to the hypervisor binary - DataPath string `toml:"data_path,omitempty"` // Optional path to the hypervisor data files (e.g. qemu bios stuff) - Vhost bool `toml:"vhost,omitempty"` // Optional: enable vhost for network performance optimization + BinaryPath string `toml:"path,omitempty"` // Optional path to the hypervisor binary + DataPath string `toml:"data_path,omitempty"` // Optional path to the hypervisor data files (e.g. qemu bios stuff) + Vhost bool `toml:"vhost,omitempty"` // Optional: enable vhost for network performance optimization + SocketPath string `toml:"socket_path,omitempty"` // Optional path for the monitor's control socket (falls back to a per-container default) + BootMode string `toml:"boot_mode,omitempty"` // Optional boot mode for the monitor. "api" drives the guest's start over the monitor's control socket; any other value boots the monitor from its command line as before. } diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index 84172aca4..1649aa24e 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -16,6 +16,7 @@ package unikontainers import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -402,6 +403,8 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { defaultVCPUs = 1 } defaultMemSizeMB := u.UruncCfg.Monitors[vmmType].DefaultMemoryMB + socketPath := u.UruncCfg.Monitors[vmmType].SocketPath + bootMode := u.UruncCfg.Monitors[vmmType].BootMode // ExecArgs vmmArgs := types.ExecArgs{ @@ -412,6 +415,8 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { MemSizeB: uint64(defaultMemSizeMB * 1024 * 1024), VCPUs: uint(defaultVCPUs), Environment: os.Environ(), + SocketPath: socketPath, + BootMode: bootMode, } // ExecArgs @@ -637,6 +642,27 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { // ExecArgs vmmArgs.Command = unikernelCmd + // api boot mode: Cloud Hypervisor is spawned bare right after changeRoot + // below, so the monitor and its API socket are confined inside the + // monitor rootfs. The whole VM configuration is sent over the socket, + // and the guest only boots when vm.boot is sent after the start-success + // handshake, preserving OCI start ordering. + isAPIBoot := bootMode == "api" && vmmType == string(hypervisors.CloudHypervisorVmm) + if isAPIBoot && vmmArgs.Sharedfs.Type == "virtiofs" { + return fmt.Errorf("boot_mode=api does not support the virtiofs shared filesystem yet") + } + var chSession *hypervisors.CHSession + chHandedOff := false + defer func() { + // Any error return after the spawn must not leave the VMM child + // behind. Supervise never returns (os.Exit), so this only fires on + // error paths. + if chSession != nil && !chHandedOff { + chSession.Kill() + } + }() + ctx := context.Background() + // pivot _, err = findNS(u.Spec.Linux.Namespaces, specs.MountNamespace) // We just want to check if a mount namespace was define din the list @@ -648,6 +674,32 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return err } + if hypervisors.UsesControlSocket(hypervisors.VmmType(vmmType)) { + sockDir := filepath.Dir(hypervisors.ResolveSocketPath(vmmArgs)) + if err = os.MkdirAll(sockDir, 0o755); err != nil { + return fmt.Errorf("failed to create control socket directory %q: %w", sockDir, err) + } + } + + // urunc is still privileged at this point; the child is started directly + // under the container user's credentials, and urunc drops its own + // privileges right after (setupUser below). + if isAPIBoot { + ch, ok := vmm.(*hypervisors.CloudHypervisor) + if !ok { + return fmt.Errorf("boot_mode=api is only supported for the cloud-hypervisor monitor") + } + chSession, err = ch.SpawnSocketVMM(vmmArgs, procAttrs.UID, procAttrs.GID) + if err != nil { + uniklog.Errorf("failed to spawn cloud-hypervisor: %v", err) + return err + } + if err = chSession.ConfigureVM(ctx, vmmArgs, unikernel); err != nil { + uniklog.Errorf("failed to configure the vm over the socket: %v", err) + return err + } + } + // uid/gid // Setup uid, gid and additional groups for the monitor process err = setupUser(u.Spec.Process.User) @@ -671,15 +723,20 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return err } - uniklog.Debug("calling vmm execve") + uniklog.Debug("preparing to start the vmm") metrics.Capture(m.TS18) // Build the VMM command once and verify it can be constructed successfully. // This ensures we don't report the container as started if command building fails. - execCmd, err := vmm.BuildExecCmd(vmmArgs, unikernel) - if err != nil { - uniklog.WithError(err).Error("failed to build VMM command") - return err + // For the api boot mode the VMM is already running and fully configured + // (the equivalent validation), so there is no command to build. + var execCmd []string + if !isAPIBoot { + execCmd, err = vmm.BuildExecCmd(vmmArgs, unikernel) + if err != nil { + uniklog.WithError(err).Error("failed to build VMM command") + return err + } } // Notify urunc start that the monitor is ready to execute. @@ -700,6 +757,18 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return err } + if isAPIBoot { + // The VMM is up and configured; boot the guest and hand this process + // over to supervising the child. This process must not exit before + // the child, since it is the container's init process. + if err = chSession.BootVM(ctx); err != nil { + uniklog.Errorf("failed to boot the guest: %v", err) + return err + } + chHandedOff = true + return chSession.Supervise() + } + // Execute the VMM using the command we built earlier. uniklog.WithField("command", execCmd).Debug("Ready to execve VMM") return syscall.Exec(vmm.Path(), execCmd, vmmArgs.Environment) //nolint: gosec diff --git a/pkg/unikontainers/urunc_config.go b/pkg/unikontainers/urunc_config.go index 22573f43c..277a82299 100644 --- a/pkg/unikontainers/urunc_config.go +++ b/pkg/unikontainers/urunc_config.go @@ -145,6 +145,8 @@ func (p *UruncConfig) Map() map[string]string { cfgMap[prefix+"binary_path"] = hvCfg.BinaryPath cfgMap[prefix+"data_path"] = hvCfg.DataPath cfgMap[prefix+"vhost"] = strconv.FormatBool(hvCfg.Vhost) + cfgMap[prefix+"socket_path"] = hvCfg.SocketPath + cfgMap[prefix+"boot_mode"] = hvCfg.BootMode } for eb, ebCfg := range p.ExtraBins { prefix := "urunc_config.extra_binaries." + eb + "." @@ -198,6 +200,10 @@ func UruncConfigFromMap(cfgMap map[string]string) *UruncConfig { } else { hvCfg.Vhost = boolVal } + case "socket_path": + hvCfg.SocketPath = val + case "boot_mode": + hvCfg.BootMode = val } cfg.Monitors[hv] = hvCfg }