From 811a6062b225063646373efe7ed479ae2ee284cc Mon Sep 17 00:00:00 2001 From: Anamika AggarwaL Date: Wed, 8 Jul 2026 15:36:01 +0530 Subject: [PATCH 01/17] feat(firecracker): launch with control socket instead of config file Firecracker currently boots via --no-api --config-file, which disables its control socket entirely. Switch to --api-sock so the socket stays open and reachable after the guest starts, instead of driving everything through a static config file. Signed-off-by: Anamika AggarwaL --- pkg/unikontainers/hypervisors/firecracker.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/unikontainers/hypervisors/firecracker.go b/pkg/unikontainers/hypervisors/firecracker.go index 9588a45bf..128012ab2 100644 --- a/pkg/unikontainers/hypervisors/firecracker.go +++ b/pkg/unikontainers/hypervisors/firecracker.go @@ -108,9 +108,9 @@ func (fc *Firecracker) BuildExecCmd(args types.ExecArgs, ukernel types.Unikernel // options in FC, since the string return value of the Monitor related // functions in the unikernel interface do not integrate well with FC's // json configuration. - cmdString := fc.Path() + " --no-api --config-file " + apiSockPath := filepath.Join("/tmp/", args.ContainerID+".sock") + cmdString := fc.Path() + " --api-sock " + apiSockPath JSONConfigFile := filepath.Join("/tmp/", FCJsonFilename) - cmdString += JSONConfigFile if !args.Seccomp { cmdString += " --no-seccomp" } From ae3bf54a287bc021fc42be2803ca633d03c159d7 Mon Sep 17 00:00:00 2001 From: Anamika AggarwaL Date: Sat, 11 Jul 2026 13:04:41 +0530 Subject: [PATCH 02/17] feat(firecracker): allow configuring the control socket path Add an optional socket_path field under a monitor's configuration section so users can override where the control socket is created. Falls back to today's default path when unset. The field is added to the shared MonitorConfig type, so it is available to any monitor, but only Firecracker's BuildExecCmd reads it in this change, since it is the only monitor with a control socket right now. Signed-off-by: Anamika AggarwaL --- pkg/unikontainers/hypervisors/firecracker.go | 5 ++++- pkg/unikontainers/types/types.go | 8 +++++--- pkg/unikontainers/unikontainers.go | 2 ++ pkg/unikontainers/urunc_config.go | 3 +++ 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/pkg/unikontainers/hypervisors/firecracker.go b/pkg/unikontainers/hypervisors/firecracker.go index 128012ab2..638fd6c64 100644 --- a/pkg/unikontainers/hypervisors/firecracker.go +++ b/pkg/unikontainers/hypervisors/firecracker.go @@ -108,7 +108,10 @@ func (fc *Firecracker) BuildExecCmd(args types.ExecArgs, ukernel types.Unikernel // options in FC, since the string return value of the Monitor related // functions in the unikernel interface do not integrate well with FC's // json configuration. - apiSockPath := filepath.Join("/tmp/", args.ContainerID+".sock") + apiSockPath := args.SocketPath + if apiSockPath == "" { + apiSockPath = filepath.Join("/tmp/", args.ContainerID+".sock") + } cmdString := fc.Path() + " --api-sock " + apiSockPath JSONConfigFile := filepath.Join("/tmp/", FCJsonFilename) if !args.Seccomp { diff --git a/pkg/unikontainers/types/types.go b/pkg/unikontainers/types/types.go index c6388e2cc..e496b6493 100644 --- a/pkg/unikontainers/types/types.go +++ b/pkg/unikontainers/types/types.go @@ -96,6 +96,7 @@ type UnikernelParams struct { // FIXME: add extra fields if required by additional VMM's type ExecArgs struct { ContainerID string // The container ID + SocketPath string // Optional user-configured path for the monitor's control socket. Empty means the monitor uses its default. Environment []string // The environment variables of the monitor Command string // The unikernel's command line Seccomp bool // Enable or disable seccomp filters for the VMM @@ -133,7 +134,8 @@ type ExtraBinConfig struct { type MonitorConfig struct { DefaultMemoryMB uint `toml:"default_memory_mb"` DefaultVCPUs uint `toml:"default_vcpus"` - BinaryPath string `toml:"path,omitempty"` // Optional path to the hypervisor binary - DataPath string `toml:"data_path,omitempty"` // Optional path to the hypervisor data files (e.g. qemu bios stuff) - Vhost bool `toml:"vhost,omitempty"` // Optional: enable vhost for network performance optimization + BinaryPath string `toml:"path,omitempty"` // Optional path to the hypervisor binary + DataPath string `toml:"data_path,omitempty"` // Optional path to the hypervisor data files (e.g. qemu bios stuff) + Vhost bool `toml:"vhost,omitempty"` // Optional: enable vhost for network performance optimization + SocketPath string `toml:"socket_path,omitempty"` // Optional path for the monitor's control socket. If not specified, the monitor uses its default. } diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index fc297f937..cc347ebb2 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -354,10 +354,12 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { defaultVCPUs = 1 } defaultMemSizeMB := u.UruncCfg.Monitors[vmmType].DefaultMemoryMB + socketPath := u.UruncCfg.Monitors[vmmType].SocketPath // ExecArgs vmmArgs := types.ExecArgs{ ContainerID: u.State.ID, + SocketPath: socketPath, UnikernelPath: unikernelPath, InitrdPath: initrdPath, Seccomp: true, // Enable Seccomp by default diff --git a/pkg/unikontainers/urunc_config.go b/pkg/unikontainers/urunc_config.go index 5f21d106e..17724c030 100644 --- a/pkg/unikontainers/urunc_config.go +++ b/pkg/unikontainers/urunc_config.go @@ -127,6 +127,7 @@ func (p *UruncConfig) Map() map[string]string { cfgMap[prefix+"binary_path"] = hvCfg.BinaryPath cfgMap[prefix+"data_path"] = hvCfg.DataPath cfgMap[prefix+"vhost"] = strconv.FormatBool(hvCfg.Vhost) + cfgMap[prefix+"socket_path"] = hvCfg.SocketPath } for eb, ebCfg := range p.ExtraBins { prefix := "urunc_config.extra_binaries." + eb + "." @@ -180,6 +181,8 @@ func UruncConfigFromMap(cfgMap map[string]string) *UruncConfig { } else { hvCfg.Vhost = boolVal } + case "socket_path": + hvCfg.SocketPath = val } cfg.Monitors[hv] = hvCfg } From 9b4f4ffe5fc953d6aaf768fe7e45ec9d8722a2be Mon Sep 17 00:00:00 2001 From: Anamika AggarwaL Date: Sat, 11 Jul 2026 13:11:37 +0530 Subject: [PATCH 03/17] docs(configuration): document the socket_path monitor option Add the new socket_path field to the Monitor Options table and the Firecracker example, matching the existing style for path/data_path. Signed-off-by: Anamika AggarwaL --- docs/configuration.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index a2daa4ba4..670fc536e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -112,6 +112,7 @@ Each monitor subsection supports the following options: | `default_vcpus` | integer | `1` | Default number of virtual CPUs | | `path` | string | (empty) | Optional custom path to the monitor binary. If not specified, urunc will search for the binary in PATH | | `data_path` | string | (empty) | Optional custom path for the monitor's data file directory | +| `socket_path` | string | (empty) | Optional custom path for the monitor's control socket. If not specified, the monitor uses its default. Currently only used by Firecracker. | Since Qemu is the only currently supported monitor which requires extra data to boot a VM, `urunc` will first check `/usr/local/share` and then `/usr/share` for @@ -130,6 +131,7 @@ data_path = "/usr/local/share/" default_memory_mb = 512 default_vcpus = 2 path = "/opt/firecracker/firecracker" +socket_path = "/run/urunc/fc.sock" ``` ### Extra binaries Configuration From 7efd0be897ec5165b57f22cf2dc7547885b81708 Mon Sep 17 00:00:00 2001 From: Anamika AggarwaL Date: Mon, 13 Jul 2026 12:20:40 +0530 Subject: [PATCH 04/17] feat(firecracker): add boot_mode to choose config-file-based or API-based boot Add an optional boot_mode field under a monitor's configuration. "api" (default) keeps today's behavior: drive the monitor's boot over its control socket. "config-file" lets Firecracker boot itself from the JSON config file, using the socket only for communication after the guest is running. Currently only used by Firecracker. Signed-off-by: Anamika AggarwaL --- docs/configuration.md | 2 ++ pkg/unikontainers/hypervisors/firecracker.go | 5 +++++ pkg/unikontainers/types/types.go | 2 ++ pkg/unikontainers/unikontainers.go | 2 ++ pkg/unikontainers/urunc_config.go | 3 +++ 5 files changed, 14 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 670fc536e..689a6bdc1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -113,6 +113,7 @@ Each monitor subsection supports the following options: | `path` | string | (empty) | Optional custom path to the monitor binary. If not specified, urunc will search for the binary in PATH | | `data_path` | string | (empty) | Optional custom path for the monitor's data file directory | | `socket_path` | string | (empty) | Optional custom path for the monitor's control socket. If not specified, the monitor uses its default. Currently only used by Firecracker. | +| `boot_mode` | string | `api` | Optional: `api` drives the monitor's boot over its control socket, `config-file` lets the monitor boot itself from a config file (socket only used afterward). Currently only used by Firecracker. | Since Qemu is the only currently supported monitor which requires extra data to boot a VM, `urunc` will first check `/usr/local/share` and then `/usr/share` for @@ -132,6 +133,7 @@ default_memory_mb = 512 default_vcpus = 2 path = "/opt/firecracker/firecracker" socket_path = "/run/urunc/fc.sock" +boot_mode = "config-file" ``` ### Extra binaries Configuration diff --git a/pkg/unikontainers/hypervisors/firecracker.go b/pkg/unikontainers/hypervisors/firecracker.go index 638fd6c64..cd7f0b8a5 100644 --- a/pkg/unikontainers/hypervisors/firecracker.go +++ b/pkg/unikontainers/hypervisors/firecracker.go @@ -114,6 +114,11 @@ func (fc *Firecracker) BuildExecCmd(args types.ExecArgs, ukernel types.Unikernel } cmdString := fc.Path() + " --api-sock " + apiSockPath JSONConfigFile := filepath.Join("/tmp/", FCJsonFilename) + if args.BootMode == "config-file" { + // config-file-based: Firecracker boots itself from the JSON config file + // below; the socket stays open only for use after the guest is running. + cmdString += " --config-file " + JSONConfigFile + } if !args.Seccomp { cmdString += " --no-seccomp" } diff --git a/pkg/unikontainers/types/types.go b/pkg/unikontainers/types/types.go index e496b6493..5ef6faffb 100644 --- a/pkg/unikontainers/types/types.go +++ b/pkg/unikontainers/types/types.go @@ -97,6 +97,7 @@ type UnikernelParams struct { type ExecArgs struct { ContainerID string // The container ID SocketPath string // Optional user-configured path for the monitor's control socket. Empty means the monitor uses its default. + BootMode string // "api" (default) drives the monitor over its control socket. "config-file" lets the monitor boot itself from a config file, socket only for later use. Environment []string // The environment variables of the monitor Command string // The unikernel's command line Seccomp bool // Enable or disable seccomp filters for the VMM @@ -138,4 +139,5 @@ type MonitorConfig struct { DataPath string `toml:"data_path,omitempty"` // Optional path to the hypervisor data files (e.g. qemu bios stuff) Vhost bool `toml:"vhost,omitempty"` // Optional: enable vhost for network performance optimization SocketPath string `toml:"socket_path,omitempty"` // Optional path for the monitor's control socket. If not specified, the monitor uses its default. + BootMode string `toml:"boot_mode,omitempty"` // Optional: "api" (default) or "config-file". Currently only used by Firecracker. } diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index cc347ebb2..542764525 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -355,11 +355,13 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { } defaultMemSizeMB := u.UruncCfg.Monitors[vmmType].DefaultMemoryMB socketPath := u.UruncCfg.Monitors[vmmType].SocketPath + bootMode := u.UruncCfg.Monitors[vmmType].BootMode // ExecArgs vmmArgs := types.ExecArgs{ ContainerID: u.State.ID, SocketPath: socketPath, + BootMode: bootMode, UnikernelPath: unikernelPath, InitrdPath: initrdPath, Seccomp: true, // Enable Seccomp by default diff --git a/pkg/unikontainers/urunc_config.go b/pkg/unikontainers/urunc_config.go index 17724c030..52a028464 100644 --- a/pkg/unikontainers/urunc_config.go +++ b/pkg/unikontainers/urunc_config.go @@ -128,6 +128,7 @@ func (p *UruncConfig) Map() map[string]string { cfgMap[prefix+"data_path"] = hvCfg.DataPath cfgMap[prefix+"vhost"] = strconv.FormatBool(hvCfg.Vhost) cfgMap[prefix+"socket_path"] = hvCfg.SocketPath + cfgMap[prefix+"boot_mode"] = hvCfg.BootMode } for eb, ebCfg := range p.ExtraBins { prefix := "urunc_config.extra_binaries." + eb + "." @@ -183,6 +184,8 @@ func UruncConfigFromMap(cfgMap map[string]string) *UruncConfig { } case "socket_path": hvCfg.SocketPath = val + case "boot_mode": + hvCfg.BootMode = val } cfg.Monitors[hv] = hvCfg } From a361b4ca4cc6efcc302ad921bf60680b750c8cb5 Mon Sep 17 00:00:00 2001 From: Anamika AggarwaL Date: Mon, 13 Jul 2026 13:33:22 +0530 Subject: [PATCH 05/17] feat(firecracker): drive API-based boot as a supervised child process Stop syscall.Exec-ing into Firecracker for boot_mode=api. Instead, spawn it as a child, configure it over its control socket using the existing config-building logic (now shared with the config-file path), start the guest, then supervise the child: forward SIGTERM/SIGINT and exit with its status once it exits. The child inherits whatever confinement changeRoot already set up on this process (pivot_root or chroot), so no separate confinement step is needed here. Add HasNetwork() as a cheap, early check for whether a container has a network interface, separate from the full (expensive) NetworkSetup. Rootfs prep only needs this cheap fact (see prepareMonRootfs's needsTAP argument), not the finished network, so for boot_mode=api, rootfs prep now runs concurrently with the real network setup, joining just before unikernel.Init (every supported unikernel type needs the resolved network result to build the guest's boot command line). Also fixes a real bug this surfaced: createTapDevice's file descriptor was opened O_CLOEXEC and relied on syscall.Exec to close it as a side effect of replacing the process. Since urunc no longer execs away for boot_mode=api, that fd was staying open indefinitely, blocking Firecracker's own attempt to attach to the same single-queue tap device ("Resource busy"). The device is created with TUNSETPERSIST, so it's safe to close the fd explicitly once setup is done. Signed-off-by: Anamika AggarwaL --- pkg/network/network.go | 17 +++ pkg/network/network_dynamic.go | 11 ++ pkg/network/network_static.go | 11 ++ pkg/unikontainers/hypervisors/firecracker.go | 126 +++++++++++++++-- .../hypervisors/firecracker_client.go | 132 ++++++++++++++++++ pkg/unikontainers/unikontainers.go | 84 +++++++++-- 6 files changed, 357 insertions(+), 24 deletions(-) create mode 100644 pkg/unikontainers/hypervisors/firecracker_client.go diff --git a/pkg/network/network.go b/pkg/network/network.go index 375af431b..67fad11ba 100644 --- a/pkg/network/network.go +++ b/pkg/network/network.go @@ -38,6 +38,11 @@ type UnikernelNetworkInfo struct { EthDevice Interface } type Manager interface { + // HasNetwork does a cheap check for whether this container has a network + // interface to configure, without doing the more expensive tap device + // creation and configuration. It lets callers learn this fact early, + // before the full NetworkSetup (which does the expensive work) finishes. + HasNetwork() (bool, error) NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error) } @@ -113,6 +118,18 @@ func createTapDevice(name string, mtu int, ownerUID, ownerGID uint32) (netlink.L } } + // LinkAdd opens the tap device's file descriptor with O_CLOEXEC and sets + // TUNSETPERSIST, so the interface survives independent of any open fd. + // We don't need to keep it open past this point (the rest of setup, and + // whatever process later attaches to this tap by name, uses netlink/its + // own independent open, not this fd) - closing it now, rather than + // relying on a future exec to do it, matters because a process that + // keeps this fd open (instead of exec-ing away) would otherwise block + // anything else from opening the same single-queue tap device. + for _, tapFd := range tapLink.Fds { + _ = tapFd.Close() + } + err = netlink.LinkSetMTU(tapLink, mtu) if err != nil { return nil, fmt.Errorf("failed to set tap device MTU to %d: %w", mtu, err) diff --git a/pkg/network/network_dynamic.go b/pkg/network/network_dynamic.go index 1d6a2237b..f1f5c991b 100644 --- a/pkg/network/network_dynamic.go +++ b/pkg/network/network_dynamic.go @@ -32,6 +32,17 @@ type DynamicNetwork struct { // FIXME: CUrrently only one tap device per netns can provide functional networking. We need to find a proper way to handle networking // for multiple unikernels in the same pod/network namespace. // See: https://github.com/urunc-dev/urunc/issues/13 +// HasNetwork checks, cheaply, whether a container network interface exists +// in the current netns, without creating the tap device or doing any of the +// other, more expensive setup NetworkSetup does. +func (n DynamicNetwork) HasNetwork() (bool, error) { + _, err := discoverContainerIface() + if err != nil { + return false, nil + } + return true, nil +} + func (n DynamicNetwork) NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error) { tapIndex, err := getTapIndex() if err != nil { diff --git a/pkg/network/network_static.go b/pkg/network/network_static.go index fa46608cc..dee813680 100644 --- a/pkg/network/network_static.go +++ b/pkg/network/network_static.go @@ -88,6 +88,17 @@ func setNATRule(iface string, sourceIP string) error { return nil } +// HasNetwork checks, cheaply, whether a container network interface exists +// in the current netns, without creating the tap device or doing any of the +// other, more expensive setup NetworkSetup does. +func (n StaticNetwork) HasNetwork() (bool, error) { + _, err := discoverContainerIface() + if err != nil { + return false, nil + } + return true, nil +} + func (n StaticNetwork) NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error) { newTapName := strings.ReplaceAll(DefaultTap, "X", "0") addTCRules := false diff --git a/pkg/unikontainers/hypervisors/firecracker.go b/pkg/unikontainers/hypervisors/firecracker.go index cd7f0b8a5..d775fc3ee 100644 --- a/pkg/unikontainers/hypervisors/firecracker.go +++ b/pkg/unikontainers/hypervisors/firecracker.go @@ -15,11 +15,17 @@ package hypervisors import ( + "context" "encoding/json" + "errors" "fmt" "os" + "os/exec" + "os/signal" "path/filepath" "strings" + "syscall" + "time" "github.com/urunc-dev/urunc/pkg/unikontainers/types" "golang.org/x/sys/unix" @@ -123,6 +129,31 @@ func (fc *Firecracker) BuildExecCmd(args types.ExecArgs, ukernel types.Unikernel cmdString += " --no-seccomp" } + FCConfig := buildFirecrackerConfig(args, ukernel) + FCConfigJSON, err := json.Marshal(FCConfig) + if err != nil { + return nil, fmt.Errorf("failed to marshal Firecracker config: %w", err) + } + if err = os.WriteFile(JSONConfigFile, FCConfigJSON, 0o644); err != nil { //nolint: gosec + return nil, fmt.Errorf("failed to save Firecracker json config: %w", err) + } + vmmLog.WithField("Json", string(FCConfigJSON)).Debug("Firecracker json config") + + exArgs := strings.Split(cmdString, " ") + return exArgs, nil +} + +// PreExec performs pre-execution setup. Firecracker has no special pre-exec requirements. +func (fc *Firecracker) PreExec(_ types.ExecArgs) error { + return nil +} + +// buildFirecrackerConfig builds the microVM configuration from args and +// ukernel. Used both to write the JSON config file (config-file-based boot) +// and to drive the same configuration over the API socket (API-based boot), +// so both paths always agree on what the guest actually gets configured +// with. +func buildFirecrackerConfig(args types.ExecArgs, ukernel types.Unikernel) *FirecrackerConfig { // VM config for Firecracker fcMem := DefaultMemory if args.MemSizeB != 0 { @@ -192,27 +223,96 @@ func (fc *Firecracker) BuildExecCmd(args types.ExecArgs, ukernel types.Unikernel } } - FCConfig := &FirecrackerConfig{ + return &FirecrackerConfig{ Source: FCSource, Machine: FCMachine, Drives: FCDrives, NetIfs: FCNet, VSock: FCVSockDev, } - FCConfigJSON, err := json.Marshal(FCConfig) - if err != nil { - return nil, fmt.Errorf("failed to marshal Firecracker config: %w", err) +} + +// RunSocketBoot starts Firecracker as a child process instead of replacing +// the current process via exec. Since nothing here changes the process's +// root (no SysProcAttr.Chroot is set), the child simply inherits whatever +// confinement changeRoot already established on the caller earlier in Exec +// (pivot_root or chroot, whichever the container spec calls for) - so it +// ends up exactly as confined as the exec path would have made it, with no +// separate confinement step needed here. +// +// It configures the guest over the API socket using the same config +// BuildExecCmd would have written to a file, starts the guest, then +// supervises the child until it exits: forwarding SIGTERM/SIGINT, and +// exiting this process with the child's exit code once it's done, mirroring +// the semantics syscall.Exec would have had. +// +// On success this function does not return: it calls os.Exit with the +// child's exit status once the child exits. It returns an error only if +// startup or configuration fails before the guest could ever run. +func (fc *Firecracker) RunSocketBoot(args types.ExecArgs, ukernel types.Unikernel, execCmd []string) error { + cfg := buildFirecrackerConfig(args, ukernel) + + cmd := exec.Command(execCmd[0], execCmd[1:]...) //nolint: gosec + cmd.Env = args.Environment + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + return fmt.Errorf("failed to start firecracker: %w", err) } - if err = os.WriteFile(JSONConfigFile, FCConfigJSON, 0o644); err != nil { //nolint: gosec - return nil, fmt.Errorf("failed to save Firecracker json config: %w", err) + + socketPath := args.SocketPath + if socketPath == "" { + socketPath = filepath.Join("/tmp/", args.ContainerID+".sock") } - vmmLog.WithField("Json", string(FCConfigJSON)).Debug("Firecracker json config") + client := newFirecrackerClient(socketPath) + ctx := context.Background() - exArgs := strings.Split(cmdString, " ") - return exArgs, nil -} + if err := client.waitForSocket(5 * time.Second); err != nil { + _ = cmd.Process.Kill() + _, _ = cmd.Process.Wait() + return fmt.Errorf("firecracker socket never became ready: %w", err) + } + if err := client.configure(ctx, cfg); err != nil { + _ = cmd.Process.Kill() + _, _ = cmd.Process.Wait() + return fmt.Errorf("failed to configure firecracker over the socket: %w", err) + } + if err := client.startGuest(ctx); err != nil { + _ = cmd.Process.Kill() + _, _ = cmd.Process.Wait() + return fmt.Errorf("failed to start the guest: %w", err) + } -// PreExec performs pre-execution setup. Firecracker has no special pre-exec requirements. -func (fc *Firecracker) PreExec(_ types.ExecArgs) error { - return nil + // Forward the signals containerd would send to stop the container. + // SIGKILL cannot be caught, so it is not listed here: if it arrives, + // this process dies immediately and the child is left running, a known + // gap for this bounded experiment, not yet handled. + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT) + go func() { + sig, ok := <-sigCh + if !ok { + return + } + if s, ok := sig.(syscall.Signal); ok { + _ = cmd.Process.Signal(s) + } + }() + + waitErr := cmd.Wait() + signal.Stop(sigCh) + close(sigCh) + + exitCode := 0 + if waitErr != nil { + var exitErr *exec.ExitError + if errors.As(waitErr, &exitErr) { + exitCode = exitErr.ExitCode() + } else { + vmmLog.WithError(waitErr).Error("firecracker exited with an unexpected error") + exitCode = 1 + } + } + os.Exit(exitCode) + return nil // unreachable } diff --git a/pkg/unikontainers/hypervisors/firecracker_client.go b/pkg/unikontainers/hypervisors/firecracker_client.go new file mode 100644 index 000000000..097ba9297 --- /dev/null +++ b/pkg/unikontainers/hypervisors/firecracker_client.go @@ -0,0 +1,132 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hypervisors + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "time" +) + +// firecrackerClient drives a running Firecracker process over its HTTP-over-Unix +// API socket: it waits for the socket, configures the microVM and starts the +// guest. +// +// This type is the API driver only; starting (and owning) the Firecracker +// process is handled by the caller (see RunSocketBoot). +type firecrackerClient struct { + socketPath string + httpClient *http.Client +} + +// newFirecrackerClient returns a client that talks to the Firecracker API socket +// at socketPath. The HTTP transport dials that Unix socket for every request, so +// "http://localhost/" requests actually travel over the socket. +func newFirecrackerClient(socketPath string) *firecrackerClient { + transport := &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "unix", socketPath) + }, + } + return &firecrackerClient{ + socketPath: socketPath, + httpClient: &http.Client{Transport: transport}, + } +} + +// waitForSocket blocks until the Firecracker API socket exists and accepts +// connections, or the timeout elapses. Firecracker creates the socket shortly +// after it starts, so the caller polls here before sending any configuration. +func (c *firecrackerClient) waitForSocket(timeout time.Duration) error { + deadline := time.Now().Add(timeout) + var lastErr error + for time.Now().Before(deadline) { + conn, err := net.DialTimeout("unix", c.socketPath, 50*time.Millisecond) + if err == nil { + _ = conn.Close() + return nil + } + lastErr = err + time.Sleep(10 * time.Millisecond) + } + if lastErr == nil { + lastErr = context.DeadlineExceeded + } + return fmt.Errorf("firecracker API socket %q not ready within %s: %w", c.socketPath, timeout, lastErr) +} + +// configure sends the full microVM configuration over the API socket, in the +// order Firecracker expects, before the guest is started. +func (c *firecrackerClient) configure(ctx context.Context, cfg *FirecrackerConfig) error { + if err := c.put(ctx, "/machine-config", cfg.Machine); err != nil { + return err + } + if err := c.put(ctx, "/boot-source", cfg.Source); err != nil { + return err + } + for _, drive := range cfg.Drives { + if err := c.put(ctx, "/drives/"+drive.DriveID, drive); err != nil { + return err + } + } + for _, iface := range cfg.NetIfs { + if err := c.put(ctx, "/network-interfaces/"+iface.IfaceID, iface); err != nil { + return err + } + } + // vsock is only configured when vAccel-over-vsock is enabled (uds_path set). + if cfg.VSock.UDSPath != "" { + if err := c.put(ctx, "/vsock", cfg.VSock); err != nil { + return err + } + } + return nil +} + +// startGuest issues the InstanceStart action, which powers on the microVM and +// boots the guest. Firecracker allows this to succeed only once. +func (c *firecrackerClient) startGuest(ctx context.Context) error { + return c.put(ctx, "/actions", map[string]string{"action_type": "InstanceStart"}) +} + +// put marshals body to JSON and sends it as an HTTP PUT to the given API path +// over the Unix socket, returning an error for any non-2xx response. +func (c *firecrackerClient) put(ctx context.Context, path string, body any) error { + payload, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("marshal %s request: %w", path, err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPut, "http://localhost"+path, bytes.NewReader(payload)) + if err != nil { + return fmt.Errorf("build %s request: %w", path, err) + } + req.Header.Set("Content-Type", "application/json") + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("send %s request: %w", path, err) + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + respBody, _ := io.ReadAll(resp.Body) + return fmt.Errorf("%s returned HTTP %d: %s", path, resp.StatusCode, string(respBody)) + } + return nil +} diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index 542764525..8eaae7dd3 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -231,6 +231,19 @@ func (u *Unikontainer) SetupNet() (types.NetDevParams, error) { return netArgs, nil } +// HasNetwork does a cheap check for whether this container has a network +// interface to configure, without doing the more expensive tap device +// creation and configuration that SetupNet does. It exists so callers can +// learn this fact early, while the full SetupNet runs concurrently. +func (u *Unikontainer) HasNetwork() (bool, error) { + networkType := u.getNetworkType() + netManager, err := network.NewNetworkManager(networkType) + if err != nil { + return false, fmt.Errorf("failed to create network manager for %s type: %v", networkType, err) + } + return netManager.HasNetwork() +} + // chooseRootfs determines the best rootfs configuration based on available options // Priority order: // 1. Initrd (if specified) @@ -406,19 +419,45 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { } // handle network - netArgs, err := u.SetupNet() - if err != nil { - uniklog.Errorf("failed to setup network: %v", err) - return err + // + // For Firecracker's API-based boot mode, rootfs prep (below) only needs + // the cheap "does this container have a network interface" fact, not the + // fully finished network setup (see prepareMonRootfs's needsTAP argument + // below) - the expensive part of network setup (creating the tap device, + // TC rules, fetching interface info) doesn't need to block it. So in + // that case we run the real SetupNet concurrently with rootfs prep, and + // only wait for it once we actually need the resolved IP/gateway/MAC - + // which is right before unikernel.Init, since every supported unikernel + // type bakes the resolved network info into the guest's boot command + // line. For every other case, this stays exactly as sequential as today. + isAPIBoot := vmmType == string(hypervisors.FirecrackerVmm) && bootMode != "config-file" + + var netArgs types.NetDevParams + var netSetupErr error + var netWG sync.WaitGroup + var withTUNTAP bool + + if isAPIBoot { + hasNet, err := u.HasNetwork() + if err != nil { + uniklog.Errorf("failed to check for a container network: %v", err) + return err + } + withTUNTAP = hasNet + netWG.Add(1) + go func() { + defer netWG.Done() + netArgs, netSetupErr = u.SetupNet() + }() + } else { + netArgs, err = u.SetupNet() + if err != nil { + uniklog.Errorf("failed to setup network: %v", err) + return err + } + withTUNTAP = netArgs.IP != "" } metrics.Capture(m.TS16) - withTUNTAP := netArgs.IP != "" - - // UnikernelParams - unikernelParams.Net = netArgs - - // ExecArgs - vmmArgs.Net = netArgs // virtiofsd config virtiofsdConfig := u.UruncCfg.ExtraBins["virtiofsd"] @@ -583,6 +622,20 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { vmmArgs.VSockDevID = idToGuestCID(u.State.ID) } + // If network setup is still running concurrently (isAPIBoot), this is + // where we actually need its result: every supported unikernel type + // bakes the resolved IP/gateway/MAC into the guest's boot command line + // inside Init below. + if isAPIBoot { + netWG.Wait() + if netSetupErr != nil { + uniklog.Errorf("failed to setup network: %v", netSetupErr) + return netSetupErr + } + } + unikernelParams.Net = netArgs + vmmArgs.Net = netArgs + // unikernel err = unikernel.Init(unikernelParams) if errors.Is(err, unikernels.ErrUndefinedVersion) || @@ -665,6 +718,15 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return err } + if isAPIBoot { + fc, ok := vmm.(*hypervisors.Firecracker) + if !ok { + return fmt.Errorf("boot_mode=api is only supported for the firecracker monitor") + } + uniklog.WithField("command", execCmd).Debug("Starting Firecracker as a supervised child, driving it over its control socket") + return fc.RunSocketBoot(vmmArgs, unikernel, execCmd) + } + // Execute the VMM using the command we built earlier. uniklog.WithField("command", execCmd).Debug("Ready to execve VMM") return syscall.Exec(vmm.Path(), execCmd, vmmArgs.Environment) //nolint: gosec From 095f46f24240269d18a9f3b06d01e24f221aba9c Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Mon, 20 Jul 2026 12:17:57 +0530 Subject: [PATCH 06/17] feat(firecracker): configure the VMM in stages while urunc sets up Spawn Firecracker at the start of Exec and send each piece of its configuration as soon as urunc produces it: machine-config immediately, the network interface once the tap device exists, drives and boot source right after unikernel.Init, and InstanceStart only after the start-success handshake. The API client now holds one persistent connection, established before changeRoot, since the socket path is not resolvable after the root changes. Signed-off-by: Anamika Aggarwal --- pkg/unikontainers/hypervisors/firecracker.go | 171 ++++++++++++++---- .../hypervisors/firecracker_client.go | 112 ++++++++---- pkg/unikontainers/unikontainers.go | 95 ++++++++-- 3 files changed, 282 insertions(+), 96 deletions(-) diff --git a/pkg/unikontainers/hypervisors/firecracker.go b/pkg/unikontainers/hypervisors/firecracker.go index d775fc3ee..8df46d4fa 100644 --- a/pkg/unikontainers/hypervisors/firecracker.go +++ b/pkg/unikontainers/hypervisors/firecracker.go @@ -232,57 +232,150 @@ func buildFirecrackerConfig(args types.ExecArgs, ukernel types.Unikernel) *Firec } } -// RunSocketBoot starts Firecracker as a child process instead of replacing -// the current process via exec. Since nothing here changes the process's -// root (no SysProcAttr.Chroot is set), the child simply inherits whatever -// confinement changeRoot already established on the caller earlier in Exec -// (pivot_root or chroot, whichever the container spec calls for) - so it -// ends up exactly as confined as the exec path would have made it, with no -// separate confinement step needed here. -// -// It configures the guest over the API socket using the same config -// BuildExecCmd would have written to a file, starts the guest, then -// supervises the child until it exits: forwarding SIGTERM/SIGINT, and -// exiting this process with the child's exit code once it's done, mirroring -// the semantics syscall.Exec would have had. +// FirecrackerSession is a Firecracker child process being configured over its +// API socket in stages, while the caller performs its own setup work between +// the stages. Create it with SpawnSocketVMM, feed it configuration with the +// Configure* methods as each piece becomes available, boot the guest with +// StartGuest, then hand the calling process over with Supervise. +type FirecrackerSession struct { + cmd *exec.Cmd + client *firecrackerClient +} + +// SpawnSocketVMM starts Firecracker as a child process with only its API +// socket enabled, and establishes the single persistent connection all later +// configuration stages use (it survives a later changeRoot; see +// firecrackerClient). // -// On success this function does not return: it calls os.Exit with the -// child's exit status once the child exits. It returns an error only if -// startup or configuration fails before the guest could ever run. -func (fc *Firecracker) RunSocketBoot(args types.ExecArgs, ukernel types.Unikernel, execCmd []string) error { - cfg := buildFirecrackerConfig(args, ukernel) +// This is called early in Exec, before the monitor rootfs is prepared and +// before changeRoot, so unlike the exec path the child keeps the caller's +// current (pre-pivot) mount view for its lifetime. It does inherit the +// sandbox's network namespace, which Exec has already joined. When uid/gid +// are non-zero the child is started directly under that credential, since +// the caller only drops its own privileges (setupUser) much later. +func (fc *Firecracker) SpawnSocketVMM(args types.ExecArgs, uid, gid uint32) (*FirecrackerSession, error) { + socketPath := args.SocketPath + if socketPath == "" { + socketPath = filepath.Join("/tmp/", args.ContainerID+".sock") + } + execCmd := []string{fc.Path(), "--api-sock", socketPath} + if !args.Seccomp { + execCmd = append(execCmd, "--no-seccomp") + } cmd := exec.Command(execCmd[0], execCmd[1:]...) //nolint: gosec cmd.Env = args.Environment cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr + if uid != 0 || gid != 0 { + cmd.SysProcAttr = &syscall.SysProcAttr{ + Credential: &syscall.Credential{Uid: uid, Gid: gid}, + } + } + vmmLog.WithField("command", execCmd).Debug("starting Firecracker as a supervised child") if err := cmd.Start(); err != nil { - return fmt.Errorf("failed to start firecracker: %w", err) + return nil, fmt.Errorf("failed to start firecracker: %w", err) } - socketPath := args.SocketPath - if socketPath == "" { - socketPath = filepath.Join("/tmp/", args.ContainerID+".sock") - } client := newFirecrackerClient(socketPath) - ctx := context.Background() + if err := client.connect(5 * time.Second); err != nil { + s := &FirecrackerSession{cmd: cmd, client: client} + s.Kill() + return nil, fmt.Errorf("firecracker socket never became ready: %w", err) + } + return &FirecrackerSession{cmd: cmd, client: client}, nil +} + +// ConfigureMachine sends the vCPU/memory configuration. Nothing but the +// container spec is needed for this, so it is the first stage, sent right +// after the spawn. +func (s *FirecrackerSession) ConfigureMachine(ctx context.Context, args types.ExecArgs) error { + fcMem := DefaultMemory + if args.MemSizeB != 0 { + fcMem = bytesToMiB(args.MemSizeB) + if fcMem == 0 { + fcMem = DefaultMemory + } + } + machine := FirecrackerMachine{ + VcpuCount: args.VCPUs, + MemSizeMiB: fcMem, + Smt: false, + TrackDirtyPages: false, + } + vmmLog.Debug("staged boot: sending machine-config") + return s.client.putMachineConfig(ctx, machine) +} - if err := client.waitForSocket(5 * time.Second); err != nil { - _ = cmd.Process.Kill() - _, _ = cmd.Process.Wait() - return fmt.Errorf("firecracker socket never became ready: %w", err) +// ConfigureNetwork attaches the container's network interface. Firecracker +// opens the tap device during this call, so it must only run once the +// network setup that creates the tap has finished. No-op without a tap. +func (s *FirecrackerSession) ConfigureNetwork(ctx context.Context, net types.NetDevParams) error { + if net.TapDev == "" { + return nil } - if err := client.configure(ctx, cfg); err != nil { - _ = cmd.Process.Kill() - _, _ = cmd.Process.Wait() - return fmt.Errorf("failed to configure firecracker over the socket: %w", err) + iface := FirecrackerNet{ + IfaceID: "net1", + GuestMAC: net.MAC, + HostIF: net.TapDev, } - if err := client.startGuest(ctx); err != nil { - _ = cmd.Process.Kill() - _, _ = cmd.Process.Wait() - return fmt.Errorf("failed to start the guest: %w", err) + vmmLog.Debug("staged boot: sending network-interfaces") + return s.client.putNetworkIface(ctx, iface) +} + +// ConfigureGuest sends everything that depends on unikernel.Init having run: +// the block devices (their IDs/paths come from the unikernel's +// MonitorBlockCli), the boot source (its boot args are the unikernel command +// line, which bakes in the resolved network), and the vsock device if any. +// +// The child was spawned before changeRoot, so it resolves paths in the +// pre-pivot mount view; monRootfs (the directory the caller will pivot into) +// is therefore prefixed onto every path the child has to open. +func (s *FirecrackerSession) ConfigureGuest(ctx context.Context, args types.ExecArgs, ukernel types.Unikernel, monRootfs string) error { + cfg := buildFirecrackerConfig(args, ukernel) + + cfg.Source.ImagePath = filepath.Join(monRootfs, cfg.Source.ImagePath) + if cfg.Source.InitrdPath != "" { + cfg.Source.InitrdPath = filepath.Join(monRootfs, cfg.Source.InitrdPath) } + for i := range cfg.Drives { + cfg.Drives[i].HostPath = filepath.Join(monRootfs, cfg.Drives[i].HostPath) + } + if cfg.VSock.UDSPath != "" { + cfg.VSock.UDSPath = filepath.Join(monRootfs, cfg.VSock.UDSPath) + } + + vmmLog.Debug("staged boot: sending drives, boot-source and vsock") + if err := s.client.putDrives(ctx, cfg.Drives); err != nil { + return err + } + if err := s.client.putBootSource(ctx, cfg.Source); err != nil { + return err + } + return s.client.putVSock(ctx, cfg.VSock) +} +// StartGuest powers on the configured microVM. +func (s *FirecrackerSession) StartGuest(ctx context.Context) error { + vmmLog.Debug("staged boot: sending InstanceStart") + return s.client.startGuest(ctx) +} + +// Kill terminates the child and reaps it. For error paths before Supervise. +func (s *FirecrackerSession) Kill() { + _ = s.cmd.Process.Kill() + _, _ = s.cmd.Process.Wait() +} + +// Supervise hands the calling process over to the child for the rest of its +// life: it forwards SIGTERM/SIGINT and, once the child exits, exits this +// process with the child's exit code, mirroring the semantics syscall.Exec +// would have had. The caller must not exit before the child, since it is +// the container's init process. +// +// On success this function does not return: it calls os.Exit with the +// child's exit status once the child exits. +func (s *FirecrackerSession) Supervise() error { // Forward the signals containerd would send to stop the container. // SIGKILL cannot be caught, so it is not listed here: if it arrives, // this process dies immediately and the child is left running, a known @@ -294,12 +387,12 @@ func (fc *Firecracker) RunSocketBoot(args types.ExecArgs, ukernel types.Unikerne if !ok { return } - if s, ok := sig.(syscall.Signal); ok { - _ = cmd.Process.Signal(s) + if sg, ok := sig.(syscall.Signal); ok { + _ = s.cmd.Process.Signal(sg) } }() - waitErr := cmd.Wait() + waitErr := s.cmd.Wait() signal.Stop(sigCh) close(sigCh) diff --git a/pkg/unikontainers/hypervisors/firecracker_client.go b/pkg/unikontainers/hypervisors/firecracker_client.go index 097ba9297..f57c049cc 100644 --- a/pkg/unikontainers/hypervisors/firecracker_client.go +++ b/pkg/unikontainers/hypervisors/firecracker_client.go @@ -22,46 +22,77 @@ import ( "io" "net" "net/http" + "sync" "time" ) // firecrackerClient drives a running Firecracker process over its HTTP-over-Unix -// API socket: it waits for the socket, configures the microVM and starts the -// guest. +// API socket, one configuration resource at a time, so the caller can send each +// piece of configuration as soon as it becomes available. +// +// The client establishes a single connection in connect() and keeps it alive +// for all requests. This matters because the caller may change its root +// (pivot_root/chroot) between configuration stages: the socket path stops +// being resolvable from the new root, but the already-open connection keeps +// working, since open file descriptors survive a root change. // // This type is the API driver only; starting (and owning) the Firecracker -// process is handled by the caller (see RunSocketBoot). +// process is handled by the caller (see SpawnSocketVMM). type firecrackerClient struct { socketPath string httpClient *http.Client + + dialMu sync.Mutex + heldConn net.Conn } // newFirecrackerClient returns a client that talks to the Firecracker API socket -// at socketPath. The HTTP transport dials that Unix socket for every request, so -// "http://localhost/" requests actually travel over the socket. +// at socketPath. "http://localhost/" requests actually travel over the +// socket. Call connect() before issuing any request. func newFirecrackerClient(socketPath string) *firecrackerClient { + c := &firecrackerClient{socketPath: socketPath} transport := &http.Transport{ - DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { - var d net.Dialer - return d.DialContext(ctx, "unix", socketPath) - }, + DialContext: c.dialContext, + // Keep the single connection alive for the whole staged boot: it is + // established before the caller may change root, and the socket path + // is not resolvable afterwards, so an idle close between stages + // would break every later stage. + IdleConnTimeout: 0, + MaxIdleConns: 1, + MaxIdleConnsPerHost: 1, } - return &firecrackerClient{ - socketPath: socketPath, - httpClient: &http.Client{Transport: transport}, + c.httpClient = &http.Client{Transport: transport} + return c +} + +// dialContext hands the transport the connection pre-established by connect(), +// and falls back to dialing the socket path directly if that connection was +// already consumed (which only works while the path is still resolvable). +func (c *firecrackerClient) dialContext(ctx context.Context, _, _ string) (net.Conn, error) { + c.dialMu.Lock() + defer c.dialMu.Unlock() + if c.heldConn != nil { + conn := c.heldConn + c.heldConn = nil + return conn, nil } + var d net.Dialer + return d.DialContext(ctx, "unix", c.socketPath) } -// waitForSocket blocks until the Firecracker API socket exists and accepts +// connect blocks until the Firecracker API socket exists and accepts // connections, or the timeout elapses. Firecracker creates the socket shortly -// after it starts, so the caller polls here before sending any configuration. -func (c *firecrackerClient) waitForSocket(timeout time.Duration) error { +// after it starts. The successful connection is kept and reused for all +// requests (see the type comment for why). +func (c *firecrackerClient) connect(timeout time.Duration) error { deadline := time.Now().Add(timeout) var lastErr error for time.Now().Before(deadline) { conn, err := net.DialTimeout("unix", c.socketPath, 50*time.Millisecond) if err == nil { - _ = conn.Close() + c.dialMu.Lock() + c.heldConn = conn + c.dialMu.Unlock() return nil } lastErr = err @@ -73,34 +104,41 @@ func (c *firecrackerClient) waitForSocket(timeout time.Duration) error { return fmt.Errorf("firecracker API socket %q not ready within %s: %w", c.socketPath, timeout, lastErr) } -// configure sends the full microVM configuration over the API socket, in the -// order Firecracker expects, before the guest is started. -func (c *firecrackerClient) configure(ctx context.Context, cfg *FirecrackerConfig) error { - if err := c.put(ctx, "/machine-config", cfg.Machine); err != nil { - return err - } - if err := c.put(ctx, "/boot-source", cfg.Source); err != nil { - return err - } - for _, drive := range cfg.Drives { +// putMachineConfig configures vCPUs and memory. This needs nothing but the +// container spec, so it can be sent as the very first stage. +func (c *firecrackerClient) putMachineConfig(ctx context.Context, machine FirecrackerMachine) error { + return c.put(ctx, "/machine-config", machine) +} + +// putNetworkIface attaches one network interface. Firecracker opens the tap +// device during this call, so it must not be sent before the tap exists. +func (c *firecrackerClient) putNetworkIface(ctx context.Context, iface FirecrackerNet) error { + return c.put(ctx, "/network-interfaces/"+iface.IfaceID, iface) +} + +// putDrives attaches the guest's block devices. +func (c *firecrackerClient) putDrives(ctx context.Context, drives []FirecrackerDrive) error { + for _, drive := range drives { if err := c.put(ctx, "/drives/"+drive.DriveID, drive); err != nil { return err } } - for _, iface := range cfg.NetIfs { - if err := c.put(ctx, "/network-interfaces/"+iface.IfaceID, iface); err != nil { - return err - } - } - // vsock is only configured when vAccel-over-vsock is enabled (uds_path set). - if cfg.VSock.UDSPath != "" { - if err := c.put(ctx, "/vsock", cfg.VSock); err != nil { - return err - } - } return nil } +// putBootSource configures the kernel image, boot arguments and initrd. +func (c *firecrackerClient) putBootSource(ctx context.Context, source FirecrackerBootSource) error { + return c.put(ctx, "/boot-source", source) +} + +// putVSock configures the vsock device. No-op when unset (empty uds_path). +func (c *firecrackerClient) putVSock(ctx context.Context, vsock FirecrackerVSockDev) error { + if vsock.UDSPath == "" { + return nil + } + return c.put(ctx, "/vsock", vsock) +} + // startGuest issues the InstanceStart action, which powers on the microVM and // boots the guest. Firecracker allows this to succeed only once. func (c *firecrackerClient) startGuest(ctx context.Context) error { diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index 8eaae7dd3..7650bdc85 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -16,6 +16,7 @@ package unikontainers import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -420,16 +421,21 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { // handle network // - // For Firecracker's API-based boot mode, rootfs prep (below) only needs - // the cheap "does this container have a network interface" fact, not the - // fully finished network setup (see prepareMonRootfs's needsTAP argument - // below) - the expensive part of network setup (creating the tap device, - // TC rules, fetching interface info) doesn't need to block it. So in - // that case we run the real SetupNet concurrently with rootfs prep, and - // only wait for it once we actually need the resolved IP/gateway/MAC - - // which is right before unikernel.Init, since every supported unikernel - // type bakes the resolved network info into the guest's boot command - // line. For every other case, this stays exactly as sequential as today. + // For Firecracker's API-based boot mode, the VMM is spawned right here, + // before any of the expensive setup work, and configured in stages while + // that work runs: machine-config immediately (it needs nothing but the + // spec), the network interface as soon as network setup finishes, and + // everything unikernel-dependent (drives, boot source) right after + // unikernel.Init. The guest itself is only started (InstanceStart) after + // the start-success handshake, preserving OCI start ordering. + // + // Rootfs prep (below) only needs the cheap "does this container have a + // network interface" fact, not the fully finished network setup (see + // prepareMonRootfs's needsTAP argument below), so the real SetupNet runs + // concurrently with rootfs prep, joined right before unikernel.Init, + // since every supported unikernel type bakes the resolved IP/gateway/MAC + // into the guest's boot command line. For every other case, this stays + // exactly as sequential as today. isAPIBoot := vmmType == string(hypervisors.FirecrackerVmm) && bootMode != "config-file" var netArgs types.NetDevParams @@ -437,7 +443,33 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { var netWG sync.WaitGroup var withTUNTAP bool + var fcSession *hypervisors.FirecrackerSession + fcHandedOff := false + defer func() { + // Any error return after the spawn must not leave the VMM child + // behind. Supervise never returns (os.Exit), so this only fires on + // error paths. + if fcSession != nil && !fcHandedOff { + fcSession.Kill() + } + }() + ctx := context.Background() + if isAPIBoot { + fc, ok := vmm.(*hypervisors.Firecracker) + if !ok { + return fmt.Errorf("boot_mode=api is only supported for the firecracker monitor") + } + fcSession, err = fc.SpawnSocketVMM(vmmArgs, procAttrs.UID, procAttrs.GID) + if err != nil { + uniklog.Errorf("failed to spawn firecracker: %v", err) + return err + } + if err = fcSession.ConfigureMachine(ctx, vmmArgs); err != nil { + uniklog.Errorf("failed to configure the machine over the socket: %v", err) + return err + } + hasNet, err := u.HasNetwork() if err != nil { uniklog.Errorf("failed to check for a container network: %v", err) @@ -625,13 +657,18 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { // If network setup is still running concurrently (isAPIBoot), this is // where we actually need its result: every supported unikernel type // bakes the resolved IP/gateway/MAC into the guest's boot command line - // inside Init below. + // inside Init below. The tap device exists from this point on, so the + // network interface is also sent to the VMM right away. if isAPIBoot { netWG.Wait() if netSetupErr != nil { uniklog.Errorf("failed to setup network: %v", netSetupErr) return netSetupErr } + if err = fcSession.ConfigureNetwork(ctx, netArgs); err != nil { + uniklog.Errorf("failed to configure the network over the socket: %v", err) + return err + } } unikernelParams.Net = netArgs vmmArgs.Net = netArgs @@ -655,6 +692,16 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { // ExecArgs vmmArgs.Command = unikernelCmd + // Everything unikernel-dependent is known now, so it goes to the VMM + // before changeRoot below: the child kept the pre-pivot mount view, so + // ConfigureGuest prefixes the monitor rootfs onto the paths it sends. + if isAPIBoot { + if err = fcSession.ConfigureGuest(ctx, vmmArgs, unikernel, rootfsParams.MonRootfs); err != nil { + uniklog.Errorf("failed to configure the guest over the socket: %v", err) + return err + } + } + // pivot _, err = findNS(u.Spec.Linux.Namespaces, specs.MountNamespace) // We just want to check if a mount namespace was define din the list @@ -694,10 +741,15 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { // Build the VMM command once and verify it can be constructed successfully. // This ensures we don't report the container as started if command building fails. - execCmd, err := vmm.BuildExecCmd(vmmArgs, unikernel) - if err != nil { - uniklog.WithError(err).Error("failed to build VMM command") - return err + // For the API-based boot the VMM is already running and fully configured + // (the equivalent validation), so there is no command to build. + var execCmd []string + if !isAPIBoot { + execCmd, err = vmm.BuildExecCmd(vmmArgs, unikernel) + if err != nil { + uniklog.WithError(err).Error("failed to build VMM command") + return err + } } // Notify urunc start that the monitor is ready to execute. @@ -719,12 +771,15 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { } if isAPIBoot { - fc, ok := vmm.(*hypervisors.Firecracker) - if !ok { - return fmt.Errorf("boot_mode=api is only supported for the firecracker monitor") + // The VMM is configured; boot the guest and hand this process over + // to supervising the child. This process must not exit before the + // child, since it is the container's init process. + if err = fcSession.StartGuest(ctx); err != nil { + uniklog.Errorf("failed to start the guest: %v", err) + return err } - uniklog.WithField("command", execCmd).Debug("Starting Firecracker as a supervised child, driving it over its control socket") - return fc.RunSocketBoot(vmmArgs, unikernel, execCmd) + fcHandedOff = true + return fcSession.Supervise() } // Execute the VMM using the command we built earlier. From 8a32f41f3a2d8d77960315a355324c6807cfffda Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Tue, 21 Jul 2026 21:26:11 +0530 Subject: [PATCH 07/17] perf(firecracker): poll the control socket every 1ms instead of 10ms Firecracker binds its API socket within ~1ms of starting, but connect() polled every 10ms, so it usually slept several ms past the moment the socket was ready before noticing. Poll every 1ms: this recovers almost all of that wasted wait (~6ms less socket-wait measured in isolation), while a 1ms sleep stays well clear of a CPU-burning busy-loop. The end-to-end boot time is unchanged (guest boot dominates it), so this is a small efficiency fix, not a boot-time speedup. Signed-off-by: Anamika Aggarwal --- pkg/unikontainers/hypervisors/firecracker_client.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/unikontainers/hypervisors/firecracker_client.go b/pkg/unikontainers/hypervisors/firecracker_client.go index f57c049cc..77bbd26fd 100644 --- a/pkg/unikontainers/hypervisors/firecracker_client.go +++ b/pkg/unikontainers/hypervisors/firecracker_client.go @@ -96,7 +96,10 @@ func (c *firecrackerClient) connect(timeout time.Duration) error { return nil } lastErr = err - time.Sleep(10 * time.Millisecond) + // Firecracker binds the socket within ~1ms of starting, so poll + // tightly: a 1ms interval captures nearly all of the readiness + // latency a coarser interval would waste, without busy-spinning. + time.Sleep(1 * time.Millisecond) } if lastErr == nil { lastErr = context.DeadlineExceeded From 9837cf2a0a92c0013e53cad94fbd84ec13a132c3 Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Tue, 21 Jul 2026 21:26:11 +0530 Subject: [PATCH 08/17] fix(firecracker): remove a stale control socket before spawning Firecracker creates its API socket with bind(), which fails if a file already exists at that path. A crash or a reused container id can leave a stale socket behind, blocking the next start. Remove any leftover socket before spawning, ignoring the not-found case. Signed-off-by: Anamika Aggarwal --- pkg/unikontainers/hypervisors/firecracker.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/unikontainers/hypervisors/firecracker.go b/pkg/unikontainers/hypervisors/firecracker.go index 8df46d4fa..eec005d21 100644 --- a/pkg/unikontainers/hypervisors/firecracker.go +++ b/pkg/unikontainers/hypervisors/firecracker.go @@ -258,6 +258,12 @@ func (fc *Firecracker) SpawnSocketVMM(args types.ExecArgs, uid, gid uint32) (*Fi if socketPath == "" { socketPath = filepath.Join("/tmp/", args.ContainerID+".sock") } + // Firecracker binds this path itself; a stale socket file left from an + // earlier run (e.g. a crash) would make its bind fail with "address + // already in use", so remove any leftover first. + if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) { + return nil, fmt.Errorf("failed to remove stale socket %q: %w", socketPath, err) + } execCmd := []string{fc.Path(), "--api-sock", socketPath} if !args.Seccomp { execCmd = append(execCmd, "--no-seccomp") From 745e80e82bde5abf10549db2158a22571a0405a3 Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Sat, 25 Jul 2026 16:28:28 +0530 Subject: [PATCH 09/17] refactor(firecracker): resolve the socket path via a shared helper Signed-off-by: Anamika Aggarwal --- pkg/unikontainers/hypervisors/firecracker.go | 10 ++-------- pkg/unikontainers/hypervisors/utils.go | 17 +++++++++++++++++ pkg/unikontainers/hypervisors/utils_test.go | 12 ++++++++++++ 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/pkg/unikontainers/hypervisors/firecracker.go b/pkg/unikontainers/hypervisors/firecracker.go index eec005d21..c43532769 100644 --- a/pkg/unikontainers/hypervisors/firecracker.go +++ b/pkg/unikontainers/hypervisors/firecracker.go @@ -114,10 +114,7 @@ func (fc *Firecracker) BuildExecCmd(args types.ExecArgs, ukernel types.Unikernel // options in FC, since the string return value of the Monitor related // functions in the unikernel interface do not integrate well with FC's // json configuration. - apiSockPath := args.SocketPath - if apiSockPath == "" { - apiSockPath = filepath.Join("/tmp/", args.ContainerID+".sock") - } + apiSockPath := ResolveSocketPath(args) cmdString := fc.Path() + " --api-sock " + apiSockPath JSONConfigFile := filepath.Join("/tmp/", FCJsonFilename) if args.BootMode == "config-file" { @@ -254,10 +251,7 @@ type FirecrackerSession struct { // are non-zero the child is started directly under that credential, since // the caller only drops its own privileges (setupUser) much later. func (fc *Firecracker) SpawnSocketVMM(args types.ExecArgs, uid, gid uint32) (*FirecrackerSession, error) { - socketPath := args.SocketPath - if socketPath == "" { - socketPath = filepath.Join("/tmp/", args.ContainerID+".sock") - } + socketPath := ResolveSocketPath(args) // Firecracker binds this path itself; a stale socket file left from an // earlier run (e.g. a crash) would make its bind fail with "address // already in use", so remove any leftover first. diff --git a/pkg/unikontainers/hypervisors/utils.go b/pkg/unikontainers/hypervisors/utils.go index 1bee33104..fe6f5d1da 100644 --- a/pkg/unikontainers/hypervisors/utils.go +++ b/pkg/unikontainers/hypervisors/utils.go @@ -17,11 +17,14 @@ package hypervisors import ( "errors" "fmt" + "path/filepath" "runtime" "strconv" "time" "golang.org/x/sys/unix" + + "github.com/urunc-dev/urunc/pkg/unikontainers/types" ) func cpuArch() string { @@ -66,6 +69,20 @@ func BytesToStringMB(argMem uint64) string { return stringMem } +// DefaultSocketDir is the directory used for a monitor's control socket when +// no socket_path is configured. It always exists inside the monitor rootfs. +const DefaultSocketDir = "/tmp" + +// ResolveSocketPath returns the path for a monitor's control socket: the +// configured SocketPath when set, otherwise a per-container default under +// DefaultSocketDir. +func ResolveSocketPath(args types.ExecArgs) string { + if args.SocketPath != "" { + return args.SocketPath + } + return filepath.Join(DefaultSocketDir, args.ContainerID+".sock") +} + func killProcess(pid int) error { const timeout = 2 * time.Second err := unix.Kill(pid, unix.SIGKILL) diff --git a/pkg/unikontainers/hypervisors/utils_test.go b/pkg/unikontainers/hypervisors/utils_test.go index ab4f73738..8afdc62ea 100644 --- a/pkg/unikontainers/hypervisors/utils_test.go +++ b/pkg/unikontainers/hypervisors/utils_test.go @@ -18,8 +18,20 @@ import ( "testing" "github.com/stretchr/testify/assert" + + "github.com/urunc-dev/urunc/pkg/unikontainers/types" ) +func TestResolveSocketPath(t *testing.T) { + t.Parallel() + if got := ResolveSocketPath(types.ExecArgs{SocketPath: "/run/urunc/x.sock"}); got != "/run/urunc/x.sock" { + t.Fatalf("configured path: got %q", got) + } + if got := ResolveSocketPath(types.ExecArgs{ContainerID: "abc"}); got != "/tmp/abc.sock" { + t.Fatalf("default path: got %q", got) + } +} + func TestBytesToMiB(t *testing.T) { t.Parallel() From 180d70db825d8fc284539becac82ece109c22b5c Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Sat, 25 Jul 2026 16:31:58 +0530 Subject: [PATCH 10/17] feat(firecracker): add UsesControlSocket to gate control-socket setup Signed-off-by: Anamika Aggarwal --- pkg/unikontainers/hypervisors/vmm.go | 11 +++++++++++ pkg/unikontainers/hypervisors/vmm_test.go | 10 ++++++++++ 2 files changed, 21 insertions(+) diff --git a/pkg/unikontainers/hypervisors/vmm.go b/pkg/unikontainers/hypervisors/vmm.go index c8600957b..7ec57c85e 100644 --- a/pkg/unikontainers/hypervisors/vmm.go +++ b/pkg/unikontainers/hypervisors/vmm.go @@ -27,6 +27,17 @@ const DefaultMemory uint64 = 256 // The default memory for every hypervisor: 256 type VmmType string +// UsesControlSocket reports whether a monitor exposes a control socket whose +// path (socket_path) urunc must make reachable before the monitor launches. +func UsesControlSocket(vmmType VmmType) bool { + switch vmmType { + case FirecrackerVmm: + return true + default: + return false + } +} + var ErrVMMNotInstalled = errors.New("vmm not found") var vmmLog = logrus.WithField("subsystem", "monitors") diff --git a/pkg/unikontainers/hypervisors/vmm_test.go b/pkg/unikontainers/hypervisors/vmm_test.go index 6d0f7ebdb..00725ee47 100644 --- a/pkg/unikontainers/hypervisors/vmm_test.go +++ b/pkg/unikontainers/hypervisors/vmm_test.go @@ -69,3 +69,13 @@ func TestVMMFactoryNonQemuIgnoresVhost(t *testing.T) { _, ok = vmm.(*Firecracker) assert.True(t, ok, "factory should return *Firecracker") } + +func TestUsesControlSocket(t *testing.T) { + t.Parallel() + if !UsesControlSocket(FirecrackerVmm) { + t.Fatal("firecracker should use a control socket") + } + if UsesControlSocket(HvtVmm) { + t.Fatal("hvt should not use a control socket") + } +} From c743614a6338d8ad542965c86cbb714863c8213e Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Sat, 25 Jul 2026 16:41:33 +0530 Subject: [PATCH 11/17] feat(firecracker): create the control socket directory Signed-off-by: Anamika Aggarwal --- pkg/unikontainers/unikontainers.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index 7650bdc85..f258cd784 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -460,6 +460,9 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { if !ok { return fmt.Errorf("boot_mode=api is only supported for the firecracker monitor") } + if err = ensureSocketDir(vmmType, vmmArgs); err != nil { + return err + } fcSession, err = fc.SpawnSocketVMM(vmmArgs, procAttrs.UID, procAttrs.GID) if err != nil { uniklog.Errorf("failed to spawn firecracker: %v", err) @@ -713,6 +716,12 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return err } + if !isAPIBoot { + if err = ensureSocketDir(vmmType, vmmArgs); err != nil { + return err + } + } + // uid/gid // Setup uid, gid and additional groups for the monitor process err = setupUser(u.Spec.Process.User) @@ -787,6 +796,22 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return syscall.Exec(vmm.Path(), execCmd, vmmArgs.Environment) //nolint: gosec } +// ensureSocketDir creates the directory of the monitor's control socket so the +// monitor can bind its socket there. For a custom socket_path this may be a +// directory that does not exist yet; MkdirAll fails only if the location is +// invalid (e.g. a file already exists on the path). No-op for monitors without +// a control socket. +func ensureSocketDir(vmmType string, vmmArgs types.ExecArgs) error { + if !hypervisors.UsesControlSocket(hypervisors.VmmType(vmmType)) { + return nil + } + sockDir := filepath.Dir(hypervisors.ResolveSocketPath(vmmArgs)) + if err := os.MkdirAll(sockDir, 0o755); err != nil { + return fmt.Errorf("failed to create control socket directory %q: %w", sockDir, err) + } + return nil +} + func setupUser(user specs.User) error { runtime.LockOSThread() // Set the user for the current go routine to exec the Monitor From 9b3883385a452b3fd83cdf48df4fa1b920b474c6 Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Sat, 25 Jul 2026 17:56:15 +0530 Subject: [PATCH 12/17] docs(configuration): describe socket_path directory creation Signed-off-by: Anamika Aggarwal --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 689a6bdc1..cdccd993b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -112,7 +112,7 @@ Each monitor subsection supports the following options: | `default_vcpus` | integer | `1` | Default number of virtual CPUs | | `path` | string | (empty) | Optional custom path to the monitor binary. If not specified, urunc will search for the binary in PATH | | `data_path` | string | (empty) | Optional custom path for the monitor's data file directory | -| `socket_path` | string | (empty) | Optional custom path for the monitor's control socket. If not specified, the monitor uses its default. Currently only used by Firecracker. | +| `socket_path` | string | (empty) | Optional path for the monitor's control socket. If not specified, urunc uses a per-container default under `/tmp`. When a custom path is set, urunc creates its parent directory; setting it to an invalid location (a file already exists on the path) makes the monitor fail to start. Currently only used by Firecracker. | | `boot_mode` | string | `api` | Optional: `api` drives the monitor's boot over its control socket, `config-file` lets the monitor boot itself from a config file (socket only used afterward). Currently only used by Firecracker. | Since Qemu is the only currently supported monitor which requires extra data to From a44a435f95ed7aaddca3ffecb56e6cefa5eed459 Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Mon, 27 Jul 2026 19:32:38 +0530 Subject: [PATCH 13/17] fix(firecracker): confine api-mode monitor in the monitor rootfs Spawn the api-mode Firecracker child after changeRoot, so it inherits the pivoted root: the control socket and every path it opens live inside the monitor rootfs, matching the confinement of the config-file mode. A custom socket_path is now created inside the monitor rootfs in both boot modes, so the directory-creation feature applies uniformly and no path can land on the host filesystem. Since the child now shares urunc's mount view, the monitor-rootfs prefixing in ConfigureGuest is removed and the guest paths are sent exactly as the exec path uses them. The SetupNet/prepareMonRootfs overlap is unaffected; only the VMM configuration calls move after the pivot, and InstanceStart still waits for the start handshake. Signed-off-by: Anamika Aggarwal --- pkg/unikontainers/hypervisors/firecracker.go | 35 ++-- .../hypervisors/firecracker_client.go | 14 +- .../hypervisors/firecracker_session_test.go | 179 ++++++++++++++++++ pkg/unikontainers/unikontainers.go | 70 ++++--- 4 files changed, 229 insertions(+), 69 deletions(-) create mode 100644 pkg/unikontainers/hypervisors/firecracker_session_test.go diff --git a/pkg/unikontainers/hypervisors/firecracker.go b/pkg/unikontainers/hypervisors/firecracker.go index c43532769..93fcaaf05 100644 --- a/pkg/unikontainers/hypervisors/firecracker.go +++ b/pkg/unikontainers/hypervisors/firecracker.go @@ -241,15 +241,15 @@ type FirecrackerSession struct { // SpawnSocketVMM starts Firecracker as a child process with only its API // socket enabled, and establishes the single persistent connection all later -// configuration stages use (it survives a later changeRoot; see -// firecrackerClient). +// configuration stages use. // -// This is called early in Exec, before the monitor rootfs is prepared and -// before changeRoot, so unlike the exec path the child keeps the caller's -// current (pre-pivot) mount view for its lifetime. It does inherit the -// sandbox's network namespace, which Exec has already joined. When uid/gid -// are non-zero the child is started directly under that credential, since -// the caller only drops its own privileges (setupUser) much later. +// This is called after changeRoot, so the child inherits the already-pivoted +// root: its control socket and every path it opens live inside the monitor +// rootfs, the same confinement the exec path gets. It also inherits the +// sandbox's network namespace, which Exec has already joined. The caller is +// still privileged here and only drops its own privileges just after +// (setupUser), so when uid/gid are non-zero the child is started directly +// under that credential. func (fc *Firecracker) SpawnSocketVMM(args types.ExecArgs, uid, gid uint32) (*FirecrackerSession, error) { socketPath := ResolveSocketPath(args) // Firecracker binds this path itself; a stale socket file left from an @@ -327,24 +327,11 @@ func (s *FirecrackerSession) ConfigureNetwork(ctx context.Context, net types.Net // the block devices (their IDs/paths come from the unikernel's // MonitorBlockCli), the boot source (its boot args are the unikernel command // line, which bakes in the resolved network), and the vsock device if any. -// -// The child was spawned before changeRoot, so it resolves paths in the -// pre-pivot mount view; monRootfs (the directory the caller will pivot into) -// is therefore prefixed onto every path the child has to open. -func (s *FirecrackerSession) ConfigureGuest(ctx context.Context, args types.ExecArgs, ukernel types.Unikernel, monRootfs string) error { +// The child runs inside the monitor rootfs, so every path is sent exactly as +// the exec path would use it. +func (s *FirecrackerSession) ConfigureGuest(ctx context.Context, args types.ExecArgs, ukernel types.Unikernel) error { cfg := buildFirecrackerConfig(args, ukernel) - cfg.Source.ImagePath = filepath.Join(monRootfs, cfg.Source.ImagePath) - if cfg.Source.InitrdPath != "" { - cfg.Source.InitrdPath = filepath.Join(monRootfs, cfg.Source.InitrdPath) - } - for i := range cfg.Drives { - cfg.Drives[i].HostPath = filepath.Join(monRootfs, cfg.Drives[i].HostPath) - } - if cfg.VSock.UDSPath != "" { - cfg.VSock.UDSPath = filepath.Join(monRootfs, cfg.VSock.UDSPath) - } - vmmLog.Debug("staged boot: sending drives, boot-source and vsock") if err := s.client.putDrives(ctx, cfg.Drives); err != nil { return err diff --git a/pkg/unikontainers/hypervisors/firecracker_client.go b/pkg/unikontainers/hypervisors/firecracker_client.go index 77bbd26fd..f6b2e625a 100644 --- a/pkg/unikontainers/hypervisors/firecracker_client.go +++ b/pkg/unikontainers/hypervisors/firecracker_client.go @@ -31,10 +31,8 @@ import ( // piece of configuration as soon as it becomes available. // // The client establishes a single connection in connect() and keeps it alive -// for all requests. This matters because the caller may change its root -// (pivot_root/chroot) between configuration stages: the socket path stops -// being resolvable from the new root, but the already-open connection keeps -// working, since open file descriptors survive a root change. +// for all requests, so every configuration stage reuses the connection whose +// readiness connect() already waited for, instead of re-dialing. // // This type is the API driver only; starting (and owning) the Firecracker // process is handled by the caller (see SpawnSocketVMM). @@ -53,10 +51,8 @@ func newFirecrackerClient(socketPath string) *firecrackerClient { c := &firecrackerClient{socketPath: socketPath} transport := &http.Transport{ DialContext: c.dialContext, - // Keep the single connection alive for the whole staged boot: it is - // established before the caller may change root, and the socket path - // is not resolvable afterwards, so an idle close between stages - // would break every later stage. + // Keep the single connection alive for the whole staged boot so an + // idle close between stages does not force a re-dial. IdleConnTimeout: 0, MaxIdleConns: 1, MaxIdleConnsPerHost: 1, @@ -67,7 +63,7 @@ func newFirecrackerClient(socketPath string) *firecrackerClient { // dialContext hands the transport the connection pre-established by connect(), // and falls back to dialing the socket path directly if that connection was -// already consumed (which only works while the path is still resolvable). +// already consumed. func (c *firecrackerClient) dialContext(ctx context.Context, _, _ string) (net.Conn, error) { c.dialMu.Lock() defer c.dialMu.Unlock() diff --git a/pkg/unikontainers/hypervisors/firecracker_session_test.go b/pkg/unikontainers/hypervisors/firecracker_session_test.go new file mode 100644 index 000000000..1c1f67b90 --- /dev/null +++ b/pkg/unikontainers/hypervisors/firecracker_session_test.go @@ -0,0 +1,179 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hypervisors + +import ( + "context" + "encoding/json" + "net" + "net/http" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/urunc-dev/urunc/pkg/unikontainers/types" +) + +// recordedRequest is one PUT the fake Firecracker API received. +type recordedRequest struct { + path string + body map[string]any +} + +// fakeFirecrackerAPI records every request the client sends, in order. +type fakeFirecrackerAPI struct { + mu sync.Mutex + requests []recordedRequest +} + +func (f *fakeFirecrackerAPI) recorded() []recordedRequest { + f.mu.Lock() + defer f.mu.Unlock() + out := make([]recordedRequest, len(f.requests)) + copy(out, f.requests) + return out +} + +// startFakeFirecrackerAPI serves a fake Firecracker API on a Unix socket in a +// temporary directory and returns the recorder plus the socket path. +func startFakeFirecrackerAPI(t *testing.T) (*fakeFirecrackerAPI, string) { + t.Helper() + sockPath := filepath.Join(t.TempDir(), "fc.sock") + ln, err := net.Listen("unix", sockPath) + if err != nil { + t.Fatalf("failed to listen on %s: %v", sockPath, err) + } + + api := &fakeFirecrackerAPI{} + srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body map[string]any + _ = json.NewDecoder(r.Body).Decode(&body) + api.mu.Lock() + api.requests = append(api.requests, recordedRequest{path: r.URL.Path, body: body}) + api.mu.Unlock() + w.WriteHeader(http.StatusNoContent) + })} + go func() { _ = srv.Serve(ln) }() + t.Cleanup(func() { _ = srv.Close() }) + return api, sockPath +} + +// newTestSession connects a client to the fake API and wraps it in a session. +func newTestSession(t *testing.T, sockPath string) *FirecrackerSession { + t.Helper() + client := newFirecrackerClient(sockPath) + if err := client.connect(2 * time.Second); err != nil { + t.Fatalf("connect failed: %v", err) + } + return &FirecrackerSession{client: client} +} + +// TestConfigureGuestSendsUnprefixedPaths verifies that the session sends the +// kernel, initrd and drive paths exactly as given: the child runs inside the +// monitor rootfs, so no path may be prefixed with a host-side rootfs +// directory. +func TestConfigureGuestSendsUnprefixedPaths(t *testing.T) { + api, sockPath := startFakeFirecrackerAPI(t) + session := newTestSession(t, sockPath) + + args := types.ExecArgs{ + ContainerID: "testct", + Command: "app -arg", + UnikernelPath: "/unikernel/app.bin", + InitrdPath: "/unikernel/initrd", + MemSizeB: 256 * 1024 * 1024, + VCPUs: 1, + } + ukernel := &fakeUnikernel{ + blockCli: []types.MonitorBlockArgs{ + {ID: "rootfs", Path: "/dev/mapper/test-snap-1"}, + }, + } + + if err := session.ConfigureGuest(context.Background(), args, ukernel); err != nil { + t.Fatalf("ConfigureGuest failed: %v", err) + } + + reqs := api.recorded() + if len(reqs) != 2 { + t.Fatalf("expected 2 requests (drive, boot-source), got %d: %+v", len(reqs), reqs) + } + + drive := reqs[0] + if drive.path != "/drives/rootfs" { + t.Errorf("expected first PUT to /drives/rootfs, got %s", drive.path) + } + if got := drive.body["path_on_host"]; got != "/dev/mapper/test-snap-1" { + t.Errorf("drive path_on_host = %v, want unprefixed /dev/mapper/test-snap-1", got) + } + + boot := reqs[1] + if boot.path != "/boot-source" { + t.Errorf("expected second PUT to /boot-source, got %s", boot.path) + } + if got := boot.body["kernel_image_path"]; got != "/unikernel/app.bin" { + t.Errorf("kernel_image_path = %v, want unprefixed /unikernel/app.bin", got) + } + if got := boot.body["initrd_path"]; got != "/unikernel/initrd" { + t.Errorf("initrd_path = %v, want unprefixed /unikernel/initrd", got) + } +} + +// TestSessionSendsStagesInOrder verifies the full post-pivot sequence lands on +// the API in the order Firecracker requires: machine-config, network +// interface, boot source, and InstanceStart last. +func TestSessionSendsStagesInOrder(t *testing.T) { + api, sockPath := startFakeFirecrackerAPI(t) + session := newTestSession(t, sockPath) + ctx := context.Background() + + args := types.ExecArgs{ + ContainerID: "testct", + Command: "app", + UnikernelPath: "/unikernel/app.bin", + MemSizeB: 256 * 1024 * 1024, + VCPUs: 1, + } + if err := session.ConfigureMachine(ctx, args); err != nil { + t.Fatalf("ConfigureMachine failed: %v", err) + } + netParams := types.NetDevParams{TapDev: "tap0", MAC: "aa:bb:cc:dd:ee:ff"} + if err := session.ConfigureNetwork(ctx, netParams); err != nil { + t.Fatalf("ConfigureNetwork failed: %v", err) + } + if err := session.ConfigureGuest(ctx, args, &fakeUnikernel{}); err != nil { + t.Fatalf("ConfigureGuest failed: %v", err) + } + if err := session.StartGuest(ctx); err != nil { + t.Fatalf("StartGuest failed: %v", err) + } + + want := []string{ + "/machine-config", + "/network-interfaces/net1", + "/boot-source", + "/actions", + } + reqs := api.recorded() + if len(reqs) != len(want) { + t.Fatalf("expected %d requests, got %d: %+v", len(want), len(reqs), reqs) + } + for i, w := range want { + if reqs[i].path != w { + t.Errorf("request %d: got %s, want %s", i, reqs[i].path, w) + } + } +} diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index f258cd784..48ca92f0e 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -421,12 +421,11 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { // handle network // - // For Firecracker's API-based boot mode, the VMM is spawned right here, - // before any of the expensive setup work, and configured in stages while - // that work runs: machine-config immediately (it needs nothing but the - // spec), the network interface as soon as network setup finishes, and - // everything unikernel-dependent (drives, boot source) right after - // unikernel.Init. The guest itself is only started (InstanceStart) after + // For Firecracker's API-based boot mode, the VMM is spawned right after + // changeRoot below, so the monitor and its control socket are confined + // inside the monitor rootfs, exactly like the exec path. Everything the + // VMM needs to be told is known by that point, so its configuration is + // sent over the socket in one sequence; only InstanceStart waits for // the start-success handshake, preserving OCI start ordering. // // Rootfs prep (below) only needs the cheap "does this container have a @@ -455,23 +454,13 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { }() ctx := context.Background() + var fcVmm *hypervisors.Firecracker if isAPIBoot { fc, ok := vmm.(*hypervisors.Firecracker) if !ok { return fmt.Errorf("boot_mode=api is only supported for the firecracker monitor") } - if err = ensureSocketDir(vmmType, vmmArgs); err != nil { - return err - } - fcSession, err = fc.SpawnSocketVMM(vmmArgs, procAttrs.UID, procAttrs.GID) - if err != nil { - uniklog.Errorf("failed to spawn firecracker: %v", err) - return err - } - if err = fcSession.ConfigureMachine(ctx, vmmArgs); err != nil { - uniklog.Errorf("failed to configure the machine over the socket: %v", err) - return err - } + fcVmm = fc hasNet, err := u.HasNetwork() if err != nil { @@ -660,18 +649,13 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { // If network setup is still running concurrently (isAPIBoot), this is // where we actually need its result: every supported unikernel type // bakes the resolved IP/gateway/MAC into the guest's boot command line - // inside Init below. The tap device exists from this point on, so the - // network interface is also sent to the VMM right away. + // inside Init below. if isAPIBoot { netWG.Wait() if netSetupErr != nil { uniklog.Errorf("failed to setup network: %v", netSetupErr) return netSetupErr } - if err = fcSession.ConfigureNetwork(ctx, netArgs); err != nil { - uniklog.Errorf("failed to configure the network over the socket: %v", err) - return err - } } unikernelParams.Net = netArgs vmmArgs.Net = netArgs @@ -695,16 +679,6 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { // ExecArgs vmmArgs.Command = unikernelCmd - // Everything unikernel-dependent is known now, so it goes to the VMM - // before changeRoot below: the child kept the pre-pivot mount view, so - // ConfigureGuest prefixes the monitor rootfs onto the paths it sends. - if isAPIBoot { - if err = fcSession.ConfigureGuest(ctx, vmmArgs, unikernel, rootfsParams.MonRootfs); err != nil { - uniklog.Errorf("failed to configure the guest over the socket: %v", err) - return err - } - } - // pivot _, err = findNS(u.Spec.Linux.Namespaces, specs.MountNamespace) // We just want to check if a mount namespace was define din the list @@ -716,8 +690,32 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return err } - if !isAPIBoot { - if err = ensureSocketDir(vmmType, vmmArgs); err != nil { + if err = ensureSocketDir(vmmType, vmmArgs); err != nil { + return err + } + + // For the API-based boot the monitor is spawned here, after changeRoot, + // so the child inherits the pivoted root: its control socket and every + // path it opens (kernel, initrd, drives, vsock) resolve inside the + // monitor rootfs. urunc is still privileged at this point — the child + // is started directly under the container user's credentials, and urunc + // drops its own privileges right after (setupUser below). + if isAPIBoot { + fcSession, err = fcVmm.SpawnSocketVMM(vmmArgs, procAttrs.UID, procAttrs.GID) + if err != nil { + uniklog.Errorf("failed to spawn firecracker: %v", err) + return err + } + if err = fcSession.ConfigureMachine(ctx, vmmArgs); err != nil { + uniklog.Errorf("failed to configure the machine over the socket: %v", err) + return err + } + if err = fcSession.ConfigureNetwork(ctx, netArgs); err != nil { + uniklog.Errorf("failed to configure the network over the socket: %v", err) + return err + } + if err = fcSession.ConfigureGuest(ctx, vmmArgs, unikernel); err != nil { + uniklog.Errorf("failed to configure the guest over the socket: %v", err) return err } } From 3bf7f50fe44977d15c48ce56d3c19821f99f7042 Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Mon, 27 Jul 2026 19:52:23 +0530 Subject: [PATCH 14/17] test(firecracker): fix lint findings in the session test Add a read-header timeout to the fake API server (gosec G112) and rename the placeholder container ID to a dictionary word (cspell). Signed-off-by: Anamika Aggarwal --- .../hypervisors/firecracker_session_test.go | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/pkg/unikontainers/hypervisors/firecracker_session_test.go b/pkg/unikontainers/hypervisors/firecracker_session_test.go index 1c1f67b90..acf30c7e7 100644 --- a/pkg/unikontainers/hypervisors/firecracker_session_test.go +++ b/pkg/unikontainers/hypervisors/firecracker_session_test.go @@ -58,14 +58,17 @@ func startFakeFirecrackerAPI(t *testing.T) (*fakeFirecrackerAPI, string) { } api := &fakeFirecrackerAPI{} - srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - var body map[string]any - _ = json.NewDecoder(r.Body).Decode(&body) - api.mu.Lock() - api.requests = append(api.requests, recordedRequest{path: r.URL.Path, body: body}) - api.mu.Unlock() - w.WriteHeader(http.StatusNoContent) - })} + srv := &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body map[string]any + _ = json.NewDecoder(r.Body).Decode(&body) + api.mu.Lock() + api.requests = append(api.requests, recordedRequest{path: r.URL.Path, body: body}) + api.mu.Unlock() + w.WriteHeader(http.StatusNoContent) + }), + ReadHeaderTimeout: time.Second, + } go func() { _ = srv.Serve(ln) }() t.Cleanup(func() { _ = srv.Close() }) return api, sockPath @@ -90,7 +93,7 @@ func TestConfigureGuestSendsUnprefixedPaths(t *testing.T) { session := newTestSession(t, sockPath) args := types.ExecArgs{ - ContainerID: "testct", + ContainerID: "test-container", Command: "app -arg", UnikernelPath: "/unikernel/app.bin", InitrdPath: "/unikernel/initrd", @@ -141,7 +144,7 @@ func TestSessionSendsStagesInOrder(t *testing.T) { ctx := context.Background() args := types.ExecArgs{ - ContainerID: "testct", + ContainerID: "test-container", Command: "app", UnikernelPath: "/unikernel/app.bin", MemSizeB: 256 * 1024 * 1024, From 1bfc0313bcac71a31469f904c719a7f453bb40d6 Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Mon, 27 Jul 2026 20:20:23 +0530 Subject: [PATCH 15/17] chore(firecracker): use plain ASCII in an exec comment Signed-off-by: Anamika Aggarwal --- pkg/unikontainers/unikontainers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index 48ca92f0e..707f7eee5 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -697,7 +697,7 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { // For the API-based boot the monitor is spawned here, after changeRoot, // so the child inherits the pivoted root: its control socket and every // path it opens (kernel, initrd, drives, vsock) resolve inside the - // monitor rootfs. urunc is still privileged at this point — the child + // monitor rootfs. urunc is still privileged at this point; the child // is started directly under the container user's credentials, and urunc // drops its own privileges right after (setupUser below). if isAPIBoot { From 4f9cf291483edea416de20aa898b0e575ae0e413 Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Mon, 27 Jul 2026 20:35:14 +0530 Subject: [PATCH 16/17] test(firecracker): cover the vsock config and refresh comments Signed-off-by: Anamika Aggarwal --- pkg/unikontainers/hypervisors/firecracker.go | 10 ++--- .../hypervisors/firecracker_client.go | 3 +- .../hypervisors/firecracker_session_test.go | 37 +++++++++++++++++++ 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/pkg/unikontainers/hypervisors/firecracker.go b/pkg/unikontainers/hypervisors/firecracker.go index 93fcaaf05..362074f1f 100644 --- a/pkg/unikontainers/hypervisors/firecracker.go +++ b/pkg/unikontainers/hypervisors/firecracker.go @@ -229,11 +229,11 @@ func buildFirecrackerConfig(args types.ExecArgs, ukernel types.Unikernel) *Firec } } -// FirecrackerSession is a Firecracker child process being configured over its -// API socket in stages, while the caller performs its own setup work between -// the stages. Create it with SpawnSocketVMM, feed it configuration with the -// Configure* methods as each piece becomes available, boot the guest with -// StartGuest, then hand the calling process over with Supervise. +// FirecrackerSession is a Firecracker child process configured over its API +// socket, one resource at a time. Create it with SpawnSocketVMM, send the +// configuration with the Configure* methods, boot the guest with StartGuest +// once the caller's start handshake allows it, then hand the calling process +// over with Supervise. type FirecrackerSession struct { cmd *exec.Cmd client *firecrackerClient diff --git a/pkg/unikontainers/hypervisors/firecracker_client.go b/pkg/unikontainers/hypervisors/firecracker_client.go index f6b2e625a..f115d6221 100644 --- a/pkg/unikontainers/hypervisors/firecracker_client.go +++ b/pkg/unikontainers/hypervisors/firecracker_client.go @@ -27,8 +27,7 @@ import ( ) // firecrackerClient drives a running Firecracker process over its HTTP-over-Unix -// API socket, one configuration resource at a time, so the caller can send each -// piece of configuration as soon as it becomes available. +// API socket, one configuration resource at a time. // // The client establishes a single connection in connect() and keeps it alive // for all requests, so every configuration stage reuses the connection whose diff --git a/pkg/unikontainers/hypervisors/firecracker_session_test.go b/pkg/unikontainers/hypervisors/firecracker_session_test.go index acf30c7e7..a8b9b3ada 100644 --- a/pkg/unikontainers/hypervisors/firecracker_session_test.go +++ b/pkg/unikontainers/hypervisors/firecracker_session_test.go @@ -135,6 +135,43 @@ func TestConfigureGuestSendsUnprefixedPaths(t *testing.T) { } } +// TestConfigureGuestSendsUnprefixedVSockPath verifies the vsock device, when +// enabled, is sent with its socket path exactly as given, with no rootfs +// prefixing, and after the boot source. +func TestConfigureGuestSendsUnprefixedVSockPath(t *testing.T) { + api, sockPath := startFakeFirecrackerAPI(t) + session := newTestSession(t, sockPath) + + args := types.ExecArgs{ + ContainerID: "test-container", + Command: "app", + UnikernelPath: "/unikernel/app.bin", + MemSizeB: 256 * 1024 * 1024, + VCPUs: 1, + VAccelType: "vsock", + VSockDevPath: "/tmp/vaccel", + VSockDevID: 3, + } + if err := session.ConfigureGuest(context.Background(), args, &fakeUnikernel{}); err != nil { + t.Fatalf("ConfigureGuest failed: %v", err) + } + + reqs := api.recorded() + if len(reqs) != 2 { + t.Fatalf("expected 2 requests (boot-source, vsock), got %d: %+v", len(reqs), reqs) + } + vsock := reqs[1] + if vsock.path != "/vsock" { + t.Errorf("expected last PUT to /vsock, got %s", vsock.path) + } + if got := vsock.body["uds_path"]; got != "/tmp/vaccel/vaccel.sock" { + t.Errorf("uds_path = %v, want unprefixed /tmp/vaccel/vaccel.sock", got) + } + if got := vsock.body["guest_cid"]; got != float64(3) { + t.Errorf("guest_cid = %v, want 3", got) + } +} + // TestSessionSendsStagesInOrder verifies the full post-pivot sequence lands on // the API in the order Firecracker requires: machine-config, network // interface, boot source, and InstanceStart last. From 03896d2758d970ec2966721cc1fa4de8431d05f0 Mon Sep 17 00:00:00 2001 From: Anamika Aggarwal Date: Tue, 28 Jul 2026 10:14:28 +0530 Subject: [PATCH 17/17] fix(firecracker): pass stdin to the api-mode monitor In the api boot mode the monitor is a supervised child, and an unset Stdin on exec.Cmd connects it to /dev/null, so the guest's serial console could produce output but never receive input. The exec-based config-file mode does not have this problem, since the monitor inherits the runtime's stdin when it replaces the process. Wire the caller's stdin through to the child, restoring input for interactive guests (e.g. a container run with -i whose command line is a shell) and matching the behavior of the config-file mode. Signed-off-by: Anamika Aggarwal --- pkg/unikontainers/hypervisors/firecracker.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/unikontainers/hypervisors/firecracker.go b/pkg/unikontainers/hypervisors/firecracker.go index 362074f1f..da9402a2b 100644 --- a/pkg/unikontainers/hypervisors/firecracker.go +++ b/pkg/unikontainers/hypervisors/firecracker.go @@ -265,6 +265,7 @@ func (fc *Firecracker) SpawnSocketVMM(args types.ExecArgs, uid, gid uint32) (*Fi cmd := exec.Command(execCmd[0], execCmd[1:]...) //nolint: gosec cmd.Env = args.Environment + cmd.Stdin = os.Stdin cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if uid != 0 || gid != 0 {