From 2d17dde724a28f888e2abb50c6d1e27e622fb0c1 Mon Sep 17 00:00:00 2001 From: Charalampos Mainas Date: Tue, 14 Jul 2026 14:11:49 +0200 Subject: [PATCH 1/3] refactor(mounts): Gather all mounts instead of mounting one by one Gather all the mounts for the monitor execution environment and the contianer in a single list which is later given as an argument in applyMOunts to perform all the necessary mounts. PR: https://github.com/urunc-dev/urunc/pull/835 Signed-off-by: Charalampos Mainas Reviewed-by: Anastassios Nanos Approved-by: Anastassios Nanos --- pkg/unikontainers/block.go | 13 +- pkg/unikontainers/initrd_rootfs.go | 13 +- pkg/unikontainers/mount.go | 138 +++++++++++++------ pkg/unikontainers/rootfs.go | 213 ++++++++++++++--------------- pkg/unikontainers/shared_fs.go | 55 ++++---- pkg/unikontainers/unikontainers.go | 18 ++- 6 files changed, 261 insertions(+), 189 deletions(-) diff --git a/pkg/unikontainers/block.go b/pkg/unikontainers/block.go index 1b7bf7892..d284dcca2 100644 --- a/pkg/unikontainers/block.go +++ b/pkg/unikontainers/block.go @@ -22,8 +22,6 @@ import ( "path/filepath" "strings" - "golang.org/x/sys/unix" - "github.com/moby/sys/mount" "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" @@ -258,14 +256,11 @@ func (b blockRootfs) postSetup() error { } } - err := createTmpfs(b.monRootfs, "/tmp", - unix.MS_NOSUID|unix.MS_NOEXEC|unix.MS_STRICTATIME, - "1777", tmpfsSizeForBlockRootfs) - if err != nil { - err = fmt.Errorf("failed to create tmpfs for monitor's execution environment: %w", err) - } + return nil +} - return err +func (b blockRootfs) getMounts() ([]specs.Mount, error) { + return []specs.Mount{tmpfsMount("/tmp", tmpfsSizeForBlockRootfs)}, nil } func (b blockRootfs) getBlockDevs() ([]types.BlockDevParams, error) { diff --git a/pkg/unikontainers/initrd_rootfs.go b/pkg/unikontainers/initrd_rootfs.go index 3f899fd3d..ee70d9da2 100644 --- a/pkg/unikontainers/initrd_rootfs.go +++ b/pkg/unikontainers/initrd_rootfs.go @@ -17,8 +17,6 @@ package unikontainers import ( "fmt" - "golang.org/x/sys/unix" - "github.com/opencontainers/runtime-spec/specs-go" "github.com/urunc-dev/urunc/pkg/unikontainers/initrd" "github.com/urunc-dev/urunc/pkg/unikontainers/types" @@ -43,14 +41,11 @@ func (i initrdRootfs) postSetup() error { return fmt.Errorf("failed to update guest's initrd: %w", err) } - err = createTmpfs(i.monRootfs, "/tmp", - unix.MS_NOSUID|unix.MS_NOEXEC|unix.MS_STRICTATIME, - "1777", tmpfsSizeForInitrdRootfs) - if err != nil { - err = fmt.Errorf("failed to create tmpfs for monitor's execution environment: %w", err) - } + return nil +} - return err +func (i initrdRootfs) getMounts() ([]specs.Mount, error) { + return []specs.Mount{tmpfsMount("/tmp", tmpfsSizeForInitrdRootfs)}, nil } func (i initrdRootfs) getBlockDevs() ([]types.BlockDevParams, error) { diff --git a/pkg/unikontainers/mount.go b/pkg/unikontainers/mount.go index a4be7cd5e..17967f585 100644 --- a/pkg/unikontainers/mount.go +++ b/pkg/unikontainers/mount.go @@ -24,6 +24,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "github.com/moby/sys/userns" "golang.org/x/sys/unix" @@ -42,17 +43,16 @@ type mountFlagStruct struct { // In particular, it is used for the creation of /tmp and /dev. // This is necessary to create the required devices for the monitor execution, // such as KVM, null, urandom etc. -func createTmpfs(monRootfs string, path string, flags uint64, mode string, size string) error { +func createTmpfs(monRootfs string, path string, flags uintptr, data string) error { dstPath := filepath.Join(monRootfs, path) mountType := "tmpfs" - data := "mode=" + mode + ",size=" + size err := os.MkdirAll(dstPath, 0755) if err != nil { return fmt.Errorf("failed to create %s dir: %w", path, err) } - err = unix.Mount(mountType, dstPath, mountType, uintptr(flags), data) + err = unix.Mount(mountType, dstPath, mountType, flags, data) if err != nil { return fmt.Errorf("failed to mount %s tmpfs: %w", path, err) } @@ -63,7 +63,7 @@ func createTmpfs(monRootfs string, path string, flags uint64, mode string, size return fmt.Errorf("failed to create %s tmpfs: %w", path, err) } - if mode == "1777" { + if strings.Contains(","+data+",", ",mode=1777,") { // sonarcloud:go:S2612 -- This is a tmpfs mount point, sticky bit 1777 is required (like /tmp), controlled path, safe by design err := os.Chmod(dstPath, 01777) // NOSONAR if err != nil { @@ -327,53 +327,113 @@ func prepareRoot(path string, rootfsPropagation string) error { return unix.Mount(path, path, "bind", unix.MS_BIND|unix.MS_REC, "") } -func mountVolumes(rootfsPath string, mounts []specs.Mount) error { +func applyMounts(rootfsPath string, mounts []specs.Mount) error { for _, m := range mounts { - // Skip non-bind mounts - // TODO handle other types of mounts too - if m.Type != "bind" { + switch m.Type { + case "tmpfs": + // TODO: replace createTmpfs with a mount package (e.g. + // containerd's). For the time being keep the compatibility + // with createTmpfs. + flags, data := splitMountOptions(m.Options) + err := createTmpfs(rootfsPath, m.Destination, flags, data) + if err != nil { + return fmt.Errorf("failed to create tmpfs at %s: %w", m.Destination, err) + } + case "proc", "devpts": + err := applySpecialFsMount(rootfsPath, m) + if err != nil { + return fmt.Errorf("failed to create %s at %s: %w", m.Type, m.Destination, err) + } + case "bind": + err := applyBindMount(rootfsPath, m) + if err != nil { + return fmt.Errorf("failed to bind mount %s at %s: %w", m.Source, m.Destination, err) + } + default: + // Skip unknown mount types + // TODO handle other types of mounts too continue } - var mountFlags int - var mountClearedFlags int - var propFlag []int - mountFlags = 0 - mountClearedFlags = 0 - for _, o := range m.Options { - f, exists := mapMountFlag(o) - if exists { - if f.clear { - mountFlags &= ^f.flag - mountClearedFlags |= f.flag - } else { - mountFlags |= f.flag - mountClearedFlags &= ^f.flag - } - continue - } - fprop, err := mapRootfsPropagationFlag(o) - if err == nil { - propFlag = append(propFlag, fprop) + } + + return nil +} + +// applySpecialFsMount handles pseudo filesystem (e.g. proc, devpts) mounts +func applySpecialFsMount(rootfsPath string, m specs.Mount) error { + dstPath := filepath.Join(rootfsPath, m.Destination) + err := os.MkdirAll(dstPath, 0755) + if err != nil { + return fmt.Errorf("failed to create %s dir: %w", m.Destination, err) + } + + flags, data := splitMountOptions(m.Options) + err = unix.Mount(m.Source, dstPath, m.Type, flags, data) + if err != nil { + return fmt.Errorf("failed to mount %s: %w", m.Destination, err) + } + + return nil +} + +// applyBindMount handles the bind mounts for the monitor rootfs +func applyBindMount(rootfsPath string, m specs.Mount) error { + var mountFlags int + var propFlag []int + for _, o := range m.Options { + f, exists := mapMountFlag(o) + if exists { + if f.clear { + mountFlags &= ^f.flag + } else { + mountFlags |= f.flag } - // Ignore unknown flags - // TODO: Handle unknown flags. These can be mount attribute flags - // or specific flags for a particular fs type. + continue + } + fprop, err := mapRootfsPropagationFlag(o) + if err == nil { + propFlag = append(propFlag, fprop) } - err := fileFromHost(rootfsPath, m.Source, m.Destination, mountFlags, false) + // Ignore unknown flags + // TODO: Handle unknown flags. These can be mount attribute flags + // or specific flags for a particular fs type. + } + err := fileFromHost(rootfsPath, m.Source, m.Destination, mountFlags, false) + if err != nil { + return err + } + + dstPath := filepath.Join(rootfsPath, m.Destination) + for _, pFlag := range propFlag { + err = unix.Mount(dstPath, dstPath, "", uintptr(pFlag), "") if err != nil { - return err + return fmt.Errorf("failed to set propagation flag for %s: %w", m.Source, err) } + } - dstPath := filepath.Join(rootfsPath, m.Destination) - for _, pFlag := range propFlag { - err = unix.Mount(dstPath, dstPath, "", uintptr(pFlag), "") - if err != nil { - return fmt.Errorf("failed to set propagation flag for %s: %w", m.Source, err) + return nil +} + +// splitMountOptions separates mount options into the corresponding mount flags +// and the remaining options, which are returned as a comma-separated data +// string to be passed to mount(2). +func splitMountOptions(options []string) (uintptr, string) { + var flags int + var data []string + for _, o := range options { + f, exists := mapMountFlag(o) + if exists { + if f.clear { + flags &= ^f.flag + } else { + flags |= f.flag } + continue } + data = append(data, o) } - return nil + return uintptr(flags), strings.Join(data, ",") } // mapMountFlag retrieves the mount flags of a mount entry diff --git a/pkg/unikontainers/rootfs.go b/pkg/unikontainers/rootfs.go index 0d7c80c0f..c43dbeb6a 100644 --- a/pkg/unikontainers/rootfs.go +++ b/pkg/unikontainers/rootfs.go @@ -20,6 +20,7 @@ import ( "path/filepath" "strconv" + "github.com/opencontainers/runtime-spec/specs-go" "golang.org/x/sys/unix" "github.com/urunc-dev/urunc/pkg/unikontainers/types" @@ -35,11 +36,38 @@ const annotRootfsParams = "com.urunc.internal.rootfs.params" type rootfsBuilder interface { preSetup() error postSetup() error + getMounts() ([]specs.Mount, error) getBlockDevs() ([]types.BlockDevParams, error) getSharedDirs() (types.SharedfsParams, error) preStart() error } +// tmpfsMount creates a mount for a tmpfs in the form of "/tmp" at target +func tmpfsMount(target string, size string) specs.Mount { + return specs.Mount{ + Type: "tmpfs", + Source: "tmpfs", + Destination: target, + Options: []string{ + "nosuid", + "noexec", + "strictatime", + "mode=1777", + "size=" + size, + }, + } +} + +// bindMount builds a private, non-recursive bind mount of source at target +func bindMount(source string, target string) specs.Mount { + return specs.Mount{ + Type: "bind", + Source: source, + Destination: target, + Options: []string{"bind", "private"}, + } +} + // rootfsSelector encapsulates the context for rootfs selection type rootfsSelector struct { bundle string @@ -61,14 +89,11 @@ func (n noRootfs) preSetup() error { } func (n noRootfs) postSetup() error { - err := createTmpfs(n.monRootfs, "/tmp", - unix.MS_NOSUID|unix.MS_NOEXEC|unix.MS_STRICTATIME, - "1777", tmpfsSizeForNoRootfs) - if err != nil { - err = fmt.Errorf("failed to create tmpfs for monitor's execution environment: %w", err) - } + return nil +} - return err +func (n noRootfs) getMounts() ([]specs.Mount, error) { + return []specs.Mount{tmpfsMount("/tmp", tmpfsSizeForNoRootfs)}, nil } func (n noRootfs) getBlockDevs() ([]types.BlockDevParams, error) { @@ -326,94 +351,11 @@ func changeRoot(rootfsDir string, pivot bool) error { return nil } -// nolint:gocyclo // prepareMonRootfs prepares the rootfs where the monitor will execute. It // essentially sets up the devices (KVM, snapshotter block device) that are required -// for the guest execution and any other files (e.g. binaries). -func prepareMonRootfs(monRootfs string, monitorPath string, monitorDataPath string, needsKVM bool, needsTAP bool) error { - err := fileFromHost(monRootfs, monitorPath, "", unix.MS_BIND|unix.MS_PRIVATE, false) - if err != nil { - return err - } - - // TODO: Remove these when we switch to static binaries - monitorName := filepath.Base(monitorPath) - if monitorName != "firecracker" { - err = fileFromHost(monRootfs, "/lib", "", unix.MS_BIND|unix.MS_PRIVATE, false) - if err != nil { - return err - } - - err = fileFromHost(monRootfs, "/lib64", "", unix.MS_BIND|unix.MS_PRIVATE, false) - if err != nil { - // If the file does not exist, just ignore it - if !os.IsNotExist(err) { - return err - } - } - - err = fileFromHost(monRootfs, "/usr/lib", "", unix.MS_BIND|unix.MS_PRIVATE, false) - if err != nil { - return err - } - } - - // TODO: Remove these when we switch to static binaries - if len(monitorName) >= 4 && monitorName[:4] == "qemu" { - var qDataPath string - var sBiosPath string - var err error - if monitorDataPath == "" { - qDataPath, err = findQemuDataDir("qemu") - } else { - qDataPath = filepath.Join(monitorDataPath, "qemu") - err = nil - } - if err != nil { - return err - } - - err = fileFromHost(monRootfs, qDataPath, "/usr/share/qemu", unix.MS_BIND|unix.MS_PRIVATE, false) - if err != nil { - return err - } - - if monitorDataPath == "" { - sBiosPath, err = findQemuDataDir("seabios") - } else { - sBiosPath = filepath.Join(monitorDataPath, "seabios") - err = nil - } - if err != nil { - return fmt.Errorf("failed to get info of seabios directory: %w", err) - } - err = fileFromHost(monRootfs, sBiosPath, "/usr/share/seabios", unix.MS_BIND|unix.MS_PRIVATE, false) - if err != nil { - // In urunc-deploy and in some distros seabios does not exist and - // we do not need it. So if we could not find it, just ignore it. - if !os.IsNotExist(err) { - return err - } - } - } - - newProcDir := filepath.Join(monRootfs, "/proc") - err = os.MkdirAll(newProcDir, 0555) - if err != nil { - return err - } - - err = unix.Mount("proc", newProcDir, "proc", 0, "") - if err != nil { - return err - } - - err = createTmpfs(monRootfs, "/dev", unix.MS_NOSUID|unix.MS_STRICTATIME, "755", "65536k") - if err != nil { - return err - } - - err = setupDev(monRootfs, "/dev/null") +// for the monitor execution +func prepareMonRootfs(monRootfs string, monitorPath string, needsKVM bool, needsTAP bool) error { + err := setupDev(monRootfs, "/dev/null") if err != nil { return err } @@ -437,21 +379,6 @@ func prepareMonRootfs(monRootfs string, monitorPath string, monitorDataPath stri } } - // Setup /dev/pts for PTY support (needed for console and debugging tools like cntr) - // This allows tools like cntr to attach to the container with a shell - devPtsDir := filepath.Join(monRootfs, "/dev/pts") - err = os.MkdirAll(devPtsDir, 0755) - if err != nil { - return fmt.Errorf("failed to create /dev/pts directory: %w", err) - } - - // Mount devpts filesystem - // Using newinstance creates an isolated pts namespace for this container - err = unix.Mount("devpts", devPtsDir, "devpts", unix.MS_NOSUID|unix.MS_NOEXEC, "newinstance,ptmxmode=0666,mode=0620,gid=5") - if err != nil { - return fmt.Errorf("failed to mount devpts: %w", err) - } - // Create /dev/ptmx as a symlink to /dev/pts/ptmx // This is the standard way to provide the PTY master device ptmxPath := filepath.Join(monRootfs, "/dev/ptmx") @@ -475,3 +402,73 @@ func prepareMonRootfs(monRootfs string, monitorPath string, monitorDataPath stri return nil } + +// mountsForMonitor returns all the required mounts from the host for the +// monitor execution: the binary, supporting libraries and data directories. +// These are the host files that need to be mirrored inside the monitor rootfs. +// In addition, it also includes the special filesystems required for the monitor +// execution (e.g. procfs) +func mountsForMonitor(monitorPath string, monitorDataPath string) ([]specs.Mount, error) { + procMount := specs.Mount{ + Type: "proc", + Source: "proc", + Destination: "/proc", + } + devMount := specs.Mount{ + Type: "tmpfs", + Source: "tmpfs", + Destination: "/dev", + Options: []string{"nosuid", "strictatime", "mode=755", "size=65536k"}, + } + devPtsMount := specs.Mount{ + Type: "devpts", + Source: "devpts", + Destination: "/dev/pts", + Options: []string{"nosuid", "noexec", "newinstance", "ptmxmode=0666", "mode=0620"}, + } + + mounts := []specs.Mount{procMount, devMount, devPtsMount, bindMount(monitorPath, monitorPath)} + + monitorName := filepath.Base(monitorPath) + // TODO: Remove most of these when we switch to static binaries. + if monitorName != "firecracker" { + mounts = append(mounts, bindMount("/lib", "/lib")) + + // If /lib64 does not exist, just ignore it + if _, err := os.Stat("/lib64"); err == nil { + mounts = append(mounts, bindMount("/lib64", "/lib64")) + } else if !os.IsNotExist(err) { + return nil, err + } + + mounts = append(mounts, bindMount("/usr/lib", "/usr/lib")) + } + + if len(monitorName) >= 4 && monitorName[:4] == "qemu" { + qDataPath := filepath.Join(monitorDataPath, "qemu") + sBiosPath := filepath.Join(monitorDataPath, "seabios") + if monitorDataPath == "" { + var err error + qDataPath, err = findQemuDataDir("qemu") + if err != nil { + return nil, err + } + sBiosPath, err = findQemuDataDir("seabios") + if err != nil { + return nil, fmt.Errorf("failed to get info of seabios directory: %w", err) + } + } + + mounts = append(mounts, bindMount(qDataPath, "/usr/share/qemu")) + + // In urunc-deploy and in some distros seabios does not exist and + // we do not need it. So if we could not find it, just ignore it. + if _, err := os.Stat(sBiosPath); err == nil { + mounts = append(mounts, bindMount(sBiosPath, "/usr/share/seabios")) + } else if !os.IsNotExist(err) { + return nil, err + } + } + + return mounts, nil +} diff --git a/pkg/unikontainers/shared_fs.go b/pkg/unikontainers/shared_fs.go index 9ac86e8b0..33f91fbd7 100644 --- a/pkg/unikontainers/shared_fs.go +++ b/pkg/unikontainers/shared_fs.go @@ -19,8 +19,6 @@ import ( "path/filepath" "strings" - "golang.org/x/sys/unix" - "github.com/opencontainers/runtime-spec/specs-go" "github.com/urunc-dev/urunc/pkg/unikontainers/hypervisors" "github.com/urunc-dev/urunc/pkg/unikontainers/types" @@ -44,35 +42,24 @@ func (s sharedfsRootfs) preSetup() error { } func (s sharedfsRootfs) postSetup() error { - // Mount the container's rootfs inside the monitor rootfs - err := fileFromHost(s.monRootfs, s.mountedPath, containerRootfsMountPath, unix.MS_BIND|unix.MS_PRIVATE, false) - if err != nil { - return fmt.Errorf("failed to mount container's rootfs in monitor rootfs; %w", err) - } + return nil +} - newCntrRootfs := filepath.Join(s.monRootfs, containerRootfsMountPath) - err = mountVolumes(newCntrRootfs, s.mounts) - if err != nil { - return fmt.Errorf("failed to mount volumes in container's rootfs; %w", err) - } +func (s sharedfsRootfs) getMounts() ([]specs.Mount, error) { + // Mount the container's rootfs inside the monitor rootfs and then the + // container's volumes on top of it. + mounts := []specs.Mount{bindMount(s.mountedPath, containerRootfsMountPath)} if s.sfsType == "virtiofs" { // Get the virtiofsd binary from host in monRootfs - err = fileFromHost(s.monRootfs, s.vfsdConfig.Path, "", unix.MS_BIND|unix.MS_PRIVATE, false) - if err != nil { - return fmt.Errorf("could not bind mount %s: %w", s.vfsdConfig.Path, err) - } + mounts = append(mounts, bindMount(s.vfsdConfig.Path, s.vfsdConfig.Path)) } tmpfsSize := chooseTmpfsSize(s.sfsType, s.memory) - err = createTmpfs(s.monRootfs, "/tmp", - unix.MS_NOSUID|unix.MS_NOEXEC|unix.MS_STRICTATIME, - "1777", tmpfsSize) - if err != nil { - err = fmt.Errorf("failed to create tmpfs for monitor's execution environment: %w", err) - } + mounts = append(mounts, tmpfsMount("/tmp", tmpfsSize)) + mounts = append(mounts, filterBindMounts(s.mounts)...) - return err + return mounts, nil } func (s sharedfsRootfs) getBlockDevs() ([]types.BlockDevParams, error) { @@ -134,3 +121,25 @@ func adjustPathsForSharedfs(path string) string { return path } + +// filterBindMounts filters the mounts form the container's spec keeping only the +// bind mounts and adjusts the Destination path to the mountpoint of the +// container's rootfs inside the monitor rootfs. +func filterBindMounts(mounts []specs.Mount) []specs.Mount { + var result []specs.Mount + for _, m := range mounts { + // Skip non-bind mounts + // TODO handle other types of mounts too + if m.Type != "bind" { + continue + } + result = append(result, specs.Mount{ + Type: "bind", + Source: m.Source, + Destination: filepath.Join(containerRootfsMountPath, m.Destination), + Options: m.Options, + }) + } + + return result +} diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index d3de43dfb..886d5c34a 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -557,9 +557,25 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return err } + monitorMounts, err := mountsForMonitor(vmm.Path(), u.UruncCfg.Monitors[vmmType].DataPath) + if err != nil { + return err + } + + rootfsMounts, err := rfsBuilder.getMounts() + if err != nil { + return fmt.Errorf("failed to get mounts for monitor rootfs: %w", err) + } + rootfsMounts = append(monitorMounts, rootfsMounts...) + + err = applyMounts(rootfsParams.MonRootfs, rootfsMounts) + if err != nil { + return fmt.Errorf("failed to apply rootfs mounts: %w", err) + } + // Setup the rootfs for the monitor execution, creating necessary // devices and the monitor's binary. - err = prepareMonRootfs(rootfsParams.MonRootfs, vmm.Path(), u.UruncCfg.Monitors[vmmType].DataPath, vmm.UsesKVM(), withTUNTAP) + err = prepareMonRootfs(rootfsParams.MonRootfs, vmm.Path(), vmm.UsesKVM(), withTUNTAP) if err != nil { return err } From 8837c26e8d2537adc99a77bdfc9d81e9016e3654 Mon Sep 17 00:00:00 2001 From: Charalampos Mainas Date: Wed, 15 Jul 2026 08:54:51 +0200 Subject: [PATCH 2/3] refactor(devices): Gather all devices and then create them Similarly with mounts, gather all devices in a single slice and then recreate them inside the monitor's execution environment. The only exception is vAccel related devices. We need to handle the case of vAccel separately for the libcontainer migration. PR: https://github.com/urunc-dev/urunc/pull/835 Signed-off-by: Charalampos Mainas Reviewed-by: Anastassios Nanos Approved-by: Anastassios Nanos --- pkg/unikontainers/block.go | 41 +++++++++++----- pkg/unikontainers/mount.go | 78 +++++++++++++++++++----------- pkg/unikontainers/rootfs.go | 68 ++++++++++++++++++++------ pkg/unikontainers/unikontainers.go | 48 ++++++++++++------ pkg/unikontainers/vaccel.go | 18 ++++--- 5 files changed, 174 insertions(+), 79 deletions(-) diff --git a/pkg/unikontainers/block.go b/pkg/unikontainers/block.go index d284dcca2..43c78cda5 100644 --- a/pkg/unikontainers/block.go +++ b/pkg/unikontainers/block.go @@ -187,7 +187,7 @@ func handleExplicitBlockImage(blockImg string, mountPoint string) (types.BlockDe // Search all the mount entries in the container's config and // find the ones that come from a block. -func getBlockVolumes(monRootfs string, mounts []specs.Mount, ukernel types.Unikernel) ([]types.BlockDevParams, error) { +func getBlockVolumes(mounts []specs.Mount, ukernel types.Unikernel) ([]types.BlockDevParams, error) { blkImgs := []types.BlockDevParams{} for i, m := range mounts { // We check only bind mounts @@ -210,10 +210,6 @@ func getBlockVolumes(monRootfs string, mounts []specs.Mount, ukernel types.Unike if err != nil { return nil, err } - err = setupDev(monRootfs, mInfo.Source) - if err != nil { - return nil, err - } mInfo.ID = fmt.Sprintf("vol%d", i) mInfo.MountPoint = m.Destination blkImgs = append(blkImgs, mInfo) @@ -223,6 +219,32 @@ func getBlockVolumes(monRootfs string, mounts []specs.Mount, ukernel types.Unike return blkImgs, nil } +// blockDevNodes transforms a list of types.BlockDevParams to a list of +// specs.LinuxDevice which cna be then used for replicating these block +// devices form the host to the monitor's execution environment. +func blockDevNodes(blockArgs []types.BlockDevParams, rootfs types.RootfsParams) ([]specs.LinuxDevice, error) { + if rootfs.Type != "block" { + return nil, nil + } + + var blockDevs []specs.LinuxDevice + for _, b := range blockArgs { + // The rootfs device is a real host device only when a container rootfs + // was converted to a block device (MountedPath set). When it is an + // explicit block image referenced by an annotation, no node is created. + if b.Source == rootfs.Path && rootfs.MountedPath == "" { + continue + } + bDev, err := deviceFromHost(b.Source) + if err != nil { + return nil, err + } + blockDevs = append(blockDevs, bDev) + } + + return blockDevs, nil +} + func (b blockRootfs) preSetup() error { if b.mountedPath == "" { return nil @@ -249,13 +271,6 @@ func (b blockRootfs) preSetup() error { } func (b blockRootfs) postSetup() error { - if b.mountedPath != "" { - err := setupDev(b.monRootfs, b.path) - if err != nil { - return err - } - } - return nil } @@ -280,7 +295,7 @@ func (b blockRootfs) getBlockDevs() ([]types.BlockDevParams, error) { } blockArgs = append(blockArgs, rootfsBlock) - blockFromMounts, err := getBlockVolumes(b.monRootfs, b.mounts, b.guest) + blockFromMounts, err := getBlockVolumes(b.mounts, b.guest) if err != nil { return nil, err } diff --git a/pkg/unikontainers/mount.go b/pkg/unikontainers/mount.go index 17967f585..9a1b5b33a 100644 --- a/pkg/unikontainers/mount.go +++ b/pkg/unikontainers/mount.go @@ -22,6 +22,7 @@ package unikontainers import ( "errors" "fmt" + "math" "os" "path/filepath" "strings" @@ -73,48 +74,50 @@ func createTmpfs(monRootfs string, path string, flags uintptr, data string) erro return nil } -// SetupDev set ups one new device in the container's rootfs. +// setupDevices creates every device from the list inside monitor's rootfs +func setupDevices(monRootfs string, devices []specs.LinuxDevice) error { + for _, dev := range devices { + err := setupDev(monRootfs, dev) + if err != nil { + return err + } + } + + return nil +} + +// setupDev set ups one new device in the container's rootfs. // This function will get the major and minor number of // the device from the host's rootfs and it will replicate the device // inside the container's rootfs. -// When running in a user namespace, urunc bind-mounts the host device node to avoid EPERM from mknod. -// When not running in a user namespace, urunc creates the device node with mknod and adds rw for other -// users so non-root monitors can access it. -func setupDev(monRootfs string, devPath string) error { +// When running in a user namespace, urunc bind-mounts the host device node +// instead of using mknod. +func setupDev(monRootfs string, dev specs.LinuxDevice) error { // In a user namespace, always bind-mount the existing host device node. // Only MS_BIND is used here (no extra flags) to mirror runc's device handling. if userns.RunningInUserNS() { - return fileFromHost(monRootfs, devPath, "", unix.MS_BIND, false) + return fileFromHost(monRootfs, dev.Path, "", unix.MS_BIND, false) } - // Get info of the original file - var devStat unix.Stat_t - err := unix.Stat(devPath, &devStat) - if err != nil { - return fmt.Errorf("failed to stat dev %s: %w", devPath, err) + var devType uint32 + switch dev.Type { + case "c": + devType = unix.S_IFCHR + case "b": + devType = unix.S_IFBLK + default: + return fmt.Errorf("%s is not a device node", dev.Path) } - // mask file's mode - mode := devStat.Mode & unix.S_IFMT - if mode != unix.S_IFCHR && mode != unix.S_IFBLK { - return fmt.Errorf("%s is not a device node", devPath) - } - // Get minor,major numbers - rdev := devStat.Rdev - major := unix.Major(uint64(rdev)) - minor := unix.Minor(uint64(rdev)) - - newDev := unix.Mkdev(major, minor) - // Set the correct target path - relHostPath, err := filepath.Rel("/", devPath) + relHostPath, err := filepath.Rel("/", dev.Path) if err != nil { - return fmt.Errorf("failed to get relative path of %s to /: %w", devPath, err) + return fmt.Errorf("failed to get relative path of %s to /: %w", dev.Path, err) } dstPath := filepath.Join(monRootfs, relHostPath) // If the device is not at /dev but further down the tree, create // the necessary directories - if filepath.Dir(devPath) != "/dev" { + if filepath.Dir(dev.Path) != "/dev" { dstDir := filepath.Dir(dstPath) err = os.MkdirAll(dstDir, 0755) if err != nil { @@ -122,8 +125,20 @@ func setupDev(monRootfs string, devPath string) error { } } + if dev.FileMode == nil { + return fmt.Errorf("reference of %s FileMode is nil", dev.Path) + } // Create the new device node - err = unix.Mknod(dstPath, devStat.Mode, int(newDev)) //nolint: gosec + permBits := uint32(*dev.FileMode) & 0o777 + if dev.Major < 0 || dev.Major > math.MaxUint32 || + dev.Minor < 0 || dev.Minor > math.MaxUint32 { + return fmt.Errorf("device number out of range for %s: major=%d minor=%d", dstPath, dev.Major, dev.Minor) + } + newDev := unix.Mkdev(uint32(dev.Major), uint32(dev.Minor)) //#nosec G115 -- device numbers validated previously + if newDev > uint64(math.MaxInt) { + return fmt.Errorf("device number for %s too large: %d", dstPath, newDev) + } + err = unix.Mknod(dstPath, devType|permBits, int(newDev)) //#nosec G115 -- device numbers validated previously if err != nil { return fmt.Errorf("failed to make device node %s: %w", dstPath, err) } @@ -131,15 +146,20 @@ func setupDev(monRootfs string, devPath string) error { // Set up permissions, adding rw for others to ensure that any user can // read/write them. This is helpful for non-root monitor execution and // removes the burdain of getting kvm/block group id - permBits := devStat.Mode & 0o777 permBits |= 0o006 err = unix.Chmod(dstPath, permBits) if err != nil { return fmt.Errorf("failed to chmod %s: %w", dstPath, err) } + if dev.UID == nil { + return fmt.Errorf("reference of %s UID is nil", dev.Path) + } + if dev.GID == nil { + return fmt.Errorf("reference of %s GID is nil", dev.Path) + } // Set the owner as in the original file - err = os.Chown(dstPath, int(devStat.Uid), int(devStat.Gid)) + err = os.Chown(dstPath, int(*dev.UID), int(*dev.GID)) if err != nil { return fmt.Errorf("failed to chown %s: %w", dstPath, err) } diff --git a/pkg/unikontainers/rootfs.go b/pkg/unikontainers/rootfs.go index c43dbeb6a..274077737 100644 --- a/pkg/unikontainers/rootfs.go +++ b/pkg/unikontainers/rootfs.go @@ -68,6 +68,40 @@ func bindMount(source string, target string) specs.Mount { } } +// deviceFromHost finds a device in the host from the path and returns its info +// in the form of specs.LinuxDevice +func deviceFromHost(path string) (specs.LinuxDevice, error) { + var devStat unix.Stat_t + err := unix.Stat(path, &devStat) + if err != nil { + return specs.LinuxDevice{}, fmt.Errorf("failed to stat dev %s: %w", path, err) + } + + var devType string + switch devStat.Mode & unix.S_IFMT { + case unix.S_IFCHR: + devType = "c" + case unix.S_IFBLK: + devType = "b" + default: + return specs.LinuxDevice{}, fmt.Errorf("%s is not a device node", path) + } + + mode := os.FileMode(devStat.Mode & 0o777) + uid := devStat.Uid + gid := devStat.Gid + + return specs.LinuxDevice{ + Path: path, + Type: devType, + Major: int64(unix.Major(uint64(devStat.Rdev))), + Minor: int64(unix.Minor(uint64(devStat.Rdev))), + FileMode: &mode, + UID: &uid, + GID: &gid, + }, nil +} + // rootfsSelector encapsulates the context for rootfs selection type rootfsSelector struct { bundle string @@ -351,38 +385,44 @@ func changeRoot(rootfsDir string, pivot bool) error { return nil } -// prepareMonRootfs prepares the rootfs where the monitor will execute. It -// essentially sets up the devices (KVM, snapshotter block device) that are required -// for the monitor execution -func prepareMonRootfs(monRootfs string, monitorPath string, needsKVM bool, needsTAP bool) error { - err := setupDev(monRootfs, "/dev/null") +// getMonitorDevices returns the devices which are needed for the execution of +// the monitor process +func getMonitorDevices(needsKVM bool, needsTAP bool) ([]specs.LinuxDevice, error) { + nullDev, err := deviceFromHost("/dev/null") if err != nil { - return err + return nil, fmt.Errorf("could not get host device /dev/null: %w", err) } - - err = setupDev(monRootfs, "/dev/urandom") + randomDev, err := deviceFromHost("/dev/urandom") if err != nil { - return err + return nil, fmt.Errorf("could not get host device /dev/urandom: %w", err) } + devices := []specs.LinuxDevice{nullDev, randomDev} if needsTAP { - err = setupDev(monRootfs, "/dev/net/tun") + tunDev, err := deviceFromHost("/dev/net/tun") if err != nil { - return err + return nil, fmt.Errorf("could not get host device /dev/net/tun: %w", err) } + devices = append(devices, tunDev) } if needsKVM { - err = setupDev(monRootfs, "/dev/kvm") + kvmDev, err := deviceFromHost("/dev/kvm") if err != nil { - return err + return nil, fmt.Errorf("could not get host device /dev/kvm: %w", err) } + devices = append(devices, kvmDev) } + return devices, nil +} + +// setupConsole setups a /dev/console file for the monitor's execution environment +func setupConsole(monRootfs string) error { // Create /dev/ptmx as a symlink to /dev/pts/ptmx // This is the standard way to provide the PTY master device ptmxPath := filepath.Join(monRootfs, "/dev/ptmx") - err = os.Symlink("pts/ptmx", ptmxPath) + err := os.Symlink("pts/ptmx", ptmxPath) if err != nil && !os.IsExist(err) { return fmt.Errorf("failed to create /dev/ptmx symlink: %w", err) } diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index 886d5c34a..0e40a2523 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -573,16 +573,14 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return fmt.Errorf("failed to apply rootfs mounts: %w", err) } - // Setup the rootfs for the monitor execution, creating necessary - // devices and the monitor's binary. - err = prepareMonRootfs(rootfsParams.MonRootfs, vmm.Path(), vmm.UsesKVM(), withTUNTAP) + monitorDevs, err := getMonitorDevices(vmm.UsesKVM(), withTUNTAP) if err != nil { - return err + return fmt.Errorf("failed to get host devices for monitor: %w", err) } err = rfsBuilder.postSetup() if err != nil { - return fmt.Errorf("post setup step for block based rootfs failed: %w", err) + return fmt.Errorf("post setup step for rootfs failed: %w", err) } blockArgs, err := rfsBuilder.getBlockDevs() @@ -590,22 +588,30 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return fmt.Errorf("failed to get block devices to attach in sandbox: %w", err) } + blockDevs, err := blockDevNodes(blockArgs, rootfsParams) + if err != nil { + return fmt.Errorf("failed to get block devices which should be replicated inside monitor's execution environment: %w", err) + } + monitorDevs = append(monitorDevs, blockDevs...) + + err = setupDevices(rootfsParams.MonRootfs, monitorDevs) + if err != nil { + return fmt.Errorf("failed to create devices in monitor rootfs: %w", err) + } + + err = setupConsole(rootfsParams.MonRootfs) + if err != nil { + return err + } + sharedfsArgs, err := rfsBuilder.getSharedDirs() if err != nil { uniklog.Errorf("failed to get directories to share with sandbox: %v", err) return err } - unikernelParams.Rootfs = rootfsParams - metrics.Capture(m.TS17) - // unikernelParams - unikernelParams.Block = blockArgs - - // ExecArgs - vmmArgs.Sharedfs = sharedfsArgs - // vAccel setup vAccelType, vsockSocketPath, rpcAddress, err := resolveVAccelConfig(u.State.Annotations[annotHypervisor], u.Spec.Annotations) if err != nil { @@ -625,9 +631,13 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { unikernelParams.EnvVars = append(unikernelParams.EnvVars, "VACCEL_RPC_ADDRESS="+rpcAddress) // Prepare the guest environment for vAccel vsock communication - err = prepareVSockEnvironment(rootfsParams.MonRootfs, u.State.Annotations[annotHypervisor], vsockSocketPath) + vaccelDevices, err := prepareVSockEnvironment(rootfsParams.MonRootfs, u.State.Annotations[annotHypervisor], vsockSocketPath) if err != nil { - uniklog.Debugf("failed to prepare all required vsock mounts: %v", err) + uniklog.Debugf("failed to prepare get required vsock devices: %v", err) + } + err = setupDevices(rootfsParams.MonRootfs, vaccelDevices) + if err != nil { + return fmt.Errorf("failed to create devices in monitor rootfs: %w", err) } vmmArgs.VAccelType = vAccelType @@ -635,6 +645,14 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { vmmArgs.VSockDevID = idToGuestCID(u.State.ID) } + unikernelParams.Rootfs = rootfsParams + + // unikernelParams + unikernelParams.Block = blockArgs + + // ExecArgs + vmmArgs.Sharedfs = sharedfsArgs + // unikernel err = unikernel.Init(unikernelParams) if errors.Is(err, unikernels.ErrUndefinedVersion) || diff --git a/pkg/unikontainers/vaccel.go b/pkg/unikontainers/vaccel.go index f6905696f..b640b1ea3 100644 --- a/pkg/unikontainers/vaccel.go +++ b/pkg/unikontainers/vaccel.go @@ -19,6 +19,7 @@ import ( "fmt" "regexp" + "github.com/opencontainers/runtime-spec/specs-go" "golang.org/x/sys/unix" ) @@ -109,23 +110,24 @@ func resolveVAccelConfig(hypervisor string, annotations map[string]string) (stri // prepareVSockEnvironment prepares all required vsock devices and mounts // for vAccel execution inside the guest. This includes /dev/vsock, // /dev/vhost-vsock, and (for Firecracker) binding the host unix socket. -func prepareVSockEnvironment(monRootfs string, hypervisor string, vsockSocketPath string) error { - err := setupDev(monRootfs, "/dev/vsock") +func prepareVSockEnvironment(monRootfs string, hypervisor string, vsockSocketPath string) ([]specs.LinuxDevice, error) { + vsockDev, err := deviceFromHost("/dev/vsock") if err != nil { - return err + return nil, fmt.Errorf("could not get host device /dev/vsock: %w", err) } - - err = setupDev(monRootfs, "/dev/vhost-vsock") + vhostSockDev, err := deviceFromHost("/dev/vhost-vsock") if err != nil { - return err + return nil, fmt.Errorf("could not get host device /dev/vhost-vsock: %w", err) } + devices := []specs.LinuxDevice{vsockDev, vhostSockDev} // bind mount the unix socket directory if hypervisor == "firecracker" { err = fileFromHost(monRootfs, vsockSocketPath, "", unix.MS_BIND|unix.MS_PRIVATE, false) if err != nil { - return err + return nil, err } } - return nil + + return devices, nil } From 4e65f25a359be9ee002c6409559a17ae9ea56331 Mon Sep 17 00:00:00 2001 From: Charalampos Mainas Date: Wed, 15 Jul 2026 13:24:20 +0200 Subject: [PATCH 3/3] refactor: Move the mounts and device collection to InitialSetup Move the logic that gathers all needed devices and mounts for monitor execution environment in InitialSetup during "urunc create" and then write the information to a file (monitor.json) which Exec will read and apply the mounts, plus create the devices. THis is necessary for the libcontainer refactor. One important note, we always include the /dev/net/tun device from "urunc create" because ein that phase we have no information about the networking of the container. Therefore, we include the device and then let Exec (which is part of "urunc reexec" and CNI hooks have been executed) to choose if it will create or not the device. The fact that we now unmount the block-based mounts in the create phase also has the side effect that we mess up with the mount of someone else's. These mounts do not belong to urunc and we should restore them back. Furthermore, as shown from the CI testing, block-based mounts which are backed from a disk file (e.g. a custom made ext2 file image) are mounted as loop devices and hence unmounting them might remove the loop device. For that reason, we unset the autoclear flag of the loop device to ensure that the loop device will remain and we will be able to attach it later in the sandbox. We bring back the original flag during cleanup. PR: https://github.com/urunc-dev/urunc/pull/835 Signed-off-by: Charalampos Mainas Reviewed-by: Anastassios Nanos Approved-by: Anastassios Nanos --- go.mod | 2 +- pkg/unikontainers/block.go | 102 +++++++++++- pkg/unikontainers/mount.go | 5 +- pkg/unikontainers/rootfs.go | 23 ++- pkg/unikontainers/types/types.go | 11 +- pkg/unikontainers/unikontainers.go | 252 +++++++++++++++++------------ pkg/unikontainers/utils.go | 39 +++++ 7 files changed, 312 insertions(+), 122 deletions(-) diff --git a/go.mod b/go.mod index ee01504d4..4844abe06 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/hashicorp/go-version v1.9.0 github.com/jackpal/gateway v1.2.0 github.com/moby/sys/mount v0.3.5 + github.com/moby/sys/mountinfo v0.7.2 github.com/moby/sys/userns v0.1.0 github.com/nubificus/hedge_cli v0.0.3 github.com/onsi/ginkgo/v2 v2.28.1 @@ -62,7 +63,6 @@ require ( github.com/klauspost/compress v1.18.6 // indirect github.com/mattn/go-colorable v0.1.15 // indirect github.com/mattn/go-isatty v0.0.22 // indirect - github.com/moby/sys/mountinfo v0.7.2 // indirect github.com/moby/sys/sequential v0.7.0 // indirect github.com/moby/sys/user v0.4.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect diff --git a/pkg/unikontainers/block.go b/pkg/unikontainers/block.go index 43c78cda5..6914586c6 100644 --- a/pkg/unikontainers/block.go +++ b/pkg/unikontainers/block.go @@ -23,9 +23,11 @@ import ( "strings" "github.com/moby/sys/mount" + "github.com/moby/sys/mountinfo" "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" "github.com/urunc-dev/urunc/pkg/unikontainers/types" + "golang.org/x/sys/unix" ) // TODO: Find and set the correct size for the tmpfs in the host @@ -75,7 +77,7 @@ func getMountInfo(path string) (types.BlockDevParams, error) { } preDash := strings.Fields(parts[0]) - if len(preDash) < 5 { + if len(preDash) < 6 { continue } postDash := strings.Fields(parts[1]) @@ -87,11 +89,15 @@ func getMountInfo(path string) (types.BlockDevParams, error) { "mounted at": path, "device": postDash[1], "fstype": postDash[0], + "options": preDash[5], }).Debug("Found block device") blockDev.Source = postDash[1] blockDev.FsType = postDash[0] blockDev.MountPoint = path + // Keep the-mount VFS options (field 6 of mountinfo) + // to restore them later in the delete path. + blockDev.MountOptions = preDash[5] blockDev.ID = "" continue } @@ -206,11 +212,31 @@ func getBlockVolumes(mounts []specs.Mount, ukernel types.Unikernel) ([]types.Blo return nil, err } if ukernel.SupportsFS(mInfo.FsType) { + // So, there was an issue which was manifested from the testing. + // If we have a file (e.g. ext2) and mount it, then we use + // a loop device for the mount and this is what is shown + // in the mount list. However, since we perform the unmount + // the device might also get removed. See + // https://www.kernel.org/doc/Documentation/ABI/testing/sysfs-block-loop + // If the device gets removed, then we attach nothing to the + // sandbox. To resolve this we remove the autoclear flag + // and therefore the device will persist. + // NOTE: Although we restore the autoclear flag in the delete path, + // if delete is never called then the autoclear flag will never + // get restored.and remounted + // TODO; Add the above note in a documentation for storage + // handling + cleared, err := setLoopAutoclear(mInfo.Source, false) + if err != nil { + return nil, err + } + mInfo.LoopAutoclear = cleared err = mount.Unmount(mInfo.MountPoint) if err != nil { return nil, err } mInfo.ID = fmt.Sprintf("vol%d", i) + mInfo.HostMountPoint = mInfo.MountPoint mInfo.MountPoint = m.Destination blkImgs = append(blkImgs, mInfo) } @@ -219,6 +245,80 @@ func getBlockVolumes(mounts []specs.Mount, ukernel types.Unikernel) ([]types.Blo return blkImgs, nil } +// restoreBlockVolumes mounts the block volume sources that were unmounted +// during create +func restoreBlockVolumes(blockArgs []types.BlockDevParams) error { + for _, b := range blockArgs { + // Only volumes gathered from the container's mounts carry the + // host mountpoint where their source was originally mounted. + if b.HostMountPoint == "" { + continue + } + mounted, err := mountinfo.Mounted(b.HostMountPoint) + if err != nil { + return fmt.Errorf("failed to check if %s is a mountpoint: %w", b.HostMountPoint, err) + } + if mounted { + continue + } + + flags, _ := splitMountOptions(strings.Split(b.MountOptions, ",")) + err = unix.Mount(b.Source, b.HostMountPoint, b.FsType, flags, "") + if err != nil { + return fmt.Errorf("failed to remount %s at %s: %w", b.Source, b.HostMountPoint, err) + } + + if b.LoopAutoclear { + _, err = setLoopAutoclear(b.Source, true) + if err != nil { + return err + } + } + } + + return nil +} + +// setLoopAutoclear sets or clears the autoclear flag of a loop device and +// returns true when the flag was changed. It returns false without an error +// when the path is not a loop device or the flag already has the wanted value. +func setLoopAutoclear(devPath string, autoclear bool) (bool, error) { + _, err := os.Stat(filepath.Join("/sys/class/block", filepath.Base(devPath), "loop")) + if os.IsNotExist(err) { + // Not a loop device + return false, nil + } + if err != nil { + return false, err + } + + dev, err := os.OpenFile(devPath, os.O_RDWR, 0) + if err != nil { + return false, fmt.Errorf("failed to open %s: %w", devPath, err) + } + defer dev.Close() + + info, err := unix.IoctlLoopGetStatus64(int(dev.Fd())) + if err != nil { + return false, fmt.Errorf("failed to get the status of %s: %w", devPath, err) + } + if autoclear == (info.Flags&unix.LO_FLAGS_AUTOCLEAR != 0) { + return false, nil + } + + if autoclear { + info.Flags |= unix.LO_FLAGS_AUTOCLEAR + } else { + info.Flags &^= unix.LO_FLAGS_AUTOCLEAR + } + err = unix.IoctlLoopSetStatus64(int(dev.Fd()), info) + if err != nil { + return false, fmt.Errorf("failed to set the status of %s: %w", devPath, err) + } + + return true, nil +} + // blockDevNodes transforms a list of types.BlockDevParams to a list of // specs.LinuxDevice which cna be then used for replicating these block // devices form the host to the monitor's execution environment. diff --git a/pkg/unikontainers/mount.go b/pkg/unikontainers/mount.go index 9a1b5b33a..290e4e3f2 100644 --- a/pkg/unikontainers/mount.go +++ b/pkg/unikontainers/mount.go @@ -75,8 +75,11 @@ func createTmpfs(monRootfs string, path string, flags uintptr, data string) erro } // setupDevices creates every device from the list inside monitor's rootfs -func setupDevices(monRootfs string, devices []specs.LinuxDevice) error { +func setupDevices(monRootfs string, devices []specs.LinuxDevice, needsTAP bool) error { for _, dev := range devices { + if !needsTAP && dev.Path == "/dev/net/tun" { + continue + } err := setupDev(monRootfs, dev) if err != nil { return err diff --git a/pkg/unikontainers/rootfs.go b/pkg/unikontainers/rootfs.go index 274077737..546684ba8 100644 --- a/pkg/unikontainers/rootfs.go +++ b/pkg/unikontainers/rootfs.go @@ -15,6 +15,7 @@ package unikontainers import ( + "errors" "fmt" "os" "path/filepath" @@ -74,7 +75,7 @@ func deviceFromHost(path string) (specs.LinuxDevice, error) { var devStat unix.Stat_t err := unix.Stat(path, &devStat) if err != nil { - return specs.LinuxDevice{}, fmt.Errorf("failed to stat dev %s: %w", path, err) + return specs.LinuxDevice{}, err } var devType string @@ -387,7 +388,7 @@ func changeRoot(rootfsDir string, pivot bool) error { // getMonitorDevices returns the devices which are needed for the execution of // the monitor process -func getMonitorDevices(needsKVM bool, needsTAP bool) ([]specs.LinuxDevice, error) { +func getMonitorDevices(needsKVM bool) ([]specs.LinuxDevice, error) { nullDev, err := deviceFromHost("/dev/null") if err != nil { return nil, fmt.Errorf("could not get host device /dev/null: %w", err) @@ -396,15 +397,21 @@ func getMonitorDevices(needsKVM bool, needsTAP bool) ([]specs.LinuxDevice, error if err != nil { return nil, fmt.Errorf("could not get host device /dev/urandom: %w", err) } - devices := []specs.LinuxDevice{nullDev, randomDev} - - if needsTAP { - tunDev, err := deviceFromHost("/dev/net/tun") - if err != nil { + // The tun device is always included because in urunc create we can not know + // if there will be a virtual ethernet device or not, since the CNI hooks + // have not executed yet. Therefore, the decision about whether it is actually + // created in the monitor rootfs is decided in Exec, which checks the + // network configuration. However, if the host does not have the "/dev/net/tun" + // device then simply skip it and fail later if the container requires network + tunDev, err := deviceFromHost("/dev/net/tun") + if err != nil { + if errors.Is(err, unix.ENOENT) { + uniklog.Warnf("could not find /dev/net/tun in host, skipping it") + } else { return nil, fmt.Errorf("could not get host device /dev/net/tun: %w", err) } - devices = append(devices, tunDev) } + devices := []specs.LinuxDevice{nullDev, randomDev, tunDev} if needsKVM { kvmDev, err := deviceFromHost("/dev/kvm") diff --git a/pkg/unikontainers/types/types.go b/pkg/unikontainers/types/types.go index 04f5204b4..bbfde55cd 100644 --- a/pkg/unikontainers/types/types.go +++ b/pkg/unikontainers/types/types.go @@ -54,10 +54,13 @@ type NetDevParams struct { } type BlockDevParams struct { - Source string - MountPoint string - FsType string - ID string + Source string + MountPoint string + FsType string + ID string + HostMountPoint string + LoopAutoclear bool + MountOptions string } type SharedfsParams struct { diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index 0e40a2523..bc43bcace 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -197,22 +197,24 @@ func (u *Unikontainer) InitialSetup() error { "mountedPath": rootfsParams.MountedPath, }).Debug("guest rootfs params") - if rootfsParams.Type == "block" { - rfsBuilder := blockRootfs{ - mounts: u.Spec.Mounts, - monRootfs: rootfsParams.MonRootfs, - mountedPath: rootfsParams.MountedPath, - path: rootfsParams.Path, - kernelPath: unikernelPath, - initrdPath: initrdPath, - uruncJSONPath: uruncJSONFilename, - guestType: unikernelType, - guest: unikernel, - } - err := rfsBuilder.preSetup() - if err != nil { - return fmt.Errorf("pre setup step for rootfs failed: %w", err) - } + vmmType := u.State.Annotations[annotHypervisor] + vmm, err := hypervisors.NewVMM(hypervisors.VmmType(vmmType), u.UruncCfg.Monitors) + if err != nil { + return err + } + + defaultMemSizeMB := u.UruncCfg.Monitors[vmmType].DefaultMemoryMB + memory := monitorMemoryBytes(defaultMemSizeMB, u.Spec.Linux.Resources) + rfsBuilder := u.newRootfsBuilder(rootfsParams, unikernel, unikernelPath, initrdPath, memory) + + err = rfsBuilder.preSetup() + if err != nil { + return fmt.Errorf("pre setup step for rootfs failed: %w", err) + } + + monRes, err := getMonitorResources(rfsBuilder, rootfsParams, vmm, u.UruncCfg.Monitors[vmmType].DataPath) + if err != nil { + return err } u.State.Status = specs.StateCreating @@ -221,6 +223,12 @@ func (u *Unikontainer) InitialSetup() error { if err != nil { return err } + + err = saveMonitorResources(u.BaseDir, monRes) + if err != nil { + return fmt.Errorf("failed to store monitor resources: %w", err) + } + return u.saveContainerState() } @@ -350,6 +358,94 @@ func ChooseRootfs(bundle, specRoot string, annot map[string]string, cfg *UruncCo return result, nil } +// getMonitorResources collects every mount and device required for the monitor's +// execution environment, plus the block parameters passed to the guest. +func getMonitorResources(rfs rootfsBuilder, rootfsParams types.RootfsParams, vmm types.VMM, monitorDataPath string) (monitorResources, error) { + var res monitorResources + var err error + + res.Mounts, err = mountsForMonitor(vmm.Path(), monitorDataPath) + if err != nil { + return res, err + } + rootfsMounts, err := rfs.getMounts() + if err != nil { + return res, fmt.Errorf("failed to get mounts for monitor rootfs: %w", err) + } + res.Mounts = append(res.Mounts, rootfsMounts...) + + res.Devices, err = getMonitorDevices(vmm.UsesKVM()) + if err != nil { + return res, fmt.Errorf("failed to get host devices for monitor: %w", err) + } + res.BlockArgs, err = rfs.getBlockDevs() + if err != nil { + return res, fmt.Errorf("failed to get block devices to attach in sandbox: %w", err) + } + blockDevs, err := blockDevNodes(res.BlockArgs, rootfsParams) + if err != nil { + return res, fmt.Errorf("failed to get block devices which should be replicated inside monitor's execution environment: %w", err) + } + res.Devices = append(res.Devices, blockDevs...) + + return res, nil +} + +// newRootfsBuilder constructs the rootfsBuilder matching the selected guest +// rootfs. It is shared by InitialSetup (which gathers the monitor resources) and +// Exec (which performs the per-rootfs actions). +func (u *Unikontainer) newRootfsBuilder(rootfsParams types.RootfsParams, unikernel types.Unikernel, unikernelPath string, initrdPath string, memory uint64) rootfsBuilder { + switch rootfsParams.Type { + case "block": + return blockRootfs{ + mounts: u.Spec.Mounts, + monRootfs: rootfsParams.MonRootfs, + mountedPath: rootfsParams.MountedPath, + path: rootfsParams.Path, + kernelPath: unikernelPath, + initrdPath: initrdPath, + uruncJSONPath: uruncJSONFilename, + guestType: u.State.Annotations[annotType], + guest: unikernel, + } + case "initrd": + return initrdRootfs{ + mounts: u.Spec.Mounts, + initrdHostFullPath: filepath.Join(rootfsParams.MonRootfs, rootfsParams.Path), + monRootfs: rootfsParams.MonRootfs, + } + case "virtiofs", "9pfs": + return sharedfsRootfs{ + mounts: u.Spec.Mounts, + monRootfs: rootfsParams.MonRootfs, + mountedPath: rootfsParams.MountedPath, + sfsType: rootfsParams.Type, + vfsdConfig: u.UruncCfg.ExtraBins["virtiofsd"], + sharedPath: containerRootfsMountPath, + memory: memory, + } + default: + return noRootfs{ + monRootfs: rootfsParams.MonRootfs, + annotBlockPath: u.State.Annotations[annotBlock], + annotBlockMountPoint: u.State.Annotations[annotBlockMntPoint], + } + } +} + +// monitorMemoryBytes returns the guest memory size in bytes, honoring a memory +// limit from the OCI spec and falling back to the monitor's configured default. +func monitorMemoryBytes(defaultMem uint, resources *specs.LinuxResources) uint64 { + mem := uint64(defaultMem * 1024 * 1024) + if resources != nil && resources.Memory != nil { + if resources.Memory.Limit != nil && *resources.Memory.Limit > 0 { + mem = uint64(*resources.Memory.Limit) // nolint:gosec + } + } + + return mem +} + // nolint:gocyclo func (u *Unikontainer) Exec(metrics m.Writer) error { metrics.Capture(m.TS15) @@ -409,21 +505,11 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { UnikernelPath: unikernelPath, InitrdPath: initrdPath, Seccomp: true, // Enable Seccomp by default - MemSizeB: uint64(defaultMemSizeMB * 1024 * 1024), + MemSizeB: monitorMemoryBytes(defaultMemSizeMB, u.Spec.Linux.Resources), VCPUs: uint(defaultVCPUs), Environment: os.Environ(), } - // ExecArgs - // If memory limit is set in spec, use it instead of the config default value - if u.Spec.Linux.Resources != nil && u.Spec.Linux.Resources.Memory != nil { - if u.Spec.Linux.Resources.Memory.Limit != nil { - if *u.Spec.Linux.Resources.Memory.Limit > 0 { - vmmArgs.MemSizeB = uint64(*u.Spec.Linux.Resources.Memory.Limit) // nolint:gosec - } - } - } - // ExecArgs // Check if container is set to unconfined -- disable seccomp if u.Spec.Linux.Seccomp == nil { @@ -465,9 +551,6 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { // ExecArgs vmmArgs.Net = netArgs - // virtiofsd config - virtiofsdConfig := u.UruncCfg.ExtraBins["virtiofsd"] - // guest rootfs // block // handle guest's rootfs. @@ -498,51 +581,11 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { "mon_rootfs": rootfsParams.MonRootfs, }).Debug("guest rootfs params") - // TODO: Add support for using both an existing - // block based snapshot of the container's rootfs - // and an auxiliary block image placed in the container's image - // Currently if a block Image is present in the container's image, then - // we will just use this image. - var rfsBuilder rootfsBuilder - switch rootfsParams.Type { - case "block": - rfsBuilder = blockRootfs{ - mounts: u.Spec.Mounts, - monRootfs: rootfsParams.MonRootfs, - mountedPath: rootfsParams.MountedPath, - path: rootfsParams.Path, - kernelPath: unikernelPath, - initrdPath: initrdPath, - uruncJSONPath: uruncJSONFilename, - guestType: unikernelType, - guest: unikernel, - } - case "initrd": - rfsBuilder = initrdRootfs{ - mounts: u.Spec.Mounts, - initrdHostFullPath: filepath.Join(rootfsParams.MonRootfs, rootfsParams.Path), - monRootfs: rootfsParams.MonRootfs, - } - case "virtiofs", "9pfs": - rfsBuilder = sharedfsRootfs{ - mounts: u.Spec.Mounts, - monRootfs: rootfsParams.MonRootfs, - mountedPath: rootfsParams.MountedPath, - sfsType: rootfsParams.Type, - vfsdConfig: virtiofsdConfig, - sharedPath: containerRootfsMountPath, - memory: vmmArgs.MemSizeB, - } + rfsBuilder := u.newRootfsBuilder(rootfsParams, unikernel, unikernelPath, initrdPath, vmmArgs.MemSizeB) + if rootfsParams.Type == "virtiofs" || rootfsParams.Type == "9pfs" { // Update the paths of the files we need to pass in the monitor process. vmmArgs.UnikernelPath = adjustPathsForSharedfs(vmmArgs.UnikernelPath) vmmArgs.InitrdPath = adjustPathsForSharedfs(vmmArgs.InitrdPath) - default: - uniklog.Debug("No rootfs for guest") - rfsBuilder = noRootfs{ - monRootfs: rootfsParams.MonRootfs, - annotBlockPath: u.State.Annotations[annotBlock], - annotBlockMountPoint: u.State.Annotations[annotBlockMntPoint], - } } if err = os.MkdirAll(rootfsParams.MonRootfs, 0o755); err != nil { @@ -557,44 +600,26 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return err } - monitorMounts, err := mountsForMonitor(vmm.Path(), u.UruncCfg.Monitors[vmmType].DataPath) - if err != nil { - return err - } - - rootfsMounts, err := rfsBuilder.getMounts() + // The monitor mounts and devices were gathered and stored in a file during + // InitialSetup; here we just apply them. postSetup applies the container's own + // bind mounts on top of the shared rootfs, and setupDevices decides whether to + // create the TUN/TAP device based on the container's network configuration. + monRes, err := loadMonitorResources(u.BaseDir) if err != nil { - return fmt.Errorf("failed to get mounts for monitor rootfs: %w", err) + return fmt.Errorf("failed to load monitor resources: %w", err) } - rootfsMounts = append(monitorMounts, rootfsMounts...) - err = applyMounts(rootfsParams.MonRootfs, rootfsMounts) + err = applyMounts(rootfsParams.MonRootfs, monRes.Mounts) if err != nil { return fmt.Errorf("failed to apply rootfs mounts: %w", err) } - monitorDevs, err := getMonitorDevices(vmm.UsesKVM(), withTUNTAP) - if err != nil { - return fmt.Errorf("failed to get host devices for monitor: %w", err) - } - err = rfsBuilder.postSetup() if err != nil { return fmt.Errorf("post setup step for rootfs failed: %w", err) } - blockArgs, err := rfsBuilder.getBlockDevs() - if err != nil { - return fmt.Errorf("failed to get block devices to attach in sandbox: %w", err) - } - - blockDevs, err := blockDevNodes(blockArgs, rootfsParams) - if err != nil { - return fmt.Errorf("failed to get block devices which should be replicated inside monitor's execution environment: %w", err) - } - monitorDevs = append(monitorDevs, blockDevs...) - - err = setupDevices(rootfsParams.MonRootfs, monitorDevs) + err = setupDevices(rootfsParams.MonRootfs, monRes.Devices, withTUNTAP) if err != nil { return fmt.Errorf("failed to create devices in monitor rootfs: %w", err) } @@ -604,14 +629,7 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return err } - sharedfsArgs, err := rfsBuilder.getSharedDirs() - if err != nil { - uniklog.Errorf("failed to get directories to share with sandbox: %v", err) - return err - } - metrics.Capture(m.TS17) - // vAccel setup vAccelType, vsockSocketPath, rpcAddress, err := resolveVAccelConfig(u.State.Annotations[annotHypervisor], u.Spec.Annotations) if err != nil { @@ -635,7 +653,7 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { if err != nil { uniklog.Debugf("failed to prepare get required vsock devices: %v", err) } - err = setupDevices(rootfsParams.MonRootfs, vaccelDevices) + err = setupDevices(rootfsParams.MonRootfs, vaccelDevices, false) if err != nil { return fmt.Errorf("failed to create devices in monitor rootfs: %w", err) } @@ -648,9 +666,15 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { unikernelParams.Rootfs = rootfsParams // unikernelParams - unikernelParams.Block = blockArgs + // The block parameters were gathered in InitialSetup and stored in the + // monitor resources file. + unikernelParams.Block = monRes.BlockArgs // ExecArgs + sharedfsArgs, err := rfsBuilder.getSharedDirs() + if err != nil { + return fmt.Errorf("failed to get directories to share with sandbox: %w", err) + } vmmArgs.Sharedfs = sharedfsArgs // unikernel @@ -821,6 +845,20 @@ func (u *Unikontainer) Delete() error { return fmt.Errorf("cannot delete running container: %s", u.State.ID) } + // Restore the block volume mounts that were unmounted during create, + // so their sources become discoverable by future containers. Do it in + // a best-effort way, since a failure to restore a mount should not + // prevent the deletion of the container. + monRes, err := loadMonitorResources(u.BaseDir) + if err != nil { + uniklog.Errorf("failed to load monitor resources: %v", err) + } + + err = restoreBlockVolumes(monRes.BlockArgs) + if err != nil { + uniklog.Errorf("failed to restore block volume mounts: %v", err) + } + // get a monitor instance of the running monitor vmmType := u.State.Annotations[annotHypervisor] vmm, err := hypervisors.NewVMM(hypervisors.VmmType(vmmType), u.UruncCfg.Monitors) diff --git a/pkg/unikontainers/utils.go b/pkg/unikontainers/utils.go index dd9fd0b58..83c9d72af 100644 --- a/pkg/unikontainers/utils.go +++ b/pkg/unikontainers/utils.go @@ -32,16 +32,55 @@ import ( "github.com/opencontainers/runtime-spec/specs-go" "github.com/urunc-dev/urunc/internal/constants" + "github.com/urunc-dev/urunc/pkg/unikontainers/types" ) const ( configFilename = "config.json" stateFilename = "state.json" + monitorFilename = "monitor.json" initPidFilename = "init.pid" uruncJSONFilename = "urunc.json" rootfsDirName = "rootfs" ) +// monitorResources holds the mounts and devices that must be replicated inside +// the monitor's rootfs, along with the block parameters the guest needs. +type monitorResources struct { + Mounts []specs.Mount `json:"mounts"` + Devices []specs.LinuxDevice `json:"devices"` + BlockArgs []types.BlockDevParams `json:"blockArgs"` +} + +// saveMonitorResources stores the monitorResources passed as an argument in a JSON +// file under the container's base +// directory, so that the reexec's Exec can read it back. +func saveMonitorResources(baseDir string, res monitorResources) error { + data, err := json.Marshal(res) + if err != nil { + return err + } + + path := filepath.Join(baseDir, monitorFilename) + + return os.WriteFile(path, data, 0o644) //nolint: gosec +} + +// loadMonitorResources reads the monitor resources file stored by InitialSetup. +func loadMonitorResources(baseDir string) (monitorResources, error) { + var res monitorResources + + path := filepath.Join(baseDir, monitorFilename) + data, err := os.ReadFile(path) + if err != nil { + return res, err + } + + err = json.Unmarshal(data, &res) + + return res, err +} + // copy sourceFile to targetDir // creates targetDir and all necessary parent directories func copyFile(sourceFile string, targetPath string) error {