rootfs-config.toml

# fcvm Configuration
#
# This file configures both runtime paths and rootfs setup.
# The rootfs SHA is computed from [base], [kernel], [packages], [services], etc.
# The [paths] section does NOT affect rootfs SHA.

[paths]
# Directory for mutable VM data (vm-disks, state, snapshots)
# Override per nesting level (e.g., /mnt/fcvm-btrfs/level-1)
data_dir = "/mnt/fcvm-btrfs"

# Directory for shared content-addressed assets (kernels, rootfs, initrd, image-cache)
# All nesting levels share this for deduplication via btrfs reflinks
assets_dir = "/mnt/fcvm-btrfs"

# Size of the btrfs loopback filesystem (sparse file, only written blocks use real space)
# Only used on non-btrfs hosts where a loopback image must be created.
# Ignored if the host filesystem is already btrfs.
btrfs_size = "60G"

# Default Firecracker binary
# Build from fork with diff snapshot fix for multi-slot VMs (>3GB RAM).
# `fcvm setup` builds this automatically; `find_firecracker()` uses it.
# Kernel profiles can override with their own firecracker_repo/branch.
# Upstream PR: https://github.com/firecracker-microvm/firecracker/pull/5696
[firecracker]
repo = "ejc3/firecracker"
branch = "bump-vsock-max-connections"

# Rootfs Modification Plan
#
# The SHA256 of the generated setup script determines the image name: layer2-{sha}.raw
# If base, kernel, packages, services, files, fstab, or cleanup change, Layer 2 is rebuilt.
#
# fc-agent is NOT in Layer 2 at all (neither binary nor service).
# Both are injected per-VM at boot time via initrd.
# This allows updating fc-agent without rebuilding Layer 2.

[base]
# Ubuntu 24.04 LTS (Noble Numbat) cloud images
# Using "current" for latest updates - URL changes trigger plan SHA change
version = "24.04"
# Codename used to download packages from correct Ubuntu release
codename = "noble"

[base.arm64]
url = "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-arm64.img"

[base.amd64]
url = "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img"

[kernel]
# Kata Containers kernel with FUSE support built-in
# Firecracker's official kernel lacks FUSE, but Kata's has it
# URL hash is included in Layer 2 SHA calculation

[kernel.arm64]
# Kata 3.24.0 release - kernel 6.12.47 with CONFIG_FUSE_FS=y
url = "https://github.com/kata-containers/kata-containers/releases/download/3.24.0/kata-static-3.24.0-arm64.tar.zst"
# Path within the tarball to extract
path = "opt/kata/share/kata-containers/vmlinux-6.12.47-173"

[kernel.amd64]
url = "https://github.com/kata-containers/kata-containers/releases/download/3.24.0/kata-static-3.24.0-amd64.tar.zst"
path = "opt/kata/share/kata-containers/vmlinux-6.12.47-173"

[packages]
# Container runtime
runtime = ["podman", "crun", "fuse-overlayfs", "skopeo", "uidmap"]

# FUSE support for overlay filesystem
fuse = ["fuse3"]

# System services and disk/filesystem tools for nested --disk-dir and --nfs
# passt: provides pasta for rootless networking (default --network rootless backend)
system = ["haveged", "chrony", "rsync", "nfs-common", "iptables", "passt", "btrfs-progs", "iputils-arping"]

# Debugging and networking tools
debug = ["strace", "netcat-openbsd"]

[services]
# Services to enable
# NOTE: fc-agent is NOT enabled here - it's injected per-VM via initrd
# NOTE: systemd-resolved is NOT enabled - DNS comes from kernel cmdline via fc-agent
enable = [
    "haveged",
    "chrony",
    "systemd-networkd",
]

# Services to disable
# podman services race with fc-agent: they run `podman start --all` at boot,
# which initializes db.sql before fc-agent writes storage.conf, causing
# "database graph driver does not match" errors.
disable = [
    "multipathd",
    "snapd",
    "cloud-init",
    "cloud-config",
    "cloud-final",
    "podman.service",
    "podman.socket",
    "podman-restart.service",
    "podman-auto-update.service",
    "podman-auto-update.timer",
    "podman-clean-transient.service",
]

[files]
# Files to create/modify in the rootfs

[files."/etc/resolv.conf"]
content = """
# Placeholder - fc-agent configures DNS at boot from kernel cmdline
nameserver 127.0.0.53
"""

[files."/etc/chrony/chrony.conf"]
content = """
# NTP servers from pool.ntp.org
pool pool.ntp.org iburst

# Allow clock to be stepped (not slewed) for large time differences
makestep 1.0 3

# Directory for drift and other runtime files
driftfile /var/lib/chrony/drift
"""

[files."/etc/systemd/network/10-eth0.network"]
content = """
[Match]
Name=eth0

[Network]
# Keep kernel IP configuration from ip= boot parameter
KeepConfiguration=yes
"""

[files."/etc/systemd/network/10-eth0.network.d/mmds.conf"]
content = """
[Route]
Destination=169.254.169.254/32
Scope=link
"""

# NOTE: fc-agent.service is NOT defined here - it's injected per-VM via initrd

[fstab]
# Lines to remove from /etc/fstab (patterns to filter out)
remove_patterns = ["LABEL=BOOT", "LABEL=UEFI"]

[cleanup]
# Patterns to remove for smaller image
remove_dirs = [
    "/usr/share/doc/*",
    "/usr/share/man/*",
    "/var/cache/apt/archives/*",
]

# Kernel Profiles
#
# Each profile defines a kernel + optional firecracker override + runtime config.
# Select via FCVM_KERNEL_PROFILE=<name> or auto-detected from kernel filename.
#
# Profile config is applied at runtime:
# - firecracker_repo/branch: build firecracker from source (content-addressed in ~/.local/share/fcvm/firecracker/)
# - firecracker_args: extra CLI args for firecracker
# - boot_args: extra kernel boot parameters
# - fuse_readers: override FUSE reader count

# arm64 nested profile (ARM64 NV2 with custom kernel and firecracker)
[kernel_profiles.nested.arm64]
description = "Nested virtualization support for running VMs inside VMs"
kernel_version = "6.18.3"
kernel_repo = "ejc3/fcvm"

# Build configuration - these files determine when kernel needs rebuilding
# SHA is computed from contents of all files matching these patterns
# NOTE: build script is generated by Rust, not in source control
build_inputs = [
    "kernel/nested.conf",
    "kernel/patches-arm64/*.patch",
]

# Build paths (relative to repo root)
kernel_config = "kernel/nested.conf"
patches_dir = "kernel/patches-arm64"

# Base config for VM kernel (Firecracker's microvm config)
base_config_url = "https://raw.githubusercontent.com/firecracker-microvm/firecracker/main/resources/guest_configs/microvm-kernel-ci-aarch64-6.1.config"

# Firecracker fork: diff snapshot fix + NV2 nested virt (ARM64)
# PR: https://github.com/firecracker-microvm/firecracker/pull/5696
firecracker_repo = "ejc3/firecracker"
firecracker_branch = "nv2-on-main"
firecracker_args = "--enable-nv2"

# Boot args for nested KVM
boot_args = "kvm-arm.mode=nested numa=off arm64.nv2"
fuse_readers = 64

# Host kernel configuration for nested profile
# Uses the running kernel's config (/boot/config-$(uname -r)) as base,
# which includes all EC2/AWS modules (ENA networking, NVMe, etc.)
# Then applies fcvm patches for NV2 cache coherency (DSB barriers)
[kernel_profiles.nested.arm64.host_kernel]
kernel_version = "6.18.3"
patches_dir = "kernel/patches-arm64"

# Build inputs for SHA calculation (patches only, skip *.vm.patch for host)
# .vm.patch files are only applied to the nested VM kernel, not host
build_inputs = [
    "kernel/patches-arm64/*.patch",
]

# x86_64 nested profile (Intel VT-x / AMD-V)
# Builds custom kernel with CONFIG_KVM_INTEL and CONFIG_KVM_AMD for nested virtualization.
# x86 nested virtualization is more mature and doesn't require special patches like ARM NV2.
# Requires CONFIG_PCI=y for GENERIC_MSI_IRQ (virtio-mmio IRQ infrastructure).
[kernel_profiles.nested.amd64]
description = "Nested virtualization support for running VMs inside VMs (x86_64)"
kernel_version = "6.18.3"
kernel_repo = "ejc3/fcvm"

# Build configuration - these files determine when kernel needs rebuilding
build_inputs = [
    "kernel/nested-x86.conf",
    "kernel/patches-x86/*.patch",
]

# Build paths (relative to repo root)
kernel_config = "kernel/nested-x86.conf"
patches_dir = "kernel/patches-x86"  # FUSE remap_file_range patch for reflink support

# Base config for VM kernel (Firecracker's microvm config)
base_config_url = "https://raw.githubusercontent.com/firecracker-microvm/firecracker/main/resources/guest_configs/microvm-kernel-ci-x86_64-6.1.config"

# No special firecracker fork needed for x86 (standard firecracker supports Intel VT-x/AMD-V)
# No special boot_args needed for x86

# Host kernel configuration for x86 nested profile
# Uses the running kernel's config (/boot/config-$(uname -r)) as base,
# then applies FUSE remap_file_range patch for reflink support in nested VMs
[kernel_profiles.nested.amd64.host_kernel]
kernel_version = "6.18.3"
patches_dir = "kernel/patches-x86"

# Build inputs for SHA calculation
build_inputs = [
    "kernel/patches-x86/*.patch",
]

# arm64 btrfs profile (FUSE + btrfs, no KVM/nested)
# Minimal kernel for running containers with btrfs storage driver.
# When the kernel supports btrfs, fc-agent auto-detects it and configures
# podman to use btrfs instead of overlay (avoids idmapped mount issues).
[kernel_profiles.btrfs.arm64]
rootfs_type = "btrfs"
description = "Kernel with FUSE and btrfs support for efficient container storage"
kernel_version = "6.18.3"
kernel_repo = "ejc3/fcvm"

build_inputs = [
    "kernel/btrfs.conf",
]

kernel_config = "kernel/btrfs.conf"
# No patches_dir — no FUSE patches needed for btrfs profile

base_config_url = "https://raw.githubusercontent.com/firecracker-microvm/firecracker/main/resources/guest_configs/microvm-kernel-ci-aarch64-6.1.config"

# x86_64 btrfs profile (FUSE + btrfs, no KVM/nested)
[kernel_profiles.btrfs.amd64]
rootfs_type = "btrfs"
description = "Kernel with FUSE and btrfs support for efficient container storage (x86_64)"
kernel_version = "6.18.3"
kernel_repo = "ejc3/fcvm"

build_inputs = [
    "kernel/btrfs-x86.conf",
]

kernel_config = "kernel/btrfs-x86.conf"
# No patches_dir — no FUSE patches needed for btrfs profile

base_config_url = "https://raw.githubusercontent.com/firecracker-microvm/firecracker/main/resources/guest_configs/microvm-kernel-ci-x86_64-6.1.config"