From 9cd0f95196b87a0231a0a9223c09a46d87b07755 Mon Sep 17 00:00:00 2001 From: Beniamin Sandu Date: Wed, 12 Aug 2026 23:23:18 +0300 Subject: [PATCH] make cpio reproducible Signed-off-by: Beniamin Sandu --- src/commands/initramfs/image.rs | 157 ++++++++++++++++++++++++++++++-- src/commands/rootfs/image.rs | 5 +- src/commands/rootfs/install.rs | 5 +- src/commands/sdk/install.rs | 4 +- src/utils/kernel_resolver.rs | 2 +- src/utils/stamps.rs | 2 +- 6 files changed, 163 insertions(+), 12 deletions(-) diff --git a/src/commands/initramfs/image.rs b/src/commands/initramfs/image.rs index 8c7b3e43..54658e9a 100644 --- a/src/commands/initramfs/image.rs +++ b/src/commands/initramfs/image.rs @@ -86,8 +86,14 @@ if [ -d "$INITRAMFS_SYSROOT/usr" ]; then {post_install_block} - # Compute deterministic build ID for initramfs - INITRAMFS_PKG_NEVRA=$(rpm -qa --queryformat '%{{NEVRA}}\n' --root "$INITRAMFS_SYSROOT" | sort) + # Compute deterministic build ID for initramfs. + # + # LC_ALL=C for the same reason as the cpio pipeline below: this hash becomes + # $INITRAMFS_BUILD_ID, which is appended to initrd-release and + # os-release-initrd *inside* the archive. Collation drift would reorder the + # NEVRA list, change the hash, and so change the archive contents for an + # otherwise unchanged package set. + INITRAMFS_PKG_NEVRA=$(rpm -qa --queryformat '%{{NEVRA}}\n' --root "$INITRAMFS_SYSROOT" | LC_ALL=C sort) INITRAMFS_PKG_HASH=$(echo "$INITRAMFS_PKG_NEVRA" | sha256sum | awk '{{print $1}}') INITRAMFS_BUILD_ID=$(python3 -c "import uuid; print(uuid.uuid5(uuid.UUID('{namespace_uuid}'), '$INITRAMFS_PKG_HASH'))") @@ -99,22 +105,54 @@ if [ -d "$INITRAMFS_SYSROOT/usr" ]; then echo "AVOCADO_OS_BUILD_ID=$INITRAMFS_BUILD_ID" >> "$INITRAMFS_WORK/usr/lib/os-release-initrd" fi - # Build initramfs image using configured filesystem format + # Normalize mtimes across the staged tree so the cpio is reproducible. + # + # `cpio --reproducible` is only --ignore-devno --ignore-dirnlink + # --renumber-inodes; it passes mtime straight through from the + # filesystem. Most file mtimes come from RPM payloads and are already + # stable, but everything *created* during install is stamped with + # wall-clock time: every directory, the `usr/lib/opkg/alternatives/*` + # links, depmod's `modules.*` output, systemd preset `*.wants/*` links, + # plus the usrmerge symlinks and the release files this script appends + # to above. A runtime build and a standalone build install at different + # times, so those mtimes differ and the same package set produces a + # different cpio — which kos_boot then reports as initramfs drift. + # + # -h stamps symlinks themselves instead of dereferencing to their + # targets. touch on an existing entry doesn't perturb its parent + # directory's mtime, so a single unordered pass is sufficient. + echo "Normalizing initramfs mtimes to SOURCE_DATE_EPOCH=${{SOURCE_DATE_EPOCH:-0}}" + find "$INITRAMFS_WORK" -print0 \ + | xargs -0r touch -h -d "@${{SOURCE_DATE_EPOCH:-0}}" + + # Build initramfs image using configured filesystem format. + # + # Reproducibility notes for the pipeline below: + # * `LC_ALL=C sort` — entry order is archive order (and, with + # --renumber-inodes, decides the inode numbers), so collation must not + # drift. Today's SDK ships only the C/POSIX locales, which makes this a + # no-op, but it stops the archive from changing if the image ever gains + # real locales or the CLI starts forwarding the host's LC_* vars. + # * `gzip -n` — belt-and-braces. gzip already writes MTIME=0 and no FNAME + # when it reads stdin (there is no input file to take them from), so + # this only matters if the pipeline is ever refactored to compress a + # file in place. + # * zstd/lz4 embed no timestamp, and both run single-threaded here. INITRAMFS_FS="{initramfs_filesystem}" INITRAMFS_OUTPUT="$OUTPUT_DIR/avocado-image-initramfs-$TARGET_ARCH.$INITRAMFS_FS" echo "Building initramfs image: $INITRAMFS_FS" case "$INITRAMFS_FS" in cpio) - (cd "$INITRAMFS_WORK" && find . | sort | cpio --reproducible -o -H newc --quiet > "$INITRAMFS_OUTPUT") + (cd "$INITRAMFS_WORK" && find . | LC_ALL=C sort | cpio --reproducible -o -H newc --quiet > "$INITRAMFS_OUTPUT") ;; cpio.zst) - (cd "$INITRAMFS_WORK" && find . | sort | cpio --reproducible -o -H newc --quiet | zstd -3 -f -o "$INITRAMFS_OUTPUT") + (cd "$INITRAMFS_WORK" && find . | LC_ALL=C sort | cpio --reproducible -o -H newc --quiet | zstd -3 -f -o "$INITRAMFS_OUTPUT") ;; cpio.lz4) - (cd "$INITRAMFS_WORK" && find . | sort | cpio --reproducible -o -H newc --quiet | lz4 -l -f - "$INITRAMFS_OUTPUT") + (cd "$INITRAMFS_WORK" && find . | LC_ALL=C sort | cpio --reproducible -o -H newc --quiet | lz4 -l -f - "$INITRAMFS_OUTPUT") ;; cpio.gz) - (cd "$INITRAMFS_WORK" && find . | sort | cpio --reproducible -o -H newc --quiet | gzip -9 > "$INITRAMFS_OUTPUT") + (cd "$INITRAMFS_WORK" && find . | LC_ALL=C sort | cpio --reproducible -o -H newc --quiet | gzip -9 -n > "$INITRAMFS_OUTPUT") ;; *) echo "ERROR: unsupported initramfs filesystem format: $INITRAMFS_FS" @@ -416,3 +454,108 @@ export AVOCADO_OS_VERSION_ID Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + + /// The staged tree's mtimes must be normalized before the cpio is created. + /// + /// Regression: `cpio --reproducible` only covers devno / dirnlink / inode + /// renumbering — it passes mtime through from the filesystem. Entries created + /// during install (directories, `opkg/alternatives` links, depmod output, + /// systemd preset links, the usrmerge symlinks) carry wall-clock mtimes, so a + /// runtime build and a standalone build of the same package set produced + /// byte-different cpios and kos_boot reported initramfs drift. + #[test] + fn test_initramfs_mtimes_normalized_before_cpio() { + let script = generate_initramfs_build_script( + "7488fa35-6390-425b-bbbf-b156cfe1eed2", + "cpio.zst", + None, + "", + ); + + // Normalization is emitted, pinned to SOURCE_DATE_EPOCH (default 0 so it + // is stable when the caller doesn't set one). + assert!(script.contains(r#"touch -h -d "@${SOURCE_DATE_EPOCH:-0}""#)); + // -h so symlinks are stamped rather than their targets. + assert!(script.contains("touch -h")); + + // It must run BEFORE the archive is created, or it normalizes nothing. + let touch_at = script.find("touch -h").expect("normalization step present"); + let cpio_at = script + .find("find . | LC_ALL=C sort | cpio --reproducible") + .expect("cpio step present"); + assert!( + touch_at < cpio_at, + "mtime normalization must precede cpio creation" + ); + + // And after the release-file injection, so those appends are covered too. + let inject_at = script + .find("AVOCADO_OS_BUILD_ID=$INITRAMFS_BUILD_ID") + .expect("build-id injection present"); + assert!( + inject_at < touch_at, + "normalization must follow the release-file appends" + ); + } + + /// Archive order is byte order, not locale collation. Entry order *is* + /// archive order, and with `--renumber-inodes` it also decides the inode + /// numbers, so a collation change would rewrite the whole archive. + #[test] + fn test_cpio_entry_order_is_locale_independent() { + for fs in ["cpio", "cpio.zst", "cpio.lz4", "cpio.gz"] { + let script = generate_initramfs_build_script("ns", fs, None, ""); + assert!( + script.contains("find . | LC_ALL=C sort | cpio --reproducible"), + "{fs}: sort must be pinned to the C locale" + ); + assert!( + !script.contains("find . | sort |"), + "{fs}: unpinned sort left in the pipeline" + ); + } + } + + /// The build-ID hash is derived from a sorted NEVRA list, and the resulting + /// $INITRAMFS_BUILD_ID is appended to release files *inside* the archive — + /// so that sort needs the same C-locale pin as the cpio pipeline. Pinning + /// only the cpio sort would leave the archive contents locale-sensitive. + #[test] + fn test_build_id_nevra_sort_is_locale_independent() { + let script = generate_initramfs_build_script("ns", "cpio.zst", None, ""); + assert!( + script.contains(r#"--root "$INITRAMFS_SYSROOT" | LC_ALL=C sort)"#), + "the NEVRA sort feeding INITRAMFS_BUILD_ID must be pinned to LC_ALL=C" + ); + // No unpinned `| sort` anywhere in the generated script. + assert!( + !script.contains("| sort)") && !script.contains("| sort |"), + "an unpinned sort remains in the generated initramfs script" + ); + } + + /// gzip already omits MTIME/FNAME when reading stdin, but `-n` keeps that + /// true if the pipeline is ever changed to compress a file in place. + #[test] + fn test_gzip_omits_timestamp() { + let script = generate_initramfs_build_script("ns", "cpio.gz", None, ""); + assert!(script.contains("gzip -9 -n")); + } + + /// Every compressed variant goes through the same normalized tree. + #[test] + fn test_all_cpio_formats_get_normalized_tree() { + for fs in ["cpio", "cpio.zst", "cpio.lz4", "cpio.gz"] { + let script = generate_initramfs_build_script("ns", fs, None, ""); + let touch_at = script.find("touch -h").expect("normalization present"); + let cpio_at = script + .find("find . | LC_ALL=C sort | cpio --reproducible") + .expect("cpio present"); + assert!(touch_at < cpio_at, "{fs}: normalization must precede cpio"); + } + } +} diff --git a/src/commands/rootfs/image.rs b/src/commands/rootfs/image.rs index bbe9f6a0..efa404eb 100644 --- a/src/commands/rootfs/image.rs +++ b/src/commands/rootfs/image.rs @@ -157,7 +157,10 @@ if [ -d "$ROOTFS_SYSROOT/usr" ]; then {post_install_block} # Compute deterministic AVOCADO_OS_BUILD_ID from installed packages - PKG_NEVRA=$(rpm --dbpath /var/lib/rpm -qa --queryformat '%{{NEVRA}}\n' --root "$ROOTFS_SYSROOT" | sort) + # LC_ALL=C so NEVRA ordering — and therefore this hash and the + # AVOCADO_OS_BUILD_ID injected into os-release inside the image — can't + # shift with the container's collation. + PKG_NEVRA=$(rpm --dbpath /var/lib/rpm -qa --queryformat '%{{NEVRA}}\n' --root "$ROOTFS_SYSROOT" | LC_ALL=C sort) PKG_HASH=$(echo "$PKG_NEVRA" | sha256sum | awk '{{print $1}}') OS_BUILD_ID=$(python3 -c "import uuid; print(uuid.uuid5(uuid.UUID('{namespace_uuid}'), '$PKG_HASH'))") diff --git a/src/commands/rootfs/install.rs b/src/commands/rootfs/install.rs index 56639d3b..5608a5a1 100644 --- a/src/commands/rootfs/install.rs +++ b/src/commands/rootfs/install.rs @@ -210,11 +210,14 @@ fi # *-{kver}.gz that isn't a known non-bootable kernel-base artifact qualifies. # (The RPM sysroot db is not queryable here — packages installed via dnf # --installroot are not tracked in the sysroot's own rpmdb.) +# LC_ALL=C: this order is not cosmetic — the 'Image' symlink block below walks +# boot_files and takes the FIRST uncompressed variant (falling back to [0]), so +# collation decides which kernel binary 'Image' ends up pointing at. mapfile -t boot_files < <( find "$ROOTFS_ROOT/boot" -maxdepth 1 -type f \ \( -name "*-{kver}" -o -name "*-{kver}.gz" \) \ ! -name "System.map-*" ! -name "config-*" \ - | sort + | LC_ALL=C sort ) if [ "${{#boot_files[@]}}" -eq 0 ]; then echo "[ERROR] No bootable kernel image found for {kver} in rootfs /boot" >&2 diff --git a/src/commands/sdk/install.rs b/src/commands/sdk/install.rs index 190e5bae..2c88d066 100644 --- a/src/commands/sdk/install.rs +++ b/src/commands/sdk/install.rs @@ -1312,7 +1312,9 @@ load_presets() { fi for dir in "${preset_dirs[@]}"; do [ -d "$dir" ] || continue - for f in $(ls "$dir"/*.preset 2>/dev/null | sort); do + # LC_ALL=C to match systemd's own C-collation ordering: preset rules are + # first-match-wins, so this order decides which units end up enabled. + for f in $(ls "$dir"/*.preset 2>/dev/null | LC_ALL=C sort); do while IFS= read -r line || [[ -n "$line" ]]; do line="${line%%#*}" line="${line#"${line%%[![:space:]]*}"}" diff --git a/src/utils/kernel_resolver.rs b/src/utils/kernel_resolver.rs index 3614d4c0..a1e3cf0d 100644 --- a/src/utils/kernel_resolver.rs +++ b/src/utils/kernel_resolver.rs @@ -254,7 +254,7 @@ set -eo pipefail { $DNF_SDK_HOST $DNF_SDK_TARGET_REPO_CONF repoquery --whatprovides 'avocado-kernel-*' --provides 2>/dev/null || true $DNF_SDK_HOST $DNF_SDK_TARGET_REPO_CONF repoquery 'kernel-*' --qf '%{NAME}\n' 2>/dev/null || true -} | sort -u +} | LC_ALL=C sort -u "# .to_string(); diff --git a/src/utils/stamps.rs b/src/utils/stamps.rs index 8bf09b03..ce6db0b7 100644 --- a/src/utils/stamps.rs +++ b/src/utils/stamps.rs @@ -1731,7 +1731,7 @@ pub fn validate_stamps_parsed( #[allow(unused)] pub fn generate_package_hash_script(installroot: &str) -> String { format!( - r#"rpm --root={installroot} -qa --queryformat '%{{NAME}}-%{{VERSION}}-%{{RELEASE}}\n' 2>/dev/null | sort | sha256sum | cut -d' ' -f1"# + r#"rpm --root={installroot} -qa --queryformat '%{{NAME}}-%{{VERSION}}-%{{RELEASE}}\n' 2>/dev/null | LC_ALL=C sort | sha256sum | cut -d' ' -f1"# ) }