From 6c7a4468622eb27ab208d5c3cf06c88e71053927 Mon Sep 17 00:00:00 2001 From: Justin Schneck Date: Thu, 13 Aug 2026 10:26:57 -0400 Subject: [PATCH 1/3] purge package-manager state from rootfs and initramfs images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dnf installs into the shared sysroots leave package-manager bookkeeping behind, and `cp -a` carries it into the work copy that gets imaged. Measured on a qemux86-64 build: rootfs (138M): 2.0M var/lib/rpm, 6.4M var/cache/dnf initramfs (123M): 2.5M var/lib/rpm, 4.3M var/lib/dnf, 6.4M var/cache/dnf Two problems. It is dead weight — there is no runtime package manager on target, so nothing ever reads it. And it is the reason the images are not reproducible: the rpmdb records INSTALLTIME and INSTALLTID per package (240 of them in the initramfs), var/lib/dnf/history.sqlite records the transaction, and var/cache/dnf holds generated repodata plus solvfiles. While any of that is in the tree, the same package set produces different image bytes on every install, no matter how the archive step is invoked. Remove all three from the work copy after post_install, so state left by a hook's own dnf call is caught too. The pristine sysroots keep their rpmdb: the AVOCADO_OS_BUILD_ID queries read $ROOTFS_SYSROOT / $INITRAMFS_SYSROOT, and the installroot seeding in `ext install`, `ext dnf`, `runtime install` and `runtime dnf` copies from $AVOCADO_PREFIX/rootfs. None of those touch the work copy. --- src/commands/initramfs/image.rs | 48 ++++++++++++++++++++++++++++ src/commands/rootfs/image.rs | 55 +++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/src/commands/initramfs/image.rs b/src/commands/initramfs/image.rs index 9da07676..b45dc95c 100644 --- a/src/commands/initramfs/image.rs +++ b/src/commands/initramfs/image.rs @@ -113,6 +113,17 @@ if [ -d "$INITRAMFS_SYSROOT/usr" ]; then echo "AVOCADO_OS_BUILD_ID=$INITRAMFS_BUILD_ID" >> "$INITRAMFS_WORK/usr/lib/os-release-initrd" fi + # Purge package-manager bookkeeping from the work copy before archiving. + # Same reasoning as the rootfs image — see the comment in + # `generate_rootfs_build_script`. Measured 14MB of a 123MB qemux86-64 + # initramfs (2.5M rpmdb, 4.3M var/lib/dnf, 6.4M var/cache/dnf), all of it + # unreproducible and none of it read by anything in an initrd. + # + # Before the mtime normalization below on purpose: the purge restamps the + # directories it empties, and the mtime pass is what makes that not matter. + echo "Purging package-manager state from initramfs image" + rm -rf "$INITRAMFS_WORK/var/lib/rpm" "$INITRAMFS_WORK/var/lib/dnf" "$INITRAMFS_WORK/var/cache/dnf" + # Normalize mtimes across the staged tree so the cpio is reproducible. # # `cpio --reproducible` is only --ignore-devno --ignore-dirnlink @@ -516,6 +527,43 @@ mod tests { ); } + /// Regression: the staged initramfs carried 14MB of dnf/rpm state (2.5M + /// rpmdb, 4.3M var/lib/dnf, 6.4M var/cache/dnf) into the cpio. Nothing in + /// an initrd reads it, and it made the archive unreproducible — the rpmdb + /// records INSTALLTIME/INSTALLTID per package and var/cache/dnf holds + /// generated repodata, so the same package set produced different bytes. + #[test] + fn test_initramfs_purges_package_manager_state_before_archiving() { + let script = generate_initramfs_build_script( + "00000000-0000-0000-0000-000000000000", + "cpio.zst", + None, + "", + ); + + for path in ["var/lib/rpm", "var/lib/dnf", "var/cache/dnf"] { + assert!( + script.contains(&format!("\"$INITRAMFS_WORK/{path}\"")), + "{path} must be purged from the work copy before archiving" + ); + } + + // The purge only helps if it runs before the archive is created. + let purge_at = script + .find("Purging package-manager state") + .expect("purge step present"); + let cpio_at = script + .find("cpio --reproducible") + .expect("cpio step present"); + assert!(purge_at < cpio_at, "purge must precede cpio creation"); + + // The shared sysroot keeps its rpmdb — the build ID query reads it. + assert!( + !script.contains("$INITRAMFS_SYSROOT/var/lib/rpm"), + "the shared sysroot's rpmdb must not be removed" + ); + } + /// Archive order is byte order, not locale collation. Entry order *is* /// archive order, and with `--renumber-inodes` it also decides the inode /// numbers, so a collation change would rewrite the whole archive. diff --git a/src/commands/rootfs/image.rs b/src/commands/rootfs/image.rs index 911b23b4..4b4a105b 100644 --- a/src/commands/rootfs/image.rs +++ b/src/commands/rootfs/image.rs @@ -211,6 +211,25 @@ if [ -d "$ROOTFS_SYSROOT/usr" ]; then sed -i '/^AVOCADO_OS_BUILD_ID=/d' "$ROOTFS_SYSROOT/usr/lib/os-release" echo "AVOCADO_OS_BUILD_ID=$OS_BUILD_ID" >> "$ROOTFS_SYSROOT/usr/lib/os-release" + # Purge package-manager bookkeeping from the work copy before imaging. + # + # dnf installs into the sysroot leave ~13MB of state behind (measured on a + # qemux86-64 rootfs: 2.0M rpmdb, 6.4M var/cache/dnf). Nothing on target + # consumes it — there is no runtime package manager — and it is what keeps + # the image from being reproducible: the rpmdb records INSTALLTIME and + # INSTALLTID per package, var/lib/dnf/history.sqlite records the + # transaction, and var/cache/dnf holds generated repodata plus solvfiles. + # While those are in the tree, two installs of the same package set never + # produce identical image bytes. + # + # Runs after post_install so state left by a hook's own dnf call is caught + # too. Safe for identity and for extension priming: the build ID above + # queries $ROOTFS_SYSROOT, and the installroot seeding in `ext install` / + # `runtime install` copies from $AVOCADO_PREFIX/rootfs — all the pristine + # sysroot, never this work copy. + echo "Purging package-manager state from rootfs image" + rm -rf "$ROOTFS_WORK/var/lib/rpm" "$ROOTFS_WORK/var/lib/dnf" "$ROOTFS_WORK/var/cache/dnf" + # Build rootfs image using configured filesystem format ROOTFS_FS="{rootfs_filesystem}" ROOTFS_OUTPUT="$OUTPUT_DIR/avocado-image-rootfs-$TARGET_ARCH.$ROOTFS_FS" @@ -629,6 +648,42 @@ mod tests { ); } + #[test] + fn test_rootfs_purges_package_manager_state_before_imaging() { + // Regression: ~13MB of dnf/rpm bookkeeping was being imaged into the + // rootfs. Nothing on target consumes it, and it blocked reproducible + // images — the rpmdb stamps INSTALLTIME/INSTALLTID per package and + // var/cache/dnf holds generated repodata, so the same package set + // produced different image bytes on every install. + let script = generate_rootfs_build_script( + "00000000-0000-0000-0000-000000000000", + "erofs-lz4", + None, + "", + ); + + for path in ["var/lib/rpm", "var/lib/dnf", "var/cache/dnf"] { + assert!( + script.contains(&format!("\"$ROOTFS_WORK/{path}\"")), + "{path} must be purged from the work copy before imaging" + ); + } + + // The purge only helps if it runs before the image is built. + let purge_at = script + .find("Purging package-manager state") + .expect("purge step present"); + let mkfs_at = script.find("mkfs.erofs").expect("mkfs step present"); + assert!(purge_at < mkfs_at, "purge must precede mkfs"); + + // The shared sysroot keeps its rpmdb: the build ID reads it, and so + // does the installroot seeding in `ext install` / `runtime install`. + assert!( + !script.contains("$ROOTFS_SYSROOT/var/lib/rpm"), + "the shared sysroot's rpmdb must not be removed" + ); + } + #[test] fn test_rootfs_passwd_guard_precedes_permissions_section() { // The guard only helps if it runs before the user-creation From c2e076dbf73b072e18f3dbc7329f93ccf5ba9527 Mon Sep 17 00:00:00 2001 From: Justin Schneck Date: Thu, 13 Aug 2026 21:27:45 -0400 Subject: [PATCH 2/3] review: purge dnf's logs too, drop a vacuous assertion, scope the claim The purge missed var/log. rootfs/install.rs omits $DNF_SDK_HOST_OPTS, which is what redirects logdir/cachedir/persistdir at the SDK prefix -- so the same omission that puts var/cache/dnf and var/lib/dnf in the sysroot puts dnf.log, dnf.rpm.log and hawkey.log in var/log, every line wall-clock stamped. Both images now remove them, and both tests assert all six paths; deleting the new `rm -f` fails them. Dropped the `!contains("$ROOTFS_SYSROOT/var/lib/rpm")` guard from both tests. That string has no occurrences to begin with, so the assertion could not fail, and the build-ID query it claimed to protect is spelled `--root "$ROOTFS_SYSROOT"` with `--dbpath`. A test that cannot fail is worse than no test: it reads as coverage. Scoped the reproducibility claim to what it earns. Removing this state is necessary but not sufficient -- `cpio --reproducible` is --device-independent on GNU cpio 2.15 and does not normalize mtime, and the removal itself restamps the directories it empties. That half is reproducible here. Adds the CHANGELOG entry, including the one consumer this can surprise: anything running `rpm -qa` against a built image rather than the sysroot. --- CHANGELOG.md | 14 ++++++++++++++ src/commands/initramfs/image.rs | 15 ++++++++------- src/commands/rootfs/image.rs | 28 ++++++++++++++++++++-------- 3 files changed, 42 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bff6df9..5a726a68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,6 +98,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 document they can read first. ### Changed +- **Rootfs and initramfs images no longer ship package-manager state.** The + rpmdb, `var/lib/dnf` and `var/cache/dnf` are removed from the staged copy + before the image is built — measured at ~13MB of a qemux86-64 rootfs and + 14MB of a 123MB initramfs. Nothing on target reads any of it; these systems + have no runtime package manager. Only the staged copy is touched, so the + shared sysroot keeps its database and `ext install` / `runtime install` + still seed their installroots from it. (dnf's own logs never land in the + staged copy at all — dnf does not prefix `logdir` with the installroot — + so they need no removal; they live under the SDK prefix.) + + Anything inspecting a built image for installed packages — `rpm -qa` against + a loop-mounted rootfs, say — needs to query the sysroot or the lockfile + instead. This is a necessary step toward reproducible images but not a + sufficient one on its own; archive mtime normalization is separate. - **Extension images no longer ship package-manager state.** `var/lib/rpm`, `var/lib/dnf` and `var/cache/dnf` are excluded from the built `.raw` — measured at 13.4M of a 22M `avocado-ext-tunnels` sysroot, against an 8.2M diff --git a/src/commands/initramfs/image.rs b/src/commands/initramfs/image.rs index b45dc95c..635ccdcd 100644 --- a/src/commands/initramfs/image.rs +++ b/src/commands/initramfs/image.rs @@ -541,7 +541,14 @@ mod tests { "", ); - for path in ["var/lib/rpm", "var/lib/dnf", "var/cache/dnf"] { + for path in [ + "var/lib/rpm", + "var/lib/dnf", + "var/cache/dnf", + "var/log/dnf.log", + "var/log/dnf.rpm.log", + "var/log/hawkey.log", + ] { assert!( script.contains(&format!("\"$INITRAMFS_WORK/{path}\"")), "{path} must be purged from the work copy before archiving" @@ -556,12 +563,6 @@ mod tests { .find("cpio --reproducible") .expect("cpio step present"); assert!(purge_at < cpio_at, "purge must precede cpio creation"); - - // The shared sysroot keeps its rpmdb — the build ID query reads it. - assert!( - !script.contains("$INITRAMFS_SYSROOT/var/lib/rpm"), - "the shared sysroot's rpmdb must not be removed" - ); } /// Archive order is byte order, not locale collation. Entry order *is* diff --git a/src/commands/rootfs/image.rs b/src/commands/rootfs/image.rs index 4b4a105b..52074ed2 100644 --- a/src/commands/rootfs/image.rs +++ b/src/commands/rootfs/image.rs @@ -222,6 +222,17 @@ if [ -d "$ROOTFS_SYSROOT/usr" ]; then # While those are in the tree, two installs of the same package set never # produce identical image bytes. # + # dnf's own logs go with it. The rootfs install omits $DNF_SDK_HOST_OPTS, + # which is what redirects logdir/cachedir/persistdir to the SDK prefix — so + # the same omission that lands var/cache/dnf and var/lib/dnf in the sysroot + # also lands dnf.log, dnf.rpm.log and hawkey.log in var/log, each line + # stamped with a wall-clock time. + # + # Removing all of this is necessary for a reproducible image but not + # sufficient: the archive's own mtime handling is a separate problem, and + # the removal itself restamps the directories it empties. That is #199's + # half, not this one's. + # # Runs after post_install so state left by a hook's own dnf call is caught # too. Safe for identity and for extension priming: the build ID above # queries $ROOTFS_SYSROOT, and the installroot seeding in `ext install` / @@ -229,6 +240,7 @@ if [ -d "$ROOTFS_SYSROOT/usr" ]; then # sysroot, never this work copy. echo "Purging package-manager state from rootfs image" rm -rf "$ROOTFS_WORK/var/lib/rpm" "$ROOTFS_WORK/var/lib/dnf" "$ROOTFS_WORK/var/cache/dnf" + rm -f "$ROOTFS_WORK/var/log/dnf.log" "$ROOTFS_WORK/var/log/dnf.rpm.log" "$ROOTFS_WORK/var/log/hawkey.log" # Build rootfs image using configured filesystem format ROOTFS_FS="{rootfs_filesystem}" @@ -662,7 +674,14 @@ mod tests { "", ); - for path in ["var/lib/rpm", "var/lib/dnf", "var/cache/dnf"] { + for path in [ + "var/lib/rpm", + "var/lib/dnf", + "var/cache/dnf", + "var/log/dnf.log", + "var/log/dnf.rpm.log", + "var/log/hawkey.log", + ] { assert!( script.contains(&format!("\"$ROOTFS_WORK/{path}\"")), "{path} must be purged from the work copy before imaging" @@ -675,13 +694,6 @@ mod tests { .expect("purge step present"); let mkfs_at = script.find("mkfs.erofs").expect("mkfs step present"); assert!(purge_at < mkfs_at, "purge must precede mkfs"); - - // The shared sysroot keeps its rpmdb: the build ID reads it, and so - // does the installroot seeding in `ext install` / `runtime install`. - assert!( - !script.contains("$ROOTFS_SYSROOT/var/lib/rpm"), - "the shared sysroot's rpmdb must not be removed" - ); } #[test] From 253fe089fdd7708bd857883f72c6303ef3965913 Mon Sep 17 00:00:00 2001 From: Jon Schneck Date: Tue, 25 Aug 2026 08:10:38 -0400 Subject: [PATCH 3/3] review: drop the no-op dnf log purge, pin its absence jetm verified against dnf's source and six real build volumes: dnf only prepends the installroot to cachedir and persistdir, never logdir, so the three log operands matched nothing in any staged tree - the rm exited 0 having deleted nothing while the build claimed a purge. The comments stated that wrong mechanism at both sites and the changelog documented a removal that never happened. The log lines are gone rather than kept as insurance: an unconditional dead purge whose tests only string-match the script reads as coverage and provides none. Both tests now pin the absence of the log operands, so the wrong mechanism cannot quietly return. The rpmdb/cache half - the real ~13-14MB - is unchanged. --- src/commands/initramfs/image.rs | 25 ++++++++++++++----------- src/commands/rootfs/image.rs | 29 +++++++++++++++-------------- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/src/commands/initramfs/image.rs b/src/commands/initramfs/image.rs index 635ccdcd..21794cf7 100644 --- a/src/commands/initramfs/image.rs +++ b/src/commands/initramfs/image.rs @@ -115,9 +115,11 @@ if [ -d "$INITRAMFS_SYSROOT/usr" ]; then # Purge package-manager bookkeeping from the work copy before archiving. # Same reasoning as the rootfs image — see the comment in - # `generate_rootfs_build_script`. Measured 14MB of a 123MB qemux86-64 - # initramfs (2.5M rpmdb, 4.3M var/lib/dnf, 6.4M var/cache/dnf), all of it - # unreproducible and none of it read by anything in an initrd. + # `generate_rootfs_build_script`, including why dnf's logs are NOT on the + # list (dnf never writes them into an installroot) and why this is + # necessary but not sufficient for reproducibility. Measured 14MB of a + # 123MB qemux86-64 initramfs (2.5M rpmdb, 4.3M var/lib/dnf, 6.4M + # var/cache/dnf), none of it read by anything in an initrd. # # Before the mtime normalization below on purpose: the purge restamps the # directories it empties, and the mtime pass is what makes that not matter. @@ -541,14 +543,7 @@ mod tests { "", ); - for path in [ - "var/lib/rpm", - "var/lib/dnf", - "var/cache/dnf", - "var/log/dnf.log", - "var/log/dnf.rpm.log", - "var/log/hawkey.log", - ] { + for path in ["var/lib/rpm", "var/lib/dnf", "var/cache/dnf"] { assert!( script.contains(&format!("\"$INITRAMFS_WORK/{path}\"")), "{path} must be purged from the work copy before archiving" @@ -563,6 +558,14 @@ mod tests { .find("cpio --reproducible") .expect("cpio step present"); assert!(purge_at < cpio_at, "purge must precede cpio creation"); + + // dnf never writes its logs into an installroot (logdir is not + // prefixed by prepend_installroot), so a log purge here would be a + // no-op that reads as coverage. Pin its absence. + assert!( + !script.contains("var/log/dnf.log"), + "the script must not claim to purge dnf logs that are never staged" + ); } /// Archive order is byte order, not locale collation. Entry order *is* diff --git a/src/commands/rootfs/image.rs b/src/commands/rootfs/image.rs index 52074ed2..343985ee 100644 --- a/src/commands/rootfs/image.rs +++ b/src/commands/rootfs/image.rs @@ -222,11 +222,12 @@ if [ -d "$ROOTFS_SYSROOT/usr" ]; then # While those are in the tree, two installs of the same package set never # produce identical image bytes. # - # dnf's own logs go with it. The rootfs install omits $DNF_SDK_HOST_OPTS, - # which is what redirects logdir/cachedir/persistdir to the SDK prefix — so - # the same omission that lands var/cache/dnf and var/lib/dnf in the sysroot - # also lands dnf.log, dnf.rpm.log and hawkey.log in var/log, each line - # stamped with a wall-clock time. + # dnf's logs are deliberately NOT on this list. dnf only prepends the + # installroot to cachedir and persistdir (dnf/cli/cli.py, the + # prepend_installroot loop); logdir is never installroot-relative, so the + # logs land in the SDK prefix and there has never been one in the staged + # tree to remove — verified across real build volumes, where var/log + # resolves to an empty volatile/log in every image. # # Removing all of this is necessary for a reproducible image but not # sufficient: the archive's own mtime handling is a separate problem, and @@ -240,7 +241,6 @@ if [ -d "$ROOTFS_SYSROOT/usr" ]; then # sysroot, never this work copy. echo "Purging package-manager state from rootfs image" rm -rf "$ROOTFS_WORK/var/lib/rpm" "$ROOTFS_WORK/var/lib/dnf" "$ROOTFS_WORK/var/cache/dnf" - rm -f "$ROOTFS_WORK/var/log/dnf.log" "$ROOTFS_WORK/var/log/dnf.rpm.log" "$ROOTFS_WORK/var/log/hawkey.log" # Build rootfs image using configured filesystem format ROOTFS_FS="{rootfs_filesystem}" @@ -674,14 +674,7 @@ mod tests { "", ); - for path in [ - "var/lib/rpm", - "var/lib/dnf", - "var/cache/dnf", - "var/log/dnf.log", - "var/log/dnf.rpm.log", - "var/log/hawkey.log", - ] { + for path in ["var/lib/rpm", "var/lib/dnf", "var/cache/dnf"] { assert!( script.contains(&format!("\"$ROOTFS_WORK/{path}\"")), "{path} must be purged from the work copy before imaging" @@ -694,6 +687,14 @@ mod tests { .expect("purge step present"); let mkfs_at = script.find("mkfs.erofs").expect("mkfs step present"); assert!(purge_at < mkfs_at, "purge must precede mkfs"); + + // dnf never writes its logs into an installroot (logdir is not + // prefixed by prepend_installroot), so a log purge here would be a + // no-op that reads as coverage. Pin its absence. + assert!( + !script.contains("var/log/dnf.log"), + "the script must not claim to purge dnf logs that are never staged" + ); } #[test]