From 572d6ee8f4f2a5f0f5d571617c8c7f2f1d452525 Mon Sep 17 00:00:00 2001 From: Justin Schneck Date: Thu, 13 Aug 2026 11:19:27 -0400 Subject: [PATCH 1/2] exclude package-manager state from extension images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verified in shipped images, not just in the sysroot: `rpmdb.sqlite` and the SQLite file magic both grep out of config-dev-0.1.0.raw and avocado-ext-tunnels-2024.1.0.raw. No excludes were being applied, because --exclude-path is only emitted for user-configured var_files patterns and get_ext_var_files returns an empty vec when the key is absent. Measured on a qemux86-64 build, avocado-ext-tunnels: 13.4M of package-manager state (2.8M var/lib/rpm, 4.2M var/lib/dnf, 6.4M var/cache/dnf) in a 22M sysroot whose actual /usr payload is 8.2M. config-dev-0.1.0.raw and avocado-bsp-qemux86-64-2024.1.0.raw are both exactly 868352 bytes because both are dominated by the same seeded rpmdb rather than their own content. Every extension carries a floor of it: `ext install` and `ext dnf` seed each installroot with `cp -rf $AVOCADO_PREFIX/rootfs/var/lib/rpm` so dependencies resolve against what the rootfs already provides, and nothing removes it before the sysroot becomes an image. None of it is readable on target — systemd-sysext/confext merge /usr, /opt and /etc, never /var — and it is what stops the images being reproducible across a reinstall, since the rpmdb stamps INSTALLTIME/INSTALLTID per package and var/cache/dnf holds generated repodata and solvfiles. Excluded at image time rather than deleted: `ext image` runs mkfs directly against the live sysroot, with no work copy, and later `ext dnf` / `ext install` calls still resolve against that rpmdb. --- src/commands/ext/image.rs | 135 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 129 insertions(+), 6 deletions(-) diff --git a/src/commands/ext/image.rs b/src/commands/ext/image.rs index 8982d2b7..fd420adb 100644 --- a/src/commands/ext/image.rs +++ b/src/commands/ext/image.rs @@ -868,6 +868,29 @@ impl ExtImageCommand { }) .collect::>(); + // Package-manager bookkeeping, always excluded, ahead of the var_files + // patterns. dnf leaves this in every extension installroot, and `ext + // install` / `ext dnf` additionally seed var/lib/rpm from the rootfs so + // dependencies resolve against what the rootfs already provides. On a + // qemux86-64 build that is 13.4M of a 22M sysroot for + // avocado-ext-tunnels, against an 8.2M /usr payload. + // + // Nothing on target can read it: systemd-sysext/confext merge /usr, + // /opt and /etc, never /var. And it is what stops the images being + // reproducible across a reinstall — the rpmdb stamps INSTALLTIME and + // INSTALLTID per package, var/lib/dnf/history.sqlite records the + // transaction, and var/cache/dnf holds generated repodata and solvfiles. + // + // Excluded at image time rather than deleted, because `ext image` runs + // mkfs directly against the live sysroot (no work copy, unlike the + // rootfs and initramfs paths) and later `ext dnf` / `ext install` calls + // still resolve against that rpmdb. + let excludes: Vec = ["var/lib/rpm", "var/lib/dnf", "var/cache/dnf"] + .iter() + .map(|s| (*s).to_string()) + .chain(var_excludes) + .collect(); + let mkfs_command = match filesystem { "erofs" | "erofs-lz4" | "erofs-zst" => { let compress_flag = match filesystem { @@ -875,7 +898,7 @@ impl ExtImageCommand { "erofs-zst" => "\n -z zstd \\", _ => "", }; - let exclude_flags = var_excludes + let exclude_flags = excludes .iter() .map(|p| format!(" --exclude-path={p} \\")) .collect::>() @@ -897,7 +920,7 @@ mkfs.erofs \ ) } _ => { - let exclude_flags = var_excludes + let exclude_flags = excludes .iter() .map(|p| format!(" -e \"{p}\"")) .collect::>() @@ -1129,6 +1152,101 @@ mod tests { ); } + /// Package-manager state must never reach an extension image. + /// + /// Regression: verified present in shipped images — `rpmdb.sqlite` and the + /// SQLite file magic both grep out of `config-dev-0.1.0.raw` and + /// `avocado-ext-tunnels-2024.1.0.raw`. It was 13.4M of a 22M sysroot against + /// an 8.2M /usr payload, unreadable on target (sysext/confext merge /usr, + /// /opt and /etc, never /var), and unreproducible across a reinstall because + /// the rpmdb stamps INSTALLTIME/INSTALLTID per package. + /// + /// Excluded rather than deleted: `ext image` runs mkfs against the live + /// sysroot, which later `ext dnf` / `ext install` calls resolve against. + #[test] + fn test_pkg_state_excluded_from_erofs_image() { + let cmd = make_cmd("my-ext"); + let script = cmd.create_build_script("1.0.0", "sysext", 0, "erofs", &[], "raw", None); + + for path in ["var/lib/rpm", "var/lib/dnf", "var/cache/dnf"] { + assert!( + script.contains(&format!("--exclude-path={path}")), + "{path} must be excluded from the erofs image" + ); + } + } + + #[test] + fn test_pkg_state_excluded_from_squashfs_image() { + let cmd = make_cmd("my-ext"); + let script = cmd.create_build_script("1.0.0", "sysext", 0, "squashfs", &[], "raw", None); + + for path in ["var/lib/rpm", "var/lib/dnf", "var/cache/dnf"] { + assert!( + script.contains(&format!("-e \"{path}\"")), + "{path} must be excluded from the squashfs image" + ); + } + } + + /// The always-excluded paths are additive — a project's own `var_files` + /// patterns must still be excluded alongside them, not replaced by them. + #[test] + fn test_var_files_excludes_survive_alongside_pkg_state() { + let cmd = make_cmd("my-ext"); + let var_files = vec!["var/lib/myapp/**".to_string()]; + let script = + cmd.create_build_script("1.0.0", "sysext", 0, "erofs", &var_files, "raw", None); + + assert!( + script.contains("--exclude-path=var/lib/myapp"), + "configured var_files patterns must still be excluded" + ); + assert!( + script.contains("--exclude-path=var/lib/rpm"), + "package-manager excludes must not displace var_files excludes" + ); + } + + /// The rest of the erofs reproducibility contract, which nothing pinned. + /// Each of these silently reintroduces per-build variance if dropped: the + /// UUID would be randomized, and ownership would come from whoever ran the + /// build instead of being normalized to root. + #[test] + fn test_erofs_image_reproducibility_flags_are_pinned() { + let cmd = make_cmd("my-ext"); + let script = cmd.create_build_script("1.0.0", "sysext", 0, "erofs", &[], "raw", None); + + assert!( + script.contains("-U 00000000-0000-0000-0000-000000000000"), + "erofs UUID must be pinned, or every build gets a fresh one" + ); + assert!( + script.contains("--all-root"), + "ownership must be normalized to root, not the build user" + ); + assert!( + script.contains("-T \"$SOURCE_DATE_EPOCH\""), + "timestamps must come from SOURCE_DATE_EPOCH" + ); + } + + /// squashfs has no `-T`; `-reproducible` is what makes its output stable. + #[test] + fn test_squashfs_image_is_built_reproducibly() { + let cmd = make_cmd("my-ext"); + let script = cmd.create_build_script("1.0.0", "sysext", 0, "squashfs", &[], "raw", None); + + assert!( + script.contains("-reproducible"), + "mksquashfs must be invoked with -reproducible" + ); + assert!( + script.contains("-no-xattrs"), + "xattrs must be dropped, they carry build-host state" + ); + } + #[test] fn test_create_build_script_source_date_epoch_default() { let cmd = make_cmd("my-ext"); @@ -1220,13 +1338,18 @@ mod tests { } #[test] - fn test_create_build_script_no_var_files_no_excludes() { + fn test_no_var_files_leaves_only_the_pkg_state_excludes() { + // Was `test_create_build_script_no_var_files_no_excludes`, which asserted + // that no var_files meant no excludes at all. The package-manager paths + // are now always excluded, so the surviving guarantee is the narrower + // one: nothing beyond them is excluded when no var_files are configured. let cmd = make_cmd("my-ext"); let script = cmd.create_build_script("1.0.0", "sysext", 0, "squashfs", &[], "raw", None); - assert!( - !script.contains("-e \"var/"), - "script should not contain exclude flags when no var_files" + assert_eq!( + script.matches("-e \"").count(), + 3, + "only the three package-manager paths should be excluded" ); } } From 1b773dad3f6beb8d8e0aaf4d842cef82f6863d00 Mon Sep 17 00:00:00 2001 From: Justin Schneck Date: Thu, 13 Aug 2026 21:35:22 -0400 Subject: [PATCH 2/2] review: delete two duplicate tests, drop dead branches, correct a claim The two reproducibility tests added here asserted what test_create_build_script_erofs_contains_reproducible_flags and test_create_build_script_squashfs_contains_reproducible_flags already assert -- same flags, same script. Their doc comment claimed nothing pinned these, which was simply false. Deleted rather than kept: a test that duplicates its neighbour teaches the next reader that this area is covered here, when the coverage is twenty lines up. The four package-state tests stay; those are the ones that go red when the exclude array is emptied. exclude_section's is_empty() arm is unreachable in both branches now that the three package-state paths are unconditional, so both collapse to a single map+collect. Output is byte-identical -- same leading separator, same joins -- and the existing exclude assertions cover both shapes. Recorded two things at the mksquashfs site that were only in review: the repeated `-e` form works because mksquashfs reads everything after the first `-e` as filenames and silently ignores the ones that do not exist, and `-reproducible` does not normalize ownership the way erofs's --all-root does, so squashfs ext images still vary by build user. That last one wants `-all-root`, which changes image contents and is not this change. Adds the CHANGELOG entry. --- CHANGELOG.md | 10 +++++ src/commands/ext/image.rs | 78 +++++++++++---------------------------- 2 files changed, 31 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57007d78..31a0597c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 second field. ### Changed +- **Extension images no longer ship package-manager state.** `var/lib/rpm`, + `var/lib/dnf` and `var/cache/dnf` are excluded from the built `.raw` — + measured at 13.4M of a 22M `avocado-ext-tunnels` sysroot, against an 8.2M + `/usr` payload. Nothing on target can read it: sysext and confext merge + `/usr`, `/opt` and `/etc`, never `/var`. + + Excluded at image time rather than deleted, because `ext image` runs mkfs + against the live sysroot with no work copy, and later `ext dnf` / `ext + install` calls still resolve against that database. Configured `var_files` + patterns are unaffected. - **A skipped remote version check no longer reports as a passed one.** When the remote's `--version` output cannot be parsed, `--runs-on` used to print `[SUCCESS] Remote avocado version: `, which is a green line diff --git a/src/commands/ext/image.rs b/src/commands/ext/image.rs index fd420adb..302977bb 100644 --- a/src/commands/ext/image.rs +++ b/src/commands/ext/image.rs @@ -898,16 +898,12 @@ impl ExtImageCommand { "erofs-zst" => "\n -z zstd \\", _ => "", }; - let exclude_flags = excludes + // Always non-empty: the package-state paths above are + // unconditional, so there is no no-excludes case to handle. + let exclude_section = excludes .iter() - .map(|p| format!(" --exclude-path={p} \\")) - .collect::>() - .join("\n"); - let exclude_section = if exclude_flags.is_empty() { - String::new() - } else { - format!("\n{exclude_flags}") - }; + .map(|p| format!("\n --exclude-path={p} \\")) + .collect::(); format!( r#"# Create erofs image mkfs.erofs \ @@ -920,16 +916,23 @@ mkfs.erofs \ ) } _ => { - let exclude_flags = excludes + // Always non-empty, as in the erofs arm. + // + // mksquashfs takes everything after the first `-e` as + // filenames, so the later `-e` tokens are read as paths that do + // not exist and ignored. It works, but not for the reason the + // shape suggests. + // + // Note `-reproducible` does NOT normalize ownership the way + // erofs's `--all-root` does -- a live run reports + // `Number of uids 1: `. So squashfs ext images still + // vary by build user. Fixing that means adding `-all-root` + // here, which changes image contents and belongs in its own + // change. + let exclude_section = excludes .iter() - .map(|p| format!(" -e \"{p}\"")) - .collect::>() - .join(" \\\n"); - let exclude_section = if exclude_flags.is_empty() { - String::new() - } else { - format!(" \\\n{exclude_flags}") - }; + .map(|p| format!(" \\\n -e \"{p}\"")) + .collect::(); format!( r#"# Create squashfs image mksquashfs \ @@ -1208,45 +1211,6 @@ mod tests { ); } - /// The rest of the erofs reproducibility contract, which nothing pinned. - /// Each of these silently reintroduces per-build variance if dropped: the - /// UUID would be randomized, and ownership would come from whoever ran the - /// build instead of being normalized to root. - #[test] - fn test_erofs_image_reproducibility_flags_are_pinned() { - let cmd = make_cmd("my-ext"); - let script = cmd.create_build_script("1.0.0", "sysext", 0, "erofs", &[], "raw", None); - - assert!( - script.contains("-U 00000000-0000-0000-0000-000000000000"), - "erofs UUID must be pinned, or every build gets a fresh one" - ); - assert!( - script.contains("--all-root"), - "ownership must be normalized to root, not the build user" - ); - assert!( - script.contains("-T \"$SOURCE_DATE_EPOCH\""), - "timestamps must come from SOURCE_DATE_EPOCH" - ); - } - - /// squashfs has no `-T`; `-reproducible` is what makes its output stable. - #[test] - fn test_squashfs_image_is_built_reproducibly() { - let cmd = make_cmd("my-ext"); - let script = cmd.create_build_script("1.0.0", "sysext", 0, "squashfs", &[], "raw", None); - - assert!( - script.contains("-reproducible"), - "mksquashfs must be invoked with -reproducible" - ); - assert!( - script.contains("-no-xattrs"), - "xattrs must be dropped, they carry build-host state" - ); - } - #[test] fn test_create_build_script_source_date_epoch_default() { let cmd = make_cmd("my-ext");