Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 45 additions & 2 deletions src/commands/initramfs/image.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@ use crate::utils::{
target::resolve_target_required,
};

use crate::commands::rootfs::image::{render_hook_block, resolve_install_hooks, NAMESPACE_UUID};
use crate::commands::rootfs::image::{
render_auth_files_hash, render_hook_block, resolve_install_hooks, NAMESPACE_UUID,
};

/// Default post-install commands for the initramfs build. Same shape as
/// `DEFAULT_ROOTFS_POST_INSTALL` but for `$INITRAMFS_WORK`, plus the
Expand Down Expand Up @@ -95,7 +97,13 @@ if [ -d "$INITRAMFS_SYSROOT/usr" ]; then
# otherwise unchanged package set.
INITRAMFS_PKG_NEVRA=$(rpm -qa --queryformat '%{{NEVRA}}\n' --root "$INITRAMFS_SYSROOT" | LC_ALL=C sort)
INITRAMFS_PKG_HASH=$(echo "$INITRAMFS_PKG_NEVRA" | sha256sum | awk '{{print $1}}')
INITRAMFS_BUILD_ID=$(python3 -c "import uuid; print(uuid.uuid5(uuid.UUID('{namespace_uuid}'), '$INITRAMFS_PKG_HASH'))")

# Fold the assembled auth files into the build id so a `permissions:`-only
# change — which the NEVRA set above is blind to — still moves the id and
# OTAs (ENG-2437). See render_auth_files_hash.
{auth_hash_block}

INITRAMFS_BUILD_ID=$(python3 -c "import uuid; print(uuid.uuid5(uuid.UUID('{namespace_uuid}'), '$INITRAMFS_PKG_HASH:$INITRAMFS_AUTH_HASH'))")

# Inject identity into initrd-release and os-release-initrd
if [ -f "$INITRAMFS_WORK/usr/lib/initrd-release" ]; then
Expand Down Expand Up @@ -172,6 +180,7 @@ fi"#,
initramfs_filesystem = initramfs_filesystem,
post_install_block = post_install_block,
permissions_section = permissions_section,
auth_hash_block = render_auth_files_hash("INITRAMFS_WORK", "INITRAMFS_AUTH_HASH"),
)
}

Expand Down Expand Up @@ -546,6 +555,40 @@ mod tests {
assert!(script.contains("gzip -9 -n"));
}

/// ENG-2437: a `permissions:`-only change must move the initramfs build
/// id too, so the id has to fold the auth-file hash into its uuid5 input —
/// the same way the rootfs build id does.
#[test]
fn test_build_id_folds_auth_files() {
let marker = "# permissions placeholder";
let script = generate_initramfs_build_script("ns", "cpio.zst", None, marker);

assert!(
script.contains("INITRAMFS_AUTH_HASH="),
"build id must incorporate a hash of the auth files"
);
for f in ["passwd", "shadow", "group", "gshadow"] {
assert!(
script.contains(&format!("$INITRAMFS_WORK/etc/{f}")),
"auth hash must cover /etc/{f}"
);
}
assert!(
script.contains("'$INITRAMFS_PKG_HASH:$INITRAMFS_AUTH_HASH'"),
"uuid5 input must combine the package hash and the auth hash"
);

// The auth hash must be computed after the permissions section runs.
let perms_pos = script.find(marker).expect("permissions section present");
let auth_pos = script
.find("INITRAMFS_AUTH_HASH=")
.expect("auth hash present");
assert!(
perms_pos < auth_pos,
"auth hash must be computed after the permissions section runs"
);
}

/// Every compressed variant goes through the same normalized tree.
#[test]
fn test_all_cpio_formats_get_normalized_tree() {
Expand Down
108 changes: 107 additions & 1 deletion src/commands/rootfs/image.rs
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,36 @@ echo \"Applied systemd presets\"; fi",
"echo \"Generated ld.so.cache\"",
];

/// Render the shell snippet that hashes an image's identity/auth files
/// (`/etc/{passwd,shadow,group,gshadow}`) into `out_var`.
///
/// Folded into the build id alongside the package NEVRA hash so a
/// `permissions:`-only change — which rewrites these files but never the
/// rpmdb — still moves the build id and therefore OTAs (ENG-2437). Shared
/// by the rootfs and initramfs build scripts so both derive their build id
/// the same way.
///
/// `work_var` names the image work-dir shell variable (e.g. `ROOTFS_WORK`,
/// `INITRAMFS_WORK`). `LC_ALL=C sort` makes the hash independent of the

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

LC_ALL=C sort can't make this deterministic - the inputs it's sorting are already order-dependent. sort normalizes line order only. It can't normalize the uid/gid values baked into each line, or the member order within one /etc/group line. PermissionsConfig.users/groups are HashMap<String, serde_yaml::Value> (config.rs:840-841), and mapping_from_hashmap (permissions.rs:380) copies them via for (k, v) in map with no sort - Rust's HashMap iteration order is randomized per process. stamps.rs:1283-1292 documents exactly this failure mode for the same reason ("HashMap iteration order varies per process... so the keys have to be sorted here or the hash is unstable between runs") and sorts before hashing; mapping_from_hashmap doesn't.

Ran the real mapping_from_hashmap -> render_users_groups_script path in 4 separate processes with identical input: 3 distinct user-emission orders. For a user declared without an explicit uid: (optional per configs/default.yaml:82-86), that changes the assigned uid/gid, and for two users sharing a supplementary group, it changes the member order within one /etc/group line - both survive the line-level sort and change AUTH_HASH.

Before this PR the build id was PKG_HASH-only and stable across rebuilds. After, a rebuild that changed nothing in the config can OTA the whole fleet, contradicting "deterministic build ID" at line 138. Fix upstream of the hash - sort the keys in mapping_from_hashmap, the same way packages_for_hash already does.

/// order the permissions section appended entries in (the `users:`/`groups:`
/// maps have no guaranteed iteration order); missing files (e.g. a minimal
/// initramfs with no `/etc/shadow`) contribute nothing rather than aborting.
///
/// The `cat` is wrapped in a `{ …; } || true` brace group so a missing file
/// doesn't fail the pipeline: the standalone `avocado rootfs image` /
/// `avocado initramfs image` scripts run under `set -euo pipefail`, where an
/// unguarded `cat` failure would take down the `$(…)` assignment — and with
/// its only diagnostic sent to `/dev/null`, kill the build with no output.
pub fn render_auth_files_hash(work_var: &str, out_var: &str) -> String {
format!(
r#" {out_var}=$({{ cat \
"${work_var}/etc/passwd" \
"${work_var}/etc/shadow" \
"${work_var}/etc/group" \
"${work_var}/etc/gshadow" 2>/dev/null || true; }} | LC_ALL=C sort | sha256sum | awk '{{print $1}}')"#
)
}

/// Render a list of user-supplied shell commands as an indented block,
/// preceded by a one-line "Running … hooks" echo for log clarity. Empty
/// input returns an empty string so the surrounding script stays clean.
Expand Down Expand Up @@ -162,7 +192,13 @@ if [ -d "$ROOTFS_SYSROOT/usr" ]; then
# shift with the container's collation.
PKG_NEVRA=$(rpm --dbpath /var/lib/rpm -qa --queryformat '%{{NEVRA}}\n' --root "$ROOTFS_SYSROOT" | LC_ALL=C sort)
PKG_HASH=$(echo "$PKG_NEVRA" | sha256sum | awk '{{print $1}}')
OS_BUILD_ID=$(python3 -c "import uuid; print(uuid.uuid5(uuid.UUID('{namespace_uuid}'), '$PKG_HASH'))")

# Fold the assembled auth files into the build id so a `permissions:`-only
# change — which the NEVRA set above is blind to — still moves the id and
# OTAs (ENG-2437). See render_auth_files_hash.
{auth_hash_block}

OS_BUILD_ID=$(python3 -c "import uuid; print(uuid.uuid5(uuid.UUID('{namespace_uuid}'), '$PKG_HASH:$AUTH_HASH'))")

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This rotates the build id for every project, even one with no permissions: block at all. The uuid5 input changes unconditionally from '$PKG_HASH' to '$PKG_HASH:$AUTH_HASH', and AUTH_HASH is the digest of the base packages' own passwd/shadow/group - never empty, even with zero permissions: config. The id is appended to os-release before mkfs.erofs runs (line 206 vs 220), so this changes the image bytes, sha256, image_id, spot hashes, and the re-signed AMF for every build across the merge boundary.

First build after this merges, zero config change, zero package change: rootfs + initramfs + os-bundle all re-upload and every device takes a full OS OTA for nothing. Worth a CHANGELOG rollout note - this repo already carries one for smaller changes (the avocado --version entry under [Unreleased]) - so downstream fleets know to expect one unscheduled full OTA rather than reading it as a regression.


# Inject identity into os-release (work copy for the image, sysroot for stone)
# Strip any prior injected fields from the work copy before appending
Expand Down Expand Up @@ -218,6 +254,7 @@ fi"#,
rootfs_filesystem = rootfs_filesystem,
post_install_block = post_install_block,
permissions_section = permissions_section,
auth_hash_block = render_auth_files_hash("ROOTFS_WORK", "AUTH_HASH"),
)
}

Expand Down Expand Up @@ -570,4 +607,73 @@ mod tests {
"the /etc/passwd guard must run before the permissions/user-creation section"
);
}

#[test]
fn test_render_auth_files_hash_snippet() {
// The snippet hashes the four auth files under the given work dir into
// the given var, tolerates missing files, and pins the sort to the C
// locale so the digest is independent of append order.
assert_eq!(
render_auth_files_hash("ROOTFS_WORK", "AUTH_HASH"),
r#" AUTH_HASH=$({ cat \
"$ROOTFS_WORK/etc/passwd" \
"$ROOTFS_WORK/etc/shadow" \
"$ROOTFS_WORK/etc/group" \
"$ROOTFS_WORK/etc/gshadow" 2>/dev/null || true; } | LC_ALL=C sort | sha256sum | awk '{print $1}')"#
);
// Work dir and output var are both substituted, so the initramfs build
// reuses it verbatim with its own names.
let initramfs = render_auth_files_hash("INITRAMFS_WORK", "INITRAMFS_AUTH_HASH");
assert!(initramfs.contains("INITRAMFS_AUTH_HASH=$({ cat"));
assert!(initramfs.contains("\"$INITRAMFS_WORK/etc/passwd\""));
}

#[test]
fn test_render_auth_files_hash_survives_pipefail() {
// Standalone `avocado rootfs image` / `avocado initramfs image` run the
// fragment under `set -euo pipefail`. A missing file (e.g. a minimal
// initramfs with no /etc/shadow) must not fail the pipeline and abort
// the build — the `{ …; } || true` brace group absorbs cat's status.
assert!(render_auth_files_hash("ROOTFS_WORK", "AUTH_HASH").contains("|| true; }"));
}

#[test]
fn test_build_id_folds_auth_files() {
// Regression for ENG-2437: a permissions-only change must move
// AVOCADO_OS_BUILD_ID, so the build id has to hash the assembled
// auth files — not just the package NEVRA set — and feed both into
// the uuid5 derivation.
let script = generate_rootfs_build_script(
"00000000-0000-0000-0000-000000000000",
"erofs-lz4",
None,
"# permissions placeholder\n",
);

assert!(
script.contains("AUTH_HASH="),
"build id must incorporate a hash of the auth files"
);
for f in ["passwd", "shadow", "group", "gshadow"] {
assert!(
script.contains(&format!("$ROOTFS_WORK/etc/{f}")),
"auth hash must cover /etc/{f}"
);
}
assert!(
script.contains("'$PKG_HASH:$AUTH_HASH'"),
"uuid5 input must combine the package hash and the auth hash"
);

// The auth hash must be computed after the permissions section has
// written the files, otherwise it can't observe the change.
let perms_pos = script
.find("# permissions placeholder")
.expect("permissions section present");
let auth_pos = script.find("AUTH_HASH=").expect("auth hash present");
assert!(
perms_pos < auth_pos,
"auth hash must be computed after the permissions section runs"
);
}
}
Loading