diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a57d447..794de39 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -156,7 +156,9 @@ jobs: run: | mkdir -p /tmp/debmagic current="$(dpkg-parsechangelog -SSource)-$(dpkg-parsechangelog -SVersion)" - find /tmp/debmagic -mindepth 1 -maxdepth 1 -type d ! -name "$current" -exec rm -rf {} + + find /tmp/debmagic -mindepth 1 -maxdepth 1 -type d \ + ! -name "$current" ! -name "${current}-test" \ + -exec rm -rf {} + - name: Resolve CI image tag id: image run: | @@ -181,6 +183,11 @@ jobs: --persistent \ --incremental \ --driver-docker-base-image="${{ steps.image.outputs.tag }}" + - name: Run Debmagic test on ourself + run: | + cargo run --locked -p debmagic -- test \ + --driver=docker \ + --driver-docker-base-image="${{ steps.image.outputs.tag }}" - name: Push the CI image if: github.ref == 'refs/heads/main' && env.image_built == 'true' run: | diff --git a/README.md b/README.md index 9eb0082..1302807 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ To learn packaging with debmagic, follow **[the documentation!](https://debmagic | - | - | | `debmagic build binary` | Build a binary package in a container | | `debmagic build source` | Create a source package for upload | -| `debmagic test` | Run package tests | +| `debmagic test` | Run Debian autopkgtest tests (`debian/tests/`) against a prior build | | `debmagic check` | Lint the package | > [!TIP] diff --git a/debian/control b/debian/control index f06c3bf..fad3b26 100644 --- a/debian/control +++ b/debian/control @@ -1,4 +1,5 @@ Source: debmagic +Testsuite: autopkgtest Section: devel Maintainer: Debmagic Maintainers Uploaders: diff --git a/debian/tests/control b/debian/tests/control new file mode 100644 index 0000000..5794aa8 --- /dev/null +++ b/debian/tests/control @@ -0,0 +1,2 @@ +Tests: smoke +Depends: @ diff --git a/debian/tests/smoke b/debian/tests/smoke new file mode 100755 index 0000000..97c7a91 --- /dev/null +++ b/debian/tests/smoke @@ -0,0 +1,5 @@ +#!/bin/sh +set -e + +debmagic --version +debmagic test --help diff --git a/docs/index.md b/docs/index.md index 7a0b0dd..6e2e4f1 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,6 +8,7 @@ usage/getting-started.md usage/build.md +usage/test.md usage/source.md usage/config.md usage/modules/index.md diff --git a/docs/usage/test.md b/docs/usage/test.md new file mode 100644 index 0000000..ec84f86 --- /dev/null +++ b/docs/usage/test.md @@ -0,0 +1,88 @@ +# Running package tests + +Quick reference for running a package's declared Debian autopkgtest tests with `debmagic test`. + +## TL;DR + +- Entry point: `debmagic test` — runs tests from `debian/tests/control` against the `.debs` of a prior `debmagic build` +- Requires a completed build in the same build root (or pass `--changes` to point at exported artifacts) + +```shell +cd your-package +debmagic build binary --driver docker +debmagic test --driver docker +``` + +## What it does + +`debmagic test` installs the binary packages from a prior build and runs the package's declared autopkgtest tests (`debian/tests/control`) +inside a **fresh, separate** driver-managed environment. +The test environment is never the build environment — even when `--persistent` reuses a container across runs, +the test tree is reset and the `.debs` are reinstalled each time. + +The driver *is* the testbed: `autopkgtest` runs with the `null` backend inside the container (or on the host for the bare driver). No `autopkgtest-virt-*` backends are used. + +## Available options + +| Option | Description | +|---|---| +| `--driver <...>` | Test environment driver (defaults to the driver recorded in the prior build's `build.json`) | +| `--persistent` | Retain the test environment after the run for reattach/debug | +| `--strict` | Treat skipped tests and "no tests declared" as failures (exit code 2) | +| `--changes ` | Path to a `.changes` file whose directory supplies the built `.debs` (for pipeline use) | +| `--distro ` | Override the target distro for the test environment (defaults to the prior build's distro from `build.json`, not the changelog) | +| `--proposed` | Enable the `-proposed` pocket in the test environment | +| `--apt-mirror ` | Mirror URL (same as [`debmagic build`](build.md)) | +| `--source-dir ` | Directory containing the `debian/` package directory | +| `--allow-host-test` | Allow the bare driver, which runs autopkgtest as root on the host | + +[`debmagic shell`](#inspecting-a-failed-test-run) — attach an interactive shell to a test environment + +Driver-specific flags (`--driver-docker-base-image`, `--driver-lxd-*`) mirror `debmagic build`. + +## Picking a driver + +Use the same drivers as for builds. Pass `--driver` explicitly (or rely on the driver recorded in the prior build's `build.json`): + +| Driver | Isolation | +|---|---| +| `lxd` / `incus` | Full container isolation | +| `docker` | Full container isolation | +| `bare` | None — tests run as root on the host; requires `--allow-host-test` | + +## Exit codes + +| Code | Meaning | +|---|---| +| `0` | All tests passed, or skips/no-tests were allowed | +| `1` | Test failure, testbed error, or other autopkgtest error | +| `2` | Strict-only failure: skipped tests or no tests declared under `--strict` | + +autopkgtest skips tests whose `Restrictions:` the `null` backend cannot satisfy (e.g. `isolation-container`, `isolation-machine`). Skips are reported loudly; use `--strict` to escalate them to exit code 2. + +If no `debian/tests/control` exists (or it declares no tests), the run exits 0 with a notice — or exit 2 under `--strict`. + +## Inspecting a failed test run + +On failure with a TTY, `debmagic test` offers an interactive shell inside the test environment (destroyed on shell exit unless `--persistent` was used). With a persistent test environment: + +```shell +# if you're in the package still +debmagic shell +# from the outside: +debmagic shell --source-dir /path/to/parent/of/debian/dir +``` + +Test output and logs are exported to a `test/` subdirectory of the build root; the path is printed at the end of the run. + +## Prior build required + +By default `debmagic test` resolves the prior build from the build root (same layout as `debmagic shell`). If no build artifacts are found: +run `debmagic build` first + +Use `--changes` to supply a `.changes` file from an exported output directory instead. + +## Bare driver + +The bare driver runs autopkgtest as root directly on the host. +This violates the no-leak principle for normal use — pass `--allow-host-test` to opt in explicitly. diff --git a/packages/debmagic/README.md b/packages/debmagic/README.md index 8deaf3b..07ab197 100644 --- a/packages/debmagic/README.md +++ b/packages/debmagic/README.md @@ -3,7 +3,8 @@ Modern, robust & easy tooling for building and packaging [Debian](https://debian.org)/[Ubuntu](https://ubuntu.com) packages — while staying backwards compatible. - **Build any package** in an isolated container environment with `debmagic build` -- **Test and lint** with `debmagic test` and `debmagic check` +- **Run Debian autopkgtest tests** against built packages with `debmagic test` +- **Lint** with `debmagic check` - **Debug** build environments interactively with `debmagic shell` ## Installation @@ -42,6 +43,15 @@ Create a source package (`.dsc`) without compilation: debmagic build source ``` +Run the package's declared Debian autopkgtest tests against a prior build: + +```shell +debmagic build binary --driver docker +debmagic test --driver docker +``` + +Use `--strict` to fail on skipped or undeclared tests (exit code 2). The bare driver requires `--allow-host-test`. + ### Useful options - `--distro ` — select the target distro/release (e.g. `trixie`, `noble`) if the changelog is ambiguous diff --git a/packages/debmagic/src/build/artifacts.rs b/packages/debmagic/src/build/artifacts.rs index d91c520..08b1474 100644 --- a/packages/debmagic/src/build/artifacts.rs +++ b/packages/debmagic/src/build/artifacts.rs @@ -7,7 +7,8 @@ use std::{ use anyhow::{Context, anyhow, bail}; use debian_control::lossless::changes::Changes; -fn changes_file_in(build_dir: &Path) -> anyhow::Result { +/// Locate the single `.changes` file in a build work directory. +pub fn find_changes_file(build_dir: &Path) -> anyhow::Result { let mut paths = fs::read_dir(build_dir) .with_context(|| { format!( @@ -54,7 +55,7 @@ pub fn export_build_artifacts(build_dir: &Path, output_dir: &Path) -> anyhow::Re fs::create_dir_all(output_dir) .with_context(|| format!("failed to create output directory {}", output_dir.display()))?; - let changes_path = changes_file_in(build_dir)?; + let changes_path = find_changes_file(build_dir)?; let changes_metadata = fs::symlink_metadata(&changes_path)?; if !changes_metadata.file_type().is_file() { bail!( @@ -106,6 +107,59 @@ pub fn export_build_artifacts(build_dir: &Path, output_dir: &Path) -> anyhow::Re Ok(exported_changes) } +/// Copy a `.changes` file and every artifact it references into `dest_dir`. +pub fn copy_changes_artifacts(changes_path: &Path, dest_dir: &Path) -> anyhow::Result<()> { + fs::create_dir_all(dest_dir) + .with_context(|| format!("failed to create directory {}", dest_dir.display()))?; + + let changes_metadata = fs::symlink_metadata(changes_path)?; + if !changes_metadata.file_type().is_file() { + bail!( + "changes file {} is not a regular file", + changes_path.display() + ); + } + let source_dir = changes_path.parent().ok_or_else(|| { + anyhow!( + "changes file {} has no parent directory", + changes_path.display() + ) + })?; + let changes = Changes::from_file(changes_path) + .with_context(|| format!("failed to parse {}", changes_path.display()))?; + let files = changes + .files() + .ok_or_else(|| anyhow!("{} has no Files field", changes_path.display()))?; + + for file in files { + let filename = artifact_filename(&file.filename)?; + let source = source_dir.join(filename); + let metadata = fs::symlink_metadata(&source).with_context(|| { + format!( + "artifact {} referenced by {} does not exist", + source.display(), + changes_path.display() + ) + })?; + if !metadata.file_type().is_file() { + bail!("build artifact {} is not a regular file", source.display()); + } + let destination = dest_dir.join(filename); + reject_destination_symlink(&destination)?; + fs::copy(&source, destination) + .with_context(|| format!("failed to copy build artifact {}", source.display()))?; + } + + let changes_filename = changes_path + .file_name() + .ok_or_else(|| anyhow!("invalid .changes path: {}", changes_path.display()))?; + let destination = dest_dir.join(changes_filename); + reject_destination_symlink(&destination)?; + fs::copy(changes_path, &destination) + .with_context(|| format!("failed to copy {}", changes_path.display()))?; + Ok(()) +} + #[cfg(test)] mod tests { use std::os::unix::fs::symlink; diff --git a/packages/debmagic/src/build/common.rs b/packages/debmagic/src/build/common.rs index 796e708..b4a8b34 100644 --- a/packages/debmagic/src/build/common.rs +++ b/packages/debmagic/src/build/common.rs @@ -116,6 +116,24 @@ pub enum SourceSyncMode { pub type DriverSpecificBuildMetadata = HashMap; +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EnvironmentPurpose { + #[default] + Build, + Test, +} + +impl EnvironmentPurpose { + /// Extra part for environment fingerprints when purpose is not [`Self::Build`]. + pub fn fingerprint_part(self) -> Option<&'static str> { + match self { + Self::Build => None, + Self::Test => Some("test"), + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct BuildMetadata { pub config: BuildConfig, @@ -153,14 +171,20 @@ pub struct BuildConfig { /// Which source files are staged into the build tree. #[serde(default)] pub source_sync_mode: SourceSyncMode, + #[serde(default)] + pub purpose: EnvironmentPurpose, } impl BuildConfig { pub fn build_identifier(&self) -> String { - format!( + let base = format!( "{}-{}-{}", self.package_identifier, self.distro.distro, self.distro.codename - ) + ); + match self.purpose { + EnvironmentPurpose::Build => base, + EnvironmentPurpose::Test => format!("{base}-test"), + } } pub fn build_work_dir(&self) -> PathBuf { @@ -194,13 +218,29 @@ pub const APT_MIRROR_SCRIPT: &str = include_str!("scripts/mirror.py"); pub trait BuildDriver { fn get_build_metadata(&self) -> DriverSpecificBuildMetadata; + fn run_command_exit_status( + &self, + cmd: &[&str], + cwd: &Path, + requires_root: bool, + env_add: &[(&str, &str)], + ) -> std::io::Result; + fn run_command_env( &self, cmd: &[&str], cwd: &Path, requires_root: bool, env_add: &[(&str, &str)], - ) -> std::io::Result<()>; + ) -> std::io::Result<()> { + let code = self.run_command_exit_status(cmd, cwd, requires_root, env_add)?; + if code != 0 { + return Err(std::io::Error::other(format!( + "Command failed with exit code: {code}" + ))); + } + Ok(()) + } fn run_command(&self, cmd: &[&str], cwd: &Path, requires_root: bool) -> std::io::Result<()> { self.run_command_env(cmd, cwd, requires_root, &[]) @@ -276,15 +316,45 @@ mod tests { sign_key: None, sign_with: crate::build::signing::SignWith::Auto, source_sync_mode: crate::build::common::SourceSyncMode::Tracked, + purpose: EnvironmentPurpose::Build, } } #[test] - fn docker_identifier_replaces_debian_prerelease_tilde() { + fn build_identifier_unchanged_for_build_purpose() { let config = sample_config("debmagic-0.0.1~alpha2"); assert_eq!( config.build_identifier(), "debmagic-0.0.1~alpha2-debian-forky" ); } + + #[test] + fn build_identifier_differs_for_test_purpose() { + let mut config = sample_config("debmagic-0.0.1~alpha2"); + config.purpose = EnvironmentPurpose::Test; + assert_eq!( + config.build_identifier(), + "debmagic-0.0.1~alpha2-debian-forky-test" + ); + assert_ne!( + config.build_identifier(), + sample_config("debmagic-0.0.1~alpha2").build_identifier() + ); + } + + #[test] + fn build_config_without_purpose_deserializes_as_build() { + let json = r#"{ + "driver": "Docker", + "package_identifier": "pkg-1.0", + "build_root_dir": "/tmp/build", + "source_dir": "/tmp/src", + "output_dir": "/tmp/out", + "distro": { "distro": "Debian", "codename": "forky", "version": "15" }, + "sign_package": false + }"#; + let config: BuildConfig = serde_json::from_str(json).unwrap(); + assert_eq!(config.purpose, EnvironmentPurpose::Build); + } } diff --git a/packages/debmagic/src/build/driver_bare.rs b/packages/debmagic/src/build/driver_bare.rs index a163023..2f2bfe8 100644 --- a/packages/debmagic/src/build/driver_bare.rs +++ b/packages/debmagic/src/build/driver_bare.rs @@ -50,13 +50,13 @@ impl BuildDriver for DriverBare { DriverSpecificBuildMetadata::from([]) } - fn run_command_env( + fn run_command_exit_status( &self, cmd: &[&str], cwd: &Path, requires_root: bool, env_add: &[(&str, &str)], - ) -> std::io::Result<()> { + ) -> std::io::Result { let mut full_cmd: Vec = Vec::new(); let is_root = unsafe { libc::geteuid() == 0 }; @@ -73,15 +73,7 @@ impl BuildDriver for DriverBare { command.envs(env_add.iter().copied()); let status = command.status()?; - - if status.success() { - Ok(()) - } else { - Err(std::io::Error::other(format!( - "Command failed with exit code: {:?}", - status.code() - ))) - } + Ok(status.code().unwrap_or(-1)) } fn cleanup(&self) -> anyhow::Result<()> { diff --git a/packages/debmagic/src/build/driver_docker.rs b/packages/debmagic/src/build/driver_docker.rs index 28d656f..9b1493f 100644 --- a/packages/debmagic/src/build/driver_docker.rs +++ b/packages/debmagic/src/build/driver_docker.rs @@ -256,8 +256,12 @@ impl DriverDocker { &uid, &gid, ]); - let desired_fingerprint = - environment_fingerprint(&["docker-container", &image_fingerprint, build_root.as_ref()]); + let mut container_fingerprint_parts = + vec!["docker-container", &image_fingerprint, build_root.as_ref()]; + if let Some(purpose) = config.purpose.fingerprint_part() { + container_fingerprint_parts.push(purpose); + } + let desired_fingerprint = environment_fingerprint(&container_fingerprint_parts); let container_name = resource_name( "debmagic", &config.package_name, @@ -319,8 +323,10 @@ impl DriverDocker { created_container = true; } + // cwd is the build root (the bind mount itself), not the source dir: + // create() must not assume the source tree has been staged yet. let update_result = driver - .run_command(&["apt-get", "update"], &config.build_source_dir(), true) + .run_command(&["apt-get", "update"], &config.build_root_dir, true) .map_err(|error| anyhow!("Error running apt-get update in container: {error}")); if let Err(error) = update_result { if created_container && let Err(cleanup_error) = driver.container_remove_force() { @@ -360,13 +366,13 @@ impl BuildDriver for DriverDocker { container_name_metadata(&self.container_name) } - fn run_command_env( + fn run_command_exit_status( &self, cmd: &[&str], cwd: &Path, requires_root: bool, env_add: &[(&str, &str)], - ) -> std::io::Result<()> { + ) -> std::io::Result { let container_path = self .translate_path_in_container(cwd) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?; @@ -389,10 +395,7 @@ impl BuildDriver for DriverDocker { exec_cmd.args(cmd); let status = exec_cmd.status()?; - if !status.success() { - return Err(std::io::Error::other("Docker exec failed")); - } - Ok(()) + Ok(status.code().unwrap_or(-1)) } fn cleanup(&self) -> anyhow::Result<()> { diff --git a/packages/debmagic/src/build/driver_lxd.rs b/packages/debmagic/src/build/driver_lxd.rs index b571303..df3ad0c 100644 --- a/packages/debmagic/src/build/driver_lxd.rs +++ b/packages/debmagic/src/build/driver_lxd.rs @@ -236,7 +236,7 @@ impl DriverLxd { let host_gid = unsafe { libc::getegid() }.to_string(); let build_root = config.build_root_dir.to_string_lossy(); let proposed_fingerprint = proposed.to_string(); - let desired_fingerprint = environment_fingerprint(&[ + let mut fingerprint_parts = vec![ variant.binary(), ENVIRONMENT_SETUP_VERSION, &base_image, @@ -246,7 +246,11 @@ impl DriverLxd { &host_uid, &host_gid, build_root.as_ref(), - ]); + ]; + if let Some(purpose) = config.purpose.fingerprint_part() { + fingerprint_parts.push(purpose); + } + let desired_fingerprint = environment_fingerprint(&fingerprint_parts); let mut base = Self { variant, @@ -475,13 +479,13 @@ impl DriverLxd { } } - fn exec_in_container( + fn exec_in_container_exit_status( &self, cmd: &[&str], workdir: Option<&Path>, as_root: bool, env_add: &[(&str, &str)], - ) -> std::io::Result<()> { + ) -> std::io::Result { println!("[{}] $ {}", self.container_name, cmd.join(" ")); let mut exec_cmd = self.lxd_cmd("exec"); @@ -506,9 +510,20 @@ impl DriverLxd { exec_cmd.args(cmd); let status = exec_cmd.status()?; - if !status.success() { + Ok(status.code().unwrap_or(-1)) + } + + fn exec_in_container( + &self, + cmd: &[&str], + workdir: Option<&Path>, + as_root: bool, + env_add: &[(&str, &str)], + ) -> std::io::Result<()> { + let code = self.exec_in_container_exit_status(cmd, workdir, as_root, env_add)?; + if code != 0 { return Err(std::io::Error::other(format!( - "{} exec failed", + "{} exec failed with exit code {code}", self.variant.binary() ))); } @@ -525,18 +540,18 @@ impl BuildDriver for DriverLxd { meta } - fn run_command_env( + fn run_command_exit_status( &self, cmd: &[&str], cwd: &Path, requires_root: bool, env_add: &[(&str, &str)], - ) -> std::io::Result<()> { + ) -> std::io::Result { let container_path = self .translate_path_in_container(cwd) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?; - self.exec_in_container(cmd, Some(&container_path), requires_root, env_add) + self.exec_in_container_exit_status(cmd, Some(&container_path), requires_root, env_add) } fn cleanup(&self) -> anyhow::Result<()> { diff --git a/packages/debmagic/src/build/mod.rs b/packages/debmagic/src/build/mod.rs index d6729b1..f4cd626 100644 --- a/packages/debmagic/src/build/mod.rs +++ b/packages/debmagic/src/build/mod.rs @@ -12,7 +12,7 @@ use crate::build::source::{source_manifest_path, stage_source_tree}; use crate::build_intent::BuildIntent; use crate::{ build::{ - common::{BuildConfig, BuildDriver, BuildDriverType, BuildMetadata}, + common::{BuildConfig, BuildDriver, BuildDriverType, BuildMetadata, EnvironmentPurpose}, config::DriverConfig, driver_bare::DriverBare, driver_docker::DriverDocker, @@ -32,6 +32,7 @@ pub mod driver_docker; pub mod driver_lxd; pub mod signing; pub mod source; +pub mod test; struct Build { config: BuildConfig, @@ -99,7 +100,7 @@ fn prepare_signing( } } -fn get_build_driver( +pub(crate) fn get_build_driver( config: &BuildConfig, driver_config: &DriverConfig, driver_overrides: &DriverOverrides, @@ -140,7 +141,7 @@ fn get_build_driver( } } -fn create_driver_from_metadata( +pub(crate) fn create_driver_from_metadata( config: &DriverConfig, metadata: &BuildMetadata, ) -> anyhow::Result> { @@ -280,6 +281,7 @@ fn prepare_build_env(intent: &BuildIntent, target: &PackageTarget) -> anyhow::Re persistent: intent.config.driver.persistent, incremental: intent.config.incremental, source_sync_mode: intent.config.source_sync_mode, + purpose: EnvironmentPurpose::Build, }; if intent.config.driver.persistent && build_root.exists() { diff --git a/packages/debmagic/src/build/source.rs b/packages/debmagic/src/build/source.rs index c1e5eba..c447e9b 100644 --- a/packages/debmagic/src/build/source.rs +++ b/packages/debmagic/src/build/source.rs @@ -463,6 +463,7 @@ mod tests { persistent: true, incremental: true, source_sync_mode: SourceSyncMode::Worktree, + purpose: crate::build::common::EnvironmentPurpose::Build, }; build_config.create_dirs()?; let initial_entries = source_tree_entries(&source_dir, SourceSyncMode::Worktree)?; diff --git a/packages/debmagic/src/build/test.rs b/packages/debmagic/src/build/test.rs new file mode 100644 index 0000000..bb57c78 --- /dev/null +++ b/packages/debmagic/src/build/test.rs @@ -0,0 +1,530 @@ +use std::{ + fs, io, + io::{BufReader, IsTerminal, stdout}, + path::{Path, PathBuf}, +}; + +use crate::build::config::DriverOverrides; +use crate::build::source::stage_source_tree; +use crate::package::PackageIdentity; +use crate::test_intent::TestIntent; +use crate::{ + build::{ + artifacts::{copy_changes_artifacts, find_changes_file}, + common::{BuildConfig, BuildDriver, BuildDriverType, BuildMetadata, EnvironmentPurpose}, + config::DriverConfig, + signing::SignWith, + }, + package::load_package_identity, +}; +use anyhow::{Context, anyhow, bail}; +use debmagic_common::distro::DistroVersion; + +/// autopkgtest(1) exit status values (Debian autopkgtest 6.x). +/// Some codes combine categories (e.g. 6 = 4|2); treat them as bitmasks where noted. +pub const AUTOPKGTEST_EXIT_PASS: i32 = 0; +pub const AUTOPKGTEST_EXIT_SKIP: i32 = 2; +pub const AUTOPKGTEST_EXIT_FAIL: i32 = 4; +pub const AUTOPKGTEST_EXIT_NO_TESTS: i32 = 8; +pub const AUTOPKGTEST_EXIT_ERRONEOUS_PKG: i32 = 12; +pub const AUTOPKGTEST_EXIT_TESTBED_FAILURE: i32 = 16; +pub const AUTOPKGTEST_EXIT_OTHER: i32 = 20; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TestOutcome { + Passed, + Failed, + StrictFailure, +} + +struct TestRun { + config: BuildConfig, + driver: Box, +} + +pub fn get_build_root_and_identifier( + temp_build_dir: &Path, + identity: &PackageIdentity, +) -> (String, PathBuf) { + let package_identifier = format!("{}-{}", identity.name, identity.version); + let build_root = temp_build_dir.join(&package_identifier); + (package_identifier, build_root) +} + +pub fn test_build_root(build_root: &Path) -> PathBuf { + let package_identifier = build_root + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or_default(); + build_root.with_file_name(format!("{package_identifier}-test")) +} + +pub fn map_autopkgtest_exit(exit_code: i32, strict: bool) -> TestOutcome { + if exit_code == AUTOPKGTEST_EXIT_PASS { + return TestOutcome::Passed; + } + if exit_code < 0 { + return TestOutcome::Failed; + } + + let has_fail = (exit_code & AUTOPKGTEST_EXIT_FAIL) != 0 + || exit_code == AUTOPKGTEST_EXIT_TESTBED_FAILURE + || exit_code == AUTOPKGTEST_EXIT_OTHER + || exit_code == AUTOPKGTEST_EXIT_ERRONEOUS_PKG; + if has_fail { + return TestOutcome::Failed; + } + + let has_skip = (exit_code & AUTOPKGTEST_EXIT_SKIP) != 0; + let has_no_tests = (exit_code & AUTOPKGTEST_EXIT_NO_TESTS) != 0; + if strict && (has_skip || has_no_tests) { + return TestOutcome::StrictFailure; + } + + TestOutcome::Passed +} + +fn lookup_distro(name: &str) -> anyhow::Result { + debmagic_common::distro::get_distro_version(name) + .ok_or_else(|| anyhow!("unknown distro codename '{name}'")) +} + +fn load_build_metadata(build_root: &Path) -> anyhow::Result { + let build_metadata_path = build_root.join("build.json"); + if !build_metadata_path.is_file() { + bail!("No build.json found"); + } + let file = fs::OpenOptions::new() + .read(true) + .open(&build_metadata_path)?; + let reader = BufReader::new(&file); + serde_json::from_reader(reader).with_context(|| { + format!( + "Failed to read build metadata from {} - invalid json", + build_metadata_path.display() + ) + }) +} + +fn get_build_driver( + config: &BuildConfig, + driver_config: &DriverConfig, + driver_overrides: &DriverOverrides, +) -> anyhow::Result> { + crate::build::get_build_driver(config, driver_config, driver_overrides) +} + +fn create_driver_from_metadata( + config: &DriverConfig, + metadata: &BuildMetadata, +) -> anyhow::Result> { + crate::build::create_driver_from_metadata(config, metadata) +} + +impl TestRun { + fn create( + config: &BuildConfig, + driver_config: &DriverConfig, + driver_overrides: &DriverOverrides, + ) -> anyhow::Result { + let driver = get_build_driver(config, driver_config, driver_overrides) + .context(format!("failed to create {:?} build driver", config.driver))?; + Ok(Self { + config: config.clone(), + driver, + }) + } + + fn write_metadata(&self) -> anyhow::Result<()> { + let metadata = BuildMetadata { + config: self.config.clone(), + driver_metadata: self.driver.get_build_metadata(), + }; + let path = self.config.build_root_dir.join("build.json"); + let json = serde_json::to_string_pretty(&metadata) + .context("Failed to serialize build metadata")?; + fs::write(path, json)?; + Ok(()) + } +} + +fn remove_tree_with_privileged_fallback( + root: &Path, + driver_config: &DriverConfig, +) -> anyhow::Result<()> { + if !root.exists() { + return Ok(()); + } + if let Err(e) = fs::remove_dir_all(root) { + if e.kind() == io::ErrorKind::PermissionDenied { + let metadata_path = root.join("build.json"); + if metadata_path.is_file() + && let Ok(file) = fs::OpenOptions::new().read(true).open(&metadata_path) + && let Ok(metadata) = + serde_json::from_reader::<_, BuildMetadata>(BufReader::new(&file)) + && let Ok(driver) = create_driver_from_metadata(driver_config, &metadata) + { + let _ = driver.reset_build_root(); + } + fs::remove_dir_all(root).with_context(|| { + format!( + "failed to remove test root {}; try: sudo rm -rf {}", + root.display(), + root.display() + ) + })?; + } else { + return Err(e.into()); + } + } + Ok(()) +} + +fn prepare_test_env( + intent: &TestIntent, + test_config: &BuildConfig, + identity: &PackageIdentity, + changes_path: &Path, +) -> anyhow::Result { + let test_root = &test_config.build_root_dir; + + if intent.config.driver.persistent && test_root.exists() { + let test_run = + TestRun::create(test_config, &intent.config.driver, &intent.driver_overrides).context( + format!("failed to create {:?} build driver", test_config.driver), + )?; + test_run + .driver + .reset_build_root() + .context("failed to reset persistent test directory")?; + test_config + .create_dirs() + .context("failed to create test directories")?; + stage_source_tree(test_config, identity)?; + copy_changes_artifacts(changes_path, &test_config.build_work_dir())?; + return Ok(test_run); + } + + remove_tree_with_privileged_fallback(test_root, &intent.config.driver)?; + + test_config + .create_dirs() + .context("failed to create test directories")?; + stage_source_tree(test_config, identity)?; + copy_changes_artifacts(changes_path, &test_config.build_work_dir())?; + + let test_run = TestRun::create(test_config, &intent.config.driver, &intent.driver_overrides)?; + Ok(test_run) +} + +fn copy_dir_all(src: &Path, dst: &Path) -> io::Result<()> { + fs::create_dir_all(dst)?; + for entry in fs::read_dir(src)? { + let entry = entry?; + let file_type = entry.file_type()?; + let dest_path = dst.join(entry.file_name()); + if file_type.is_dir() { + copy_dir_all(&entry.path(), &dest_path)?; + } else { + fs::copy(entry.path(), dest_path)?; + } + } + Ok(()) +} + +fn print_autopkgtest_notices(exit_code: i32, summary_path: &Path) { + if let Ok(summary) = fs::read_to_string(summary_path) { + for line in summary.lines() { + if line.contains(" SKIP ") { + eprintln!("autopkgtest: {line}"); + } + } + } + + if (exit_code & AUTOPKGTEST_EXIT_NO_TESTS) != 0 || exit_code == AUTOPKGTEST_EXIT_NO_TESTS { + eprintln!( + "WARNING: autopkgtest reported no tests declared in this package (exit code {exit_code})" + ); + } +} + +pub fn run_test(intent: &TestIntent) -> anyhow::Result { + let identity = load_package_identity(&intent.source_dir)?; + let (package_identifier, build_root) = + get_build_root_and_identifier(&intent.config.temp_build_dir, &identity); + + let changes_path = if let Some(ref explicit) = intent.changes { + if !explicit.is_file() { + bail!("--changes file {} does not exist", explicit.display()); + } + explicit.clone() + } else { + let build_metadata_path = build_root.join("build.json"); + if !build_metadata_path.is_file() { + bail!( + "no prior build found at {}; run `debmagic build binary` first", + build_root.display() + ); + } + find_changes_file(&build_root.join("work"))? + }; + + let prior_build = if build_root.join("build.json").is_file() { + Some(load_build_metadata(&build_root)?) + } else { + None + }; + + let driver = intent + .driver + .or_else(|| prior_build.as_ref().map(|metadata| metadata.config.driver)) + .ok_or_else(|| { + anyhow!( + "no driver specified and no prior build found; pass --driver or run `debmagic build binary` first" + ) + })?; + + if driver == BuildDriverType::Bare && !intent.allow_host_test { + bail!( + "the bare driver runs autopkgtest as root directly on the host; \ + pass --allow-host-test to opt in explicitly" + ); + } + + let distro = if let Some(ref override_distro) = intent.distro { + lookup_distro(override_distro)? + } else if let Some(ref metadata) = prior_build { + metadata.config.distro.clone() + } else { + bail!( + "no prior build metadata found; pass --distro when using --changes without a build root" + ); + }; + + let test_root = test_build_root(&build_root); + let base_config = prior_build + .as_ref() + .map(|metadata| metadata.config.clone()) + .unwrap_or_else(|| BuildConfig { + driver, + package_name: identity.name.clone(), + package_identifier: package_identifier.clone(), + source_dir: intent.source_dir.clone(), + output_dir: intent.source_dir.clone(), + build_root_dir: test_root.clone(), + distro: distro.clone(), + sign_package: false, + sign_with: SignWith::Auto, + sign_key: None, + build_debug_symbols: false, + clean: false, + persistent: intent.config.driver.persistent, + incremental: false, + source_sync_mode: intent.config.source_sync_mode, + purpose: EnvironmentPurpose::Test, + }); + + let test_config = BuildConfig { + driver, + package_name: identity.name.clone(), + package_identifier, + source_dir: intent.source_dir.clone(), + output_dir: base_config.output_dir.clone(), + build_root_dir: test_root.clone(), + distro, + sign_package: false, + sign_with: SignWith::Auto, + sign_key: None, + build_debug_symbols: false, + clean: false, + persistent: intent.config.driver.persistent, + incremental: false, + source_sync_mode: intent.config.source_sync_mode, + purpose: EnvironmentPurpose::Test, + }; + + let test_run = prepare_test_env(intent, &test_config, &identity, &changes_path) + .context("failed to prepare test environment")?; + test_run + .write_metadata() + .context("failed to write test metadata")?; + + let apt_env = [("DEBIAN_FRONTEND", "noninteractive")]; + test_run.driver.run_command_env( + &["apt-get", "update"], + &test_config.build_source_dir(), + true, + &apt_env, + )?; + test_run.driver.run_command_env( + &["apt-get", "install", "-y", "autopkgtest"], + &test_config.build_source_dir(), + true, + &apt_env, + )?; + + let work_dir = test_config.build_work_dir(); + let changes_filename = changes_path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| anyhow!("invalid .changes path: {}", changes_path.display()))?; + let source_tree_name = test_config.package_identifier.as_str(); + let autopkgtest_out_host = test_root.join("autopkgtest-out"); + if autopkgtest_out_host.exists() { + fs::remove_dir_all(&autopkgtest_out_host)?; + } + let output_dir_arg = "../autopkgtest-out"; + let summary_arg = "../autopkgtest-out/summary"; + + // Binary-only builds have no .dsc in the .changes; pass the staged source + // tree alongside the .changes so debian/tests/ is found without rebuilding + // (-B). See autopkgtest(1) "TESTING A DEBIAN PACKAGE" (.changes + tree). + let autopkgtest_cmd = [ + "autopkgtest", + "-B", + "--no-auto-control", + &format!("--output-dir={output_dir_arg}"), + &format!("--summary={summary_arg}"), + changes_filename, + &format!("{source_tree_name}/"), + "--", + "null", + ]; + + let exit_code = test_run + .driver + .run_command_exit_status(&autopkgtest_cmd, &work_dir, true, &[]) + .unwrap_or(-1); + + let summary_path = autopkgtest_out_host.join("summary"); + print_autopkgtest_notices(exit_code, &summary_path); + + let export_root = prior_build + .as_ref() + .map(|_| build_root.as_path()) + .unwrap_or_else(|| changes_path.parent().unwrap()); + let exported_test_dir = export_root.join("test"); + if exported_test_dir.exists() { + fs::remove_dir_all(&exported_test_dir)?; + } + copy_dir_all(&autopkgtest_out_host, &exported_test_dir).with_context(|| { + format!( + "failed to copy test output to {}", + exported_test_dir.display() + ) + })?; + println!("Test output written to {}", exported_test_dir.display()); + + let outcome = map_autopkgtest_exit(exit_code, intent.strict); + + if outcome == TestOutcome::Failed && stdout().is_terminal() { + eprintln!("Tests failed (autopkgtest exit code {exit_code}). Dropping into shell..."); + eprintln!("Test logs: {}", exported_test_dir.display()); + if let Err(shell_error) = test_run + .driver + .interactive_shell(&test_config.build_source_dir()) + { + eprintln!("Dropping into shell failed: {shell_error}"); + } + } + + if !test_config.persistent { + // Clear container-owned files from the bind mount before destroying + // the container; otherwise the host user cannot remove them later. + if let Err(e) = test_run.driver.reset_build_root() { + eprintln!("Warning: failed to reset test root before cleanup: {e}"); + } + } + + if let Err(cleanup_error) = test_run.driver.cleanup() { + eprintln!("Failed to clean up test environment: {cleanup_error}"); + } + + Ok(outcome) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::package::PackageIdentity; + use debmagic_common::debian::version::PackageVersion; + + fn sample_identity() -> PackageIdentity { + PackageIdentity { + name: "pkg".to_string(), + version: PackageVersion::new(None, "1.0".to_string(), Some("1".to_string())), + source_dir: PathBuf::from("/src"), + } + } + + #[test] + fn test_build_root_appends_test_suffix() { + let (_, build_root) = + get_build_root_and_identifier(Path::new("/tmp/debmagic"), &sample_identity()); + assert_eq!(build_root, PathBuf::from("/tmp/debmagic/pkg-1.0-1")); + assert_eq!( + test_build_root(&build_root), + PathBuf::from("/tmp/debmagic/pkg-1.0-1-test") + ); + } + + #[test] + fn map_exit_pass() { + assert_eq!( + map_autopkgtest_exit(AUTOPKGTEST_EXIT_PASS, false), + TestOutcome::Passed + ); + assert_eq!( + map_autopkgtest_exit(AUTOPKGTEST_EXIT_PASS, true), + TestOutcome::Passed + ); + } + + #[test] + fn map_exit_fail_and_testbed_failure() { + for code in [ + AUTOPKGTEST_EXIT_FAIL, + 6, + AUTOPKGTEST_EXIT_ERRONEOUS_PKG, + 14, + AUTOPKGTEST_EXIT_TESTBED_FAILURE, + AUTOPKGTEST_EXIT_OTHER, + ] { + assert_eq!( + map_autopkgtest_exit(code, false), + TestOutcome::Failed, + "code {code}" + ); + assert_eq!( + map_autopkgtest_exit(code, true), + TestOutcome::Failed, + "code {code} strict" + ); + } + } + + #[test] + fn map_exit_spawn_failure_is_failed() { + assert_eq!(map_autopkgtest_exit(-1, false), TestOutcome::Failed); + } + + #[test] + fn map_exit_skip_and_no_tests_respects_strict() { + assert_eq!( + map_autopkgtest_exit(AUTOPKGTEST_EXIT_SKIP, false), + TestOutcome::Passed + ); + assert_eq!( + map_autopkgtest_exit(AUTOPKGTEST_EXIT_SKIP, true), + TestOutcome::StrictFailure + ); + assert_eq!( + map_autopkgtest_exit(AUTOPKGTEST_EXIT_NO_TESTS, false), + TestOutcome::Passed + ); + assert_eq!( + map_autopkgtest_exit(AUTOPKGTEST_EXIT_NO_TESTS, true), + TestOutcome::StrictFailure + ); + } +} diff --git a/packages/debmagic/src/cli.rs b/packages/debmagic/src/cli.rs index 6551fb3..c3e0928 100644 --- a/packages/debmagic/src/cli.rs +++ b/packages/debmagic/src/cli.rs @@ -19,7 +19,7 @@ pub enum Commands { Build(Box), #[command(about = "Open an interactive shell to the currently active build environment")] Shell(ShellSubcommandArgs), - #[command(about = "Run tests")] + #[command(about = "Run the package's declared Debian autopkgtest tests against a prior build")] Test(TestSubcommandArgs), #[command(about = "Check the project")] Check(CheckSubcommandArgs), @@ -200,6 +200,61 @@ pub struct ShellSubcommandArgs { #[derive(Args, Debug)] pub struct TestSubcommandArgs { + #[arg( + short, + long, + help = "Build driver type for the test environment. Defaults to the driver recorded in the prior build's build.json." + )] + pub driver: Option, + + #[arg(long, action = clap::ArgAction::SetTrue, help = "Keep the test environment for reuse after the test run finishes")] + pub persistent: Option, + + #[command(flatten)] + pub docker: DockerArgs, + + #[command(flatten)] + pub lxd: LxdArgs, + + #[arg( + long = "apt-mirror", + help = "Apt mirror URL to use inside the test environment instead of the default archive mirrors. Ignored by the bare driver." + )] + pub apt_mirror: Option, + + #[arg( + long, + action = clap::ArgAction::SetTrue, + help = "Also enable the '-proposed' pocket in the test environment. Ignored by the bare driver." + )] + pub proposed: Option, + + #[arg( + long, + help = "Override the target distribution for the test environment. Defaults to the distro recorded in the prior build's build.json, not the changelog." + )] + pub distro: Option, + + #[arg( + long, + action = clap::ArgAction::SetTrue, + help = "Treat skipped tests and 'no tests declared' as failures (exit code 2)" + )] + pub strict: bool, + + #[arg( + long, + help = "Path to a .changes file whose directory supplies the built .debs (for pipeline use)" + )] + pub changes: Option, + + #[arg( + long, + action = clap::ArgAction::SetTrue, + help = "Allow running tests with the bare driver, which executes autopkgtest as root on the host" + )] + pub allow_host_test: bool, + #[command(flatten)] pub common: CommonCli, } diff --git a/packages/debmagic/src/main.rs b/packages/debmagic/src/main.rs index 5efad7d..cce0921 100644 --- a/packages/debmagic/src/main.rs +++ b/packages/debmagic/src/main.rs @@ -1,4 +1,5 @@ use std::env; +use std::process::ExitCode; use anyhow::Context; use clap::{CommandFactory, Parser}; @@ -7,11 +8,12 @@ use crate::{ build::{ build_package, build_source_package, common::BuildDriverType, config::DriverOverrides, driver_bare::DriverBareConfigOverrides, driver_docker::DriverDockerConfigOverrides, - driver_lxd::DriverLxdConfigOverrides, get_shell_in_build, + driver_lxd::DriverLxdConfigOverrides, get_shell_in_build, test::TestOutcome, }, build_intent::{BuildIntentInput, load_config, resolve_build_intent}, cli::{BuildTarget, Cli, Commands}, package::{load_package_identity, resolve_package_target}, + test_intent::{TestIntentInput, resolve_test_intent}, }; pub mod build; @@ -19,8 +21,19 @@ pub mod build_intent; pub mod cli; pub mod config; pub mod package; +pub mod test_intent; -fn main() -> anyhow::Result<()> { +fn main() -> ExitCode { + match run() { + Ok(code) => code, + Err(error) => { + eprintln!("{error:?}"); + ExitCode::FAILURE + } + } +} + +fn run() -> anyhow::Result { let cli = Cli::parse(); let current_dir = env::current_dir()?; @@ -93,8 +106,37 @@ fn main() -> anyhow::Result<()> { let identity = load_package_identity(&source_dir)?; get_shell_in_build(&config, &identity)?; } - Commands::Test(_args) => { - println!("Test subcommand! - not implemented"); + Commands::Test(args) => { + let intent = resolve_test_intent(TestIntentInput { + fallback_dir: current_dir.clone(), + source_dir: args.common.source_dir.clone(), + config_file: cli.config.clone(), + driver: args.driver, + persistent: args.persistent, + strict: args.strict, + changes: args.changes.clone(), + allow_host_test: args.allow_host_test, + distro: args.distro.clone(), + driver_overrides: DriverOverrides { + apt_mirror: args.apt_mirror.clone(), + proposed: args.proposed, + docker: DriverDockerConfigOverrides { + base_image: args.docker.base_image.clone(), + }, + bare: DriverBareConfigOverrides {}, + lxd: DriverLxdConfigOverrides { + base_image: args.lxd.base_image.clone(), + project: args.lxd.project.clone(), + }, + }, + })?; + + let outcome = crate::build::test::run_test(&intent).context("running tests failed")?; + return Ok(match outcome { + TestOutcome::Passed => ExitCode::SUCCESS, + TestOutcome::Failed => ExitCode::from(1), + TestOutcome::StrictFailure => ExitCode::from(2), + }); } Commands::Check(_args) => { println!("Check subcommand! - not implemented"); @@ -105,5 +147,5 @@ fn main() -> anyhow::Result<()> { } } - Ok(()) + Ok(ExitCode::SUCCESS) } diff --git a/packages/debmagic/src/test_intent.rs b/packages/debmagic/src/test_intent.rs new file mode 100644 index 0000000..9336607 --- /dev/null +++ b/packages/debmagic/src/test_intent.rs @@ -0,0 +1,139 @@ +use std::path::PathBuf; + +use anyhow::Context; + +use crate::{ + build::{common::BuildDriverType, config::DriverOverrides}, + build_intent::load_config, + config::Config, +}; + +/// Clap-free inputs for resolving a [`TestIntent`]. +#[derive(Debug, Clone)] +pub struct TestIntentInput { + /// Directory used when `source_dir` is unset (typically cwd). + pub fallback_dir: PathBuf, + pub source_dir: Option, + pub config_file: Option, + pub driver: Option, + pub persistent: Option, + pub strict: bool, + pub changes: Option, + pub allow_host_test: bool, + pub distro: Option, + pub driver_overrides: DriverOverrides, +} + +/// Fully resolved description of *how* a TestRun executes. +/// +/// Does not include *which* artifacts are being tested. +#[derive(Debug, Clone)] +pub struct TestIntent { + pub source_dir: PathBuf, + pub driver: Option, + pub strict: bool, + pub changes: Option, + pub allow_host_test: bool, + pub distro: Option, + pub config: Config, + pub driver_overrides: DriverOverrides, +} + +pub fn resolve_test_intent(input: TestIntentInput) -> anyhow::Result { + let source_dir = std::path::absolute(input.source_dir.unwrap_or(input.fallback_dir)) + .context("resolving source dir failed")?; + + let mut config = load_config(Some(&source_dir), input.config_file.as_deref())?; + + if let Some(persistent) = input.persistent { + config.driver.persistent = persistent; + } + + let changes = if let Some(changes) = input.changes { + Some(std::path::absolute(changes).context("resolving --changes path failed")?) + } else { + None + }; + + Ok(TestIntent { + source_dir, + driver: input.driver, + strict: input.strict, + changes, + allow_host_test: input.allow_host_test, + distro: input.distro, + config, + driver_overrides: input.driver_overrides, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::build::{ + driver_bare::DriverBareConfigOverrides, driver_docker::DriverDockerConfigOverrides, + driver_lxd::DriverLxdConfigOverrides, + }; + + fn asset_config() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests") + .join("assets") + .join("config1.toml") + } + + fn base_input(fallback: PathBuf) -> TestIntentInput { + TestIntentInput { + fallback_dir: fallback, + source_dir: None, + config_file: Some(asset_config()), + driver: None, + persistent: None, + strict: false, + changes: None, + allow_host_test: false, + distro: None, + driver_overrides: DriverOverrides { + apt_mirror: None, + proposed: None, + docker: DriverDockerConfigOverrides { base_image: None }, + bare: DriverBareConfigOverrides {}, + lxd: DriverLxdConfigOverrides { + base_image: None, + project: None, + }, + }, + } + } + + #[test] + fn resolve_applies_persistent_override() -> anyhow::Result<()> { + let dir = std::env::temp_dir(); + let mut input = base_input(dir); + input.persistent = Some(false); + + let intent = resolve_test_intent(input)?; + assert!(!intent.config.driver.persistent); + Ok(()) + } + + #[test] + fn resolve_passes_through_strict() -> anyhow::Result<()> { + let dir = std::env::temp_dir(); + let mut input = base_input(dir); + input.strict = true; + + let intent = resolve_test_intent(input)?; + assert!(intent.strict); + Ok(()) + } + + #[test] + fn resolve_absolutizes_source_dir() -> anyhow::Result<()> { + let dir = std::env::temp_dir(); + let intent = resolve_test_intent(base_input(dir.clone()))?; + assert!(intent.source_dir.is_absolute()); + assert_eq!(intent.source_dir, std::path::absolute(&dir)?); + Ok(()) + } +}