From b165b0be7a70bf733e5f252b7c98373993285ea7 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Fri, 10 Jul 2026 18:04:18 +0200 Subject: [PATCH 1/7] feat(build): add lxd/incus driver, persistent environments, and source builds Containers are identified by a fingerprint of their environment (image, apt mirror, proposed pocket, host uid/gid, build root) and named deterministically within LXD's 63-char limit, so persistent environments self-invalidate when any input changes. New build capabilities, configurable via CLI flags or debmagic.toml: --apt-mirror replaces the base image's apt sources with a managed deb822 file pointing at a given mirror (via scripts/mirror.py) --proposed enables the release's -proposed pocket --persistent keeps the build environment for reuse across invocations --incremental (binary builds) synchronizes changed source inputs while preserving build outputs, tracked by a source manifest --sign/--sign-key GPG-sign the resulting .changes/.dsc with debsign, always on the host since it needs the user's keyring --debug-symbols controls the automatic -dbgsym package via DEB_BUILD_OPTIONS --clean runs debian/rules clean before building Split 'debmagic build' into 'build binary' and 'build source' targets; source builds default to the bare driver since they need neither build-deps nor compilation. --- .github/workflows/ci.yaml | 2 +- Cargo.lock | 355 ++++++++- Cargo.toml | 3 +- README.md | 2 + debian/control | 1 + debian/debmagic.1 | 58 +- docs/conf.py | 1 - docs/index.md | 2 + docs/usage/build.md | 131 ++++ docs/usage/getting-started.md | 6 + docs/usage/source.md | 58 ++ .../debmagic-common/src/debian/version.rs | 4 + packages/debmagic-common/src/distro.rs | 189 ++--- packages/debmagic/Cargo.toml | 1 + packages/debmagic/src/build.rs | 676 +++++++++++++++--- packages/debmagic/src/build/artifacts.rs | 219 ++++++ packages/debmagic/src/build/common.rs | 186 ++++- packages/debmagic/src/build/config.rs | 10 + packages/debmagic/src/build/driver_bare.rs | 20 +- packages/debmagic/src/build/driver_docker.rs | 442 ++++++++---- packages/debmagic/src/build/driver_lxd.rs | 588 +++++++++++++++ packages/debmagic/src/build/scripts/mirror.py | 157 ++++ packages/debmagic/src/cli.rs | 122 +++- packages/debmagic/src/config.rs | 16 + packages/debmagic/src/main.rs | 97 ++- tests/integration/test_packages.py | 1 + 26 files changed, 2849 insertions(+), 498 deletions(-) create mode 100644 docs/usage/build.md create mode 100644 docs/usage/source.md create mode 100644 packages/debmagic/src/build/artifacts.rs create mode 100644 packages/debmagic/src/build/driver_lxd.rs create mode 100644 packages/debmagic/src/build/scripts/mirror.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1e01757..af777a4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -155,7 +155,7 @@ jobs: - name: Install debmagic (cli) run: uv pip install packages/debmagic - name: Run Debmagic build on ourself - run: uv run debmagic build --driver=docker + run: uv run debmagic build binary --driver=docker # TODO: integration tests currently don't work in the CI since they require running apt source on debian trixie -> CI runs on ubuntu # integration-tests: diff --git a/Cargo.lock b/Cargo.lock index aab1ddb..34ea967 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -90,7 +90,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -193,7 +193,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -319,6 +319,29 @@ dependencies = [ "typenum", ] +[[package]] +name = "deb822-derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b6e5cafe61e77421a090e2a33b8a2e4e2ff1b106fd906ebade111307064d981" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "deb822-lossless" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812bb5c8052a89edc6d45d1bc3b3400e8186dd166e9b0a9520bfa5a2bd8477ee" +dependencies = [ + "deb822-derive", + "regex", + "rowan", + "serde", +] + [[package]] name = "debian-changelog" version = "0.2.14" @@ -334,6 +357,20 @@ dependencies = [ "whoami", ] +[[package]] +name = "debian-control" +version = "0.1.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3367f1dc94222cb9cf16e07c782e442b1928a26dd8dcee3aa4d2dbd32ca11d21" +dependencies = [ + "chrono", + "deb822-lossless", + "debversion", + "regex", + "rowan", + "url", +] + [[package]] name = "debmagic" version = "0.0.1-alpha.5" @@ -342,6 +379,7 @@ dependencies = [ "clap", "config", "debian-changelog", + "debian-control", "debmagic-common", "dirs", "glob", @@ -404,6 +442,17 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "dlv-list" version = "0.5.2" @@ -445,6 +494,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -551,6 +609,108 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "ignore" version = "0.4.25" @@ -620,7 +780,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn", + "syn 2.0.111", ] [[package]] @@ -640,6 +800,12 @@ dependencies = [ "redox_syscall", ] +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + [[package]] name = "log" version = "0.4.29" @@ -714,6 +880,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + [[package]] name = "pest" version = "2.8.4" @@ -744,7 +916,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -757,6 +929,15 @@ dependencies = [ "sha2", ] +[[package]] +name = "potential_utf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +dependencies = [ + "zerovec", +] + [[package]] name = "proc-macro2" version = "1.0.103" @@ -926,7 +1107,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -951,6 +1132,12 @@ dependencies = [ "serde_core", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.10.9" @@ -968,12 +1155,24 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + [[package]] name = "smawk" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c" +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + [[package]] name = "strsim" version = "0.11.1" @@ -991,6 +1190,28 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "test-case" version = "3.3.1" @@ -1009,7 +1230,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -1020,7 +1241,7 @@ checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", "test-case-core", ] @@ -1058,7 +1279,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -1070,6 +1291,16 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinystr" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "toml" version = "0.9.10+spec-1.1.0" @@ -1143,6 +1374,24 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -1157,6 +1406,7 @@ checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" dependencies = [ "getrandom 0.3.4", "js-sys", + "sha1_smol", "wasm-bindgen", ] @@ -1229,7 +1479,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.111", "wasm-bindgen-shared", ] @@ -1282,7 +1532,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -1293,7 +1543,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -1344,6 +1594,12 @@ version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + [[package]] name = "yaml-rust2" version = "0.10.4" @@ -1355,6 +1611,83 @@ dependencies = [ "hashlink", ] +[[package]] +name = "yoke" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "zmij" version = "0.1.8" diff --git a/Cargo.toml b/Cargo.toml index 0a10d23..25c28d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,10 +20,11 @@ glob = ">=0.3.2" libc = ">=0.2.169" serde = { version = ">=1.0.217", features = ["derive"] } serde_json = ">=1.0.139" -uuid = { version = ">=1.10.0", features = ["v4"] } +uuid = { version = ">=1.10.0", features = ["v4", "v5"] } chrono = { version = ">=0.4.42" } regex = { version = ">=1.12.2" } debian-changelog = { version = ">=0.2.14" } +debian-control = { version = ">=0.1.39" } ignore = { version = ">=0.4.25" } # dev dependencies diff --git a/README.md b/README.md index ee1ca6a..502783b 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,8 @@ Included features: - debugging tools - `debmagic shell` - enter current running/finished package environment +> [!TIP] +> Want to use debmagic to build a package? See [docs/usage/build.md](docs/usage/build.md) for a quickstart. ## Debmagic packaging diff --git a/debian/control b/debian/control index 827b876..f06c3bf 100644 --- a/debian/control +++ b/debian/control @@ -26,6 +26,7 @@ Build-Depends: librust-test-case-dev (>=3.3.1), librust-pyo3-dev (>=0.27.2), librust-debian-changelog-dev (>=0.2.14), + librust-debian-control-dev (>= 0.1.39), librust-ignore-dev (>=0.4.25) Rules-Requires-Root: no X-Style: black diff --git a/debian/debmagic.1 b/debian/debmagic.1 index 4550f3e..f0d3cfe 100644 --- a/debian/debmagic.1 +++ b/debian/debmagic.1 @@ -1,43 +1,37 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.49.3. -.TH DEBMAGIC "1" "November 2025" "debmagic 0.1.0" "User Commands" +.TH DEBMAGIC "1" "July 2026" "debmagic 0.0.1-alpha1" "User Commands" .SH NAME -debmagic \- manual page for debmagic 0.1.0 +debmagic \- build Debian packages in isolated environments +.SH SYNOPSIS +.B debmagic +[\fI\,OPTIONS\/\fR] \fI\,\/\fR .SH DESCRIPTION -usage: debmagic [\-h] [\-\-version] {help,version,debuild,build} ... -.PP -Debmagic -.SS "positional arguments:" -.IP -{help,version,debuild,build} +.SS "Commands:" .TP -help -Show this help page and exit +build +Build a debian package: 'binary' (.deb) or 'source' (.dsc) packages +.TP +shell +Open an interactive shell to the currently active build environment +.TP +test +Run tests +.TP +check +Check the project .TP version -Print the version information and exit +Show version information .TP -debuild -Simply run debuild in the current working directory +help +Print this message or the help of the given subcommand(s) +.SH OPTIONS .TP -build -Buidl a debian package with the selected -containerization driver -.SS "options:" +\fB\-c\fR, \fB\-\-config\fR +Path to config file .TP \fB\-h\fR, \fB\-\-help\fR -show this help message and exit +Print help .TP -\fB\-\-version\fR -show program's version number and exit -.SH "SEE ALSO" -The full documentation for -.B debmagic -is maintained as a Texinfo manual. If the -.B info -and -.B debmagic -programs are properly installed at your site, the command -.IP -.B info debmagic -.PP -should give you access to the complete manual. +\fB\-V\fR, \fB\-\-version\fR +Print version diff --git a/docs/conf.py b/docs/conf.py index b002e1d..ffd3ac8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -30,7 +30,6 @@ # html settings html_theme = "sphinx_rtd_theme" -html_static_path = ["_static"] html_context = { "display_github": True, "github_user": "SFTtech", diff --git a/docs/index.md b/docs/index.md index ec902c9..9bca4fd 100644 --- a/docs/index.md +++ b/docs/index.md @@ -7,6 +7,8 @@ :caption: Usage usage/getting-started.md +usage/build.md +usage/source.md usage/modules/index.md ``` diff --git a/docs/usage/build.md b/docs/usage/build.md new file mode 100644 index 0000000..7ae066c --- /dev/null +++ b/docs/usage/build.md @@ -0,0 +1,131 @@ +# Building packages + +Quick reference to build a Debian/Ubuntu package with `debmagic`. +It covers `debmagic build binary` — the generic entry point that works on *any* `debian/`-packaged source tree, including ones that don't use `debmagic-pkg` to write `debian/rules`. +For building just a `.dsc`/tarball without compiling anything, see [`debmagic build source`](source.md) instead. + +## TL;DR + +```shell +debmagic build binary --driver lxd \ + --source-dir /path/to/parent/of/debian/dir \ + --output-dir /path/to/put/the/.deb/files \ + --apt-mirror http:///ubuntu +``` + +- `--source-dir` is the directory *containing* `debian/`, not `debian/` itself. + Defaults to the current directory. +- `--output-dir` is where the resulting `.deb`/`.udeb`/`.ddeb`, `.buildinfo` and `.changes` files end up. + It's created if missing. + Defaults to the current directory. +- `--driver` is required. + Use `lxd` or `incus` if available (isolated containers); fall back to `docker`, then `bare` (builds directly on the host with no isolation, so only use this if you already trust the host environment). +- Exit code is non-zero on failure; stderr/stdout carry the real `dpkg-buildpackage`/`apt-get` output, so grep that for the actual error instead of guessing from the exit code alone. + +## Picking a driver + +Check what's installed and use the first that applies, in this order: + +| Driver | Check it's available | Isolation | +|---|---|---| +| `lxd` / `incus` | `lxc list` / `incus list` | Full container isolation | +| `docker` | `docker info` | Full container isolation | +| `bare` | none (no daemon) | None — build-deps install with `sudo apt-get` directly on the host; only use in a disposable/CI environment | + +There's no auto-detection; pick one and pass it explicitly every time. + +## Speeding up builds with a mirror + +Fresh containers install their base tooling plus every `Build-Depends`, so slow mirrors directly translate into slow builds. +Pass `--apt-mirror` to use a faster mirror for build-dependency resolution after the base tooling is bootstrapped from the image's configured archives: + +```shell +debmagic build binary --driver lxd --apt-mirror http:///ubuntu ... +``` + +Notes: + +- Works for the `lxd`, `incus` and `docker` drivers. + It's a no-op for `bare`, which uses the host's own apt sources. +- The image, mirror, proposed-pocket setting and host user IDs form the build-environment identity. + Changing any of them automatically replaces an incompatible persistent container. +- Handles both the classic `sources.list` format and the deb822 `*.sources` format (Ubuntu 24.04+). +- To avoid repeating the flag, set it once in `debian/debmagic.toml` (see below) instead. + +## Iterating on a build (faster repeat runs) + +Every `debmagic build binary` invocation creates a new container by default and tears it down afterwards. +For repeated attempts against the same package and distro, add `--persistent` to retain and reuse the running environment while restaging the source tree for each build: + +```shell +debmagic build binary --driver lxd --persistent \ + --source-dir . --output-dir /tmp/out +``` + +Use `--incremental` to retain the environment and synchronize only source changes while preserving generated files and unchanged source inodes. +Incremental mode is binary-only, implies `--persistent`, and cannot be combined with `--clean yes`. + +## Inspecting a failed build + +By default a failed build tears down the container, so nothing is left to inspect. +If a build might fail and you need to inspect it afterwards, pass `--persistent` up front, then once the run finishes: + +```shell +debmagic shell --source-dir /path/to/parent/of/debian/dir +``` + +This attaches an interactive shell inside the still-running (or restartable) build environment, at the package's build directory. + +## Selecting a distro/release + +Only needed when `debian/changelog`'s top entry doesn't unambiguously determine the target: pass `--distro ` (e.g. `--distro noble`, `--distro trixie`). +If the changelog has a single unambiguous entry, omit it. + +## Building debug symbol packages + +By default `debmagic build binary` passes `DEB_BUILD_OPTIONS=noautodbgsym` to `dpkg-buildpackage`, which suppresses debhelper's automatic `-dbgsym` package (the detached debug info package debhelper otherwise builds by default from compat 9 onward). +Pass `--debug-symbols` to build it for one invocation: + +```shell +debmagic build binary --driver lxd --debug-symbols --source-dir . --output-dir /tmp/out +``` + +Or set `build_debug_symbols = true` in `debian/debmagic.toml`/`$XDG_CONFIG_HOME/debmagic/config.toml` to always build it. + +## Signing and cleaning + +`--sign yes` (plus optionally `--sign-key you@example.com`) GPG-signs the resulting `.changes`/`.dsc`/`.buildinfo` with `debsign` after building — always on the host, using your own gpg keyring, regardless of `--driver`. +This is mainly useful for [source builds destined for Launchpad](source.md#uploading-to-launchpad), but works for binary builds too. + +`--clean yes` runs `debian/rules clean` before building, like plain `dpkg-buildpackage` does unless passed `-nc`. +Non-incremental builds already stage a clean source tree, while incremental builds preserve outputs intentionally. +Enable cleaning only for packages whose `clean` target performs required setup or code generation. + +Both default to the `sign_package`/`sign_key`/`clean` settings in the config file (see below) if not passed on the CLI. + +## Persisting options in `debian/debmagic.toml` + +Instead of repeating CLI flags on every invocation, drop a config file next to `debian/rules`: + +```toml +build_debug_symbols = true +sign_package = true +sign_key = "you@example.com" +clean = false + +[driver] +persistent = true +apt_mirror = "http:///ubuntu" + +[driver.lxd] +# project = "my-project" +``` + +Config precedence (highest wins): `--config ` on the CLI > `/debian/debmagic.toml` > `$XDG_CONFIG_HOME/debmagic/config.toml`. +CLI flags like `--apt-mirror`/`--persistent`/`--sign`/`--clean` always override the matching config file value for that one invocation. + +## What NOT to expect yet + +- `debmagic test` and `debmagic check` are not implemented yet — don't rely on them for lintian/test output. + Rely on `debmagic build binary`'s own `dpkg-buildpackage` run (which already runs `dh_auto_test` unless the package's `debian/rules` disables it). +- Container/device names are derived and sanitized internally (alphanumeric + hyphen, ≤63 chars for LXD/Incus) — don't try to predict or construct them yourself; use `debmagic shell` instead of `lxc`/`docker` commands directly. diff --git a/docs/usage/getting-started.md b/docs/usage/getting-started.md index 517dc5a..9a7d9e7 100644 --- a/docs/usage/getting-started.md +++ b/docs/usage/getting-started.md @@ -20,6 +20,12 @@ uvx debmagic apt install debmagic ``` +## Building an existing package (CLI) + +`debmagic build` builds *any* Debian-packaged source tree inside a throwaway build environment, driven by a build driver. + +To learn about `debmagic build`, see the [Building packages](build.md) page. + ## Example debian/rules.py Python `debian/rules.py` equivalent of [Ubuntu 24.04 htop](https://git.launchpad.net/ubuntu/+source/htop/tree/debian/rules?h=ubuntu/noble): diff --git a/docs/usage/source.md b/docs/usage/source.md new file mode 100644 index 0000000..4b39bf0 --- /dev/null +++ b/docs/usage/source.md @@ -0,0 +1,58 @@ +# Building source packages + +`debmagic build source` runs `dpkg-buildpackage -S -d -nc` to build a `.dsc`, tarball(s), `.buildinfo` and `.changes` without building binaries. +It's a target of the same [`debmagic build`](build.md) command that builds binary packages (`debmagic build binary`). + +## TL;DR + +```shell +# be in debian package directory, output to current directory: +debmagic build source + +# or outside the debian package dir: +debmagic build source --source-dir /path/to/parent/of/debian/dir --output-dir /path/to/put/the/artifacts +``` + +- Same `--source-dir`/`--output-dir`/`--distro` semantics as [`debmagic build`](build.md). +- `--driver` defaults to `bare`: building a source package needs neither package build-dependencies nor a compiler, but the host must provide `dpkg-buildpackage` from `dpkg-dev`. + Pass `--driver lxd`/`--driver incus`/`--driver docker` when the host does not provide a usable Debian build environment. + Container drivers install their base tooling, but package build-dependencies are installed only with `--clean yes`. + Binary builds (`debmagic build binary`) still require `--driver` to be passed explicitly. + +(uploading-to-launchpad)= +## Uploading to Launchpad + +```shell +debmagic build source --sign yes --sign-key you@example.com \ + --source-dir . --output-dir /tmp/out +dput ppa:your-lp-username/your-ppa /tmp/out/*_source.changes +``` + +- `--sign yes` GPG-signs the `.dsc`/`.buildinfo`/`.changes` with `debsign` (from `devscripts`) after building. + This always runs on the host — never inside a driver's container — since it needs your own gpg keyring. +- `--sign-key` picks which key/uid to sign with (`debsign`'s `-k`); omit it to let `debsign` fall back to its own maintainer-address lookup. +- Both can be set as defaults in `debian/debmagic.toml`/`$XDG_CONFIG_HOME/debmagic/config.toml` instead of passing them every time: + + ```toml + sign_package = true + sign_key = "you@example.com" + ``` + +## What ends up in the source package + +The same file selection as `debmagic build` uses to populate the build environment: everything under `--source-dir` except files matched by `.gitignore` (build artifacts, virtualenvs, ...). +Untracked-but-not-ignored files are included, so uncommitted work-in-progress changes are packaged too — useful while iterating locally. +`debian/source/options` (`tar-ignore`/`diff-ignore` patterns, etc.) is honored as usual, since it's `dpkg-source` itself that reads it. + +## What's NOT run + +By default, `debian/rules` is never invoked (neither `dpkg-source` nor `dpkg-genchanges` need it), so this also works for source trees whose build-dependencies aren't installed anywhere. +Pass `--clean` to opt into running `debian/rules clean` first (like plain `dpkg-buildpackage` does unless passed `-nc`) — useful if a package's `clean` target is (ab)used for setup/codegen that should end up in the source package. +This also installs build-dependencies first, since the `clean` target usually needs its own tooling; a source build without `--clean` (the default) needs none of that. + +## Known limitation: `--driver bare` on a non-Debian host + +`dpkg-genbuildinfo` reads the dpkg status database (`/var/lib/dpkg/status`) to record installed package versions, unlike plain `dpkg-source`. +On a host that has `dpkg-dev` installed but isn't itself Debian/Ubuntu-based (or otherwise lacks a real dpkg database), this step fails with an error like `cannot open /var/lib/dpkg/status`. +Use `--driver lxd`/`--driver incus` in that case to build inside a proper Debian-ish container instead. + diff --git a/packages/debmagic-common/src/debian/version.rs b/packages/debmagic-common/src/debian/version.rs index b8c8b26..a6e5400 100644 --- a/packages/debmagic-common/src/debian/version.rs +++ b/packages/debmagic-common/src/debian/version.rs @@ -48,6 +48,10 @@ impl PackageVersion { self.upstream.clone() } + pub fn upstream_version(&self) -> &str { + &self.upstream + } + /// upstream version plus packaging revision pub fn upstream_revision(&self) -> String { if let Some(revision) = &self.revision { diff --git a/packages/debmagic-common/src/distro.rs b/packages/debmagic-common/src/distro.rs index a66279c..e4f18f6 100644 --- a/packages/debmagic-common/src/distro.rs +++ b/packages/debmagic-common/src/distro.rs @@ -14,6 +14,25 @@ pub struct DistroVersion { pub codename: String, /// numeric or semver version, e.g. "24.04" for ubuntu or "12" for debian pub version: String, + /// true for unreleased development releases (affects image selection) + #[serde(default)] + pub is_devel: bool, +} + +impl DistroVersion { + fn new(distro: Distro, codename: &str, version: &str) -> Self { + Self { + distro, + codename: codename.to_string(), + version: version.to_string(), + is_devel: false, + } + } + + fn devel(mut self) -> Self { + self.is_devel = true; + self + } } impl Distro { @@ -32,161 +51,35 @@ impl std::fmt::Display for Distro { } static DISTRO_INFO_MAP: LazyLock> = LazyLock::new(|| { + use Distro::{Debian, Ubuntu}; HashMap::from([ // debian ( "experimental", - DistroVersion { - distro: Distro::Debian, - codename: "experimental".to_string(), - version: "".to_string(), - }, - ), - ( - "unstable", - DistroVersion { - distro: Distro::Debian, - codename: "unstable".to_string(), - version: "".to_string(), - }, - ), - ( - "sid", - DistroVersion { - distro: Distro::Debian, - codename: "sid".to_string(), - version: "".to_string(), - }, - ), - ( - "testing", - DistroVersion { - distro: Distro::Debian, - codename: "testing".to_string(), - version: "".to_string(), - }, - ), - ( - "duke", - DistroVersion { - distro: Distro::Debian, - codename: "duke".to_string(), - version: "15".to_string(), - }, - ), - ( - "forky", - DistroVersion { - distro: Distro::Debian, - codename: "forky".to_string(), - version: "14".to_string(), - }, - ), - ( - "trixie", - DistroVersion { - distro: Distro::Debian, - codename: "trixie".to_string(), - version: "13".to_string(), - }, - ), - ( - "bookworm", - DistroVersion { - distro: Distro::Debian, - codename: "bookworm".to_string(), - version: "12".to_string(), - }, - ), - ( - "bullseye", - DistroVersion { - distro: Distro::Debian, - codename: "bullseye".to_string(), - version: "11".to_string(), - }, - ), - ( - "buster", - DistroVersion { - distro: Distro::Debian, - codename: "buster".to_string(), - version: "10".to_string(), - }, - ), - ( - "stretch", - DistroVersion { - distro: Distro::Debian, - codename: "stretch".to_string(), - version: "9".to_string(), - }, - ), + DistroVersion::new(Debian, "experimental", ""), + ), + ("unstable", DistroVersion::new(Debian, "unstable", "")), + ("sid", DistroVersion::new(Debian, "sid", "")), + ("testing", DistroVersion::new(Debian, "testing", "")), + ("duke", DistroVersion::new(Debian, "duke", "15")), + ("forky", DistroVersion::new(Debian, "forky", "14")), + ("trixie", DistroVersion::new(Debian, "trixie", "13")), + ("bookworm", DistroVersion::new(Debian, "bookworm", "12")), + ("bullseye", DistroVersion::new(Debian, "bullseye", "11")), + ("buster", DistroVersion::new(Debian, "buster", "10")), + ("stretch", DistroVersion::new(Debian, "stretch", "9")), // ubuntu ( - "resolute", - DistroVersion { - distro: Distro::Ubuntu, - codename: "resolute".to_string(), - version: "26.04".to_string(), - }, - ), - ( - "questing", - DistroVersion { - distro: Distro::Ubuntu, - codename: "questing".to_string(), - version: "25.10".to_string(), - }, - ), - ( - "noble", - DistroVersion { - distro: Distro::Ubuntu, - codename: "noble".to_string(), - version: "24.04".to_string(), - }, - ), - ( - "jammy", - DistroVersion { - distro: Distro::Ubuntu, - codename: "jammy".to_string(), - version: "22.04".to_string(), - }, - ), - ( - "focal", - DistroVersion { - distro: Distro::Ubuntu, - codename: "focal".to_string(), - version: "20.04".to_string(), - }, - ), - ( - "bionic", - DistroVersion { - distro: Distro::Ubuntu, - codename: "bionic".to_string(), - version: "18.04".to_string(), - }, - ), - ( - "xenial", - DistroVersion { - distro: Distro::Ubuntu, - codename: "xenial".to_string(), - version: "16.04".to_string(), - }, - ), - ( - "trusty", - DistroVersion { - distro: Distro::Ubuntu, - codename: "trusty".to_string(), - version: "14.04".to_string(), - }, + "stonking", + DistroVersion::new(Ubuntu, "stonking", "26.10").devel(), ), + ("resolute", DistroVersion::new(Ubuntu, "resolute", "26.04")), + ("noble", DistroVersion::new(Ubuntu, "noble", "24.04")), + ("jammy", DistroVersion::new(Ubuntu, "jammy", "22.04")), + ("focal", DistroVersion::new(Ubuntu, "focal", "20.04")), + ("bionic", DistroVersion::new(Ubuntu, "bionic", "18.04")), + ("xenial", DistroVersion::new(Ubuntu, "xenial", "16.04")), + ("trusty", DistroVersion::new(Ubuntu, "trusty", "14.04")), ]) }); diff --git a/packages/debmagic/Cargo.toml b/packages/debmagic/Cargo.toml index 5281842..6be06a1 100644 --- a/packages/debmagic/Cargo.toml +++ b/packages/debmagic/Cargo.toml @@ -19,4 +19,5 @@ serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } uuid = { workspace = true, features = ["v4"] } debian-changelog = { workspace = true } +debian-control = { workspace = true } ignore = { workspace = true } diff --git a/packages/debmagic/src/build.rs b/packages/debmagic/src/build.rs index 7bd28d9..e0a7f81 100644 --- a/packages/debmagic/src/build.rs +++ b/packages/debmagic/src/build.rs @@ -1,37 +1,45 @@ use core::time; use std::net::Shutdown; +use std::os::unix::fs::PermissionsExt; +use std::os::unix::fs::symlink; use std::os::unix::net::{UnixListener, UnixStream}; use std::sync::{Arc, Mutex}; use std::{ + cmp::Reverse, fs, io::{self, BufReader, IsTerminal, Read, Write, stdout}, - path::{Path, PathBuf}, + path::{Component, Path, PathBuf}, + process::{Command, Stdio}, thread, }; use crate::build::config::DriverOverrides; use crate::{ build::{ - common::{BuildConfig, BuildDriver, BuildDriverType, BuildMetadata}, + common::{BuildConfig, BuildDriver, BuildDriverType, BuildMetadata, run_checked}, config::DriverConfig, driver_bare::DriverBare, driver_docker::DriverDocker, + driver_lxd::{DriverLxd, LxdVariant}, }, config::Config, package::PackageDescription, }; -use anyhow::{Context, anyhow}; +use anyhow::{Context, anyhow, bail}; use debmagic_common::distro::DistroVersion; use glob::glob; +pub mod artifacts; pub mod common; pub mod config; pub mod driver_bare; pub mod driver_docker; +pub mod driver_lxd; struct Build { config: BuildConfig, pub driver: Box, + attached: bool, } fn get_build_driver( @@ -39,18 +47,39 @@ fn get_build_driver( driver_config: &DriverConfig, driver_overrides: &DriverOverrides, ) -> anyhow::Result> { + let apt_mirror = driver_overrides + .apt_mirror + .as_deref() + .or(driver_config.apt_mirror.as_deref()); + let proposed = driver_overrides.proposed.unwrap_or(driver_config.proposed); + match config.driver { BuildDriverType::Docker => Ok(Box::new(DriverDocker::create( config, driver_config, &driver_overrides.docker, + apt_mirror, + proposed, )?)), BuildDriverType::Bare => Ok(Box::new(DriverBare::create( config, driver_config, &driver_overrides.bare, ))), - // BuildDriverType::Lxd => ... + BuildDriverType::Lxd | BuildDriverType::Incus => { + let variant = match config.driver { + BuildDriverType::Lxd => LxdVariant::Lxd, + _ => LxdVariant::Incus, + }; + Ok(Box::new(DriverLxd::create( + variant, + config, + driver_config, + &driver_overrides.lxd, + apt_mirror, + proposed, + )?)) + } } } @@ -63,13 +92,23 @@ fn create_driver_from_metadata( &metadata.config, config, metadata, - ))), + )?)), BuildDriverType::Bare => Ok(Box::new(DriverBare::from_build_metadata( &metadata.config, config, metadata, ))), - // BuildDriverType::Lxd => ... + BuildDriverType::Lxd | BuildDriverType::Incus => { + let variant = match metadata.config.driver { + BuildDriverType::Lxd => LxdVariant::Lxd, + _ => LxdVariant::Incus, + }; + Ok(Box::new(DriverLxd::from_build_metadata( + variant, + &metadata.config, + metadata, + )?)) + } }; driver } @@ -85,6 +124,7 @@ impl Build { Ok(Self { config: config.clone(), driver, + attached: false, }) } @@ -115,19 +155,20 @@ impl Build { let driver = create_driver_from_metadata(driver_config, &metadata)?; - // Try to signal the main debmagic build process that a shell attached - send_socket_command(build_root, "attach") - .context("No debmagic build is currently running for this source directory")?; + let attached = send_socket_command(build_root, "attach").is_ok(); Ok(Self { config: metadata.config.clone(), driver, + attached, }) } pub fn detach(&self) -> anyhow::Result<()> { let build_root = &self.config.build_root_dir; - send_socket_command(build_root, "detach")?; + if self.attached { + send_socket_command(build_root, "detach")?; + } Ok(()) } @@ -234,35 +275,242 @@ fn send_socket_command(build_root: &Path, cmd: &str) -> anyhow::Result<()> { } fn copy_dir_all(src: impl AsRef, dst: impl AsRef) -> anyhow::Result<()> { - fs::create_dir_all(&dst)?; + let entries = source_tree_entries(src.as_ref())?; + copy_source_entries(src.as_ref(), dst.as_ref(), &entries) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "snake_case")] +enum SourcePathKind { + Directory, + File, + Symlink, +} + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +struct SourcePath { + path: PathBuf, + kind: SourcePathKind, +} - let walker = ignore::WalkBuilder::new(&src) +fn validate_source_path(path: &Path) -> anyhow::Result<()> { + if path.as_os_str().is_empty() + || !path + .components() + .all(|component| matches!(component, Component::Normal(_))) + { + bail!("invalid source manifest path: {}", path.display()); + } + Ok(()) +} + +fn source_tree_entries(src: &Path) -> anyhow::Result> { + let walker = ignore::WalkBuilder::new(src) .standard_filters(true) .hidden(false) .filter_entry(|entry| !(entry.path().is_dir() && entry.path().ends_with(".git"))) .build(); + let mut entries = Vec::new(); for entry in walker { let entry = entry?; let file_type = entry.file_type().ok_or(anyhow!( "failed to get file type of {}", entry.path().display() ))?; - - // get path of entry relative to src let relative_path = entry .path() - .strip_prefix(src.as_ref()) + .strip_prefix(src) .context("failed to get relative path")?; - - if file_type.is_dir() { - fs::create_dir_all(dst.as_ref().join(relative_path))?; + if relative_path.as_os_str().is_empty() { + continue; + } + let kind = if file_type.is_dir() { + SourcePathKind::Directory } else if file_type.is_file() { - fs::copy(entry.path(), dst.as_ref().join(relative_path)) - .context(format!("failed to copy file: {}", entry.path().display()))?; + SourcePathKind::File + } else if file_type.is_symlink() { + SourcePathKind::Symlink + } else { + return Err(anyhow!( + "unsupported file type in source tree: {}", + entry.path().display() + )); + }; + entries.push(SourcePath { + path: relative_path.to_path_buf(), + kind, + }); + } + entries.sort_by_key(|entry| entry.path.components().count()); + Ok(entries) +} + +fn remove_path(path: &Path) -> std::io::Result<()> { + match fs::symlink_metadata(path) { + Ok(metadata) if metadata.is_dir() => fs::remove_dir_all(path), + Ok(_) => fs::remove_file(path), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error), + } +} + +fn files_match(source: &Path, destination: &Path) -> std::io::Result { + let source_metadata = fs::metadata(source)?; + let destination_metadata = match fs::symlink_metadata(destination) { + Ok(metadata) if metadata.is_file() => metadata, + Ok(_) => return Ok(false), + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error), + }; + if source_metadata.len() != destination_metadata.len() + || source_metadata.permissions().mode() != destination_metadata.permissions().mode() + { + return Ok(false); + } + + let mut source = BufReader::new(fs::File::open(source)?); + let mut destination = BufReader::new(fs::File::open(destination)?); + let mut source_buffer = [0; 8192]; + let mut destination_buffer = [0; 8192]; + loop { + let source_len = source.read(&mut source_buffer)?; + let destination_len = destination.read(&mut destination_buffer)?; + if source_len != destination_len + || source_buffer[..source_len] != destination_buffer[..destination_len] + { + return Ok(false); + } + if source_len == 0 { + return Ok(true); + } + } +} + +fn copy_source_entries(src: &Path, dst: &Path, entries: &[SourcePath]) -> anyhow::Result<()> { + fs::create_dir_all(dst)?; + for entry in entries { + let source = src.join(&entry.path); + let destination = dst.join(&entry.path); + match entry.kind { + SourcePathKind::Directory => match fs::symlink_metadata(&destination) { + Ok(metadata) if metadata.is_dir() => {} + Ok(_) => { + remove_path(&destination)?; + fs::create_dir_all(&destination)?; + } + Err(error) if error.kind() == io::ErrorKind::NotFound => { + fs::create_dir_all(&destination)?; + } + Err(error) => return Err(error.into()), + }, + SourcePathKind::File => { + if !files_match(&source, &destination)? { + remove_path(&destination)?; + fs::copy(&source, &destination) + .with_context(|| format!("failed to copy file: {}", source.display()))?; + } + } + SourcePathKind::Symlink => { + let target = fs::read_link(&source)?; + if fs::read_link(&destination).ok().as_deref() != Some(target.as_path()) { + remove_path(&destination)?; + symlink(target, &destination)?; + } + } + } + } + Ok(()) +} + +fn source_manifest_path(build_config: &BuildConfig) -> PathBuf { + build_config.build_root_dir.join("source-manifest.json") +} + +fn write_source_manifest(build_config: &BuildConfig, entries: &[SourcePath]) -> anyhow::Result<()> { + let manifest_path = source_manifest_path(build_config); + let temporary_path = manifest_path.with_extension("json.tmp"); + fs::write(&temporary_path, serde_json::to_vec_pretty(entries)?)?; + fs::rename(temporary_path, manifest_path)?; + Ok(()) +} + +fn sync_source_tree(build_config: &BuildConfig) -> anyhow::Result<()> { + let manifest_path = source_manifest_path(build_config); + let previous: Vec = serde_json::from_reader(BufReader::new( + fs::File::open(&manifest_path) + .with_context(|| format!("failed to open {}", manifest_path.display()))?, + )) + .with_context(|| format!("failed to read {}", manifest_path.display()))?; + for entry in &previous { + validate_source_path(&entry.path)?; + } + let current = source_tree_entries(&build_config.source_dir)?; + + let current_kinds = current + .iter() + .map(|entry| (entry.path.as_path(), entry.kind)) + .collect::>(); + let mut stale = previous + .iter() + .filter(|entry| current_kinds.get(entry.path.as_path()) != Some(&entry.kind)) + .collect::>(); + stale.sort_by_key(|entry| Reverse(entry.path.components().count())); + for entry in stale { + let destination = build_config.build_source_dir().join(&entry.path); + if entry.kind == SourcePathKind::Directory + && !current_kinds.contains_key(entry.path.as_path()) + { + match fs::remove_dir(&destination) { + Ok(()) => {} + Err(error) + if matches!( + error.kind(), + io::ErrorKind::NotFound | io::ErrorKind::DirectoryNotEmpty + ) => {} + Err(error) => return Err(error.into()), + } + } else { + remove_path(&destination)?; } - // handle hardlinks, symlinks and similar weird filetypes } + + copy_source_entries( + &build_config.source_dir, + &build_config.build_source_dir(), + ¤t, + )?; + write_source_manifest(build_config, ¤t) +} + +fn stage_source_tree( + build_config: &BuildConfig, + package: &PackageDescription, +) -> anyhow::Result<()> { + if build_config.incremental && source_manifest_path(build_config).is_file() { + sync_source_tree(build_config).context("failed to synchronize source tree")?; + } else { + copy_dir_all(&build_config.source_dir, build_config.build_source_dir()) + .context("failed to copy source tree to build directory")?; + let entries = source_tree_entries(&build_config.source_dir)?; + write_source_manifest(build_config, &entries)?; + } + + let source_parent = build_config + .source_dir + .parent() + .ok_or_else(|| anyhow!("source directory has no parent"))?; + let prefix = format!("{}_{}", package.name, package.version.upstream_version()); + copy_glob( + source_parent, + &format!("{prefix}.orig.tar.*"), + &build_config.build_work_dir(), + )?; + copy_glob( + source_parent, + &format!("{prefix}.orig-*.tar.*"), + &build_config.build_work_dir(), + )?; Ok(()) } @@ -328,29 +576,80 @@ fn prepare_build_env( explicit_distro_version: Option<&str>, ) -> anyhow::Result { let (package_identifier, build_root) = get_build_root_and_identifier(config, package); - if build_root.exists() { - fs::remove_dir_all(&build_root)?; - } let distro_version = resolve_distro_version(&package.distro_versions, explicit_distro_version) .context("failed to determine distro version")?; let build_config = BuildConfig { driver: driver_type, + package_name: package.name.clone(), package_identifier, source_dir: package.source_dir.clone(), output_dir: output_dir.to_path_buf(), - build_root_dir: build_root, + build_root_dir: build_root.clone(), distro: distro_version.clone(), - sign_package: false, + sign_package: config.sign_package, + sign_key: config.sign_key.clone(), + build_debug_symbols: config.build_debug_symbols, + clean: config.clean, + persistent: config.driver.persistent, + incremental: config.incremental, }; + if config.driver.persistent && build_root.exists() { + // For persistent containers, starting first lets root inside delete + // container-owned files the host user can't remove. + let build = Build::create(&build_config, &config.driver, driver_overrides) + .context(format!("failed to create {:?} build driver", driver_type))?; + if !config.incremental + || !source_manifest_path(&build_config).is_file() + || !build.driver.reused_environment() + { + build + .driver + .reset_build_root() + .context("failed to reset persistent build directory")?; + } + build_config + .create_dirs() + .context("failed to create build directories")?; + stage_source_tree(&build_config, package)?; + return Ok(build); + } + + if build_root.exists() + && let Err(e) = fs::remove_dir_all(&build_root) + { + if e.kind() == io::ErrorKind::PermissionDenied { + // Some files were created by a privileged user inside a container + // and can't be deleted by the host user directly. Load the previous + // build's driver and ask it to clean up from inside. + let metadata_path = build_root.join("build.json"); + if metadata_path.is_file() + && let Ok(file) = fs::OpenOptions::new().read(true).open(&metadata_path) + && let Ok(metadata) = + serde_json::from_reader::<_, BuildMetadata>(BufReader::new(&file)) + && let Ok(driver) = create_driver_from_metadata(&config.driver, &metadata) + { + let _ = driver.reset_build_root(); + } + fs::remove_dir_all(&build_root).with_context(|| { + format!( + "failed to remove build root {}; try: sudo rm -rf {}", + build_root.display(), + build_root.display() + ) + })?; + } else { + return Err(e.into()); + } + } + build_config .create_dirs() .context("failed to create build directories")?; - copy_dir_all(&build_config.source_dir, build_config.build_source_dir()) - .context("failed to copy source tree to build directory")?; + stage_source_tree(&build_config, package)?; let build = Build::create(&build_config, &config.driver, driver_overrides)?; Ok(build) @@ -369,21 +668,46 @@ pub fn get_shell_in_build(config: &Config, package: &PackageDescription) -> anyh Ok(()) } -pub fn build_package( - config: &Config, - package: &PackageDescription, - driver_type: BuildDriverType, - driver_overrides: &DriverOverrides, - output_dir: &Path, - explicit_distro_version: Option<&str>, +fn deb_build_options(existing: Option<&str>, build_debug_symbols: bool) -> String { + let mut options = existing + .unwrap_or_default() + .split_whitespace() + .filter(|option| *option != "noautodbgsym") + .collect::>(); + if !build_debug_symbols { + options.push("noautodbgsym"); + } + options.join(" ") +} + +/// Everything needed to run one package build, independent of whether the +/// build produces binary or source packages. +pub struct BuildRequest<'a> { + pub config: &'a Config, + pub package: &'a PackageDescription, + pub driver_type: BuildDriverType, + pub driver_overrides: &'a DriverOverrides, + pub output_dir: &'a Path, + pub explicit_distro_version: Option<&'a str>, +} + +/// Shared build orchestration: prepare the environment, run `build_commands` +/// in it, export the artifacts to the output dir, sign them if requested, and +/// clean up (dropping into a shell first on failure of an interactive binary +/// build). While `shell_on_failure` is set, a socket server lets concurrent +/// `debmagic shell` sessions attach to the environment. +fn run_build( + request: &BuildRequest, + shell_on_failure: bool, + build_commands: impl FnOnce(&Build) -> anyhow::Result<()>, ) -> anyhow::Result<()> { let build = prepare_build_env( - config, - driver_overrides, - package, - driver_type, - output_dir, - explicit_distro_version, + request.config, + request.driver_overrides, + request.package, + request.driver_type, + request.output_dir, + request.explicit_distro_version, ) .context("failed to prepare build environment")?; build @@ -394,72 +718,260 @@ pub fn build_package( let socket_server_handle = start_socket_server(&build.config.build_root_dir, should_exit.clone())?; - let result = (|| -> anyhow::Result<()> { + let stop_socket_server = || { + *should_exit.lock().unwrap() = true; + if !socket_server_handle.is_finished() { + println!("Waiting for all attached shells to exit..."); + } + socket_server_handle.join().ok(); + }; + + let result = build_commands(&build).and_then(|()| { + let changes_file = artifacts::export_build_artifacts( + &build.config.build_work_dir(), + &build.config.output_dir, + )?; + if build.config.sign_package { + sign_changes_file(&changes_file, build.config.sign_key.as_deref())?; + } + Ok(()) + }); + + if let Err(error) = result { + if shell_on_failure && stdout().is_terminal() { + eprintln!("Build failed: {error}. Dropping into shell..."); + if let Err(shell_error) = build + .driver + .interactive_shell(&build.config.build_source_dir()) + { + eprintln!("Dropping into shell failed: {shell_error}"); + } + } else { + eprintln!("Build failed: {error}"); + } + if let Err(cleanup_error) = build.driver.cleanup() { + eprintln!("Failed to clean up build environment: {cleanup_error}"); + } + stop_socket_server(); + return Err(error); + } + + stop_socket_server(); + build + .driver + .cleanup() + .context("failed to clean up build environment")?; + Ok(()) +} + +pub fn build_package(request: &BuildRequest) -> anyhow::Result<()> { + run_build(request, true, |build| { build.driver.run_command( &["apt-get", "-y", "build-dep", "."], &build.config.build_source_dir(), true, )?; - build.driver.run_command( - &["dpkg-buildpackage", "-us", "-uc", "-ui", "-nc", "-b"], + let inherited_options = std::env::var("DEB_BUILD_OPTIONS").ok(); + let options = deb_build_options( + inherited_options.as_deref(), + build.config.build_debug_symbols, + ); + let env_add = [("DEB_BUILD_OPTIONS", options.as_str())]; + let mut dpkg_buildpackage_args = vec!["dpkg-buildpackage", "-us", "-uc", "-ui"]; + if !build.config.clean { + // Non-incremental builds already stage a clean source tree, while + // incremental builds preserve their outputs intentionally. + dpkg_buildpackage_args.push("-nc"); + } + dpkg_buildpackage_args.push("-b"); + build.driver.run_command_env( + &dpkg_buildpackage_args, &build.config.build_source_dir(), false, + &env_add, )?; + Ok(()) + }) +} - if build.config.sign_package { - // SIGN .changes and .dsc files - // changes = *.changes / *.dsc - // driver.run_command(&["debsign", opts, changes], &build_config.build_source_dir(), false)?; - // driver.run_command(&["debrsign", opts, username, changes], &build_config.build_source_dir(), false)?; +/// Confirm `cmd` is on `PATH`, failing with an actionable message (rather +/// than a raw "command not found") if it isn't. +fn check_command_available(cmd: &str, install_hint: &str) -> anyhow::Result<()> { + match Command::new(cmd) + .arg("--version") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + { + Ok(_) => Ok(()), + Err(e) if e.kind() == io::ErrorKind::NotFound => { + Err(anyhow!("{cmd} not found on PATH. {install_hint}")) } + Err(e) => Err(e).with_context(|| format!("failed to check for {cmd}")), + } +} - let parent_dir = build.config.build_source_dir().join(".."); - copy_glob(&parent_dir, "*.deb", &build.config.output_dir)?; - copy_glob(&parent_dir, "*.changes", &build.config.output_dir)?; - copy_glob(&parent_dir, "*.buildinfo", &build.config.output_dir)?; - copy_glob(&parent_dir, "*.dsc", &build.config.output_dir)?; +fn check_dpkg_buildpackage_available() -> anyhow::Result<()> { + check_command_available( + "dpkg-buildpackage", + "It's part of dpkg-dev; install it, or pass --driver lxd/incus/docker to build inside a Debian-ish container instead.", + ) +} - Ok(()) - })(); +/// GPG-sign every `.changes` (and its referenced `.dsc`/`.buildinfo`) in +/// `output_dir` with `debsign`. Always runs on the host regardless of the +/// build driver, since signing needs the user's own gpg keyring, which an +/// ephemeral container doesn't have access to. +fn sign_changes_file(changes_file: &Path, sign_key: Option<&str>) -> anyhow::Result<()> { + check_command_available( + "debsign", + "It's part of devscripts; install it and set up a gpg signing key to use --sign.", + )?; - if let Err(e) = result { - if stdout().is_terminal() { - eprintln!("Build failed: {e}. Dropping into shell..."); - let res = build - .driver - .interactive_shell(&build.config.build_source_dir()); - if let Err(shell_error) = res { - eprintln!("Dropping into shell failed: {shell_error}"); - } - } else { - eprintln!("Build failed: {e}"); - } - build.driver.cleanup(); - *should_exit.lock().unwrap() = true; - if !socket_server_handle.is_finished() { - println!("Waiting for all attached shells to exit..."); - } - socket_server_handle.join().ok(); - return Err(e); + let output_dir = changes_file.parent().ok_or_else(|| { + anyhow!( + "could not get output directory of {}", + changes_file.display() + ) + })?; + let filename = changes_file + .file_name() + .ok_or_else(|| anyhow!("could not get filename of {}", changes_file.display()))?; + + let mut cmd = Command::new("debsign"); + if let Some(key) = sign_key { + cmd.arg(format!("-k{key}")); } + cmd.arg(filename).current_dir(output_dir); + run_checked(&mut cmd, &format!("signing {}", changes_file.display()))?; + Ok(()) +} - // Signal the socket server to exit and wait for it to complete - *should_exit.lock().unwrap() = true; - if !socket_server_handle.is_finished() { - println!("Waiting for all attached shells to exit..."); +/// Build a `.dsc` + tarball + `.buildinfo` + `.changes` source package. +/// +/// If `config.clean` is set, build-dependencies are installed before +/// `dpkg-buildpackage` runs `debian/rules clean` once. +pub fn build_source_package(request: &BuildRequest) -> anyhow::Result<()> { + if request.driver_type == BuildDriverType::Bare { + check_dpkg_buildpackage_available()?; } - socket_server_handle.join().ok(); - build.driver.cleanup(); - Ok(()) + run_build(request, false, |build| { + let build_source_dir = build.config.build_source_dir(); + if build.config.clean { + build.driver.run_command( + &["apt-get", "-y", "build-dep", "."], + &build_source_dir, + true, + )?; + } + let mut args = vec!["dpkg-buildpackage", "-S", "-d", "-us", "-uc", "-ui"]; + if !build.config.clean { + args.push("-nc"); + } + build.driver.run_command(&args, &build_source_dir, false)?; + Ok(()) + }) + .context("failed to build source package") } #[cfg(test)] mod tests { + use std::os::unix::fs::MetadataExt; + use debmagic_common::distro::Distro; use super::*; + #[test] + fn debug_symbol_option_preserves_other_build_options() { + assert_eq!( + deb_build_options(Some("nocheck parallel=8"), false), + "nocheck parallel=8 noautodbgsym" + ); + assert_eq!( + deb_build_options(Some("nocheck noautodbgsym parallel=8"), true), + "nocheck parallel=8" + ); + } + + #[test] + fn incremental_sync_updates_sources_and_preserves_build_outputs() -> anyhow::Result<()> { + let test_root = std::env::temp_dir().join(format!( + "debmagic-incremental-test-{}", + uuid::Uuid::new_v4() + )); + let source_dir = test_root.join("source"); + let build_root_dir = test_root.join("build"); + fs::create_dir_all(source_dir.join("cache"))?; + fs::write(source_dir.join("changed.txt"), "before")?; + fs::write(source_dir.join("unchanged.txt"), "unchanged")?; + fs::write(source_dir.join("removed.txt"), "remove me")?; + fs::write(source_dir.join("cache/input.c"), "source")?; + symlink("changed.txt", source_dir.join("link"))?; + + let build_config = BuildConfig { + driver: BuildDriverType::Bare, + package_name: "example".to_string(), + package_identifier: "example-1.0".to_string(), + build_root_dir: build_root_dir.clone(), + source_dir: source_dir.clone(), + output_dir: test_root.join("output"), + distro: debmagic_common::distro::get_distro_version("trixie").unwrap(), + sign_package: false, + sign_key: None, + build_debug_symbols: false, + clean: false, + persistent: true, + incremental: true, + }; + build_config.create_dirs()?; + copy_dir_all(&source_dir, build_config.build_source_dir())?; + write_source_manifest(&build_config, &source_tree_entries(&source_dir)?)?; + let unchanged_inode = + fs::metadata(build_config.build_source_dir().join("unchanged.txt"))?.ino(); + fs::write( + build_config.build_source_dir().join("cache/output.o"), + "compiled", + )?; + + fs::write(source_dir.join("changed.txt"), "after")?; + fs::remove_file(source_dir.join("removed.txt"))?; + fs::remove_file(source_dir.join("cache/input.c"))?; + fs::remove_dir(source_dir.join("cache"))?; + fs::remove_file(source_dir.join("link"))?; + symlink("added.txt", source_dir.join("link"))?; + fs::write(source_dir.join("added.txt"), "new")?; + + sync_source_tree(&build_config)?; + + let staged = build_config.build_source_dir(); + assert_eq!(fs::read_to_string(staged.join("changed.txt"))?, "after"); + assert_eq!(fs::read_to_string(staged.join("added.txt"))?, "new"); + assert_eq!( + fs::metadata(staged.join("unchanged.txt"))?.ino(), + unchanged_inode + ); + assert_eq!(fs::read_link(staged.join("link"))?, Path::new("added.txt")); + assert!(!staged.join("removed.txt").exists()); + assert!(!staged.join("cache/input.c").exists()); + assert_eq!( + fs::read_to_string(staged.join("cache/output.o"))?, + "compiled" + ); + + fs::remove_dir_all(test_root)?; + Ok(()) + } + + #[test] + fn source_manifest_paths_must_be_relative_and_normal() { + assert!(validate_source_path(Path::new("debian/control")).is_ok()); + for path in ["", ".", "../outside", "debian/../outside", "/tmp/outside"] { + assert!(validate_source_path(Path::new(path)).is_err(), "{path}"); + } + } + #[test] fn test_resolve_distro_version_single_distro_no_explicit() { let distros = vec!["forky".to_string()]; diff --git a/packages/debmagic/src/build/artifacts.rs b/packages/debmagic/src/build/artifacts.rs new file mode 100644 index 0000000..d91c520 --- /dev/null +++ b/packages/debmagic/src/build/artifacts.rs @@ -0,0 +1,219 @@ +use std::{ + ffi::OsStr, + fs, + path::{Component, Path, PathBuf}, +}; + +use anyhow::{Context, anyhow, bail}; +use debian_control::lossless::changes::Changes; + +fn changes_file_in(build_dir: &Path) -> anyhow::Result { + let mut paths = fs::read_dir(build_dir) + .with_context(|| { + format!( + "failed to read build output directory {}", + build_dir.display() + ) + })? + .filter_map(|entry| entry.ok().map(|entry| entry.path())) + .filter(|path| path.extension() == Some(OsStr::new("changes"))); + + let path = paths + .next() + .ok_or_else(|| anyhow!("build produced no .changes file in {}", build_dir.display()))?; + if paths.next().is_some() { + bail!( + "build produced multiple .changes files in {}; refusing to guess which upload set to export", + build_dir.display() + ); + } + Ok(path) +} + +fn artifact_filename(filename: &str) -> anyhow::Result<&OsStr> { + let path = Path::new(filename); + let mut components = path.components(); + match (components.next(), components.next()) { + (Some(Component::Normal(filename)), None) => Ok(filename), + _ => bail!("invalid artifact filename in .changes: {filename:?}"), + } +} + +fn reject_destination_symlink(path: &Path) -> anyhow::Result<()> { + match fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_symlink() => { + bail!("refusing to overwrite artifact symlink {}", path.display()) + } + Ok(_) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + } +} + +pub fn export_build_artifacts(build_dir: &Path, output_dir: &Path) -> anyhow::Result { + fs::create_dir_all(output_dir) + .with_context(|| format!("failed to create output directory {}", output_dir.display()))?; + + let changes_path = changes_file_in(build_dir)?; + let changes_metadata = fs::symlink_metadata(&changes_path)?; + if !changes_metadata.file_type().is_file() { + bail!( + "changes file {} is not a regular file", + changes_path.display() + ); + } + let changes = Changes::from_file(&changes_path) + .with_context(|| format!("failed to parse {}", changes_path.display()))?; + let files = changes + .files() + .ok_or_else(|| anyhow!("{} has no Files field", changes_path.display()))?; + + for file in files { + let filename = artifact_filename(&file.filename)?; + let source = build_dir.join(filename); + let metadata = fs::symlink_metadata(&source).with_context(|| { + format!( + "artifact {} referenced by {} does not exist", + source.display(), + changes_path.display() + ) + })?; + if !metadata.file_type().is_file() { + bail!("build artifact {} is not a regular file", source.display()); + } + if metadata.len() != file.size as u64 { + bail!( + "artifact {} has size {}, but {} records {}", + source.display(), + metadata.len(), + changes_path.display(), + file.size + ); + } + let destination = output_dir.join(filename); + reject_destination_symlink(&destination)?; + fs::copy(&source, destination) + .with_context(|| format!("failed to copy build artifact {}", source.display()))?; + } + + let changes_filename = changes_path + .file_name() + .ok_or_else(|| anyhow!("invalid .changes path: {}", changes_path.display()))?; + let exported_changes = output_dir.join(changes_filename); + reject_destination_symlink(&exported_changes)?; + fs::copy(&changes_path, &exported_changes) + .with_context(|| format!("failed to copy {}", changes_path.display()))?; + Ok(exported_changes) +} + +#[cfg(test)] +mod tests { + use std::os::unix::fs::symlink; + + use super::*; + + fn test_dir(name: &str) -> PathBuf { + let path = std::env::temp_dir().join(format!("debmagic-{name}-{}", uuid::Uuid::new_v4())); + fs::create_dir(&path).unwrap(); + path + } + + fn write_changes(path: &Path, files: &[(&str, &str)]) { + let files = files + .iter() + .map(|(filename, contents)| { + fs::write(path.parent().unwrap().join(filename), contents).unwrap(); + format!( + " d41d8cd98f00b204e9800998ecf8427e {} misc optional {}", + contents.len(), + filename + ) + }) + .collect::>() + .join("\n"); + fs::write( + path, + format!("Format: 1.8\nSource: test\nFiles:\n{files}\n"), + ) + .unwrap(); + } + + #[test] + fn exports_every_file_listed_by_changes() { + let build_dir = test_dir("artifact-build"); + let output_dir = test_dir("artifact-output"); + let changes_path = build_dir.join("test_1_amd64.changes"); + let artifacts = [ + ("test_1_amd64.deb", "deb"), + ("test_1_amd64.udeb", "udeb"), + ("test-dbgsym_1_amd64.ddeb", "ddeb"), + ("test_1_amd64.buildinfo", "buildinfo"), + ]; + write_changes(&changes_path, &artifacts); + + let exported_changes = export_build_artifacts(&build_dir, &output_dir).unwrap(); + + assert_eq!(exported_changes, output_dir.join("test_1_amd64.changes")); + for (filename, contents) in artifacts { + assert_eq!( + fs::read_to_string(output_dir.join(filename)).unwrap(), + contents + ); + } + fs::remove_dir_all(build_dir).unwrap(); + fs::remove_dir_all(output_dir).unwrap(); + } + + #[test] + fn rejects_ambiguous_changes_files() { + let build_dir = test_dir("artifact-ambiguous"); + let output_dir = test_dir("artifact-ambiguous-output"); + fs::write(build_dir.join("one.changes"), "").unwrap(); + fs::write(build_dir.join("two.changes"), "").unwrap(); + + let error = export_build_artifacts(&build_dir, &output_dir).unwrap_err(); + + assert!(error.to_string().contains("multiple .changes files")); + fs::remove_dir_all(build_dir).unwrap(); + fs::remove_dir_all(output_dir).unwrap(); + } + + #[test] + fn rejects_artifact_symlinks() { + let build_dir = test_dir("artifact-symlink"); + let output_dir = test_dir("artifact-symlink-output"); + let changes_path = build_dir.join("test_1_amd64.changes"); + write_changes(&changes_path, &[("test_1_amd64.deb", "deb")]); + fs::remove_file(build_dir.join("test_1_amd64.deb")).unwrap(); + fs::write(build_dir.join("target"), "deb").unwrap(); + symlink("target", build_dir.join("test_1_amd64.deb")).unwrap(); + + let error = export_build_artifacts(&build_dir, &output_dir).unwrap_err(); + + assert!(error.to_string().contains("not a regular file")); + fs::remove_dir_all(build_dir).unwrap(); + fs::remove_dir_all(output_dir).unwrap(); + } + + #[test] + fn rejects_destination_symlinks() { + let build_dir = test_dir("artifact-destination-symlink"); + let output_dir = test_dir("artifact-destination-symlink-output"); + let changes_path = build_dir.join("test_1_amd64.changes"); + write_changes(&changes_path, &[("test_1_amd64.deb", "deb")]); + let target = output_dir.join("target"); + fs::write(&target, "unchanged").unwrap(); + symlink("target", output_dir.join("test_1_amd64.deb")).unwrap(); + + let error = export_build_artifacts(&build_dir, &output_dir).unwrap_err(); + + assert!( + error + .to_string() + .contains("refusing to overwrite artifact symlink") + ); + assert_eq!(fs::read_to_string(target).unwrap(), "unchanged"); + fs::remove_dir_all(build_dir).unwrap(); + fs::remove_dir_all(output_dir).unwrap(); + } +} diff --git a/packages/debmagic/src/build/common.rs b/packages/debmagic/src/build/common.rs index e420f3f..fae4244 100644 --- a/packages/debmagic/src/build/common.rs +++ b/packages/debmagic/src/build/common.rs @@ -3,17 +3,98 @@ use std::{ fmt::Debug, fs, path::{Path, PathBuf}, + process::Command, }; +use anyhow::Context; use clap::ValueEnum; use debmagic_common::distro::DistroVersion; use serde::{Deserialize, Serialize}; +/// Path at which the build root is bind-mounted inside container-based +/// drivers (Docker, LXD, Incus). +pub const BUILD_DIR_IN_CONTAINER: &str = "/debmagic"; + +/// Rewrite a path inside the host's build root to the equivalent path inside +/// a container that has it bind-mounted at [`BUILD_DIR_IN_CONTAINER`]. +pub fn translate_path_in_container( + build_root_dir: &Path, + path_in_source: &Path, +) -> std::io::Result { + path_in_source + .strip_prefix(build_root_dir) + .map(|rel| Path::new(BUILD_DIR_IN_CONTAINER).join(rel)) + .map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::NotFound, + "Path is not relative to build root".to_string(), + ) + }) +} + +/// Run `cmd`, failing with `context` (and, on a clean but unsuccessful exit, +/// its exit status) if it can't be spawned or exits unsuccessfully. +pub fn run_checked(cmd: &mut Command, context: &str) -> anyhow::Result<()> { + let status = cmd + .status() + .with_context(|| format!("Error running {context}"))?; + if !status.success() { + anyhow::bail!("{context} failed (exit status: {status})"); + } + Ok(()) +} + +pub fn resource_name(prefix: &str, label: &str, identifier: &str) -> String { + const MAX_LEN: usize = 63; + const HASH_LEN: usize = 16; + + let mut hash = uuid::Uuid::new_v5(&uuid::Uuid::NAMESPACE_OID, identifier.as_bytes()) + .simple() + .to_string(); + hash.truncate(HASH_LEN); + let max_label_len = MAX_LEN - prefix.len() - hash.len() - 2; + let label = label + .chars() + .map(|character| { + if character.is_ascii_alphanumeric() { + character.to_ascii_lowercase() + } else { + '-' + } + }) + .take(max_label_len) + .collect::(); + format!("{prefix}-{label}-{hash}") +} + +pub fn environment_fingerprint(parts: &[&str]) -> String { + uuid::Uuid::new_v5(&uuid::Uuid::NAMESPACE_OID, parts.join("\0").as_bytes()) + .simple() + .to_string() +} + +/// Metadata key under which container-based drivers store their container's +/// name for later reattachment via `from_build_metadata`. +const CONTAINER_NAME_KEY: &str = "container_name"; + +pub fn container_name_metadata(name: &str) -> DriverSpecificBuildMetadata { + DriverSpecificBuildMetadata::from([(CONTAINER_NAME_KEY.to_string(), name.to_string())]) +} + +pub fn container_name_from_metadata(build_metadata: &BuildMetadata) -> anyhow::Result { + build_metadata + .driver_metadata + .get(CONTAINER_NAME_KEY) + .cloned() + .context("build metadata has no container_name") +} + #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum, Serialize, Deserialize)] pub enum BuildDriverType { Docker, Bare, - // Lxd + Lxd, + Incus, } pub type DriverSpecificBuildMetadata = HashMap; @@ -28,12 +109,28 @@ pub struct BuildMetadata { pub struct BuildConfig { pub driver: BuildDriverType, + #[serde(default)] + pub package_name: String, pub package_identifier: String, pub build_root_dir: PathBuf, pub source_dir: PathBuf, pub output_dir: PathBuf, pub distro: DistroVersion, pub sign_package: bool, + /// GPG key ID/email to sign with (debsign's `-k` option). + pub sign_key: Option, + /// Build the automatic `-dbgsym` debug symbol package alongside the regular binaries. + #[serde(default)] + pub build_debug_symbols: bool, + /// Run `debian/rules clean` before building. + #[serde(default)] + pub clean: bool, + /// Keep the build environment running after the build. + #[serde(default)] + pub persistent: bool, + /// Synchronize source inputs while preserving build-generated files. + #[serde(default)] + pub incremental: bool, } impl BuildConfig { @@ -44,14 +141,6 @@ impl BuildConfig { ) } - /// Identifier safe for Docker image tags and container names. - /// - /// Debian versions may contain `~`, `+`, or `:` (e.g. `0.0.1~alpha2`), which - /// are invalid in Docker references. - pub fn docker_identifier(&self) -> String { - sanitize_docker_reference(&self.build_identifier()) - } - pub fn build_work_dir(&self) -> PathBuf { self.build_root_dir.join("work") } @@ -73,34 +162,66 @@ impl BuildConfig { } } -/// Replace characters that Docker image/container names disallow. -fn sanitize_docker_reference(name: &str) -> String { - name.chars() - .map(|c| match c { - 'a'..='z' | 'A'..='Z' | '0'..='9' | '.' | '_' | '-' => c, - _ => '-', - }) - .collect() -} +/// Source of a Python script that rewrites the default Debian/Ubuntu apt +/// sources to point at a mirror. Replaces the base image's default sources +/// file(s) outright with one debmagic owns, rather than parsing and +/// patching them in place, defaulting to the release/updates/security +/// pockets it detects from `/etc/os-release`. +pub const APT_MIRROR_SCRIPT: &str = include_str!("scripts/mirror.py"); pub trait BuildDriver { fn get_build_metadata(&self) -> DriverSpecificBuildMetadata; - fn run_command(&self, cmd: &[&str], cwd: &Path, requires_root: bool) -> std::io::Result<()>; + fn run_command_env( + &self, + cmd: &[&str], + cwd: &Path, + requires_root: bool, + env_add: &[(&str, &str)], + ) -> std::io::Result<()>; - fn cleanup(&self); + fn run_command(&self, cmd: &[&str], cwd: &Path, requires_root: bool) -> std::io::Result<()> { + self.run_command_env(cmd, cwd, requires_root, &[]) + } + + fn cleanup(&self) -> anyhow::Result<()>; fn interactive_shell(&self, cwd: &Path) -> std::io::Result<()>; fn driver_type(&self) -> BuildDriverType; + + fn reset_build_root(&self) -> std::io::Result<()>; + + fn reused_environment(&self) -> bool { + true + } } #[cfg(test)] mod tests { use super::*; + use debmagic_common::distro::{Distro, DistroVersion}; use std::path::PathBuf; + #[test] + fn resource_names_are_valid_stable_and_distinct() { + let first = resource_name("debmagic", "package", "package-1.0~beta-1:2-debian-forky"); + let second = resource_name("debmagic", "package", "package-1.0-beta-1:2-debian-forky"); + + assert_eq!( + first, + resource_name("debmagic", "package", "package-1.0~beta-1:2-debian-forky") + ); + assert_ne!(first, second); + assert!(first.starts_with("debmagic-package-")); + assert_eq!(first.len(), "debmagic-package-".len() + 16); + assert!(first.len() <= 63); + assert!(first.chars().all(|character| character.is_ascii_lowercase() + || character.is_ascii_digit() + || character == '-')); + } + fn sample_config(package_identifier: &str) -> BuildConfig { BuildConfig { driver: BuildDriverType::Docker, @@ -112,8 +233,15 @@ mod tests { distro: Distro::Debian, codename: "forky".to_string(), version: "15".to_string(), + is_devel: false, }, sign_package: false, + incremental: false, + build_debug_symbols: false, + clean: false, + persistent: false, + package_name: "debmagic".to_string(), + sign_key: None, } } @@ -124,23 +252,5 @@ mod tests { config.build_identifier(), "debmagic-0.0.1~alpha2-debian-forky" ); - assert_eq!( - config.docker_identifier(), - "debmagic-0.0.1-alpha2-debian-forky" - ); - } - - #[test] - fn docker_identifier_replaces_epoch_and_plus() { - let config = sample_config("pkg-1:2.0.0+dfsg1"); - assert_eq!(config.docker_identifier(), "pkg-1-2.0.0-dfsg1-debian-forky"); - } - - #[test] - fn sanitize_docker_reference_keeps_allowed_chars() { - assert_eq!( - sanitize_docker_reference("debmagic-0.0.1-alpha1_x86"), - "debmagic-0.0.1-alpha1_x86" - ); } } diff --git a/packages/debmagic/src/build/config.rs b/packages/debmagic/src/build/config.rs index 193d454..79a4aaf 100644 --- a/packages/debmagic/src/build/config.rs +++ b/packages/debmagic/src/build/config.rs @@ -2,17 +2,27 @@ use serde::Deserialize; use crate::build::driver_bare::{DriverBareConfig, DriverBareConfigOverrides}; use crate::build::driver_docker::{DriverDockerConfig, DriverDockerConfigOverrides}; +use crate::build::driver_lxd::{DriverLxdConfig, DriverLxdConfigOverrides}; #[derive(Deserialize, Debug, Clone, Default)] #[serde(default)] pub struct DriverConfig { pub persistent: bool, + /// Not used by the bare driver, which builds on the host's own sources. + pub apt_mirror: Option, + /// Also enable the `-proposed` pocket. Not used by the bare + /// driver, which builds on the host's own sources. + pub proposed: bool, pub docker: DriverDockerConfig, pub bare: DriverBareConfig, + pub lxd: DriverLxdConfig, } #[derive(Deserialize, Debug, Clone, Default)] pub struct DriverOverrides { + pub apt_mirror: Option, + pub proposed: Option, pub docker: DriverDockerConfigOverrides, pub bare: DriverBareConfigOverrides, + pub lxd: DriverLxdConfigOverrides, } diff --git a/packages/debmagic/src/build/driver_bare.rs b/packages/debmagic/src/build/driver_bare.rs index 647df21..b87d2f4 100644 --- a/packages/debmagic/src/build/driver_bare.rs +++ b/packages/debmagic/src/build/driver_bare.rs @@ -50,7 +50,13 @@ impl BuildDriver for DriverBare { DriverSpecificBuildMetadata::from([]) } - fn run_command(&self, cmd: &[&str], cwd: &Path, requires_root: bool) -> std::io::Result<()> { + fn run_command_env( + &self, + cmd: &[&str], + cwd: &Path, + requires_root: bool, + env_add: &[(&str, &str)], + ) -> std::io::Result<()> { let mut full_cmd: Vec = Vec::new(); let is_root = unsafe { libc::geteuid() == 0 }; @@ -64,6 +70,7 @@ impl BuildDriver for DriverBare { command.args(&full_cmd[1..]); command.current_dir(cwd); + command.envs(env_add.iter().copied()); let status = command.status()?; @@ -77,8 +84,8 @@ impl BuildDriver for DriverBare { } } - fn cleanup(&self) { - // No-op for bare driver + fn cleanup(&self) -> anyhow::Result<()> { + Ok(()) } fn interactive_shell(&self, _cwd: &Path) -> std::io::Result<()> { @@ -92,4 +99,11 @@ impl BuildDriver for DriverBare { fn driver_type(&self) -> BuildDriverType { BuildDriverType::Bare } + + fn reset_build_root(&self) -> std::io::Result<()> { + if self.config.build_root_dir.exists() { + std::fs::remove_dir_all(&self.config.build_root_dir)?; + } + Ok(()) + } } diff --git a/packages/debmagic/src/build/driver_docker.rs b/packages/debmagic/src/build/driver_docker.rs index e80c6f7..7afe789 100644 --- a/packages/debmagic/src/build/driver_docker.rs +++ b/packages/debmagic/src/build/driver_docker.rs @@ -11,7 +11,10 @@ use serde::{Deserialize, Serialize}; use crate::build::{ common::{ - BuildConfig, BuildDriver, BuildDriverType, BuildMetadata, DriverSpecificBuildMetadata, + APT_MIRROR_SCRIPT, BUILD_DIR_IN_CONTAINER, BuildConfig, BuildDriver, BuildDriverType, + BuildMetadata, DriverSpecificBuildMetadata, container_name_from_metadata, + container_name_metadata, environment_fingerprint, resource_name, run_checked, + translate_path_in_container, }, config::DriverConfig, }; @@ -37,71 +40,99 @@ pub struct DriverDockerConfigOverrides { } // Constants -const BUILD_DIR_IN_CONTAINER: &str = "/debmagic"; -const DOCKER_USER: &str = "user"; +const ENVIRONMENT_LABEL: &str = "dev.debmagic.environment"; + +fn bind_mount_arg(build_root: &Path) -> String { + format!( + "type=bind,src={},dst={}", + build_root.display(), + BUILD_DIR_IN_CONTAINER + ) +} const DOCKERFILE_TEMPLATE: &str = r#" FROM {base_image} -ARG USERNAME={docker_user} ARG USER_UID=1000 ARG USER_GID=$USER_UID -RUN apt-get update && apt-get install -y sudo dpkg-dev python3 -RUN groupadd --gid $USER_GID $USERNAME \ - && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME \ - && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \ - && chmod 0440 /etc/sudoers.d/$USERNAME -RUN mkdir -p /build/package/debian -RUN --mount=type=bind,source=debian/control,target=/build/package/debian/control apt-get -y build-dep /build/package -RUN mkdir -p {build_dir} -RUN chown $USERNAME:$USERNAME {build_dir} -USER $USERNAME +RUN apt-get update && apt-get install -y dpkg-dev python3 python3-apt +{apt_mirror_setup} +RUN set -e; \ + getent group "$USER_GID" >/dev/null || groupadd --gid "$USER_GID" debmagic; \ + getent passwd "$USER_UID" >/dev/null || useradd --uid "$USER_UID" --gid "$USER_GID" -m debmagic +RUN mkdir -p {build_dir} && chown $USER_UID:$USER_GID {build_dir} +USER $USER_UID:$USER_GID ENTRYPOINT ["sleep", "infinity"] "#; -#[derive(Debug, Serialize, Deserialize)] -pub struct DockerDriverBuildMetadata { - pub container_name: String, +// Ubuntu images ship python3 already; installed explicitly above for the +// others (e.g. Debian's slim images) before the mirror script needs it. +const APT_MIRROR_SCRIPT_FILENAME: &str = "mirror.py"; + +/// Quote `s` as a single POSIX shell word for embedding in a Dockerfile +/// `RUN` instruction (which is interpreted by `/bin/sh -c`). +fn shell_quote(s: &str) -> String { + format!("'{}'", s.replace('\'', "'\\''")) } pub struct DriverDocker { config: BuildConfig, - driver_config: DriverConfig, container_name: String, + reused_environment: bool, +} + +/// Replace characters that Docker image/container names disallow. +fn sanitize_docker_reference(name: &str) -> String { + name.chars() + .map(|c| match c { + 'a'..='z' | 'A'..='Z' | '0'..='9' | '.' | '_' | '-' => c, + _ => '-', + }) + .collect() } fn build_build_image( config: &BuildConfig, - driver_config: &DriverConfig, - overrides: &DriverDockerConfigOverrides, -) -> anyhow::Result { - let base_image = overrides - .base_image - .clone() - .unwrap_or_else(|| driver_config.docker.base_image_for_distro(&config.distro)); + base_image: &str, + apt_mirror: Option<&str>, + proposed: bool, + image_name: &str, +) -> anyhow::Result<()> { + let apt_mirror_setup = if apt_mirror.is_some() || proposed { + fs::write( + config.build_temp_dir().join(APT_MIRROR_SCRIPT_FILENAME), + APT_MIRROR_SCRIPT, + ) + .map_err(|e| anyhow!("Failed to write apt mirror script: {e}"))?; + + let mut args = vec![ + "--codename".to_string(), + shell_quote(&config.distro.codename), + ]; + if let Some(mirror) = apt_mirror { + args.extend(["--mirror".to_string(), shell_quote(mirror)]); + } + if proposed { + args.push("--proposed".to_string()); + } - let debian_control_file_path = config.build_source_dir().join("debian").join("control"); + let setup = format!( + "COPY {script} /tmp/{script}\nRUN python3 /tmp/{script} {args} && rm /tmp/{script} && apt-get update\n", + script = APT_MIRROR_SCRIPT_FILENAME, + args = args.join(" "), + ); + setup + } else { + String::new() + }; let formatted_dockerfile = DOCKERFILE_TEMPLATE - .replace("{base_image}", &base_image) - .replace("{docker_user}", DOCKER_USER) - .replace("{build_dir}", BUILD_DIR_IN_CONTAINER) - .replace( - "{debian_control_file}", - &debian_control_file_path.to_string_lossy(), - ); + .replace("{base_image}", base_image) + .replace("{apt_mirror_setup}", &apt_mirror_setup) + .replace("{build_dir}", BUILD_DIR_IN_CONTAINER); let dockerfile_path = config.build_temp_dir().join("Dockerfile"); fs::write(&dockerfile_path, formatted_dockerfile) .map_err(|e| anyhow!("Failed to write Dockerfile, {e}"))?; - fs::create_dir_all(config.build_temp_dir().join("debian")) - .map_err(|e| anyhow!("Failed to create debian directory: {e}"))?; - fs::copy( - &debian_control_file_path, - config.build_temp_dir().join("debian").join("control"), - ) - .map_err(|e| anyhow!("Failed to copy debian control file: {e}"))?; - - let docker_image_name = format!("debmagic-{}", config.docker_identifier()); let mut build_args = Vec::new(); let uid = unsafe { libc::geteuid() }; @@ -117,150 +148,227 @@ fn build_build_image( build_cmd .args(["build"]) .args(&build_args) - .args(["--tag", &docker_image_name, "-f"]) + .args(["--tag", image_name, "-f"]) .arg(dockerfile_path) .arg(config.build_temp_dir()); - let status = build_cmd - .status() - .map_err(|e| anyhow!("Error running docker build: {}", e))?; - if !status.success() { - return Err(anyhow!("Error creating docker image")); - } + run_checked(&mut build_cmd, "building docker image")?; - Ok(docker_image_name) + Ok(()) } -fn does_container_exist(container_name: &str) -> anyhow::Result { - let mut ps_cmd = Command::new("docker"); - ps_cmd.args(["ps", "--all", "--format", "json"]); - ps_cmd.stdout(Stdio::piped()); +fn does_image_exist(image_name: &str) -> anyhow::Result { + let status = Command::new("docker") + .args(["image", "inspect", image_name]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .map_err(|error| anyhow!("Failed to inspect Docker image: {error}"))?; + Ok(status.success()) +} - let output = ps_cmd +/// The environment fingerprint stored in the container's labels, or `None` +/// if the container does not exist (or has no fingerprint label). +fn container_environment_fingerprint(container_name: &str) -> anyhow::Result> { + let output = Command::new("docker") + .args(["inspect", container_name]) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) .output() - .map_err(|_| anyhow!("Failed to read docker ps output"))?; - + .map_err(|error| anyhow!("failed to inspect Docker container: {error}"))?; if !output.status.success() { - return Err(anyhow!("failed to query running docker containers")); + // docker inspect exits non-zero when the container doesn't exist. + return Ok(None); } - let stdout = String::from_utf8_lossy(&output.stdout); - for line in stdout.lines() { - if let Ok(container) = serde_json::from_str::(line) - && let Some(names) = container.get("Names") - && names == container_name - { - return Ok(true); - } - } - Ok(false) + let containers: Vec = serde_json::from_slice(&output.stdout) + .map_err(|error| anyhow!("failed to parse Docker inspect output: {error}"))?; + Ok(containers + .first() + .and_then(|container| container.pointer("/Config/Labels")) + .and_then(|labels| labels.get(ENVIRONMENT_LABEL)) + .and_then(serde_json::Value::as_str) + .map(str::to_owned)) } impl DriverDocker { + fn container_start(&self) -> anyhow::Result<()> { + run_checked( + Command::new("docker").args(["start", &self.container_name]), + "starting docker container", + ) + } + + fn container_stop(&self) -> anyhow::Result<()> { + run_checked( + Command::new("docker").args(["stop", &self.container_name]), + "stopping docker container", + ) + } + + fn container_is_running(&self) -> anyhow::Result { + let output = Command::new("docker") + .args([ + "inspect", + "--format", + "{{.State.Running}}", + &self.container_name, + ]) + .output() + .map_err(|error| anyhow!("failed to inspect Docker container: {error}"))?; + if !output.status.success() { + return Err(anyhow!( + "Docker container {} not found", + self.container_name + )); + } + Ok(String::from_utf8_lossy(&output.stdout).trim() == "true") + } + + fn container_remove_force(&self) -> anyhow::Result<()> { + run_checked( + Command::new("docker").args(["rm", "-f", &self.container_name]), + "removing docker container", + ) + } + pub fn create( config: &BuildConfig, driver_config: &DriverConfig, overrides: &DriverDockerConfigOverrides, + apt_mirror: Option<&str>, + proposed: bool, ) -> anyhow::Result { - let container_name = format!("debmagic-{}", config.docker_identifier()); - let container_exists = does_container_exist(&container_name)?; - - if driver_config.persistent && container_exists { - let mut start_cmd = Command::new("docker"); - start_cmd.args(["start", &container_name]); - let status = start_cmd - .status() - .map_err(|e| anyhow!("Error running docker start: {}", e))?; - if !status.success() { - return Err(anyhow!("Error starting docker container")); + let base_image = overrides + .base_image + .clone() + .unwrap_or_else(|| driver_config.docker.base_image_for_distro(&config.distro)); + let uid = unsafe { libc::geteuid() }.to_string(); + let gid = unsafe { libc::getegid() }.to_string(); + let build_root = config.build_root_dir.to_string_lossy(); + let proposed_fingerprint = proposed.to_string(); + let image_fingerprint = environment_fingerprint(&[ + "docker", + DOCKERFILE_TEMPLATE, + APT_MIRROR_SCRIPT, + &base_image, + &config.distro.codename, + apt_mirror.unwrap_or(""), + &proposed_fingerprint, + &uid, + &gid, + ]); + let desired_fingerprint = + environment_fingerprint(&["docker-container", &image_fingerprint, build_root.as_ref()]); + let container_name = resource_name( + "debmagic", + &config.package_name, + &sanitize_docker_reference(&config.build_identifier()), + ); + let mut driver = Self { + config: config.clone(), + container_name, + reused_environment: false, + }; + let environment_matches = container_environment_fingerprint(&driver.container_name)? + .as_deref() + == Some(&desired_fingerprint); + driver.reused_environment = config.persistent && environment_matches; + let created_container; + + if config.persistent && environment_matches { + created_container = false; + if !driver.container_is_running()? { + driver.container_start()?; } } else { - if container_exists { - let mut rm_cmd = Command::new("docker"); - rm_cmd.args(["rm", "-f", &container_name]); - let status = rm_cmd - .status() - .map_err(|e| anyhow!("Error running docker rm: {}", e))?; - if !status.success() { - return Err(anyhow!("Error removing existing docker container")); - } + // The container may not exist; removal errors don't matter here. + let _ = Command::new("docker") + .args(["rm", "-f", &driver.container_name]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + + let docker_image_name = format!("debmagic-env-{}", &image_fingerprint[..16]); + if !does_image_exist(&docker_image_name)? { + build_build_image( + config, + &base_image, + apt_mirror, + proposed, + &docker_image_name, + )?; } + run_checked( + Command::new("docker") + .args([ + "run", + "--detach", + "--name", + &driver.container_name, + "--label", + &format!("{ENVIRONMENT_LABEL}={desired_fingerprint}"), + "--mount", + ]) + .arg(bind_mount_arg(&config.build_root_dir)) + .arg(&docker_image_name), + "starting docker container", + )?; + created_container = true; + } - let docker_image_name = build_build_image(config, driver_config, overrides)?; - let mut run_cmd = Command::new("docker"); - run_cmd.args([ - "run", - "--detach", - "--name", - &container_name, - "--mount", - &format!( - "type=bind,src={},dst={}", - config.build_root_dir.display(), - BUILD_DIR_IN_CONTAINER - ), - &docker_image_name, - ]); - - let status = run_cmd - .status() - .map_err(|e| anyhow!("Error running docker run: {}", e))?; - if !status.success() { - return Err(anyhow!("Error starting docker container")); + let update_result = driver + .run_command(&["apt-get", "update"], &config.build_source_dir(), true) + .map_err(|error| anyhow!("Error running apt-get update in container: {error}")); + if let Err(error) = update_result { + if created_container && let Err(cleanup_error) = driver.container_remove_force() { + return Err(error.context(format!( + "also failed to remove Docker container: {cleanup_error}" + ))); } + return Err(error); } - Ok(Self { - config: config.clone(), - driver_config: driver_config.clone(), - container_name, - }) + Ok(driver) } pub fn from_build_metadata( config: &BuildConfig, - driver_config: &DriverConfig, + _driver_config: &DriverConfig, build_metadata: &BuildMetadata, - ) -> Self { - let container_name = build_metadata - .driver_metadata - .get("container_name") - .cloned() - .expect("Missing container_name in metadata"); - - Self { + ) -> anyhow::Result { + Ok(Self { config: config.clone(), - driver_config: driver_config.clone(), - container_name, - } + container_name: container_name_from_metadata(build_metadata)?, + reused_environment: true, + }) } + fn translate_path_in_container( &self, path_in_source: &Path, ) -> Result { - path_in_source - .strip_prefix(&self.config.build_root_dir) - .map(|rel| Path::new(BUILD_DIR_IN_CONTAINER).join(rel)) - .map_err(|_| { - std::io::Error::new( - std::io::ErrorKind::NotFound, - "Path is not relative to build root".to_string(), - ) - }) + translate_path_in_container(&self.config.build_root_dir, path_in_source) } } impl BuildDriver for DriverDocker { fn get_build_metadata(&self) -> DriverSpecificBuildMetadata { - let mut meta = DriverSpecificBuildMetadata::new(); - meta.insert("container_name".to_string(), self.container_name.clone()); - meta + container_name_metadata(&self.container_name) } - fn run_command(&self, cmd: &[&str], cwd: &Path, requires_root: bool) -> std::io::Result<()> { + fn run_command_env( + &self, + cmd: &[&str], + cwd: &Path, + requires_root: bool, + env_add: &[(&str, &str)], + ) -> std::io::Result<()> { let container_path = self .translate_path_in_container(cwd) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?; + println!("[{}] $ {}", self.container_name, cmd.join(" ")); + let mut exec_cmd = Command::new("docker"); exec_cmd.args(["exec", "--workdir"]); exec_cmd.arg(container_path); @@ -269,6 +377,10 @@ impl BuildDriver for DriverDocker { exec_cmd.args(["--user", "root"]); } + for (key, value) in env_add { + exec_cmd.args(["--env", &format!("{key}={value}")]); + } + exec_cmd.arg(&self.container_name); exec_cmd.args(cmd); @@ -279,25 +391,53 @@ impl BuildDriver for DriverDocker { Ok(()) } - fn cleanup(&self) { - if self.driver_config.persistent { - let _ = Command::new("docker") - .args(["stop", &self.container_name]) - .status(); + fn cleanup(&self) -> anyhow::Result<()> { + if self.config.persistent { + Ok(()) } else { - let _ = Command::new("docker") - .args(["rm", "-f", &self.container_name]) - .status(); + self.container_remove_force() } } + fn reset_build_root(&self) -> std::io::Result<()> { + let find_cmd = ["find", BUILD_DIR_IN_CONTAINER, "-mindepth", "1", "-delete"]; + println!("[{}] $ {}", self.container_name, find_cmd.join(" ")); + let status = Command::new("docker") + .args(["exec", "--user", "root", &self.container_name]) + .args(find_cmd) + .status()?; + if !status.success() { + return Err(std::io::Error::other( + "failed to reset Docker build directory", + )); + } + Ok(()) + } + + fn reused_environment(&self) -> bool { + self.reused_environment + } + fn interactive_shell(&self, cwd: &Path) -> std::io::Result<()> { let workdir = self.translate_path_in_container(cwd)?; - let _ = Command::new("docker") - .args(["exec", "-it", "--workdir"]) - .arg(&workdir) + let was_running = self.container_is_running().map_err(std::io::Error::other)?; + if !was_running { + self.container_start().map_err(std::io::Error::other)?; + } + + let mut command = Command::new("docker"); + command + .args(["exec", "-it", "--user", "root", "--workdir"]) + .arg(&workdir); + let status = command .args([&self.container_name, "/usr/bin/env", "bash"]) - .status()?; + .status(); + if !was_running { + self.container_stop().map_err(std::io::Error::other)?; + } + if !status?.success() { + return Err(std::io::Error::other("Docker shell failed")); + } Ok(()) } diff --git a/packages/debmagic/src/build/driver_lxd.rs b/packages/debmagic/src/build/driver_lxd.rs new file mode 100644 index 0000000..c94cb6b --- /dev/null +++ b/packages/debmagic/src/build/driver_lxd.rs @@ -0,0 +1,588 @@ +use std::{ + fs, + path::{Path, PathBuf}, + process::{Command, Stdio}, +}; + +use debmagic_common::distro::Distro; +use serde::{Deserialize, Serialize}; + +use crate::build::{ + common::{ + APT_MIRROR_SCRIPT, BUILD_DIR_IN_CONTAINER, BuildConfig, BuildDriver, BuildDriverType, + BuildMetadata, DriverSpecificBuildMetadata, container_name_from_metadata, + container_name_metadata, environment_fingerprint, resource_name, run_checked, + translate_path_in_container, + }, + config::DriverConfig, +}; + +// The binary name differs between LXD and Incus, but everything else is shared. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub enum LxdVariant { + Lxd, + Incus, +} + +impl LxdVariant { + pub fn binary(self) -> &'static str { + match self { + LxdVariant::Lxd => "lxc", + LxdVariant::Incus => "incus", + } + } +} + +// The host user is mapped to BUILD_USER_UID/GID via raw.idmap so that the +// bind-mounted source tree is writable from within the container. +const BUILD_USER_UID: u32 = 1000; +const BUILD_USER_GID: u32 = 1000; +const ENVIRONMENT_CONFIG_KEY: &str = "user.debmagic.environment"; +const ENVIRONMENT_SETUP_VERSION: &str = "dpkg-dev python3 python3-apt; build-user-v1; raw.idmap-v1"; + +// ── Config ──────────────────────────────────────────────────────────────────── + +/// Persistent configuration for both LXD and Incus drivers (from `debmagic.toml`). +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[serde(default)] +pub struct DriverLxdConfig { + /// LXD/Incus project to use. `None` means use the default project. + pub project: Option, + /// Override base image per distro, keyed by `":"`. + /// If absent, falls back to `"images:/"`. + pub base_images: std::collections::HashMap, +} + +impl DriverLxdConfig { + pub fn base_image_for_distro( + &self, + variant: LxdVariant, + distro: &debmagic_common::distro::DistroVersion, + ) -> String { + self.base_images + .get(&format!("{}:{}", distro.distro, distro.codename)) + .cloned() + .unwrap_or_else(|| default_base_image(variant, distro)) + } +} + +fn default_base_image( + variant: LxdVariant, + distro: &debmagic_common::distro::DistroVersion, +) -> String { + use debmagic_common::distro::Distro; + match (&distro.distro, variant, distro.is_devel) { + // LXD ships a dedicated ubuntu: remote; daily builds are on ubuntu-daily:. + (Distro::Ubuntu, LxdVariant::Lxd, false) => format!("ubuntu:{}", distro.version), + (Distro::Ubuntu, LxdVariant::Lxd, true) => format!("ubuntu-daily:{}", distro.version), + // Incus uses the images: remote for everything; daily via /daily variant. + (Distro::Ubuntu, LxdVariant::Incus, false) => format!("images:ubuntu/{}", distro.version), + (Distro::Ubuntu, LxdVariant::Incus, true) => { + format!("images:ubuntu/{}/daily", distro.codename) + } + // Debian images live on images: for both variants, released and devel alike. + (Distro::Debian, _, _) => { + format!("images:debian/{}", debian_image_codename(&distro.codename)) + } + } +} + +// The images remote uses real codenames; changelog aliases like "unstable" +// have no dedicated image and map to sid. +fn debian_image_codename(codename: &str) -> &str { + match codename { + "unstable" => "sid", + other => other, + } +} + +/// Per-invocation overrides (CLI flags). +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct DriverLxdConfigOverrides { + pub base_image: Option, + pub project: Option, +} + +// ── Shared implementation ───────────────────────────────────────────────────── + +pub struct DriverLxd { + variant: LxdVariant, + config: BuildConfig, + container_name: String, + /// Resolved project name (None → omit `--project` flag). + project: Option, + reused_environment: bool, +} + +impl DriverLxd { + fn project_args(&self) -> Vec { + match &self.project { + Some(p) => vec!["--project".to_string(), p.clone()], + None => vec![], + } + } + + fn lxd_cmd(&self, subcommand: &str) -> Command { + let mut cmd = Command::new(self.variant.binary()); + cmd.args(self.project_args()); + cmd.arg(subcommand); + cmd + } + + /// Query the LXD/Incus `list` entry for this container, if it exists. + fn container_list_entry(&self) -> anyhow::Result> { + let mut list_cmd = self.lxd_cmd("list"); + list_cmd.args(["--format", "json"]); + list_cmd.stdout(Stdio::piped()); + + let output = list_cmd + .output() + .map_err(|e| anyhow::anyhow!("Failed to list containers: {e}"))?; + + if !output.status.success() { + return Err(anyhow::anyhow!( + "failed to query {} containers", + self.variant.binary() + )); + } + + let stdout = String::from_utf8_lossy(&output.stdout); + let containers: Vec = serde_json::from_str(&stdout) + .map_err(|e| anyhow::anyhow!("Failed to parse container list JSON: {e}"))?; + + Ok(containers.into_iter().find(|container| { + container.get("name").and_then(|v| v.as_str()) == Some(self.container_name.as_str()) + })) + } + + fn is_entry_running(entry: &serde_json::Value) -> bool { + entry + .get("status") + .and_then(|v| v.as_str()) + .map(|s| s == "Running") + .unwrap_or(false) + } + + fn container_start(&self) -> anyhow::Result<()> { + run_checked( + self.lxd_cmd("start").arg(&self.container_name), + &format!("starting {} container", self.variant.binary()), + ) + } + + fn container_stop(&self) -> anyhow::Result<()> { + run_checked( + self.lxd_cmd("stop").arg(&self.container_name), + &format!("stopping {} container", self.variant.binary()), + ) + } + + fn container_delete_force(&self) -> anyhow::Result<()> { + run_checked( + self.lxd_cmd("delete") + .args(["--force", &self.container_name]), + &format!("removing {} container", self.variant.binary()), + ) + } + + fn container_environment_fingerprint(&self) -> anyhow::Result> { + let output = self + .lxd_cmd("config") + .args(["get", &self.container_name, ENVIRONMENT_CONFIG_KEY]) + .output() + .map_err(|error| { + anyhow::anyhow!( + "failed to inspect {} container: {error}", + self.variant.binary() + ) + })?; + if !output.status.success() { + return Err(anyhow::anyhow!( + "failed to inspect {} container {}", + self.variant.binary(), + self.container_name + )); + } + let fingerprint = String::from_utf8_lossy(&output.stdout).trim().to_string(); + Ok((!fingerprint.is_empty()).then_some(fingerprint)) + } + + pub fn create( + variant: LxdVariant, + config: &BuildConfig, + driver_config: &DriverConfig, + overrides: &DriverLxdConfigOverrides, + apt_mirror: Option<&str>, + proposed: bool, + ) -> anyhow::Result { + let container_name = + resource_name("debmagic", &config.package_name, &config.build_identifier()); + + // Project: CLI override > config file value + let project = overrides + .project + .clone() + .or_else(|| driver_config.lxd.project.clone()); + let base_image = overrides.base_image.clone().unwrap_or_else(|| { + driver_config + .lxd + .base_image_for_distro(variant, &config.distro) + }); + let host_uid = unsafe { libc::geteuid() }.to_string(); + let host_gid = unsafe { libc::getegid() }.to_string(); + let build_root = config.build_root_dir.to_string_lossy(); + let proposed_fingerprint = proposed.to_string(); + let desired_fingerprint = environment_fingerprint(&[ + variant.binary(), + ENVIRONMENT_SETUP_VERSION, + &base_image, + &config.distro.codename, + apt_mirror.unwrap_or(""), + &proposed_fingerprint, + &host_uid, + &host_gid, + build_root.as_ref(), + ]); + + let mut base = Self { + variant, + config: config.clone(), + container_name: container_name.clone(), + project, + reused_environment: false, + }; + + let container_entry = base.container_list_entry()?; + let environment_matches = container_entry.is_some() + && base.container_environment_fingerprint()?.as_deref() == Some(&desired_fingerprint); + let reusing_container = config.persistent && environment_matches; + base.reused_environment = reusing_container; + + let mut initialized_container = false; + let setup_result = (|| -> anyhow::Result<()> { + if reusing_container { + let already_running = container_entry + .as_ref() + .is_some_and(DriverLxd::is_entry_running); + if !already_running { + base.container_start()?; + } + } else { + if container_entry.is_some() { + base.container_delete_force()?; + } + + let mut init = base.lxd_cmd("init"); + if !config.persistent { + init.arg("--ephemeral"); + } + init.args([&base_image, &container_name]); + run_checked( + &mut init, + &format!("initialising {} container", variant.binary()), + )?; + initialized_container = true; + + // Map the host user's uid/gid to BUILD_USER_UID inside the container + // so that files in the bind-mounted build root are writable. + let host_uid = unsafe { libc::geteuid() }; + let host_gid = unsafe { libc::getegid() }; + if host_uid != 0 { + let idmap = format!( + "uid {} {}\ngid {} {}", + host_uid, BUILD_USER_UID, host_gid, BUILD_USER_GID + ); + run_checked( + base.lxd_cmd("config") + .arg("set") + .arg(&container_name) + .arg("raw.idmap") + .arg(&idmap), + "setting raw.idmap on container", + )?; + } + + let device_name = resource_name( + "debmagic-src", + &config.package_name, + &config.build_identifier(), + ); + run_checked( + base.lxd_cmd("config") + .arg("device") + .arg("add") + .arg(&container_name) + .arg(&device_name) + .arg("disk") + .arg(format!("source={}", config.build_root_dir.display())) + .arg(format!("path={}", BUILD_DIR_IN_CONTAINER)), + &format!("mounting build root into {} container", variant.binary()), + )?; + + run_checked( + base.lxd_cmd("start").arg(&container_name), + &format!("starting {} container", variant.binary()), + )?; + + if config.distro.distro == Distro::Ubuntu { + base.exec_in_container(&["cloud-init", "status", "--wait"], None, true, &[]) + .map_err(|e| { + anyhow::anyhow!("Error waiting for cloud-init to finish: {e}") + })?; + } + } + + // Re-run on every reuse of a persistent container too, so that a + // previous invocation that crashed before finishing this setup (or a + // long-lived incremental container with an aging package cache) + // doesn't leave `apt-get build-dep` unable to resolve anything. + base.exec_in_container(&["apt-get", "update"], None, true, &[]) + .map_err(|e| anyhow::anyhow!("Error running apt-get update in container: {e}"))?; + + if !reusing_container { + // Install the base tooling that stock images don't include. + // build-dep is intentionally omitted here: build.rs runs it for + // every driver against the real mounted source tree. + base.exec_in_container( + &[ + "apt-get", + "install", + "-y", + "dpkg-dev", + "python3", + "python3-apt", + ], + None, + true, + &[], + ) + .map_err(|e| anyhow::anyhow!("Error installing base packages in container: {e}"))?; + + let ensure_build_user = format!( + "getent group {gid} >/dev/null || groupadd --gid {gid} debmagic; \ + getent passwd {uid} >/dev/null || useradd --uid {uid} --gid {gid} -m debmagic", + uid = BUILD_USER_UID, + gid = BUILD_USER_GID, + ); + base.exec_in_container(&["sh", "-ec", &ensure_build_user], None, true, &[]) + .map_err(|e| anyhow::anyhow!("Error creating build user in container: {e}"))?; + + if apt_mirror.is_some() || proposed { + let script_path = config.build_temp_dir().join("mirror.py"); + fs::write(&script_path, APT_MIRROR_SCRIPT)?; + let container_script_path = base.translate_path_in_container(&script_path)?; + let mut args = vec![ + "python3".to_string(), + container_script_path.to_string_lossy().into_owned(), + "--codename".to_string(), + config.distro.codename.clone(), + ]; + if let Some(mirror) = apt_mirror { + args.extend(["--mirror".to_string(), mirror.to_string()]); + } + if proposed { + args.push("--proposed".to_string()); + } + let args = args.iter().map(String::as_str).collect::>(); + base.exec_in_container(&args, None, true, &[]) + .map_err(|e| anyhow::anyhow!("Error configuring apt sources: {e}"))?; + base.exec_in_container(&["apt-get", "update"], None, true, &[]) + .map_err(|e| { + anyhow::anyhow!("Error updating configured apt sources: {e}") + })?; + } + + run_checked( + base.lxd_cmd("config") + .arg("set") + .arg(&container_name) + .arg(ENVIRONMENT_CONFIG_KEY) + .arg(&desired_fingerprint), + &format!("recording {} environment", variant.binary()), + )?; + } + Ok(()) + })(); + + if let Err(error) = setup_result { + if initialized_container && let Err(cleanup_error) = base.container_delete_force() { + return Err(error.context(format!( + "also failed to remove {} container: {cleanup_error}", + variant.binary() + ))); + } + return Err(error); + } + + Ok(base) + } + + pub fn from_build_metadata( + variant: LxdVariant, + config: &BuildConfig, + build_metadata: &BuildMetadata, + ) -> anyhow::Result { + let project = build_metadata.driver_metadata.get("project").cloned(); + + Ok(Self { + variant, + config: config.clone(), + container_name: container_name_from_metadata(build_metadata)?, + project, + reused_environment: true, + }) + } + + fn translate_path_in_container( + &self, + path_in_source: &Path, + ) -> Result { + translate_path_in_container(&self.config.build_root_dir, path_in_source) + } + + /// Run `action` with the container running, restoring a previously + /// stopped container to the stopped state afterwards. + fn with_running_container( + &self, + action: impl FnOnce(&Self) -> std::io::Result<()>, + ) -> std::io::Result<()> { + let entry = self + .container_list_entry() + .map_err(std::io::Error::other)? + .ok_or_else(|| { + std::io::Error::new(std::io::ErrorKind::NotFound, "container not found") + })?; + let was_running = Self::is_entry_running(&entry); + if !was_running { + self.container_start().map_err(std::io::Error::other)?; + } + let result = action(self); + let stop_result = if was_running { + Ok(()) + } else { + self.container_stop() + }; + match (result, stop_result) { + (Ok(()), Ok(())) => Ok(()), + (Err(e), Ok(())) => Err(e), + (Ok(()), Err(e)) => Err(std::io::Error::other(e)), + (Err(e), Err(stop_error)) => Err(std::io::Error::other(format!( + "{e}; also failed to stop {} container: {stop_error}", + self.variant.binary() + ))), + } + } + + fn exec_in_container( + &self, + cmd: &[&str], + workdir: Option<&Path>, + as_root: bool, + env_add: &[(&str, &str)], + ) -> std::io::Result<()> { + println!("[{}] $ {}", self.container_name, cmd.join(" ")); + + let mut exec_cmd = self.lxd_cmd("exec"); + exec_cmd.arg(&self.container_name); + + if let Some(wd) = workdir { + exec_cmd.args(["--cwd", &wd.to_string_lossy()]); + } + + if !as_root { + // Run as BUILD_USER_UID — the host user is mapped to this uid via + // raw.idmap, so it owns the bind-mounted source tree inside. + exec_cmd.args(["--user", &BUILD_USER_UID.to_string()]); + exec_cmd.args(["--group", &BUILD_USER_GID.to_string()]); + } + + for (key, value) in env_add { + exec_cmd.args(["--env", &format!("{key}={value}")]); + } + + exec_cmd.arg("--"); + exec_cmd.args(cmd); + + let status = exec_cmd.status()?; + if !status.success() { + return Err(std::io::Error::other(format!( + "{} exec failed", + self.variant.binary() + ))); + } + Ok(()) + } +} + +impl BuildDriver for DriverLxd { + fn get_build_metadata(&self) -> DriverSpecificBuildMetadata { + let mut meta = container_name_metadata(&self.container_name); + if let Some(ref p) = self.project { + meta.insert("project".to_string(), p.clone()); + } + meta + } + + fn run_command_env( + &self, + cmd: &[&str], + cwd: &Path, + requires_root: bool, + env_add: &[(&str, &str)], + ) -> std::io::Result<()> { + let container_path = self + .translate_path_in_container(cwd) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?; + + self.exec_in_container(cmd, Some(&container_path), requires_root, env_add) + } + + fn cleanup(&self) -> anyhow::Result<()> { + if self.config.persistent { + Ok(()) + } else { + // Ephemeral containers are auto-deleted after stopping. + self.container_stop() + } + } + + fn reset_build_root(&self) -> std::io::Result<()> { + self.with_running_container(|driver| { + driver.exec_in_container( + &["find", BUILD_DIR_IN_CONTAINER, "-mindepth", "1", "-delete"], + None, + true, + &[], + ) + }) + } + + fn reused_environment(&self) -> bool { + self.reused_environment + } + + fn interactive_shell(&self, cwd: &Path) -> std::io::Result<()> { + let workdir = self.translate_path_in_container(cwd)?; + self.with_running_container(|driver| { + let mut command = driver.lxd_cmd("exec"); + command + .arg(&driver.container_name) + .args(["--cwd", &workdir.to_string_lossy()]); + let status = command.arg("--").args(["/usr/bin/env", "bash"]).status()?; + if !status.success() { + return Err(std::io::Error::other(format!( + "{} shell failed", + driver.variant.binary() + ))); + } + Ok(()) + }) + } + + fn driver_type(&self) -> BuildDriverType { + match self.variant { + LxdVariant::Lxd => BuildDriverType::Lxd, + LxdVariant::Incus => BuildDriverType::Incus, + } + } +} diff --git a/packages/debmagic/src/build/scripts/mirror.py b/packages/debmagic/src/build/scripts/mirror.py new file mode 100644 index 0000000..30be0d5 --- /dev/null +++ b/packages/debmagic/src/build/scripts/mirror.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Point apt at a mirror by replacing the base image's default sources. + +debmagic picks the base image itself, so the default sources file it ships +(classic `sources.list` or deb822 `*.sources`) is a known quantity: rather +than parsing and patching it in place, it is simply removed and replaced +with a single deb822 file that debmagic owns and fully controls. Unless +overridden, the suites/components enabled default to the release, its +`-updates` and its `-security` pockets - the same ones any base image +ships out of the box. +""" + +from __future__ import annotations + +import argparse +import os + +from aptsources.sourceslist import SourcesList + +OS_RELEASE_FILE = "/etc/os-release" + +# Default apt configuration shipped by the base images debmagic uses; +# removed unconditionally in favour of MANAGED_SOURCES_FILE below. +DEFAULT_SOURCE_FILES = [ + "/etc/apt/sources.list", + "/etc/apt/sources.list.d/ubuntu.sources", + "/etc/apt/sources.list.d/debian.sources", +] + +MANAGED_SOURCES_FILE = "/etc/apt/sources.list.d/debmagic.sources" + +# Debian codenames with no separate updates/security/backports pockets. +DEBIAN_ROLLING_CODENAMES = {"unstable", "sid", "testing", "experimental"} +DEBIAN_WITHOUT_PROPOSED = {"unstable", "sid", "experimental"} + + +def read_os_release(path: str = OS_RELEASE_FILE) -> dict[str, str]: + values: dict[str, str] = {} + with open(path, encoding="utf-8") as f: + for line in f: + stripped_line = line.strip() + if not stripped_line or stripped_line.startswith("#") or "=" not in stripped_line: + continue + key, _, value = stripped_line.partition("=") + values[key] = value.strip().strip('"') + return values + + +def default_suites(os_release: dict[str, str], codename: str) -> list[str]: + if os_release.get("ID") == "debian" and codename in DEBIAN_ROLLING_CODENAMES: + return [codename] + return [codename, f"{codename}-updates", f"{codename}-security"] + + +def default_components(os_release: dict[str, str]) -> list[str]: + if os_release.get("ID") == "ubuntu": + return ["main", "restricted", "universe", "multiverse"] + + major = int((os_release.get("VERSION_ID") or "0").split(".")[0] or 0) + if major >= 12: + return ["main", "contrib", "non-free", "non-free-firmware"] + return ["main", "contrib", "non-free"] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--mirror", + help="apt mirror base URI, e.g. http://archive.ubuntu.com/ubuntu/", + ) + parser.add_argument( + "--codename", + help="target distribution codename; defaults to VERSION_CODENAME from the base image", + ) + parser.add_argument( + "--suite", + dest="suites", + action="append", + help="suite/pocket to enable, e.g. noble or noble-security (repeatable); " + "defaults to the release plus its '-updates' and '-security' pockets", + ) + parser.add_argument( + "--component", + dest="components", + action="append", + help="component to enable, e.g. main or universe (repeatable); defaults to those enabled by the base image", + ) + parser.add_argument( + "--proposed", + action="store_true", + help="also enable the '-proposed' pocket", + ) + return parser.parse_args() + + +def configured_mirror(codename: str) -> str: + for source in SourcesList().list: + if not source.disabled and source.type in {"deb", "deb-src"} and source.dist == codename: + return source.uri + raise RuntimeError(f"could not determine the configured mirror for {codename}") + + +def proposed_suite(os_release: dict[str, str], codename: str) -> str: + if os_release.get("ID") == "debian": + if codename in DEBIAN_WITHOUT_PROPOSED: + raise ValueError(f"Debian {codename} has no proposed-updates pocket") + return f"{codename}-proposed-updates" + return f"{codename}-proposed" + + +def main() -> None: + args = parse_args() + os_release = read_os_release() + codename = args.codename or os_release["VERSION_CODENAME"] + + suites = args.suites or default_suites(os_release, codename) + components = args.components or default_components(os_release) + if args.proposed: + suites = [*suites, proposed_suite(os_release, codename)] + + if not args.mirror: + if not args.proposed: + raise ValueError("--mirror or --proposed is required") + suites = [proposed_suite(os_release, codename)] + args.mirror = configured_mirror(codename) + source_files = [] + source_groups = [(args.mirror, suites)] + else: + source_files = DEFAULT_SOURCE_FILES + if os_release.get("ID") == "debian" and codename not in DEBIAN_ROLLING_CODENAMES: + security_suite = f"{codename}-security" + source_groups = [ + (args.mirror, [suite for suite in suites if suite != security_suite]), + (configured_mirror(security_suite), [security_suite]), + ] + else: + source_groups = [(args.mirror, suites)] + + for path in source_files: + try: + os.remove(path) + except FileNotFoundError: + pass + + os.makedirs(os.path.dirname(MANAGED_SOURCES_FILE), exist_ok=True) + with open(MANAGED_SOURCES_FILE, "w", encoding="utf-8") as f: + for index, (mirror, group_suites) in enumerate(source_groups): + if index: + f.write("\n") + f.write("Types: deb deb-src\n") + f.write(f"URIs: {mirror}\n") + f.write(f"Suites: {' '.join(group_suites)}\n") + f.write(f"Components: {' '.join(components)}\n") + + +if __name__ == "__main__": + main() diff --git a/packages/debmagic/src/cli.rs b/packages/debmagic/src/cli.rs index 22eac71..5a95a69 100644 --- a/packages/debmagic/src/cli.rs +++ b/packages/debmagic/src/cli.rs @@ -1,7 +1,7 @@ use std::path::PathBuf; use crate::build::common::BuildDriverType; -use clap::{Args, Parser, Subcommand}; +use clap::{Args, Parser, Subcommand, builder::BoolishValueParser}; #[derive(Parser, Debug)] #[command(version, about, long_about = None)] @@ -15,8 +15,8 @@ pub struct Cli { #[derive(Subcommand, Debug)] pub enum Commands { - #[command(about = "Build a debian package")] - Build(BuildSubcommandArgs), + #[command(about = "Build a debian package: 'binary' (.deb) or 'source' (.dsc) packages")] + Build(Box), #[command(about = "Open an interactive shell to the currently active build environment")] Shell(ShellSubcommandArgs), #[command(about = "Run tests")] @@ -40,6 +40,7 @@ pub struct CommonCli { #[derive(Args, Debug)] pub struct DockerArgs { #[arg( + id = "docker-base-image", long = "driver-docker-base-image", help = "If passed will override the base image for the current build" )] @@ -47,25 +48,86 @@ pub struct DockerArgs { } #[derive(Args, Debug)] -pub struct BuildSubcommandArgs { - #[arg(short, long, help = "Build driver type")] - pub driver: BuildDriverType, +pub struct LxdArgs { + #[arg( + id = "lxd-base-image", + long = "driver-lxd-base-image", + help = "Override the base image (image alias) for the LXD/Incus container" + )] + pub base_image: Option, - #[arg(long, action = clap::ArgAction::SetTrue, help = "Persist the build environment after the build finished")] - pub persist_driver: Option, + #[arg( + long = "driver-lxd-project", + help = "LXD/Incus project to register the container in" + )] + pub project: Option, +} + +/// Flags shared between `debmagic build binary` and `debmagic build source`. +#[derive(Args, Debug)] +pub struct CommonBuildArgs { + #[arg( + short, + long, + help = "Build driver type. Required for binary builds; source-only builds default to 'bare', since those need no build-deps or compilation." + )] + pub driver: Option, + + #[arg(long, action = clap::ArgAction::SetTrue, help = "Keep the build environment for reuse after the build finishes")] + pub persistent: Option, #[command(flatten)] pub docker: DockerArgs, - #[arg(short, long, action = clap::ArgAction::SetTrue, help = "Enable incremental builds. This implies --persist-driver")] - pub incremental: Option, + #[command(flatten)] + pub lxd: LxdArgs, + + #[arg( + long = "apt-mirror", + help = "Apt mirror URL to use inside the build environment instead of the default archive.ubuntu.com/security.ubuntu.com/deb.debian.org, e.g. http://my-mirror.example/ubuntu. Ignored by the bare driver." + )] + pub apt_mirror: Option, + + #[arg( + long, + action = clap::ArgAction::SetTrue, + help = "Also enable the '-proposed' pocket in the build environment. Ignored by the bare driver." + )] + pub proposed: Option, #[arg( long, - help = "Select the target distribution version, only required in the debian changelog specifies multiple versions" + help = "Select the target distribution version, only required if the debian changelog specifies multiple versions" )] pub distro: Option, + #[arg( + long, + value_parser = BoolishValueParser::new(), + help = "Sign the resulting .changes/.dsc with debsign after building (yes/no). Defaults to the 'sign_package' setting in the config file (false if unset). Always runs on the host, using your own gpg keyring, regardless of --driver." + )] + pub sign: Option, + + #[arg( + long = "sign-key", + help = "GPG key ID/email to sign with, passed to debsign's -k option. Defaults to the 'sign_key' setting in the config file, or debsign's own maintainer-based key lookup if unset." + )] + pub sign_key: Option, + + #[arg( + long, + action = clap::ArgAction::SetTrue, + help = "Run 'debian/rules clean' before building, like plain dpkg-buildpackage does unless passed -nc. Defaults to the 'clean' setting in the config file (false if unset); non-incremental builds already stage a clean source tree, while incremental builds preserve outputs by design. For source builds this also installs build-dependencies first, since a clean target usually needs its own tooling." + )] + pub clean: Option, + + #[arg( + long, + action = clap::ArgAction::SetFalse, + help = "Do not run 'debian/rules clean' before building, overriding a 'clean = true' default in the config file." + )] + pub no_clean: Option, + #[command(flatten)] pub common: CommonCli, @@ -73,6 +135,44 @@ pub struct BuildSubcommandArgs { pub output_dir: Option, } +#[derive(Args, Debug)] +pub struct BuildSubcommandArgs { + #[command(subcommand)] + pub target: BuildTarget, +} + +#[derive(Subcommand, Debug)] +pub enum BuildTarget { + #[command(about = "Build binary .deb packages")] + Binary(BinaryTargetArgs), + #[command( + about = "Build a source package only (.dsc + tarball, plus .buildinfo/.changes), no build-deps or compilation required" + )] + Source(SourceTargetArgs), +} + +#[derive(Args, Debug)] +pub struct BinaryTargetArgs { + #[command(flatten)] + pub build: CommonBuildArgs, + + #[arg(short, long, action = clap::ArgAction::SetTrue, help = "Synchronize changed source inputs while preserving build outputs. Implies --persistent")] + pub incremental: Option, + + #[arg( + long = "debug-symbols", + action = clap::ArgAction::SetTrue, + help = "Also build the automatic '-dbgsym' debug symbol package" + )] + pub debug_symbols: Option, +} + +#[derive(Args, Debug)] +pub struct SourceTargetArgs { + #[command(flatten)] + pub build: CommonBuildArgs, +} + #[derive(Args, Debug)] pub struct ShellSubcommandArgs { #[command(flatten)] diff --git a/packages/debmagic/src/config.rs b/packages/debmagic/src/config.rs index 9ab1084..434ecc5 100644 --- a/packages/debmagic/src/config.rs +++ b/packages/debmagic/src/config.rs @@ -11,6 +11,18 @@ pub struct Config { pub driver: DriverConfig, pub temp_build_dir: PathBuf, pub incremental: bool, + /// Always build the automatic `-dbgsym` debug symbol package. + pub build_debug_symbols: bool, + /// Sign the resulting `.changes`/`.dsc` with `debsign` after building. + pub sign_package: bool, + /// GPG key ID/email to sign with (debsign's `-k` option). `None` lets + /// debsign fall back to its own maintainer-based key lookup. + pub sign_key: Option, + /// Run `debian/rules clean` before building (like `dpkg-buildpackage` + /// does unless passed `-nc`). Disabled by default because non-incremental + /// builds already stage a clean source tree and incremental builds preserve + /// outputs intentionally. + pub clean: bool, } impl Default for Config { @@ -19,6 +31,10 @@ impl Default for Config { driver: DriverConfig::default(), temp_build_dir: PathBuf::from("/tmp/debmagic"), incremental: false, + build_debug_symbols: false, + sign_package: false, + sign_key: None, + clean: false, } } } diff --git a/packages/debmagic/src/main.rs b/packages/debmagic/src/main.rs index 81f8781..4728e4d 100644 --- a/packages/debmagic/src/main.rs +++ b/packages/debmagic/src/main.rs @@ -8,10 +8,12 @@ use clap::{CommandFactory, Parser}; use crate::{ build::{ - build_package, config::DriverOverrides, driver_bare::DriverBareConfigOverrides, - driver_docker::DriverDockerConfigOverrides, get_shell_in_build, + BuildRequest, build_package, build_source_package, common::BuildDriverType, + config::DriverOverrides, driver_bare::DriverBareConfigOverrides, + driver_docker::DriverDockerConfigOverrides, driver_lxd::DriverLxdConfigOverrides, + get_shell_in_build, }, - cli::{Cli, Commands}, + cli::{BuildTarget, Cli, Commands, CommonBuildArgs}, config::Config, package::PackageDescription, }; @@ -54,44 +56,101 @@ fn main() -> anyhow::Result<()> { let current_dir = env::current_dir()?; match &cli.command { Commands::Build(args) => { - let source_dir = args.common.source_dir.as_deref().unwrap_or(¤t_dir); + let (build_args, debug_symbols, incremental, is_source): ( + &CommonBuildArgs, + Option, + Option, + bool, + ) = match &args.target { + BuildTarget::Binary(binary_args) => ( + &binary_args.build, + binary_args.debug_symbols, + binary_args.incremental, + false, + ), + BuildTarget::Source(source_args) => (&source_args.build, None, None, true), + }; + + let source_dir = build_args + .common + .source_dir + .as_deref() + .unwrap_or(¤t_dir); let mut config = get_config(&cli, &Some(source_dir.to_path_buf()))?; // TODO: figure out a better way to override config from CLI args - maybe more generic, if that is even possible since // we want a nice cli which somewhat matches the config structure // but some config options only make sense in some cli subcommands -> these flags don't make sense in all commands // and should only be used in some - if let Some(persist_driver) = args.persist_driver { - config.driver.persistent = persist_driver; + if let Some(persistent) = build_args.persistent { + config.driver.persistent = persistent; } - if let Some(incremental) = args.incremental { + if is_source { + config.incremental = false; + } else if let Some(incremental) = incremental { config.incremental = incremental; } + + if let Some(debug_symbols) = debug_symbols { + config.build_debug_symbols = debug_symbols; + } + if let Some(sign) = build_args.sign { + config.sign_package = sign; + } + if let Some(sign_key) = build_args.sign_key.clone() { + config.sign_key = Some(sign_key); + } + if let Some(clean) = build_args.clean { + config.clean = clean; + } + if let Some(no_clean) = build_args.no_clean { + config.clean = !no_clean; + } if config.incremental { - // TODO: investigate if this is actually needed + if config.clean { + anyhow::bail!("incremental builds are incompatible with clean builds"); + } config.driver.persistent = true; } let driver_overrides = DriverOverrides { + apt_mirror: build_args.apt_mirror.clone(), + proposed: build_args.proposed, docker: DriverDockerConfigOverrides { - base_image: args.docker.base_image.clone(), + base_image: build_args.docker.base_image.clone(), }, bare: DriverBareConfigOverrides {}, + lxd: DriverLxdConfigOverrides { + base_image: build_args.lxd.base_image.clone(), + project: build_args.lxd.project.clone(), + }, }; let package = PackageDescription::from_dir( &path::absolute(source_dir).context("resolving source dir failed")?, )?; - let output_dir = args.output_dir.as_deref().unwrap_or(¤t_dir); - build_package( - &config, - &package, - args.driver, - &driver_overrides, - &path::absolute(output_dir).context("resolving output dir failed")?, - args.distro.as_deref(), - ) - .context("Building the package failed")?; + let output_dir = build_args.output_dir.as_deref().unwrap_or(¤t_dir); + let output_dir = path::absolute(output_dir).context("resolving output dir failed")?; + + let request = BuildRequest { + config: &config, + package: &package, + driver_type: if is_source { + build_args.driver.unwrap_or(BuildDriverType::Bare) + } else { + build_args.driver.context( + "--driver is required for binary builds (docker, bare, lxd or incus)", + )? + }, + driver_overrides: &driver_overrides, + output_dir: &output_dir, + explicit_distro_version: build_args.distro.as_deref(), + }; + if is_source { + build_source_package(&request).context("Building the source package failed")?; + } else { + build_package(&request).context("Building the package failed")?; + } } Commands::Shell(args) => { let source_dir = args.common.source_dir.as_deref().unwrap_or(¤t_dir); diff --git a/tests/integration/test_packages.py b/tests/integration/test_packages.py index ea133c8..107de33 100644 --- a/tests/integration/test_packages.py +++ b/tests/integration/test_packages.py @@ -96,6 +96,7 @@ def test_build_package(test_env: Environment, package: str, version: str): "debmagic", "--", "build", + "binary", "--driver", "docker", "--driver-docker-base-image", From 9660c46ac66f4591af08642898be9775021366d4 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Fri, 31 Jul 2026 16:20:15 +0200 Subject: [PATCH 2/7] feat(build): don't rely on python3-apt --- packages/debmagic/src/build/driver_docker.rs | 2 +- packages/debmagic/src/build/driver_lxd.rs | 3 +- packages/debmagic/src/build/scripts/mirror.py | 49 +++++++++++++++++-- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/packages/debmagic/src/build/driver_docker.rs b/packages/debmagic/src/build/driver_docker.rs index 7afe789..427b719 100644 --- a/packages/debmagic/src/build/driver_docker.rs +++ b/packages/debmagic/src/build/driver_docker.rs @@ -53,7 +53,7 @@ const DOCKERFILE_TEMPLATE: &str = r#" FROM {base_image} ARG USER_UID=1000 ARG USER_GID=$USER_UID -RUN apt-get update && apt-get install -y dpkg-dev python3 python3-apt +RUN apt-get update && apt-get install -y dpkg-dev python3 {apt_mirror_setup} RUN set -e; \ getent group "$USER_GID" >/dev/null || groupadd --gid "$USER_GID" debmagic; \ diff --git a/packages/debmagic/src/build/driver_lxd.rs b/packages/debmagic/src/build/driver_lxd.rs index c94cb6b..c548f2b 100644 --- a/packages/debmagic/src/build/driver_lxd.rs +++ b/packages/debmagic/src/build/driver_lxd.rs @@ -38,7 +38,7 @@ impl LxdVariant { const BUILD_USER_UID: u32 = 1000; const BUILD_USER_GID: u32 = 1000; const ENVIRONMENT_CONFIG_KEY: &str = "user.debmagic.environment"; -const ENVIRONMENT_SETUP_VERSION: &str = "dpkg-dev python3 python3-apt; build-user-v1; raw.idmap-v1"; +const ENVIRONMENT_SETUP_VERSION: &str = "dpkg-dev python3; build-user-v1; raw.idmap-v1"; // ── Config ──────────────────────────────────────────────────────────────────── @@ -350,7 +350,6 @@ impl DriverLxd { "-y", "dpkg-dev", "python3", - "python3-apt", ], None, true, diff --git a/packages/debmagic/src/build/scripts/mirror.py b/packages/debmagic/src/build/scripts/mirror.py index 30be0d5..85239c0 100644 --- a/packages/debmagic/src/build/scripts/mirror.py +++ b/packages/debmagic/src/build/scripts/mirror.py @@ -13,10 +13,9 @@ from __future__ import annotations import argparse +import glob import os -from aptsources.sourceslist import SourcesList - OS_RELEASE_FILE = "/etc/os-release" # Default apt configuration shipped by the base images debmagic uses; @@ -93,10 +92,50 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() +def iter_sources() -> list[tuple[str, list[str]]]: + """Yield (uri, suites) from deb822 *.sources files and legacy sources.list.""" + sources: list[tuple[str, list[str]]] = [] + for path in glob.glob("/etc/apt/sources.list.d/*.sources"): + uri: str | None = None + suites: list[str] = [] + enabled = True + with open(path, encoding="utf-8") as f: + for line in [*f, "\n"]: + stripped_line = line.strip() + if not stripped_line: + if uri and suites and enabled: + sources.append((uri, suites)) + uri, suites, enabled = None, [], True + continue + key, _, value = stripped_line.partition(":") + key, value = key.strip(), value.strip() + if key == "URIs": + uri = value.split()[0] + elif key == "Suites": + suites = value.split() + elif key == "Enabled": + enabled = value.lower() != "no" + for path in ["/etc/apt/sources.list", *glob.glob("/etc/apt/sources.list.d/*.list")]: + try: + f = open(path, encoding="utf-8") + except FileNotFoundError: + continue + with f: + for line in f: + fields = line.split() + if len(fields) >= 3 and fields[0] in {"deb", "deb-src"}: + fields = fields[1:] + if fields[0].startswith("["): + fields = fields[fields.index(next(f for f in fields if f.endswith("]"))) + 1 :] + if len(fields) >= 2: + sources.append((fields[0], fields[1:])) + return sources + + def configured_mirror(codename: str) -> str: - for source in SourcesList().list: - if not source.disabled and source.type in {"deb", "deb-src"} and source.dist == codename: - return source.uri + for uri, suites in iter_sources(): + if codename in suites: + return uri raise RuntimeError(f"could not determine the configured mirror for {codename}") From 2eb8248d72a9e9ef35c7b9c53b7f3f2953d3f218 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Fri, 31 Jul 2026 16:57:33 +0200 Subject: [PATCH 3/7] feat(build): select staged source files by git tracking state add --source-sync (config key source_sync_mode) with three modes: - 'tracked' (default) stages git-tracked files including uncommitted changes and warns about untracked files left out - 'committed' fails on a dirty worktree - 'worktree' keeps staging everything not git-ignored. this makes it explicit which files end up in the build tree and in generated source packages. git submodules are skipped with a note, and both git modes fall back to 'worktree' with a warning outside a git worktree. --- docs/usage/build.md | 18 +- packages/debmagic/src/build.rs | 303 ++++++++++++++++++++-- packages/debmagic/src/build/common.rs | 20 ++ packages/debmagic/src/build/driver_lxd.rs | 8 +- packages/debmagic/src/cli.rs | 8 +- packages/debmagic/src/config.rs | 4 + packages/debmagic/src/main.rs | 3 + 7 files changed, 327 insertions(+), 37 deletions(-) diff --git a/docs/usage/build.md b/docs/usage/build.md index 7ae066c..aee9476 100644 --- a/docs/usage/build.md +++ b/docs/usage/build.md @@ -50,7 +50,23 @@ Notes: - The image, mirror, proposed-pocket setting and host user IDs form the build-environment identity. Changing any of them automatically replaces an incompatible persistent container. - Handles both the classic `sources.list` format and the deb822 `*.sources` format (Ubuntu 24.04+). -- To avoid repeating the flag, set it once in `debian/debmagic.toml` (see below) instead. +- To avoid repeating the flag, set it once in `$XDG_CONFIG_HOME/debmagic/config.toml` (see below) instead — a mirror is a property of your machine, not of a package, so it belongs in the global config rather than in the repo's `debian/debmagic.toml`. + +## Selecting which source files are staged + +Before building, debmagic stages the source tree into the build environment. +`--source-sync ` controls which files are staged, so you always know what ends up in the build and in a generated source package: + +| Mode | Stages | Notes | +|---|---|---| +| `tracked` (default) | git-tracked files, including uncommitted modifications | Untracked files are left out and listed as a warning — `git add` them or switch modes to include them | +| `committed` | the same files as `tracked` | Fails if the worktree has uncommitted changes or untracked files; use for reproducible, reviewable source packages | +| `worktree` | everything that isn't git-ignored, tracked or not | | + +If the source directory is not a git worktree, `tracked` and `committed` fall back to `worktree` with a warning. +Git submodules are skipped with a note, since their contents aren't tracked by the parent repository. + +To persist a mode, set `source_sync_mode = "committed"` in `debian/debmagic.toml`. ## Iterating on a build (faster repeat runs) diff --git a/packages/debmagic/src/build.rs b/packages/debmagic/src/build.rs index e0a7f81..63048ab 100644 --- a/packages/debmagic/src/build.rs +++ b/packages/debmagic/src/build.rs @@ -16,7 +16,9 @@ use std::{ use crate::build::config::DriverOverrides; use crate::{ build::{ - common::{BuildConfig, BuildDriver, BuildDriverType, BuildMetadata, run_checked}, + common::{ + BuildConfig, BuildDriver, BuildDriverType, BuildMetadata, SourceSyncMode, run_checked, + }, config::DriverConfig, driver_bare::DriverBare, driver_docker::DriverDocker, @@ -274,9 +276,80 @@ fn send_socket_command(build_root: &Path, cmd: &str) -> anyhow::Result<()> { Ok(()) } -fn copy_dir_all(src: impl AsRef, dst: impl AsRef) -> anyhow::Result<()> { - let entries = source_tree_entries(src.as_ref())?; - copy_source_entries(src.as_ref(), dst.as_ref(), &entries) +/// Paths of files tracked by git in `src`, as reported by `git ls-files`. +/// Returns `None` if `src` is not inside a git worktree. +fn git_tracked_paths(src: &Path) -> anyhow::Result>> { + let output = Command::new("git") + .args(["-C"]) + .arg(src) + .args(["ls-files", "-z"]) + .output() + .context("failed to run git ls-files")?; + if !output.status.success() { + return Ok(None); + } + let mut paths = Vec::new(); + for raw in output.stdout.split(|byte| *byte == 0) { + if raw.is_empty() { + continue; + } + let path = PathBuf::from(String::from_utf8(raw.to_vec()).with_context(|| { + format!("git-tracked path is not valid UTF-8 in {}", src.display()) + })?); + validate_source_path(&path)?; + paths.push(path); + } + Ok(Some(paths)) +} + +/// Paths of files git knows about but does not track (respecting ignore +/// rules), for warning about what a `tracked` sync leaves out. +fn git_untracked_paths(src: &Path) -> Vec { + let output = Command::new("git") + .args(["-C"]) + .arg(src) + .args(["ls-files", "-z", "--others", "--exclude-standard"]) + .output(); + match output { + Ok(output) if output.status.success() => output + .stdout + .split(|byte| *byte == 0) + .filter(|raw| !raw.is_empty()) + .filter_map(|raw| String::from_utf8(raw.to_vec()).ok()) + .map(PathBuf::from) + .collect(), + _ => Vec::new(), + } +} + +/// Ensure the git worktree in `src` has no uncommitted changes and no +/// untracked files, as required by `SourceSyncMode::Committed`. +fn git_ensure_clean_worktree(src: &Path) -> anyhow::Result<()> { + let output = Command::new("git") + .args(["-C"]) + .arg(src) + .args(["status", "--porcelain"]) + .output() + .context("failed to run git status")?; + if !output.status.success() { + // Not a git worktree; the caller falls back to worktree staging. + return Ok(()); + } + let stdout = String::from_utf8_lossy(&output.stdout); + let entries: Vec<&str> = stdout.lines().filter(|line| !line.is_empty()).collect(); + if entries.is_empty() { + return Ok(()); + } + let mut message = + String::from("source-sync mode 'committed' requires a clean git worktree, but found:\n"); + for entry in entries.iter().take(20) { + message.push_str(&format!(" {entry}\n")); + } + if entries.len() > 20 { + message.push_str(&format!(" ... and {} more\n", entries.len() - 20)); + } + message.push_str("commit the changes or use a different --source-sync mode"); + bail!(message) } #[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)] @@ -304,7 +377,82 @@ fn validate_source_path(path: &Path) -> anyhow::Result<()> { Ok(()) } -fn source_tree_entries(src: &Path) -> anyhow::Result> { +fn source_tree_entries(src: &Path, mode: SourceSyncMode) -> anyhow::Result> { + match mode { + SourceSyncMode::Worktree => worktree_entries(src), + SourceSyncMode::Tracked | SourceSyncMode::Committed => { + if mode == SourceSyncMode::Committed { + git_ensure_clean_worktree(src)?; + } + match git_tracked_paths(src)? { + Some(paths) => tracked_entries(src, &paths), + None => { + eprintln!( + "debmagic: warning: {} is not a git worktree, falling back to 'worktree' source sync", + src.display() + ); + worktree_entries(src) + } + } + } + } +} + +fn entry_kind(path: &Path) -> anyhow::Result { + let metadata = fs::symlink_metadata(path) + .with_context(|| format!("failed to stat source path {}", path.display()))?; + if metadata.is_dir() { + Ok(SourcePathKind::Directory) + } else if metadata.is_file() { + Ok(SourcePathKind::File) + } else if metadata.is_symlink() { + Ok(SourcePathKind::Symlink) + } else { + Err(anyhow!( + "unsupported file type in source tree: {}", + path.display() + )) + } +} + +/// Build the entry list from git-tracked paths: tracked files (plus their +/// parent directories), with submodule gitlinks skipped. +fn tracked_entries(src: &Path, paths: &[PathBuf]) -> anyhow::Result> { + let mut seen = std::collections::HashSet::new(); + let mut entries = Vec::new(); + for path in paths { + // Add parent directories first. + let mut ancestors: Vec<&Path> = path.ancestors().skip(1).collect(); + ancestors.pop(); // drop the empty "" ancestor + ancestors.reverse(); + for ancestor in ancestors { + if seen.insert(ancestor.to_path_buf()) { + entries.push(SourcePath { + path: ancestor.to_path_buf(), + kind: SourcePathKind::Directory, + }); + } + } + let full_path = src.join(path); + // Submodule gitlinks are directories; their contents are not staged + // since git does not track them as part of this repository. + if full_path.is_dir() { + eprintln!( + "debmagic: skipping git submodule {}; its contents are not staged", + path.display() + ); + continue; + } + entries.push(SourcePath { + path: path.clone(), + kind: entry_kind(&full_path)?, + }); + } + entries.sort_by_key(|entry| entry.path.components().count()); + Ok(entries) +} + +fn worktree_entries(src: &Path) -> anyhow::Result> { let walker = ignore::WalkBuilder::new(src) .standard_filters(true) .hidden(false) @@ -314,10 +462,6 @@ fn source_tree_entries(src: &Path) -> anyhow::Result> { let mut entries = Vec::new(); for entry in walker { let entry = entry?; - let file_type = entry.file_type().ok_or(anyhow!( - "failed to get file type of {}", - entry.path().display() - ))?; let relative_path = entry .path() .strip_prefix(src) @@ -325,21 +469,9 @@ fn source_tree_entries(src: &Path) -> anyhow::Result> { if relative_path.as_os_str().is_empty() { continue; } - let kind = if file_type.is_dir() { - SourcePathKind::Directory - } else if file_type.is_file() { - SourcePathKind::File - } else if file_type.is_symlink() { - SourcePathKind::Symlink - } else { - return Err(anyhow!( - "unsupported file type in source tree: {}", - entry.path().display() - )); - }; entries.push(SourcePath { path: relative_path.to_path_buf(), - kind, + kind: entry_kind(entry.path())?, }); } entries.sort_by_key(|entry| entry.path.components().count()); @@ -445,7 +577,7 @@ fn sync_source_tree(build_config: &BuildConfig) -> anyhow::Result<()> { for entry in &previous { validate_source_path(&entry.path)?; } - let current = source_tree_entries(&build_config.source_dir)?; + let current = source_tree_entries(&build_config.source_dir, build_config.source_sync_mode)?; let current_kinds = current .iter() @@ -487,12 +619,32 @@ fn stage_source_tree( build_config: &BuildConfig, package: &PackageDescription, ) -> anyhow::Result<()> { + if build_config.source_sync_mode == SourceSyncMode::Tracked { + let untracked = git_untracked_paths(&build_config.source_dir); + if !untracked.is_empty() { + eprintln!( + "debmagic: warning: {} untracked file(s) not staged into the build tree:", + untracked.len() + ); + for path in untracked.iter().take(20) { + eprintln!(" {}", path.display()); + } + if untracked.len() > 20 { + eprintln!(" ... and {} more", untracked.len() - 20); + } + eprintln!(" git add them or use --source-sync worktree to include them"); + } + } if build_config.incremental && source_manifest_path(build_config).is_file() { sync_source_tree(build_config).context("failed to synchronize source tree")?; } else { - copy_dir_all(&build_config.source_dir, build_config.build_source_dir()) - .context("failed to copy source tree to build directory")?; - let entries = source_tree_entries(&build_config.source_dir)?; + let entries = source_tree_entries(&build_config.source_dir, build_config.source_sync_mode)?; + copy_source_entries( + &build_config.source_dir, + &build_config.build_source_dir(), + &entries, + ) + .context("failed to copy source tree to build directory")?; write_source_manifest(build_config, &entries)?; } @@ -594,6 +746,7 @@ fn prepare_build_env( clean: config.clean, persistent: config.driver.persistent, incremental: config.incremental, + source_sync_mode: config.source_sync_mode, }; if config.driver.persistent && build_root.exists() { @@ -924,10 +1077,16 @@ mod tests { clean: false, persistent: true, incremental: true, + source_sync_mode: SourceSyncMode::Worktree, }; build_config.create_dirs()?; - copy_dir_all(&source_dir, build_config.build_source_dir())?; - write_source_manifest(&build_config, &source_tree_entries(&source_dir)?)?; + let initial_entries = source_tree_entries(&source_dir, SourceSyncMode::Worktree)?; + copy_source_entries( + &source_dir, + &build_config.build_source_dir(), + &initial_entries, + )?; + write_source_manifest(&build_config, &initial_entries)?; let unchanged_inode = fs::metadata(build_config.build_source_dir().join("unchanged.txt"))?.ino(); fs::write( @@ -972,6 +1131,94 @@ mod tests { } } + /// Create a git repo with one committed file in a fresh temp dir. + fn git_test_repo() -> anyhow::Result { + let repo = std::env::temp_dir().join(format!("debmagic-git-test-{}", uuid::Uuid::new_v4())); + fs::create_dir_all(repo.join("debian"))?; + fs::write(repo.join("debian/control"), "Source: example")?; + let run = |args: &[&str]| -> anyhow::Result<()> { + let status = Command::new("git") + .arg("-C") + .arg(&repo) + .args(args) + .status()?; + if status.success() { + Ok(()) + } else { + Err(anyhow!("git {:?} failed", args)) + } + }; + run(&["init", "-q"])?; + run(&["add", "debian/control"])?; + run(&[ + "-c", + "user.email=t@t", + "-c", + "user.name=t", + "commit", + "-qm", + "init", + ])?; + Ok(repo) + } + + #[test] + fn tracked_sync_stages_only_git_tracked_files() -> anyhow::Result<()> { + let repo = git_test_repo()?; + fs::write(repo.join("untracked.txt"), "not staged")?; + fs::write(repo.join("dirty.txt"), "uncommitted but tracked? no")?; + // A tracked file with uncommitted modifications is staged with its + // worktree content. + fs::write(repo.join("debian/rules"), "new content")?; + Command::new("git") + .arg("-C") + .arg(&repo) + .args(["add", "debian/rules"]) + .status()?; + + let entries = source_tree_entries(&repo, SourceSyncMode::Tracked)?; + let paths: Vec<&Path> = entries.iter().map(|e| e.path.as_path()).collect(); + assert!(paths.contains(&Path::new("debian"))); + assert!(paths.contains(&Path::new("debian/control"))); + assert!(paths.contains(&Path::new("debian/rules"))); + assert!(!paths.contains(&Path::new("untracked.txt"))); + assert!(!paths.contains(&Path::new("dirty.txt"))); + assert_eq!(git_untracked_paths(&repo).len(), 2); + + fs::remove_dir_all(repo)?; + Ok(()) + } + + #[test] + fn committed_sync_requires_clean_worktree() -> anyhow::Result<()> { + let repo = git_test_repo()?; + assert!(source_tree_entries(&repo, SourceSyncMode::Committed).is_ok()); + + fs::write(repo.join("untracked.txt"), "dirty")?; + let result = source_tree_entries(&repo, SourceSyncMode::Committed); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("requires a clean git worktree") + ); + + fs::remove_dir_all(repo)?; + Ok(()) + } + + #[test] + fn tracked_sync_falls_back_outside_git_worktree() -> anyhow::Result<()> { + let dir = std::env::temp_dir().join(format!("debmagic-nogit-{}", uuid::Uuid::new_v4())); + fs::create_dir_all(&dir)?; + fs::write(dir.join("file.txt"), "content")?; + let entries = source_tree_entries(&dir, SourceSyncMode::Tracked)?; + assert!(entries.iter().any(|e| e.path == Path::new("file.txt"))); + fs::remove_dir_all(dir)?; + Ok(()) + } + #[test] fn test_resolve_distro_version_single_distro_no_explicit() { let distros = vec!["forky".to_string()]; diff --git a/packages/debmagic/src/build/common.rs b/packages/debmagic/src/build/common.rs index fae4244..aa7ffe0 100644 --- a/packages/debmagic/src/build/common.rs +++ b/packages/debmagic/src/build/common.rs @@ -97,6 +97,23 @@ pub enum BuildDriverType { Incus, } +/// Selects which files from the source directory are staged into the build tree. +#[derive( + Debug, Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum, Serialize, Deserialize, +)] +#[serde(rename_all = "snake_case")] +pub enum SourceSyncMode { + /// Git-tracked files, including uncommitted modifications. Untracked + /// files are not staged and reported as a warning. + #[default] + Tracked, + /// Like `tracked`, but the build fails if the worktree has uncommitted + /// changes or untracked files. + Committed, + /// All files except git-ignored ones, regardless of git tracking state. + Worktree, +} + pub type DriverSpecificBuildMetadata = HashMap; #[derive(Debug, Clone, Serialize, Deserialize)] @@ -131,6 +148,9 @@ pub struct BuildConfig { /// Synchronize source inputs while preserving build-generated files. #[serde(default)] pub incremental: bool, + /// Which source files are staged into the build tree. + #[serde(default)] + pub source_sync_mode: SourceSyncMode, } impl BuildConfig { diff --git a/packages/debmagic/src/build/driver_lxd.rs b/packages/debmagic/src/build/driver_lxd.rs index c548f2b..b129304 100644 --- a/packages/debmagic/src/build/driver_lxd.rs +++ b/packages/debmagic/src/build/driver_lxd.rs @@ -344,13 +344,7 @@ impl DriverLxd { // build-dep is intentionally omitted here: build.rs runs it for // every driver against the real mounted source tree. base.exec_in_container( - &[ - "apt-get", - "install", - "-y", - "dpkg-dev", - "python3", - ], + &["apt-get", "install", "-y", "dpkg-dev", "python3"], None, true, &[], diff --git a/packages/debmagic/src/cli.rs b/packages/debmagic/src/cli.rs index 5a95a69..c7883e8 100644 --- a/packages/debmagic/src/cli.rs +++ b/packages/debmagic/src/cli.rs @@ -1,6 +1,6 @@ use std::path::PathBuf; -use crate::build::common::BuildDriverType; +use crate::build::common::{BuildDriverType, SourceSyncMode}; use clap::{Args, Parser, Subcommand, builder::BoolishValueParser}; #[derive(Parser, Debug)] @@ -76,6 +76,12 @@ pub struct CommonBuildArgs { #[arg(long, action = clap::ArgAction::SetTrue, help = "Keep the build environment for reuse after the build finishes")] pub persistent: Option, + #[arg( + long = "source-sync", + help = "Which source files are staged into the build tree: 'tracked' stages git-tracked files including uncommitted changes and warns about untracked files (default), 'committed' additionally fails if the worktree is dirty, 'worktree' stages everything that is not git-ignored. Defaults to the 'source_sync_mode' setting in the config file." + )] + pub source_sync: Option, + #[command(flatten)] pub docker: DockerArgs, diff --git a/packages/debmagic/src/config.rs b/packages/debmagic/src/config.rs index 434ecc5..84c671a 100644 --- a/packages/debmagic/src/config.rs +++ b/packages/debmagic/src/config.rs @@ -1,5 +1,6 @@ use std::path::PathBuf; +use crate::build::common::SourceSyncMode; use crate::build::config::DriverConfig; use anyhow::{Context, anyhow}; use config::{Config as ConfigBuilder, File}; @@ -11,6 +12,8 @@ pub struct Config { pub driver: DriverConfig, pub temp_build_dir: PathBuf, pub incremental: bool, + /// Which source files are staged into the build tree. + pub source_sync_mode: SourceSyncMode, /// Always build the automatic `-dbgsym` debug symbol package. pub build_debug_symbols: bool, /// Sign the resulting `.changes`/`.dsc` with `debsign` after building. @@ -31,6 +34,7 @@ impl Default for Config { driver: DriverConfig::default(), temp_build_dir: PathBuf::from("/tmp/debmagic"), incremental: false, + source_sync_mode: SourceSyncMode::default(), build_debug_symbols: false, sign_package: false, sign_key: None, diff --git a/packages/debmagic/src/main.rs b/packages/debmagic/src/main.rs index 4728e4d..a986798 100644 --- a/packages/debmagic/src/main.rs +++ b/packages/debmagic/src/main.rs @@ -107,6 +107,9 @@ fn main() -> anyhow::Result<()> { if let Some(no_clean) = build_args.no_clean { config.clean = !no_clean; } + if let Some(source_sync) = build_args.source_sync { + config.source_sync_mode = source_sync; + } if config.incremental { if config.clean { anyhow::bail!("incremental builds are incompatible with clean builds"); From f3837e446972bde811d0638469c87d63c733bcc8 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Fri, 31 Jul 2026 23:05:27 +0200 Subject: [PATCH 4/7] refactor(build): split build.rs into a module directory move source-tree staging and incremental sync into build/source.rs and the attach/detach socket server into build/attach.rs, leaving build/mod.rs with the build orchestration itself --- packages/debmagic/src/build.rs | 1303 ------------------------- packages/debmagic/src/build/attach.rs | 93 ++ packages/debmagic/src/build/mod.rs | 645 ++++++++++++ packages/debmagic/src/build/source.rs | 605 ++++++++++++ 4 files changed, 1343 insertions(+), 1303 deletions(-) delete mode 100644 packages/debmagic/src/build.rs create mode 100644 packages/debmagic/src/build/attach.rs create mode 100644 packages/debmagic/src/build/mod.rs create mode 100644 packages/debmagic/src/build/source.rs diff --git a/packages/debmagic/src/build.rs b/packages/debmagic/src/build.rs deleted file mode 100644 index 63048ab..0000000 --- a/packages/debmagic/src/build.rs +++ /dev/null @@ -1,1303 +0,0 @@ -use core::time; -use std::net::Shutdown; -use std::os::unix::fs::PermissionsExt; -use std::os::unix::fs::symlink; -use std::os::unix::net::{UnixListener, UnixStream}; -use std::sync::{Arc, Mutex}; -use std::{ - cmp::Reverse, - fs, - io::{self, BufReader, IsTerminal, Read, Write, stdout}, - path::{Component, Path, PathBuf}, - process::{Command, Stdio}, - thread, -}; - -use crate::build::config::DriverOverrides; -use crate::{ - build::{ - common::{ - BuildConfig, BuildDriver, BuildDriverType, BuildMetadata, SourceSyncMode, run_checked, - }, - config::DriverConfig, - driver_bare::DriverBare, - driver_docker::DriverDocker, - driver_lxd::{DriverLxd, LxdVariant}, - }, - config::Config, - package::PackageDescription, -}; -use anyhow::{Context, anyhow, bail}; -use debmagic_common::distro::DistroVersion; -use glob::glob; - -pub mod artifacts; -pub mod common; -pub mod config; -pub mod driver_bare; -pub mod driver_docker; -pub mod driver_lxd; - -struct Build { - config: BuildConfig, - pub driver: Box, - attached: bool, -} - -fn get_build_driver( - config: &BuildConfig, - driver_config: &DriverConfig, - driver_overrides: &DriverOverrides, -) -> anyhow::Result> { - let apt_mirror = driver_overrides - .apt_mirror - .as_deref() - .or(driver_config.apt_mirror.as_deref()); - let proposed = driver_overrides.proposed.unwrap_or(driver_config.proposed); - - match config.driver { - BuildDriverType::Docker => Ok(Box::new(DriverDocker::create( - config, - driver_config, - &driver_overrides.docker, - apt_mirror, - proposed, - )?)), - BuildDriverType::Bare => Ok(Box::new(DriverBare::create( - config, - driver_config, - &driver_overrides.bare, - ))), - BuildDriverType::Lxd | BuildDriverType::Incus => { - let variant = match config.driver { - BuildDriverType::Lxd => LxdVariant::Lxd, - _ => LxdVariant::Incus, - }; - Ok(Box::new(DriverLxd::create( - variant, - config, - driver_config, - &driver_overrides.lxd, - apt_mirror, - proposed, - )?)) - } - } -} - -fn create_driver_from_metadata( - config: &DriverConfig, - metadata: &BuildMetadata, -) -> anyhow::Result> { - let driver: anyhow::Result> = match &metadata.config.driver { - BuildDriverType::Docker => Ok(Box::new(DriverDocker::from_build_metadata( - &metadata.config, - config, - metadata, - )?)), - BuildDriverType::Bare => Ok(Box::new(DriverBare::from_build_metadata( - &metadata.config, - config, - metadata, - ))), - BuildDriverType::Lxd | BuildDriverType::Incus => { - let variant = match metadata.config.driver { - BuildDriverType::Lxd => LxdVariant::Lxd, - _ => LxdVariant::Incus, - }; - Ok(Box::new(DriverLxd::from_build_metadata( - variant, - &metadata.config, - metadata, - )?)) - } - }; - driver -} - -impl Build { - pub fn create( - config: &BuildConfig, - driver_config: &DriverConfig, - driver_overrides: &DriverOverrides, - ) -> anyhow::Result { - let driver = get_build_driver(config, driver_config, driver_overrides) - .context(format!("failed to create {:?} build driver", config.driver))?; - Ok(Self { - config: config.clone(), - driver, - attached: false, - }) - } - - pub fn from_build_root( - build_root: &Path, - driver_config: &DriverConfig, - ) -> anyhow::Result { - let build_metadata_path = build_root.join("build.json"); - if !build_metadata_path.is_file() { - return Err(anyhow!("No build.json found")); - } - // read metadata from file - let file = fs::OpenOptions::new() - .read(true) - .open(&build_metadata_path)?; - let metadata = || -> anyhow::Result { - let reader = BufReader::new(&file); - let metadata: BuildMetadata = serde_json::from_reader(reader).with_context(|| { - format!( - "Failed to read build metadata from {} - invalid json", - build_metadata_path.display() - ) - })?; - Ok(metadata) - }(); - - let metadata = metadata?; - - let driver = create_driver_from_metadata(driver_config, &metadata)?; - - let attached = send_socket_command(build_root, "attach").is_ok(); - - Ok(Self { - config: metadata.config.clone(), - driver, - attached, - }) - } - - pub fn detach(&self) -> anyhow::Result<()> { - let build_root = &self.config.build_root_dir; - if self.attached { - send_socket_command(build_root, "detach")?; - } - Ok(()) - } - - pub fn write_metadata(&self) -> anyhow::Result<()> { - let metadata = BuildMetadata { - config: self.config.clone(), - driver_metadata: self.driver.get_build_metadata(), - }; - let path = self.config.build_root_dir.join("build.json"); - let json = serde_json::to_string_pretty(&metadata) - .context("Failed to serialize build metadata")?; - fs::write(path, json)?; - Ok(()) - } -} - -fn copy_glob(src_dir: &Path, pattern: &str, dest_dir: &Path) -> anyhow::Result<()> { - let full_pattern = src_dir.join(pattern).to_string_lossy().into_owned(); - for entry in glob(&full_pattern)? { - let path = entry?; - if path.is_file() { - let filename = path.file_name().ok_or(anyhow!( - "Could not retrieve filename from {}", - path.display() - ))?; - fs::copy(&path, dest_dir.join(filename))?; - } - } - Ok(()) -} - -fn socket_path_for_build(build_root: &Path) -> PathBuf { - build_root.join("build.sock") -} - -fn start_socket_server( - build_root: &Path, - should_exit: Arc>, -) -> anyhow::Result> { - let sock = socket_path_for_build(build_root); - if sock.exists() { - // try to remove stale socket file - let _ = fs::remove_file(&sock); - } - - let listener = UnixListener::bind(&sock) - .with_context(|| format!("failed to bind unix socket {}", sock.display()))?; - - // Set non-blocking mode so we can check the exit flag - listener - .set_nonblocking(true) - .context("failed to set socket non-blocking")?; - - let handle = thread::spawn(move || { - let mut num_attached = 0u64; - loop { - // Check if we should exit - let exit_requested = *should_exit.lock().unwrap(); - if exit_requested && num_attached == 0 { - break; - } - - match listener.accept() { - Ok((mut s, _)) => { - let mut buf = String::new(); - if s.read_to_string(&mut buf).is_err() { - let _ = s.shutdown(Shutdown::Both); - continue; - } - let cmd = buf.trim(); - match cmd { - "attach" => { - num_attached += 1; - } - "detach" => { - num_attached = num_attached.saturating_sub(1); - } - _ => {} - } - let _ = s.shutdown(Shutdown::Both); - } - Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => { - // No connection available, sleep briefly to avoid busy-waiting - thread::sleep(time::Duration::from_millis(10)); - } - Err(_) => break, - } - } - let _ = fs::remove_file(&sock); - }); - - Ok(handle) -} - -fn send_socket_command(build_root: &Path, cmd: &str) -> anyhow::Result<()> { - let sock = socket_path_for_build(build_root); - let mut stream = UnixStream::connect(&sock) - .with_context(|| format!("failed to connect to socket {}", sock.display()))?; - stream - .write_all(cmd.as_bytes()) - .context("failed to send socket command")?; - stream.shutdown(Shutdown::Write).ok(); - Ok(()) -} - -/// Paths of files tracked by git in `src`, as reported by `git ls-files`. -/// Returns `None` if `src` is not inside a git worktree. -fn git_tracked_paths(src: &Path) -> anyhow::Result>> { - let output = Command::new("git") - .args(["-C"]) - .arg(src) - .args(["ls-files", "-z"]) - .output() - .context("failed to run git ls-files")?; - if !output.status.success() { - return Ok(None); - } - let mut paths = Vec::new(); - for raw in output.stdout.split(|byte| *byte == 0) { - if raw.is_empty() { - continue; - } - let path = PathBuf::from(String::from_utf8(raw.to_vec()).with_context(|| { - format!("git-tracked path is not valid UTF-8 in {}", src.display()) - })?); - validate_source_path(&path)?; - paths.push(path); - } - Ok(Some(paths)) -} - -/// Paths of files git knows about but does not track (respecting ignore -/// rules), for warning about what a `tracked` sync leaves out. -fn git_untracked_paths(src: &Path) -> Vec { - let output = Command::new("git") - .args(["-C"]) - .arg(src) - .args(["ls-files", "-z", "--others", "--exclude-standard"]) - .output(); - match output { - Ok(output) if output.status.success() => output - .stdout - .split(|byte| *byte == 0) - .filter(|raw| !raw.is_empty()) - .filter_map(|raw| String::from_utf8(raw.to_vec()).ok()) - .map(PathBuf::from) - .collect(), - _ => Vec::new(), - } -} - -/// Ensure the git worktree in `src` has no uncommitted changes and no -/// untracked files, as required by `SourceSyncMode::Committed`. -fn git_ensure_clean_worktree(src: &Path) -> anyhow::Result<()> { - let output = Command::new("git") - .args(["-C"]) - .arg(src) - .args(["status", "--porcelain"]) - .output() - .context("failed to run git status")?; - if !output.status.success() { - // Not a git worktree; the caller falls back to worktree staging. - return Ok(()); - } - let stdout = String::from_utf8_lossy(&output.stdout); - let entries: Vec<&str> = stdout.lines().filter(|line| !line.is_empty()).collect(); - if entries.is_empty() { - return Ok(()); - } - let mut message = - String::from("source-sync mode 'committed' requires a clean git worktree, but found:\n"); - for entry in entries.iter().take(20) { - message.push_str(&format!(" {entry}\n")); - } - if entries.len() > 20 { - message.push_str(&format!(" ... and {} more\n", entries.len() - 20)); - } - message.push_str("commit the changes or use a different --source-sync mode"); - bail!(message) -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)] -#[serde(rename_all = "snake_case")] -enum SourcePathKind { - Directory, - File, - Symlink, -} - -#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] -struct SourcePath { - path: PathBuf, - kind: SourcePathKind, -} - -fn validate_source_path(path: &Path) -> anyhow::Result<()> { - if path.as_os_str().is_empty() - || !path - .components() - .all(|component| matches!(component, Component::Normal(_))) - { - bail!("invalid source manifest path: {}", path.display()); - } - Ok(()) -} - -fn source_tree_entries(src: &Path, mode: SourceSyncMode) -> anyhow::Result> { - match mode { - SourceSyncMode::Worktree => worktree_entries(src), - SourceSyncMode::Tracked | SourceSyncMode::Committed => { - if mode == SourceSyncMode::Committed { - git_ensure_clean_worktree(src)?; - } - match git_tracked_paths(src)? { - Some(paths) => tracked_entries(src, &paths), - None => { - eprintln!( - "debmagic: warning: {} is not a git worktree, falling back to 'worktree' source sync", - src.display() - ); - worktree_entries(src) - } - } - } - } -} - -fn entry_kind(path: &Path) -> anyhow::Result { - let metadata = fs::symlink_metadata(path) - .with_context(|| format!("failed to stat source path {}", path.display()))?; - if metadata.is_dir() { - Ok(SourcePathKind::Directory) - } else if metadata.is_file() { - Ok(SourcePathKind::File) - } else if metadata.is_symlink() { - Ok(SourcePathKind::Symlink) - } else { - Err(anyhow!( - "unsupported file type in source tree: {}", - path.display() - )) - } -} - -/// Build the entry list from git-tracked paths: tracked files (plus their -/// parent directories), with submodule gitlinks skipped. -fn tracked_entries(src: &Path, paths: &[PathBuf]) -> anyhow::Result> { - let mut seen = std::collections::HashSet::new(); - let mut entries = Vec::new(); - for path in paths { - // Add parent directories first. - let mut ancestors: Vec<&Path> = path.ancestors().skip(1).collect(); - ancestors.pop(); // drop the empty "" ancestor - ancestors.reverse(); - for ancestor in ancestors { - if seen.insert(ancestor.to_path_buf()) { - entries.push(SourcePath { - path: ancestor.to_path_buf(), - kind: SourcePathKind::Directory, - }); - } - } - let full_path = src.join(path); - // Submodule gitlinks are directories; their contents are not staged - // since git does not track them as part of this repository. - if full_path.is_dir() { - eprintln!( - "debmagic: skipping git submodule {}; its contents are not staged", - path.display() - ); - continue; - } - entries.push(SourcePath { - path: path.clone(), - kind: entry_kind(&full_path)?, - }); - } - entries.sort_by_key(|entry| entry.path.components().count()); - Ok(entries) -} - -fn worktree_entries(src: &Path) -> anyhow::Result> { - let walker = ignore::WalkBuilder::new(src) - .standard_filters(true) - .hidden(false) - .filter_entry(|entry| !(entry.path().is_dir() && entry.path().ends_with(".git"))) - .build(); - - let mut entries = Vec::new(); - for entry in walker { - let entry = entry?; - let relative_path = entry - .path() - .strip_prefix(src) - .context("failed to get relative path")?; - if relative_path.as_os_str().is_empty() { - continue; - } - entries.push(SourcePath { - path: relative_path.to_path_buf(), - kind: entry_kind(entry.path())?, - }); - } - entries.sort_by_key(|entry| entry.path.components().count()); - Ok(entries) -} - -fn remove_path(path: &Path) -> std::io::Result<()> { - match fs::symlink_metadata(path) { - Ok(metadata) if metadata.is_dir() => fs::remove_dir_all(path), - Ok(_) => fs::remove_file(path), - Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), - Err(error) => Err(error), - } -} - -fn files_match(source: &Path, destination: &Path) -> std::io::Result { - let source_metadata = fs::metadata(source)?; - let destination_metadata = match fs::symlink_metadata(destination) { - Ok(metadata) if metadata.is_file() => metadata, - Ok(_) => return Ok(false), - Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), - Err(error) => return Err(error), - }; - if source_metadata.len() != destination_metadata.len() - || source_metadata.permissions().mode() != destination_metadata.permissions().mode() - { - return Ok(false); - } - - let mut source = BufReader::new(fs::File::open(source)?); - let mut destination = BufReader::new(fs::File::open(destination)?); - let mut source_buffer = [0; 8192]; - let mut destination_buffer = [0; 8192]; - loop { - let source_len = source.read(&mut source_buffer)?; - let destination_len = destination.read(&mut destination_buffer)?; - if source_len != destination_len - || source_buffer[..source_len] != destination_buffer[..destination_len] - { - return Ok(false); - } - if source_len == 0 { - return Ok(true); - } - } -} - -fn copy_source_entries(src: &Path, dst: &Path, entries: &[SourcePath]) -> anyhow::Result<()> { - fs::create_dir_all(dst)?; - for entry in entries { - let source = src.join(&entry.path); - let destination = dst.join(&entry.path); - match entry.kind { - SourcePathKind::Directory => match fs::symlink_metadata(&destination) { - Ok(metadata) if metadata.is_dir() => {} - Ok(_) => { - remove_path(&destination)?; - fs::create_dir_all(&destination)?; - } - Err(error) if error.kind() == io::ErrorKind::NotFound => { - fs::create_dir_all(&destination)?; - } - Err(error) => return Err(error.into()), - }, - SourcePathKind::File => { - if !files_match(&source, &destination)? { - remove_path(&destination)?; - fs::copy(&source, &destination) - .with_context(|| format!("failed to copy file: {}", source.display()))?; - } - } - SourcePathKind::Symlink => { - let target = fs::read_link(&source)?; - if fs::read_link(&destination).ok().as_deref() != Some(target.as_path()) { - remove_path(&destination)?; - symlink(target, &destination)?; - } - } - } - } - Ok(()) -} - -fn source_manifest_path(build_config: &BuildConfig) -> PathBuf { - build_config.build_root_dir.join("source-manifest.json") -} - -fn write_source_manifest(build_config: &BuildConfig, entries: &[SourcePath]) -> anyhow::Result<()> { - let manifest_path = source_manifest_path(build_config); - let temporary_path = manifest_path.with_extension("json.tmp"); - fs::write(&temporary_path, serde_json::to_vec_pretty(entries)?)?; - fs::rename(temporary_path, manifest_path)?; - Ok(()) -} - -fn sync_source_tree(build_config: &BuildConfig) -> anyhow::Result<()> { - let manifest_path = source_manifest_path(build_config); - let previous: Vec = serde_json::from_reader(BufReader::new( - fs::File::open(&manifest_path) - .with_context(|| format!("failed to open {}", manifest_path.display()))?, - )) - .with_context(|| format!("failed to read {}", manifest_path.display()))?; - for entry in &previous { - validate_source_path(&entry.path)?; - } - let current = source_tree_entries(&build_config.source_dir, build_config.source_sync_mode)?; - - let current_kinds = current - .iter() - .map(|entry| (entry.path.as_path(), entry.kind)) - .collect::>(); - let mut stale = previous - .iter() - .filter(|entry| current_kinds.get(entry.path.as_path()) != Some(&entry.kind)) - .collect::>(); - stale.sort_by_key(|entry| Reverse(entry.path.components().count())); - for entry in stale { - let destination = build_config.build_source_dir().join(&entry.path); - if entry.kind == SourcePathKind::Directory - && !current_kinds.contains_key(entry.path.as_path()) - { - match fs::remove_dir(&destination) { - Ok(()) => {} - Err(error) - if matches!( - error.kind(), - io::ErrorKind::NotFound | io::ErrorKind::DirectoryNotEmpty - ) => {} - Err(error) => return Err(error.into()), - } - } else { - remove_path(&destination)?; - } - } - - copy_source_entries( - &build_config.source_dir, - &build_config.build_source_dir(), - ¤t, - )?; - write_source_manifest(build_config, ¤t) -} - -fn stage_source_tree( - build_config: &BuildConfig, - package: &PackageDescription, -) -> anyhow::Result<()> { - if build_config.source_sync_mode == SourceSyncMode::Tracked { - let untracked = git_untracked_paths(&build_config.source_dir); - if !untracked.is_empty() { - eprintln!( - "debmagic: warning: {} untracked file(s) not staged into the build tree:", - untracked.len() - ); - for path in untracked.iter().take(20) { - eprintln!(" {}", path.display()); - } - if untracked.len() > 20 { - eprintln!(" ... and {} more", untracked.len() - 20); - } - eprintln!(" git add them or use --source-sync worktree to include them"); - } - } - if build_config.incremental && source_manifest_path(build_config).is_file() { - sync_source_tree(build_config).context("failed to synchronize source tree")?; - } else { - let entries = source_tree_entries(&build_config.source_dir, build_config.source_sync_mode)?; - copy_source_entries( - &build_config.source_dir, - &build_config.build_source_dir(), - &entries, - ) - .context("failed to copy source tree to build directory")?; - write_source_manifest(build_config, &entries)?; - } - - let source_parent = build_config - .source_dir - .parent() - .ok_or_else(|| anyhow!("source directory has no parent"))?; - let prefix = format!("{}_{}", package.name, package.version.upstream_version()); - copy_glob( - source_parent, - &format!("{prefix}.orig.tar.*"), - &build_config.build_work_dir(), - )?; - copy_glob( - source_parent, - &format!("{prefix}.orig-*.tar.*"), - &build_config.build_work_dir(), - )?; - Ok(()) -} - -fn get_build_root_and_identifier( - config: &Config, - package: &PackageDescription, -) -> (String, PathBuf) { - let package_identifier = format!("{}-{}", package.name, package.version); - let build_root = config.temp_build_dir.join(&package_identifier); - (package_identifier, build_root) -} - -/// Determine which distro version to use for the build. -/// -/// If only one distro version is specified in the changelog, it's used automatically. -/// If multiple distro versions are specified, an explicit --distro is required. -/// If --distro is provided, it's validated against the changelog versions. -fn resolve_distro_version( - changelog_distros: &[String], - explicit_distro: Option<&str>, -) -> anyhow::Result { - let resolved_codename = match (changelog_distros.len(), explicit_distro) { - (0, _) => Err(anyhow!("changelog contains no distributions")), - (1, None) => Ok(changelog_distros[0].clone()), - (1, Some(explicit)) => { - if explicit == changelog_distros[0] { - Ok(explicit.to_string()) - } else { - Err(anyhow!( - "explicit distro version '{}' conflicts with distribution specified in changelog '{}'", - explicit, - changelog_distros[0] - )) - } - } - (_, None) => Err(anyhow!( - "changelog contains multiple distributions ({}), please specify which one to build for with --distro", - changelog_distros.join(", ") - )), - (_, Some(explicit)) => { - if changelog_distros.contains(&explicit.to_string()) { - Ok(explicit.to_string()) - } else { - Err(anyhow!( - "explicit distro version '{}' not found in changelog distributions: {}", - explicit, - changelog_distros.join(", ") - )) - } - } - }?; - let resolved = debmagic_common::distro::get_distro_version(&resolved_codename) - .ok_or_else(|| anyhow!("unknown distro codename '{}'", resolved_codename))?; - Ok(resolved) -} - -fn prepare_build_env( - config: &Config, - driver_overrides: &DriverOverrides, - package: &PackageDescription, - driver_type: BuildDriverType, - output_dir: &Path, - explicit_distro_version: Option<&str>, -) -> anyhow::Result { - let (package_identifier, build_root) = get_build_root_and_identifier(config, package); - - let distro_version = resolve_distro_version(&package.distro_versions, explicit_distro_version) - .context("failed to determine distro version")?; - - let build_config = BuildConfig { - driver: driver_type, - package_name: package.name.clone(), - package_identifier, - source_dir: package.source_dir.clone(), - output_dir: output_dir.to_path_buf(), - build_root_dir: build_root.clone(), - distro: distro_version.clone(), - sign_package: config.sign_package, - sign_key: config.sign_key.clone(), - build_debug_symbols: config.build_debug_symbols, - clean: config.clean, - persistent: config.driver.persistent, - incremental: config.incremental, - source_sync_mode: config.source_sync_mode, - }; - - if config.driver.persistent && build_root.exists() { - // For persistent containers, starting first lets root inside delete - // container-owned files the host user can't remove. - let build = Build::create(&build_config, &config.driver, driver_overrides) - .context(format!("failed to create {:?} build driver", driver_type))?; - if !config.incremental - || !source_manifest_path(&build_config).is_file() - || !build.driver.reused_environment() - { - build - .driver - .reset_build_root() - .context("failed to reset persistent build directory")?; - } - build_config - .create_dirs() - .context("failed to create build directories")?; - stage_source_tree(&build_config, package)?; - return Ok(build); - } - - if build_root.exists() - && let Err(e) = fs::remove_dir_all(&build_root) - { - if e.kind() == io::ErrorKind::PermissionDenied { - // Some files were created by a privileged user inside a container - // and can't be deleted by the host user directly. Load the previous - // build's driver and ask it to clean up from inside. - let metadata_path = build_root.join("build.json"); - if metadata_path.is_file() - && let Ok(file) = fs::OpenOptions::new().read(true).open(&metadata_path) - && let Ok(metadata) = - serde_json::from_reader::<_, BuildMetadata>(BufReader::new(&file)) - && let Ok(driver) = create_driver_from_metadata(&config.driver, &metadata) - { - let _ = driver.reset_build_root(); - } - fs::remove_dir_all(&build_root).with_context(|| { - format!( - "failed to remove build root {}; try: sudo rm -rf {}", - build_root.display(), - build_root.display() - ) - })?; - } else { - return Err(e.into()); - } - } - - build_config - .create_dirs() - .context("failed to create build directories")?; - - stage_source_tree(&build_config, package)?; - - let build = Build::create(&build_config, &config.driver, driver_overrides)?; - Ok(build) -} - -pub fn get_shell_in_build(config: &Config, package: &PackageDescription) -> anyhow::Result<()> { - let (_package_identifier, build_root) = get_build_root_and_identifier(config, package); - let build = Build::from_build_root(&build_root, &config.driver)?; - let result = build - .driver - .interactive_shell(&build.config.build_source_dir()); - - build.detach()?; - - result?; - Ok(()) -} - -fn deb_build_options(existing: Option<&str>, build_debug_symbols: bool) -> String { - let mut options = existing - .unwrap_or_default() - .split_whitespace() - .filter(|option| *option != "noautodbgsym") - .collect::>(); - if !build_debug_symbols { - options.push("noautodbgsym"); - } - options.join(" ") -} - -/// Everything needed to run one package build, independent of whether the -/// build produces binary or source packages. -pub struct BuildRequest<'a> { - pub config: &'a Config, - pub package: &'a PackageDescription, - pub driver_type: BuildDriverType, - pub driver_overrides: &'a DriverOverrides, - pub output_dir: &'a Path, - pub explicit_distro_version: Option<&'a str>, -} - -/// Shared build orchestration: prepare the environment, run `build_commands` -/// in it, export the artifacts to the output dir, sign them if requested, and -/// clean up (dropping into a shell first on failure of an interactive binary -/// build). While `shell_on_failure` is set, a socket server lets concurrent -/// `debmagic shell` sessions attach to the environment. -fn run_build( - request: &BuildRequest, - shell_on_failure: bool, - build_commands: impl FnOnce(&Build) -> anyhow::Result<()>, -) -> anyhow::Result<()> { - let build = prepare_build_env( - request.config, - request.driver_overrides, - request.package, - request.driver_type, - request.output_dir, - request.explicit_distro_version, - ) - .context("failed to prepare build environment")?; - build - .write_metadata() - .context("failed to write build metadata")?; - - let should_exit = Arc::new(Mutex::new(false)); - let socket_server_handle = - start_socket_server(&build.config.build_root_dir, should_exit.clone())?; - - let stop_socket_server = || { - *should_exit.lock().unwrap() = true; - if !socket_server_handle.is_finished() { - println!("Waiting for all attached shells to exit..."); - } - socket_server_handle.join().ok(); - }; - - let result = build_commands(&build).and_then(|()| { - let changes_file = artifacts::export_build_artifacts( - &build.config.build_work_dir(), - &build.config.output_dir, - )?; - if build.config.sign_package { - sign_changes_file(&changes_file, build.config.sign_key.as_deref())?; - } - Ok(()) - }); - - if let Err(error) = result { - if shell_on_failure && stdout().is_terminal() { - eprintln!("Build failed: {error}. Dropping into shell..."); - if let Err(shell_error) = build - .driver - .interactive_shell(&build.config.build_source_dir()) - { - eprintln!("Dropping into shell failed: {shell_error}"); - } - } else { - eprintln!("Build failed: {error}"); - } - if let Err(cleanup_error) = build.driver.cleanup() { - eprintln!("Failed to clean up build environment: {cleanup_error}"); - } - stop_socket_server(); - return Err(error); - } - - stop_socket_server(); - build - .driver - .cleanup() - .context("failed to clean up build environment")?; - Ok(()) -} - -pub fn build_package(request: &BuildRequest) -> anyhow::Result<()> { - run_build(request, true, |build| { - build.driver.run_command( - &["apt-get", "-y", "build-dep", "."], - &build.config.build_source_dir(), - true, - )?; - let inherited_options = std::env::var("DEB_BUILD_OPTIONS").ok(); - let options = deb_build_options( - inherited_options.as_deref(), - build.config.build_debug_symbols, - ); - let env_add = [("DEB_BUILD_OPTIONS", options.as_str())]; - let mut dpkg_buildpackage_args = vec!["dpkg-buildpackage", "-us", "-uc", "-ui"]; - if !build.config.clean { - // Non-incremental builds already stage a clean source tree, while - // incremental builds preserve their outputs intentionally. - dpkg_buildpackage_args.push("-nc"); - } - dpkg_buildpackage_args.push("-b"); - build.driver.run_command_env( - &dpkg_buildpackage_args, - &build.config.build_source_dir(), - false, - &env_add, - )?; - Ok(()) - }) -} - -/// Confirm `cmd` is on `PATH`, failing with an actionable message (rather -/// than a raw "command not found") if it isn't. -fn check_command_available(cmd: &str, install_hint: &str) -> anyhow::Result<()> { - match Command::new(cmd) - .arg("--version") - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - { - Ok(_) => Ok(()), - Err(e) if e.kind() == io::ErrorKind::NotFound => { - Err(anyhow!("{cmd} not found on PATH. {install_hint}")) - } - Err(e) => Err(e).with_context(|| format!("failed to check for {cmd}")), - } -} - -fn check_dpkg_buildpackage_available() -> anyhow::Result<()> { - check_command_available( - "dpkg-buildpackage", - "It's part of dpkg-dev; install it, or pass --driver lxd/incus/docker to build inside a Debian-ish container instead.", - ) -} - -/// GPG-sign every `.changes` (and its referenced `.dsc`/`.buildinfo`) in -/// `output_dir` with `debsign`. Always runs on the host regardless of the -/// build driver, since signing needs the user's own gpg keyring, which an -/// ephemeral container doesn't have access to. -fn sign_changes_file(changes_file: &Path, sign_key: Option<&str>) -> anyhow::Result<()> { - check_command_available( - "debsign", - "It's part of devscripts; install it and set up a gpg signing key to use --sign.", - )?; - - let output_dir = changes_file.parent().ok_or_else(|| { - anyhow!( - "could not get output directory of {}", - changes_file.display() - ) - })?; - let filename = changes_file - .file_name() - .ok_or_else(|| anyhow!("could not get filename of {}", changes_file.display()))?; - - let mut cmd = Command::new("debsign"); - if let Some(key) = sign_key { - cmd.arg(format!("-k{key}")); - } - cmd.arg(filename).current_dir(output_dir); - run_checked(&mut cmd, &format!("signing {}", changes_file.display()))?; - Ok(()) -} - -/// Build a `.dsc` + tarball + `.buildinfo` + `.changes` source package. -/// -/// If `config.clean` is set, build-dependencies are installed before -/// `dpkg-buildpackage` runs `debian/rules clean` once. -pub fn build_source_package(request: &BuildRequest) -> anyhow::Result<()> { - if request.driver_type == BuildDriverType::Bare { - check_dpkg_buildpackage_available()?; - } - - run_build(request, false, |build| { - let build_source_dir = build.config.build_source_dir(); - if build.config.clean { - build.driver.run_command( - &["apt-get", "-y", "build-dep", "."], - &build_source_dir, - true, - )?; - } - let mut args = vec!["dpkg-buildpackage", "-S", "-d", "-us", "-uc", "-ui"]; - if !build.config.clean { - args.push("-nc"); - } - build.driver.run_command(&args, &build_source_dir, false)?; - Ok(()) - }) - .context("failed to build source package") -} - -#[cfg(test)] -mod tests { - use std::os::unix::fs::MetadataExt; - - use debmagic_common::distro::Distro; - - use super::*; - - #[test] - fn debug_symbol_option_preserves_other_build_options() { - assert_eq!( - deb_build_options(Some("nocheck parallel=8"), false), - "nocheck parallel=8 noautodbgsym" - ); - assert_eq!( - deb_build_options(Some("nocheck noautodbgsym parallel=8"), true), - "nocheck parallel=8" - ); - } - - #[test] - fn incremental_sync_updates_sources_and_preserves_build_outputs() -> anyhow::Result<()> { - let test_root = std::env::temp_dir().join(format!( - "debmagic-incremental-test-{}", - uuid::Uuid::new_v4() - )); - let source_dir = test_root.join("source"); - let build_root_dir = test_root.join("build"); - fs::create_dir_all(source_dir.join("cache"))?; - fs::write(source_dir.join("changed.txt"), "before")?; - fs::write(source_dir.join("unchanged.txt"), "unchanged")?; - fs::write(source_dir.join("removed.txt"), "remove me")?; - fs::write(source_dir.join("cache/input.c"), "source")?; - symlink("changed.txt", source_dir.join("link"))?; - - let build_config = BuildConfig { - driver: BuildDriverType::Bare, - package_name: "example".to_string(), - package_identifier: "example-1.0".to_string(), - build_root_dir: build_root_dir.clone(), - source_dir: source_dir.clone(), - output_dir: test_root.join("output"), - distro: debmagic_common::distro::get_distro_version("trixie").unwrap(), - sign_package: false, - sign_key: None, - build_debug_symbols: false, - clean: false, - persistent: true, - incremental: true, - source_sync_mode: SourceSyncMode::Worktree, - }; - build_config.create_dirs()?; - let initial_entries = source_tree_entries(&source_dir, SourceSyncMode::Worktree)?; - copy_source_entries( - &source_dir, - &build_config.build_source_dir(), - &initial_entries, - )?; - write_source_manifest(&build_config, &initial_entries)?; - let unchanged_inode = - fs::metadata(build_config.build_source_dir().join("unchanged.txt"))?.ino(); - fs::write( - build_config.build_source_dir().join("cache/output.o"), - "compiled", - )?; - - fs::write(source_dir.join("changed.txt"), "after")?; - fs::remove_file(source_dir.join("removed.txt"))?; - fs::remove_file(source_dir.join("cache/input.c"))?; - fs::remove_dir(source_dir.join("cache"))?; - fs::remove_file(source_dir.join("link"))?; - symlink("added.txt", source_dir.join("link"))?; - fs::write(source_dir.join("added.txt"), "new")?; - - sync_source_tree(&build_config)?; - - let staged = build_config.build_source_dir(); - assert_eq!(fs::read_to_string(staged.join("changed.txt"))?, "after"); - assert_eq!(fs::read_to_string(staged.join("added.txt"))?, "new"); - assert_eq!( - fs::metadata(staged.join("unchanged.txt"))?.ino(), - unchanged_inode - ); - assert_eq!(fs::read_link(staged.join("link"))?, Path::new("added.txt")); - assert!(!staged.join("removed.txt").exists()); - assert!(!staged.join("cache/input.c").exists()); - assert_eq!( - fs::read_to_string(staged.join("cache/output.o"))?, - "compiled" - ); - - fs::remove_dir_all(test_root)?; - Ok(()) - } - - #[test] - fn source_manifest_paths_must_be_relative_and_normal() { - assert!(validate_source_path(Path::new("debian/control")).is_ok()); - for path in ["", ".", "../outside", "debian/../outside", "/tmp/outside"] { - assert!(validate_source_path(Path::new(path)).is_err(), "{path}"); - } - } - - /// Create a git repo with one committed file in a fresh temp dir. - fn git_test_repo() -> anyhow::Result { - let repo = std::env::temp_dir().join(format!("debmagic-git-test-{}", uuid::Uuid::new_v4())); - fs::create_dir_all(repo.join("debian"))?; - fs::write(repo.join("debian/control"), "Source: example")?; - let run = |args: &[&str]| -> anyhow::Result<()> { - let status = Command::new("git") - .arg("-C") - .arg(&repo) - .args(args) - .status()?; - if status.success() { - Ok(()) - } else { - Err(anyhow!("git {:?} failed", args)) - } - }; - run(&["init", "-q"])?; - run(&["add", "debian/control"])?; - run(&[ - "-c", - "user.email=t@t", - "-c", - "user.name=t", - "commit", - "-qm", - "init", - ])?; - Ok(repo) - } - - #[test] - fn tracked_sync_stages_only_git_tracked_files() -> anyhow::Result<()> { - let repo = git_test_repo()?; - fs::write(repo.join("untracked.txt"), "not staged")?; - fs::write(repo.join("dirty.txt"), "uncommitted but tracked? no")?; - // A tracked file with uncommitted modifications is staged with its - // worktree content. - fs::write(repo.join("debian/rules"), "new content")?; - Command::new("git") - .arg("-C") - .arg(&repo) - .args(["add", "debian/rules"]) - .status()?; - - let entries = source_tree_entries(&repo, SourceSyncMode::Tracked)?; - let paths: Vec<&Path> = entries.iter().map(|e| e.path.as_path()).collect(); - assert!(paths.contains(&Path::new("debian"))); - assert!(paths.contains(&Path::new("debian/control"))); - assert!(paths.contains(&Path::new("debian/rules"))); - assert!(!paths.contains(&Path::new("untracked.txt"))); - assert!(!paths.contains(&Path::new("dirty.txt"))); - assert_eq!(git_untracked_paths(&repo).len(), 2); - - fs::remove_dir_all(repo)?; - Ok(()) - } - - #[test] - fn committed_sync_requires_clean_worktree() -> anyhow::Result<()> { - let repo = git_test_repo()?; - assert!(source_tree_entries(&repo, SourceSyncMode::Committed).is_ok()); - - fs::write(repo.join("untracked.txt"), "dirty")?; - let result = source_tree_entries(&repo, SourceSyncMode::Committed); - assert!(result.is_err()); - assert!( - result - .unwrap_err() - .to_string() - .contains("requires a clean git worktree") - ); - - fs::remove_dir_all(repo)?; - Ok(()) - } - - #[test] - fn tracked_sync_falls_back_outside_git_worktree() -> anyhow::Result<()> { - let dir = std::env::temp_dir().join(format!("debmagic-nogit-{}", uuid::Uuid::new_v4())); - fs::create_dir_all(&dir)?; - fs::write(dir.join("file.txt"), "content")?; - let entries = source_tree_entries(&dir, SourceSyncMode::Tracked)?; - assert!(entries.iter().any(|e| e.path == Path::new("file.txt"))); - fs::remove_dir_all(dir)?; - Ok(()) - } - - #[test] - fn test_resolve_distro_version_single_distro_no_explicit() { - let distros = vec!["forky".to_string()]; - let result = resolve_distro_version(&distros, None); - assert!(result.is_ok()); - let distro_version = result.unwrap(); - assert_eq!(distro_version.codename, "forky"); - assert_eq!(distro_version.distro, Distro::Debian); - } - - #[test] - fn test_resolve_distro_version_single_distro_matching_explicit() { - let distros = vec!["forky".to_string()]; - let result = resolve_distro_version(&distros, Some("forky")); - assert!(result.is_ok()); - let distro_version = result.unwrap(); - assert_eq!(distro_version.codename, "forky"); - assert_eq!(distro_version.distro, Distro::Debian); - } - - #[test] - fn test_resolve_distro_version_single_distro_conflicting_explicit() { - let distros = vec!["forky".to_string()]; - let result = resolve_distro_version(&distros, Some("duke")); - assert!(result.is_err()); - assert!( - result - .unwrap_err() - .to_string() - .contains("conflicts with distribution specified in changelog") - ); - } - - #[test] - fn test_resolve_distro_version_multiple_distros_no_explicit() { - let distros = vec!["forky".to_string(), "duke".to_string()]; - let result = resolve_distro_version(&distros, None); - assert!(result.is_err()); - assert!( - result - .unwrap_err() - .to_string() - .contains("multiple distributions") - ); - } - - #[test] - fn test_resolve_distro_version_multiple_distros_explicit_valid() { - let distros = vec!["forky".to_string(), "duke".to_string()]; - let result = resolve_distro_version(&distros, Some("duke")); - assert!(result.is_ok()); - let distro_version = result.unwrap(); - assert_eq!(distro_version.codename, "duke"); - assert_eq!(distro_version.distro, Distro::Debian); - } - - #[test] - fn test_resolve_distro_version_multiple_distros_explicit_invalid() { - let distros = vec!["forky".to_string(), "duke".to_string()]; - let result = resolve_distro_version(&distros, Some("trixie")); - assert!(result.is_err()); - assert!( - result - .unwrap_err() - .to_string() - .contains("not found in changelog distributions") - ); - } - - #[test] - fn test_resolve_distro_version_empty_distros() { - let distros: Vec = vec![]; - let result = resolve_distro_version(&distros, None); - assert!(result.is_err()); - assert!( - result - .unwrap_err() - .to_string() - .contains("changelog contains no distributions") - ); - } -} diff --git a/packages/debmagic/src/build/attach.rs b/packages/debmagic/src/build/attach.rs new file mode 100644 index 0000000..15110d4 --- /dev/null +++ b/packages/debmagic/src/build/attach.rs @@ -0,0 +1,93 @@ +//! Unix-socket attach/detach tracking for build environments. +//! +//! While a build keeps its environment alive after a failure, a small +//! socket server in the build root lets concurrent `debmagic shell` +//! sessions register themselves, so the environment is only torn down +//! once the last attached shell detaches. + +use core::time; +use std::net::Shutdown; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::sync::{Arc, Mutex}; +use std::{ + fs, io, + io::{Read, Write}, + path::{Path, PathBuf}, + thread, +}; + +use anyhow::Context; + +fn socket_path_for_build(build_root: &Path) -> PathBuf { + build_root.join("build.sock") +} + +pub fn start_socket_server( + build_root: &Path, + should_exit: Arc>, +) -> anyhow::Result> { + let sock = socket_path_for_build(build_root); + if sock.exists() { + // try to remove stale socket file + let _ = fs::remove_file(&sock); + } + + let listener = UnixListener::bind(&sock) + .with_context(|| format!("failed to bind unix socket {}", sock.display()))?; + + // Set non-blocking mode so we can check the exit flag + listener + .set_nonblocking(true) + .context("failed to set socket non-blocking")?; + + let handle = thread::spawn(move || { + let mut num_attached = 0u64; + loop { + // Check if we should exit + let exit_requested = *should_exit.lock().unwrap(); + if exit_requested && num_attached == 0 { + break; + } + + match listener.accept() { + Ok((mut s, _)) => { + let mut buf = String::new(); + if s.read_to_string(&mut buf).is_err() { + let _ = s.shutdown(Shutdown::Both); + continue; + } + let cmd = buf.trim(); + match cmd { + "attach" => { + num_attached += 1; + } + "detach" => { + num_attached = num_attached.saturating_sub(1); + } + _ => {} + } + let _ = s.shutdown(Shutdown::Both); + } + Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => { + // No connection available, sleep briefly to avoid busy-waiting + thread::sleep(time::Duration::from_millis(10)); + } + Err(_) => break, + } + } + let _ = fs::remove_file(&sock); + }); + + Ok(handle) +} + +pub fn send_socket_command(build_root: &Path, cmd: &str) -> anyhow::Result<()> { + let sock = socket_path_for_build(build_root); + let mut stream = UnixStream::connect(&sock) + .with_context(|| format!("failed to connect to socket {}", sock.display()))?; + stream + .write_all(cmd.as_bytes()) + .context("failed to send socket command")?; + stream.shutdown(Shutdown::Write).ok(); + Ok(()) +} diff --git a/packages/debmagic/src/build/mod.rs b/packages/debmagic/src/build/mod.rs new file mode 100644 index 0000000..0476a6e --- /dev/null +++ b/packages/debmagic/src/build/mod.rs @@ -0,0 +1,645 @@ +use std::sync::{Arc, Mutex}; +use std::{ + fs, io, + io::{BufReader, IsTerminal, stdout}, + path::{Path, PathBuf}, + process::{Command, Stdio}, +}; + +use crate::build::attach::{send_socket_command, start_socket_server}; +use crate::build::config::DriverOverrides; +use crate::build::source::{source_manifest_path, stage_source_tree}; +use crate::{ + build::{ + common::{BuildConfig, BuildDriver, BuildDriverType, BuildMetadata, run_checked}, + config::DriverConfig, + driver_bare::DriverBare, + driver_docker::DriverDocker, + driver_lxd::{DriverLxd, LxdVariant}, + }, + config::Config, + package::PackageDescription, +}; +use anyhow::{Context, anyhow}; +use debmagic_common::distro::DistroVersion; + +pub mod artifacts; +pub mod attach; +pub mod common; +pub mod config; +pub mod driver_bare; +pub mod driver_docker; +pub mod driver_lxd; +pub mod source; + +struct Build { + config: BuildConfig, + pub driver: Box, + attached: bool, +} + +fn get_build_driver( + config: &BuildConfig, + driver_config: &DriverConfig, + driver_overrides: &DriverOverrides, +) -> anyhow::Result> { + let apt_mirror = driver_overrides + .apt_mirror + .as_deref() + .or(driver_config.apt_mirror.as_deref()); + let proposed = driver_overrides.proposed.unwrap_or(driver_config.proposed); + + match config.driver { + BuildDriverType::Docker => Ok(Box::new(DriverDocker::create( + config, + driver_config, + &driver_overrides.docker, + apt_mirror, + proposed, + )?)), + BuildDriverType::Bare => Ok(Box::new(DriverBare::create( + config, + driver_config, + &driver_overrides.bare, + ))), + BuildDriverType::Lxd | BuildDriverType::Incus => { + let variant = match config.driver { + BuildDriverType::Lxd => LxdVariant::Lxd, + _ => LxdVariant::Incus, + }; + Ok(Box::new(DriverLxd::create( + variant, + config, + driver_config, + &driver_overrides.lxd, + apt_mirror, + proposed, + )?)) + } + } +} + +fn create_driver_from_metadata( + config: &DriverConfig, + metadata: &BuildMetadata, +) -> anyhow::Result> { + let driver: anyhow::Result> = match &metadata.config.driver { + BuildDriverType::Docker => Ok(Box::new(DriverDocker::from_build_metadata( + &metadata.config, + config, + metadata, + )?)), + BuildDriverType::Bare => Ok(Box::new(DriverBare::from_build_metadata( + &metadata.config, + config, + metadata, + ))), + BuildDriverType::Lxd | BuildDriverType::Incus => { + let variant = match metadata.config.driver { + BuildDriverType::Lxd => LxdVariant::Lxd, + _ => LxdVariant::Incus, + }; + Ok(Box::new(DriverLxd::from_build_metadata( + variant, + &metadata.config, + metadata, + )?)) + } + }; + driver +} + +impl Build { + pub fn create( + config: &BuildConfig, + driver_config: &DriverConfig, + driver_overrides: &DriverOverrides, + ) -> anyhow::Result { + let driver = get_build_driver(config, driver_config, driver_overrides) + .context(format!("failed to create {:?} build driver", config.driver))?; + Ok(Self { + config: config.clone(), + driver, + attached: false, + }) + } + + pub fn from_build_root( + build_root: &Path, + driver_config: &DriverConfig, + ) -> anyhow::Result { + let build_metadata_path = build_root.join("build.json"); + if !build_metadata_path.is_file() { + return Err(anyhow!("No build.json found")); + } + // read metadata from file + let file = fs::OpenOptions::new() + .read(true) + .open(&build_metadata_path)?; + let metadata = || -> anyhow::Result { + let reader = BufReader::new(&file); + let metadata: BuildMetadata = serde_json::from_reader(reader).with_context(|| { + format!( + "Failed to read build metadata from {} - invalid json", + build_metadata_path.display() + ) + })?; + Ok(metadata) + }(); + + let metadata = metadata?; + + let driver = create_driver_from_metadata(driver_config, &metadata)?; + + let attached = send_socket_command(build_root, "attach").is_ok(); + + Ok(Self { + config: metadata.config.clone(), + driver, + attached, + }) + } + + pub fn detach(&self) -> anyhow::Result<()> { + let build_root = &self.config.build_root_dir; + if self.attached { + send_socket_command(build_root, "detach")?; + } + Ok(()) + } + + pub fn write_metadata(&self) -> anyhow::Result<()> { + let metadata = BuildMetadata { + config: self.config.clone(), + driver_metadata: self.driver.get_build_metadata(), + }; + let path = self.config.build_root_dir.join("build.json"); + let json = serde_json::to_string_pretty(&metadata) + .context("Failed to serialize build metadata")?; + fs::write(path, json)?; + Ok(()) + } +} + +fn get_build_root_and_identifier( + config: &Config, + package: &PackageDescription, +) -> (String, PathBuf) { + let package_identifier = format!("{}-{}", package.name, package.version); + let build_root = config.temp_build_dir.join(&package_identifier); + (package_identifier, build_root) +} + +/// Determine which distro version to use for the build. +/// +/// If only one distro version is specified in the changelog, it's used automatically. +/// If multiple distro versions are specified, an explicit --distro is required. +/// If --distro is provided, it's validated against the changelog versions. +fn resolve_distro_version( + changelog_distros: &[String], + explicit_distro: Option<&str>, +) -> anyhow::Result { + let resolved_codename = match (changelog_distros.len(), explicit_distro) { + (0, _) => Err(anyhow!("changelog contains no distributions")), + (1, None) => Ok(changelog_distros[0].clone()), + (1, Some(explicit)) => { + if explicit == changelog_distros[0] { + Ok(explicit.to_string()) + } else { + Err(anyhow!( + "explicit distro version '{}' conflicts with distribution specified in changelog '{}'", + explicit, + changelog_distros[0] + )) + } + } + (_, None) => Err(anyhow!( + "changelog contains multiple distributions ({}), please specify which one to build for with --distro", + changelog_distros.join(", ") + )), + (_, Some(explicit)) => { + if changelog_distros.contains(&explicit.to_string()) { + Ok(explicit.to_string()) + } else { + Err(anyhow!( + "explicit distro version '{}' not found in changelog distributions: {}", + explicit, + changelog_distros.join(", ") + )) + } + } + }?; + let resolved = debmagic_common::distro::get_distro_version(&resolved_codename) + .ok_or_else(|| anyhow!("unknown distro codename '{}'", resolved_codename))?; + Ok(resolved) +} + +fn prepare_build_env( + config: &Config, + driver_overrides: &DriverOverrides, + package: &PackageDescription, + driver_type: BuildDriverType, + output_dir: &Path, + explicit_distro_version: Option<&str>, +) -> anyhow::Result { + let (package_identifier, build_root) = get_build_root_and_identifier(config, package); + + let distro_version = resolve_distro_version(&package.distro_versions, explicit_distro_version) + .context("failed to determine distro version")?; + + let build_config = BuildConfig { + driver: driver_type, + package_name: package.name.clone(), + package_identifier, + source_dir: package.source_dir.clone(), + output_dir: output_dir.to_path_buf(), + build_root_dir: build_root.clone(), + distro: distro_version.clone(), + sign_package: config.sign_package, + sign_key: config.sign_key.clone(), + build_debug_symbols: config.build_debug_symbols, + clean: config.clean, + persistent: config.driver.persistent, + incremental: config.incremental, + source_sync_mode: config.source_sync_mode, + }; + + if config.driver.persistent && build_root.exists() { + // For persistent containers, starting first lets root inside delete + // container-owned files the host user can't remove. + let build = Build::create(&build_config, &config.driver, driver_overrides) + .context(format!("failed to create {:?} build driver", driver_type))?; + if !config.incremental + || !source_manifest_path(&build_config).is_file() + || !build.driver.reused_environment() + { + build + .driver + .reset_build_root() + .context("failed to reset persistent build directory")?; + } + build_config + .create_dirs() + .context("failed to create build directories")?; + stage_source_tree(&build_config, package)?; + return Ok(build); + } + + if build_root.exists() + && let Err(e) = fs::remove_dir_all(&build_root) + { + if e.kind() == io::ErrorKind::PermissionDenied { + // Some files were created by a privileged user inside a container + // and can't be deleted by the host user directly. Load the previous + // build's driver and ask it to clean up from inside. + let metadata_path = build_root.join("build.json"); + if metadata_path.is_file() + && let Ok(file) = fs::OpenOptions::new().read(true).open(&metadata_path) + && let Ok(metadata) = + serde_json::from_reader::<_, BuildMetadata>(BufReader::new(&file)) + && let Ok(driver) = create_driver_from_metadata(&config.driver, &metadata) + { + let _ = driver.reset_build_root(); + } + fs::remove_dir_all(&build_root).with_context(|| { + format!( + "failed to remove build root {}; try: sudo rm -rf {}", + build_root.display(), + build_root.display() + ) + })?; + } else { + return Err(e.into()); + } + } + + build_config + .create_dirs() + .context("failed to create build directories")?; + + stage_source_tree(&build_config, package)?; + + let build = Build::create(&build_config, &config.driver, driver_overrides)?; + Ok(build) +} + +pub fn get_shell_in_build(config: &Config, package: &PackageDescription) -> anyhow::Result<()> { + let (_package_identifier, build_root) = get_build_root_and_identifier(config, package); + let build = Build::from_build_root(&build_root, &config.driver)?; + let result = build + .driver + .interactive_shell(&build.config.build_source_dir()); + + build.detach()?; + + result?; + Ok(()) +} + +fn deb_build_options(existing: Option<&str>, build_debug_symbols: bool) -> String { + let mut options = existing + .unwrap_or_default() + .split_whitespace() + .filter(|option| *option != "noautodbgsym") + .collect::>(); + if !build_debug_symbols { + options.push("noautodbgsym"); + } + options.join(" ") +} + +/// Everything needed to run one package build, independent of whether the +/// build produces binary or source packages. +pub struct BuildRequest<'a> { + pub config: &'a Config, + pub package: &'a PackageDescription, + pub driver_type: BuildDriverType, + pub driver_overrides: &'a DriverOverrides, + pub output_dir: &'a Path, + pub explicit_distro_version: Option<&'a str>, +} + +/// Shared build orchestration: prepare the environment, run `build_commands` +/// in it, export the artifacts to the output dir, sign them if requested, and +/// clean up (dropping into a shell first on failure of an interactive binary +/// build). While `shell_on_failure` is set, a socket server lets concurrent +/// `debmagic shell` sessions attach to the environment. +fn run_build( + request: &BuildRequest, + shell_on_failure: bool, + build_commands: impl FnOnce(&Build) -> anyhow::Result<()>, +) -> anyhow::Result<()> { + let build = prepare_build_env( + request.config, + request.driver_overrides, + request.package, + request.driver_type, + request.output_dir, + request.explicit_distro_version, + ) + .context("failed to prepare build environment")?; + build + .write_metadata() + .context("failed to write build metadata")?; + + let should_exit = Arc::new(Mutex::new(false)); + let socket_server_handle = + start_socket_server(&build.config.build_root_dir, should_exit.clone())?; + + let stop_socket_server = || { + *should_exit.lock().unwrap() = true; + if !socket_server_handle.is_finished() { + println!("Waiting for all attached shells to exit..."); + } + socket_server_handle.join().ok(); + }; + + let result = build_commands(&build).and_then(|()| { + let changes_file = artifacts::export_build_artifacts( + &build.config.build_work_dir(), + &build.config.output_dir, + )?; + if build.config.sign_package { + sign_changes_file(&changes_file, build.config.sign_key.as_deref())?; + } + Ok(()) + }); + + if let Err(error) = result { + if shell_on_failure && stdout().is_terminal() { + eprintln!("Build failed: {error}. Dropping into shell..."); + if let Err(shell_error) = build + .driver + .interactive_shell(&build.config.build_source_dir()) + { + eprintln!("Dropping into shell failed: {shell_error}"); + } + } else { + eprintln!("Build failed: {error}"); + } + if let Err(cleanup_error) = build.driver.cleanup() { + eprintln!("Failed to clean up build environment: {cleanup_error}"); + } + stop_socket_server(); + return Err(error); + } + + stop_socket_server(); + build + .driver + .cleanup() + .context("failed to clean up build environment")?; + Ok(()) +} + +pub fn build_package(request: &BuildRequest) -> anyhow::Result<()> { + run_build(request, true, |build| { + build.driver.run_command( + &["apt-get", "-y", "build-dep", "."], + &build.config.build_source_dir(), + true, + )?; + let inherited_options = std::env::var("DEB_BUILD_OPTIONS").ok(); + let options = deb_build_options( + inherited_options.as_deref(), + build.config.build_debug_symbols, + ); + let env_add = [("DEB_BUILD_OPTIONS", options.as_str())]; + let mut dpkg_buildpackage_args = vec!["dpkg-buildpackage", "-us", "-uc", "-ui"]; + if !build.config.clean { + // Non-incremental builds already stage a clean source tree, while + // incremental builds preserve their outputs intentionally. + dpkg_buildpackage_args.push("-nc"); + } + dpkg_buildpackage_args.push("-b"); + build.driver.run_command_env( + &dpkg_buildpackage_args, + &build.config.build_source_dir(), + false, + &env_add, + )?; + Ok(()) + }) +} + +/// Confirm `cmd` is on `PATH`, failing with an actionable message (rather +/// than a raw "command not found") if it isn't. +fn check_command_available(cmd: &str, install_hint: &str) -> anyhow::Result<()> { + match Command::new(cmd) + .arg("--version") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + { + Ok(_) => Ok(()), + Err(e) if e.kind() == io::ErrorKind::NotFound => { + Err(anyhow!("{cmd} not found on PATH. {install_hint}")) + } + Err(e) => Err(e).with_context(|| format!("failed to check for {cmd}")), + } +} + +fn check_dpkg_buildpackage_available() -> anyhow::Result<()> { + check_command_available( + "dpkg-buildpackage", + "It's part of dpkg-dev; install it, or pass --driver lxd/incus/docker to build inside a Debian-ish container instead.", + ) +} + +/// GPG-sign every `.changes` (and its referenced `.dsc`/`.buildinfo`) in +/// `output_dir` with `debsign`. Always runs on the host regardless of the +/// build driver, since signing needs the user's own gpg keyring, which an +/// ephemeral container doesn't have access to. +fn sign_changes_file(changes_file: &Path, sign_key: Option<&str>) -> anyhow::Result<()> { + check_command_available( + "debsign", + "It's part of devscripts; install it and set up a gpg signing key to use --sign.", + )?; + + let output_dir = changes_file.parent().ok_or_else(|| { + anyhow!( + "could not get output directory of {}", + changes_file.display() + ) + })?; + let filename = changes_file + .file_name() + .ok_or_else(|| anyhow!("could not get filename of {}", changes_file.display()))?; + + let mut cmd = Command::new("debsign"); + if let Some(key) = sign_key { + cmd.arg(format!("-k{key}")); + } + cmd.arg(filename).current_dir(output_dir); + run_checked(&mut cmd, &format!("signing {}", changes_file.display()))?; + Ok(()) +} + +/// Build a `.dsc` + tarball + `.buildinfo` + `.changes` source package. +/// +/// If `config.clean` is set, build-dependencies are installed before +/// `dpkg-buildpackage` runs `debian/rules clean` once. +pub fn build_source_package(request: &BuildRequest) -> anyhow::Result<()> { + if request.driver_type == BuildDriverType::Bare { + check_dpkg_buildpackage_available()?; + } + + run_build(request, false, |build| { + let build_source_dir = build.config.build_source_dir(); + if build.config.clean { + build.driver.run_command( + &["apt-get", "-y", "build-dep", "."], + &build_source_dir, + true, + )?; + } + let mut args = vec!["dpkg-buildpackage", "-S", "-d", "-us", "-uc", "-ui"]; + if !build.config.clean { + args.push("-nc"); + } + build.driver.run_command(&args, &build_source_dir, false)?; + Ok(()) + }) + .context("failed to build source package") +} + +#[cfg(test)] +mod tests { + use debmagic_common::distro::Distro; + + use super::*; + + #[test] + fn debug_symbol_option_preserves_other_build_options() { + assert_eq!( + deb_build_options(Some("nocheck parallel=8"), false), + "nocheck parallel=8 noautodbgsym" + ); + assert_eq!( + deb_build_options(Some("nocheck noautodbgsym parallel=8"), true), + "nocheck parallel=8" + ); + } + + #[test] + fn test_resolve_distro_version_single_distro_no_explicit() { + let distros = vec!["forky".to_string()]; + let result = resolve_distro_version(&distros, None); + assert!(result.is_ok()); + let distro_version = result.unwrap(); + assert_eq!(distro_version.codename, "forky"); + assert_eq!(distro_version.distro, Distro::Debian); + } + + #[test] + fn test_resolve_distro_version_single_distro_matching_explicit() { + let distros = vec!["forky".to_string()]; + let result = resolve_distro_version(&distros, Some("forky")); + assert!(result.is_ok()); + let distro_version = result.unwrap(); + assert_eq!(distro_version.codename, "forky"); + assert_eq!(distro_version.distro, Distro::Debian); + } + + #[test] + fn test_resolve_distro_version_single_distro_conflicting_explicit() { + let distros = vec!["forky".to_string()]; + let result = resolve_distro_version(&distros, Some("duke")); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("conflicts with distribution specified in changelog") + ); + } + + #[test] + fn test_resolve_distro_version_multiple_distros_no_explicit() { + let distros = vec!["forky".to_string(), "duke".to_string()]; + let result = resolve_distro_version(&distros, None); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("multiple distributions") + ); + } + + #[test] + fn test_resolve_distro_version_multiple_distros_explicit_valid() { + let distros = vec!["forky".to_string(), "duke".to_string()]; + let result = resolve_distro_version(&distros, Some("duke")); + assert!(result.is_ok()); + let distro_version = result.unwrap(); + assert_eq!(distro_version.codename, "duke"); + assert_eq!(distro_version.distro, Distro::Debian); + } + + #[test] + fn test_resolve_distro_version_multiple_distros_explicit_invalid() { + let distros = vec!["forky".to_string(), "duke".to_string()]; + let result = resolve_distro_version(&distros, Some("trixie")); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("not found in changelog distributions") + ); + } + + #[test] + fn test_resolve_distro_version_empty_distros() { + let distros: Vec = vec![]; + let result = resolve_distro_version(&distros, None); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("changelog contains no distributions") + ); + } +} diff --git a/packages/debmagic/src/build/source.rs b/packages/debmagic/src/build/source.rs new file mode 100644 index 0000000..f6a7cb8 --- /dev/null +++ b/packages/debmagic/src/build/source.rs @@ -0,0 +1,605 @@ +//! Staging of the source tree into the build directory. +//! +//! Which files are staged is selected by `SourceSyncMode`: git-tracked +//! files (optionally requiring a clean worktree) or everything that isn't +//! git-ignored. Staging copies the selected entries, preserving symlinks +//! and keeping unchanged files untouched so incremental builds keep their +//! outputs; a manifest of staged entries lets incremental syncs remove +//! stale paths. + +use std::cmp::Reverse; +use std::io::{self, BufReader, Read}; +use std::os::unix::fs::{PermissionsExt, symlink}; +use std::path::{Component, Path, PathBuf}; +use std::{fs, process::Command}; + +use anyhow::{Context, anyhow, bail}; +use glob::glob; + +use crate::build::common::{BuildConfig, SourceSyncMode}; +use crate::package::PackageDescription; + +/// Paths of files tracked by git in `src`, as reported by `git ls-files`. +/// Returns `None` if `src` is not inside a git worktree. +fn git_tracked_paths(src: &Path) -> anyhow::Result>> { + let output = Command::new("git") + .args(["-C"]) + .arg(src) + .args(["ls-files", "-z"]) + .output() + .context("failed to run git ls-files")?; + if !output.status.success() { + return Ok(None); + } + let mut paths = Vec::new(); + for raw in output.stdout.split(|byte| *byte == 0) { + if raw.is_empty() { + continue; + } + let path = PathBuf::from(String::from_utf8(raw.to_vec()).with_context(|| { + format!("git-tracked path is not valid UTF-8 in {}", src.display()) + })?); + validate_source_path(&path)?; + paths.push(path); + } + Ok(Some(paths)) +} + +/// Paths of files git knows about but does not track (respecting ignore +/// rules), for warning about what a `tracked` sync leaves out. +fn git_untracked_paths(src: &Path) -> Vec { + let output = Command::new("git") + .args(["-C"]) + .arg(src) + .args(["ls-files", "-z", "--others", "--exclude-standard"]) + .output(); + match output { + Ok(output) if output.status.success() => output + .stdout + .split(|byte| *byte == 0) + .filter(|raw| !raw.is_empty()) + .filter_map(|raw| String::from_utf8(raw.to_vec()).ok()) + .map(PathBuf::from) + .collect(), + _ => Vec::new(), + } +} + +/// Ensure the git worktree in `src` has no uncommitted changes and no +/// untracked files, as required by `SourceSyncMode::Committed`. +fn git_ensure_clean_worktree(src: &Path) -> anyhow::Result<()> { + let output = Command::new("git") + .args(["-C"]) + .arg(src) + .args(["status", "--porcelain"]) + .output() + .context("failed to run git status")?; + if !output.status.success() { + // Not a git worktree; the caller falls back to worktree staging. + return Ok(()); + } + let stdout = String::from_utf8_lossy(&output.stdout); + let entries: Vec<&str> = stdout.lines().filter(|line| !line.is_empty()).collect(); + if entries.is_empty() { + return Ok(()); + } + let mut message = + String::from("source-sync mode 'committed' requires a clean git worktree, but found:\n"); + for entry in entries.iter().take(20) { + message.push_str(&format!(" {entry}\n")); + } + if entries.len() > 20 { + message.push_str(&format!(" ... and {} more\n", entries.len() - 20)); + } + message.push_str("commit the changes or use a different --source-sync mode"); + bail!(message) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "snake_case")] +enum SourcePathKind { + Directory, + File, + Symlink, +} + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +struct SourcePath { + path: PathBuf, + kind: SourcePathKind, +} + +fn validate_source_path(path: &Path) -> anyhow::Result<()> { + if path.as_os_str().is_empty() + || !path + .components() + .all(|component| matches!(component, Component::Normal(_))) + { + bail!("invalid source manifest path: {}", path.display()); + } + Ok(()) +} + +fn source_tree_entries(src: &Path, mode: SourceSyncMode) -> anyhow::Result> { + match mode { + SourceSyncMode::Worktree => worktree_entries(src), + SourceSyncMode::Tracked | SourceSyncMode::Committed => { + if mode == SourceSyncMode::Committed { + git_ensure_clean_worktree(src)?; + } + match git_tracked_paths(src)? { + Some(paths) => tracked_entries(src, &paths), + None => { + eprintln!( + "debmagic: warning: {} is not a git worktree, falling back to 'worktree' source sync", + src.display() + ); + worktree_entries(src) + } + } + } + } +} + +fn entry_kind(path: &Path) -> anyhow::Result { + let metadata = fs::symlink_metadata(path) + .with_context(|| format!("failed to stat source path {}", path.display()))?; + if metadata.is_dir() { + Ok(SourcePathKind::Directory) + } else if metadata.is_file() { + Ok(SourcePathKind::File) + } else if metadata.is_symlink() { + Ok(SourcePathKind::Symlink) + } else { + Err(anyhow!( + "unsupported file type in source tree: {}", + path.display() + )) + } +} + +/// Build the entry list from git-tracked paths: tracked files (plus their +/// parent directories), with submodule gitlinks skipped. +fn tracked_entries(src: &Path, paths: &[PathBuf]) -> anyhow::Result> { + let mut seen = std::collections::HashSet::new(); + let mut entries = Vec::new(); + for path in paths { + // Add parent directories first. + let mut ancestors: Vec<&Path> = path.ancestors().skip(1).collect(); + ancestors.pop(); // drop the empty "" ancestor + ancestors.reverse(); + for ancestor in ancestors { + if seen.insert(ancestor.to_path_buf()) { + entries.push(SourcePath { + path: ancestor.to_path_buf(), + kind: SourcePathKind::Directory, + }); + } + } + let full_path = src.join(path); + // Submodule gitlinks are directories; their contents are not staged + // since git does not track them as part of this repository. + if full_path.is_dir() { + eprintln!( + "debmagic: skipping git submodule {}; its contents are not staged", + path.display() + ); + continue; + } + entries.push(SourcePath { + path: path.clone(), + kind: entry_kind(&full_path)?, + }); + } + entries.sort_by_key(|entry| entry.path.components().count()); + Ok(entries) +} + +fn worktree_entries(src: &Path) -> anyhow::Result> { + let walker = ignore::WalkBuilder::new(src) + .standard_filters(true) + .hidden(false) + .filter_entry(|entry| !(entry.path().is_dir() && entry.path().ends_with(".git"))) + .build(); + + let mut entries = Vec::new(); + for entry in walker { + let entry = entry?; + let relative_path = entry + .path() + .strip_prefix(src) + .context("failed to get relative path")?; + if relative_path.as_os_str().is_empty() { + continue; + } + entries.push(SourcePath { + path: relative_path.to_path_buf(), + kind: entry_kind(entry.path())?, + }); + } + entries.sort_by_key(|entry| entry.path.components().count()); + Ok(entries) +} + +fn remove_path(path: &Path) -> std::io::Result<()> { + match fs::symlink_metadata(path) { + Ok(metadata) if metadata.is_dir() => fs::remove_dir_all(path), + Ok(_) => fs::remove_file(path), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error), + } +} + +fn files_match(source: &Path, destination: &Path) -> std::io::Result { + let source_metadata = fs::metadata(source)?; + let destination_metadata = match fs::symlink_metadata(destination) { + Ok(metadata) if metadata.is_file() => metadata, + Ok(_) => return Ok(false), + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error), + }; + if source_metadata.len() != destination_metadata.len() + || source_metadata.permissions().mode() != destination_metadata.permissions().mode() + { + return Ok(false); + } + + let mut source = BufReader::new(fs::File::open(source)?); + let mut destination = BufReader::new(fs::File::open(destination)?); + let mut source_buffer = [0; 8192]; + let mut destination_buffer = [0; 8192]; + loop { + let source_len = source.read(&mut source_buffer)?; + let destination_len = destination.read(&mut destination_buffer)?; + if source_len != destination_len + || source_buffer[..source_len] != destination_buffer[..destination_len] + { + return Ok(false); + } + if source_len == 0 { + return Ok(true); + } + } +} + +fn copy_source_entries(src: &Path, dst: &Path, entries: &[SourcePath]) -> anyhow::Result<()> { + fs::create_dir_all(dst)?; + for entry in entries { + let source = src.join(&entry.path); + let destination = dst.join(&entry.path); + match entry.kind { + SourcePathKind::Directory => match fs::symlink_metadata(&destination) { + Ok(metadata) if metadata.is_dir() => {} + Ok(_) => { + remove_path(&destination)?; + fs::create_dir_all(&destination)?; + } + Err(error) if error.kind() == io::ErrorKind::NotFound => { + fs::create_dir_all(&destination)?; + } + Err(error) => return Err(error.into()), + }, + SourcePathKind::File => { + if !files_match(&source, &destination)? { + remove_path(&destination)?; + fs::copy(&source, &destination) + .with_context(|| format!("failed to copy file: {}", source.display()))?; + } + } + SourcePathKind::Symlink => { + let target = fs::read_link(&source)?; + if fs::read_link(&destination).ok().as_deref() != Some(target.as_path()) { + remove_path(&destination)?; + symlink(target, &destination)?; + } + } + } + } + Ok(()) +} + +fn copy_glob(src_dir: &Path, pattern: &str, dest_dir: &Path) -> anyhow::Result<()> { + let full_pattern = src_dir.join(pattern).to_string_lossy().into_owned(); + for entry in glob(&full_pattern)? { + let path = entry?; + if path.is_file() { + let filename = path.file_name().ok_or(anyhow!( + "Could not retrieve filename from {}", + path.display() + ))?; + fs::copy(&path, dest_dir.join(filename))?; + } + } + Ok(()) +} + +pub fn source_manifest_path(build_config: &BuildConfig) -> PathBuf { + build_config.build_root_dir.join("source-manifest.json") +} + +fn write_source_manifest(build_config: &BuildConfig, entries: &[SourcePath]) -> anyhow::Result<()> { + let manifest_path = source_manifest_path(build_config); + let temporary_path = manifest_path.with_extension("json.tmp"); + fs::write(&temporary_path, serde_json::to_vec_pretty(entries)?)?; + fs::rename(temporary_path, manifest_path)?; + Ok(()) +} + +fn sync_source_tree(build_config: &BuildConfig) -> anyhow::Result<()> { + let manifest_path = source_manifest_path(build_config); + let previous: Vec = serde_json::from_reader(BufReader::new( + fs::File::open(&manifest_path) + .with_context(|| format!("failed to open {}", manifest_path.display()))?, + )) + .with_context(|| format!("failed to read {}", manifest_path.display()))?; + for entry in &previous { + validate_source_path(&entry.path)?; + } + let current = source_tree_entries(&build_config.source_dir, build_config.source_sync_mode)?; + + let current_kinds = current + .iter() + .map(|entry| (entry.path.as_path(), entry.kind)) + .collect::>(); + let mut stale = previous + .iter() + .filter(|entry| current_kinds.get(entry.path.as_path()) != Some(&entry.kind)) + .collect::>(); + stale.sort_by_key(|entry| Reverse(entry.path.components().count())); + for entry in stale { + let destination = build_config.build_source_dir().join(&entry.path); + if entry.kind == SourcePathKind::Directory + && !current_kinds.contains_key(entry.path.as_path()) + { + match fs::remove_dir(&destination) { + Ok(()) => {} + Err(error) + if matches!( + error.kind(), + io::ErrorKind::NotFound | io::ErrorKind::DirectoryNotEmpty + ) => {} + Err(error) => return Err(error.into()), + } + } else { + remove_path(&destination)?; + } + } + + copy_source_entries( + &build_config.source_dir, + &build_config.build_source_dir(), + ¤t, + )?; + write_source_manifest(build_config, ¤t) +} + +pub fn stage_source_tree( + build_config: &BuildConfig, + package: &PackageDescription, +) -> anyhow::Result<()> { + if build_config.source_sync_mode == SourceSyncMode::Tracked { + let untracked = git_untracked_paths(&build_config.source_dir); + if !untracked.is_empty() { + eprintln!( + "debmagic: warning: {} untracked file(s) not staged into the build tree:", + untracked.len() + ); + for path in untracked.iter().take(20) { + eprintln!(" {}", path.display()); + } + if untracked.len() > 20 { + eprintln!(" ... and {} more", untracked.len() - 20); + } + eprintln!(" git add them or use --source-sync worktree to include them"); + } + } + if build_config.incremental && source_manifest_path(build_config).is_file() { + sync_source_tree(build_config).context("failed to synchronize source tree")?; + } else { + let entries = source_tree_entries(&build_config.source_dir, build_config.source_sync_mode)?; + copy_source_entries( + &build_config.source_dir, + &build_config.build_source_dir(), + &entries, + ) + .context("failed to copy source tree to build directory")?; + write_source_manifest(build_config, &entries)?; + } + + let source_parent = build_config + .source_dir + .parent() + .ok_or_else(|| anyhow!("source directory has no parent"))?; + let prefix = format!("{}_{}", package.name, package.version.upstream_version()); + copy_glob( + source_parent, + &format!("{prefix}.orig.tar.*"), + &build_config.build_work_dir(), + )?; + copy_glob( + source_parent, + &format!("{prefix}.orig-*.tar.*"), + &build_config.build_work_dir(), + )?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::os::unix::fs::MetadataExt; + + use crate::build::common::BuildDriverType; + + use super::*; + + #[test] + fn incremental_sync_updates_sources_and_preserves_build_outputs() -> anyhow::Result<()> { + let test_root = std::env::temp_dir().join(format!( + "debmagic-incremental-test-{}", + uuid::Uuid::new_v4() + )); + let source_dir = test_root.join("source"); + let build_root_dir = test_root.join("build"); + fs::create_dir_all(source_dir.join("cache"))?; + fs::write(source_dir.join("changed.txt"), "before")?; + fs::write(source_dir.join("unchanged.txt"), "unchanged")?; + fs::write(source_dir.join("removed.txt"), "remove me")?; + fs::write(source_dir.join("cache/input.c"), "source")?; + symlink("changed.txt", source_dir.join("link"))?; + + let build_config = BuildConfig { + driver: BuildDriverType::Bare, + package_name: "example".to_string(), + package_identifier: "example-1.0".to_string(), + build_root_dir: build_root_dir.clone(), + source_dir: source_dir.clone(), + output_dir: test_root.join("output"), + distro: debmagic_common::distro::get_distro_version("trixie").unwrap(), + sign_package: false, + sign_key: None, + build_debug_symbols: false, + clean: false, + persistent: true, + incremental: true, + source_sync_mode: SourceSyncMode::Worktree, + }; + build_config.create_dirs()?; + let initial_entries = source_tree_entries(&source_dir, SourceSyncMode::Worktree)?; + copy_source_entries( + &source_dir, + &build_config.build_source_dir(), + &initial_entries, + )?; + write_source_manifest(&build_config, &initial_entries)?; + let unchanged_inode = + fs::metadata(build_config.build_source_dir().join("unchanged.txt"))?.ino(); + fs::write( + build_config.build_source_dir().join("cache/output.o"), + "compiled", + )?; + + fs::write(source_dir.join("changed.txt"), "after")?; + fs::remove_file(source_dir.join("removed.txt"))?; + fs::remove_file(source_dir.join("cache/input.c"))?; + fs::remove_dir(source_dir.join("cache"))?; + fs::remove_file(source_dir.join("link"))?; + symlink("added.txt", source_dir.join("link"))?; + fs::write(source_dir.join("added.txt"), "new")?; + + sync_source_tree(&build_config)?; + + let staged = build_config.build_source_dir(); + assert_eq!(fs::read_to_string(staged.join("changed.txt"))?, "after"); + assert_eq!(fs::read_to_string(staged.join("added.txt"))?, "new"); + assert_eq!( + fs::metadata(staged.join("unchanged.txt"))?.ino(), + unchanged_inode + ); + assert_eq!(fs::read_link(staged.join("link"))?, Path::new("added.txt")); + assert!(!staged.join("removed.txt").exists()); + assert!(!staged.join("cache/input.c").exists()); + assert_eq!( + fs::read_to_string(staged.join("cache/output.o"))?, + "compiled" + ); + + fs::remove_dir_all(test_root)?; + Ok(()) + } + + #[test] + fn source_manifest_paths_must_be_relative_and_normal() { + assert!(validate_source_path(Path::new("debian/control")).is_ok()); + for path in ["", ".", "../outside", "debian/../outside", "/tmp/outside"] { + assert!(validate_source_path(Path::new(path)).is_err(), "{path}"); + } + } + + /// Create a git repo with one committed file in a fresh temp dir. + fn git_test_repo() -> anyhow::Result { + let repo = std::env::temp_dir().join(format!("debmagic-git-test-{}", uuid::Uuid::new_v4())); + fs::create_dir_all(repo.join("debian"))?; + fs::write(repo.join("debian/control"), "Source: example")?; + let run = |args: &[&str]| -> anyhow::Result<()> { + let status = Command::new("git") + .arg("-C") + .arg(&repo) + .args(args) + .status()?; + if status.success() { + Ok(()) + } else { + Err(anyhow!("git {:?} failed", args)) + } + }; + run(&["init", "-q"])?; + run(&["add", "debian/control"])?; + run(&[ + "-c", + "user.email=t@t", + "-c", + "user.name=t", + "commit", + "-qm", + "init", + ])?; + Ok(repo) + } + + #[test] + fn tracked_sync_stages_only_git_tracked_files() -> anyhow::Result<()> { + let repo = git_test_repo()?; + fs::write(repo.join("untracked.txt"), "not staged")?; + fs::write(repo.join("dirty.txt"), "uncommitted but tracked? no")?; + // A tracked file with uncommitted modifications is staged with its + // worktree content. + fs::write(repo.join("debian/rules"), "new content")?; + Command::new("git") + .arg("-C") + .arg(&repo) + .args(["add", "debian/rules"]) + .status()?; + + let entries = source_tree_entries(&repo, SourceSyncMode::Tracked)?; + let paths: Vec<&Path> = entries.iter().map(|e| e.path.as_path()).collect(); + assert!(paths.contains(&Path::new("debian"))); + assert!(paths.contains(&Path::new("debian/control"))); + assert!(paths.contains(&Path::new("debian/rules"))); + assert!(!paths.contains(&Path::new("untracked.txt"))); + assert!(!paths.contains(&Path::new("dirty.txt"))); + assert_eq!(git_untracked_paths(&repo).len(), 2); + + fs::remove_dir_all(repo)?; + Ok(()) + } + + #[test] + fn committed_sync_requires_clean_worktree() -> anyhow::Result<()> { + let repo = git_test_repo()?; + assert!(source_tree_entries(&repo, SourceSyncMode::Committed).is_ok()); + + fs::write(repo.join("untracked.txt"), "dirty")?; + let result = source_tree_entries(&repo, SourceSyncMode::Committed); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("requires a clean git worktree") + ); + + fs::remove_dir_all(repo)?; + Ok(()) + } + + #[test] + fn tracked_sync_falls_back_outside_git_worktree() -> anyhow::Result<()> { + let dir = std::env::temp_dir().join(format!("debmagic-nogit-{}", uuid::Uuid::new_v4())); + fs::create_dir_all(&dir)?; + fs::write(dir.join("file.txt"), "content")?; + let entries = source_tree_entries(&dir, SourceSyncMode::Tracked)?; + assert!(entries.iter().any(|e| e.path == Path::new("file.txt"))); + fs::remove_dir_all(dir)?; + Ok(()) + } +} From a7ea24d59413ad72fd2bd430e3e1f418dba74e1b Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Tue, 4 Aug 2026 16:12:06 +0200 Subject: [PATCH 5/7] feat(build): package signing in separate container environment --- docs/usage/build.md | 27 +- docs/usage/source.md | 10 +- packages/debmagic/src/build/common.rs | 12 + packages/debmagic/src/build/driver_bare.rs | 9 + packages/debmagic/src/build/driver_docker.rs | 74 +++++- packages/debmagic/src/build/driver_lxd.rs | 117 +++++++++ packages/debmagic/src/build/mod.rs | 106 +++++--- packages/debmagic/src/build/signing.rs | 256 +++++++++++++++++++ packages/debmagic/src/build/source.rs | 1 + packages/debmagic/src/cli.rs | 21 +- packages/debmagic/src/config.rs | 8 +- packages/debmagic/src/main.rs | 6 + packages/debmagic/tests/signing.rs | 240 +++++++++++++++++ 13 files changed, 831 insertions(+), 56 deletions(-) create mode 100644 packages/debmagic/src/build/signing.rs create mode 100644 packages/debmagic/tests/signing.rs diff --git a/docs/usage/build.md b/docs/usage/build.md index aee9476..39b0a02 100644 --- a/docs/usage/build.md +++ b/docs/usage/build.md @@ -110,22 +110,37 @@ Or set `build_debug_symbols = true` in `debian/debmagic.toml`/`$XDG_CONFIG_HOME/ ## Signing and cleaning -`--sign yes` (plus optionally `--sign-key you@example.com`) GPG-signs the resulting `.changes`/`.dsc`/`.buildinfo` with `debsign` after building — always on the host, using your own gpg keyring, regardless of `--driver`. +`--sign` (plus optionally `--sign-key you@example.com`) GPG-signs the resulting `.changes`/`.dsc`/`.buildinfo` with `debsign` after building. This is mainly useful for [source builds destined for Launchpad](source.md#uploading-to-launchpad), but works for binary builds too. +If your config file defaults to signing, pass `--no-sign` to skip it for one invocation. -`--clean yes` runs `debian/rules clean` before building, like plain `dpkg-buildpackage` does unless passed `-nc`. +Where `debsign` runs is selected by `--sign-with` (config: `sign_with`): + +- `auto` (default): sign on the host if `debsign` is installed there, otherwise in a container (requires a container driver). +- `host`: always sign on the host, using your own gpg keyring — requires `devscripts` installed locally. +- `same`: sign inside a minimal same-distro container, forwarding the host's gpg-agent socket (`gpgconf --list-dirs agent-extra-socket`) into it. + Only signing *operations* cross the socket; private key material never enters the container, and only the public key is imported into its throwaway keyring. + Container signing requires an explicit `--sign-key`, since debsign's maintainer-based key lookup only works on the host. + +Signing prerequisites (agent running, secret key available) are validated before the build starts, so a broken gpg setup fails fast instead of after the build. +`--clean` runs `debian/rules clean` before building, like plain `dpkg-buildpackage` does unless passed `-nc`; `--no-clean` skips it even if the config file defaults to cleaning. Non-incremental builds already stage a clean source tree, while incremental builds preserve outputs intentionally. Enable cleaning only for packages whose `clean` target performs required setup or code generation. -Both default to the `sign_package`/`sign_key`/`clean` settings in the config file (see below) if not passed on the CLI. +Both default to the `sign_package`/`sign_with`/`sign_key`/`clean` settings in the config file (see below) if not passed on the CLI. + +## Persisting options in a config file -## Persisting options in `debian/debmagic.toml` +Instead of repeating CLI flags on every invocation, drop a config file. +There are two locations, with different scopes: -Instead of repeating CLI flags on every invocation, drop a config file next to `debian/rules`: +- `$XDG_CONFIG_HOME/debmagic/config.toml` — your machine-wide defaults (mirror, signing key, persistent driver, ...). +- `/debian/debmagic.toml` — per-package defaults, committed next to `debian/rules` (e.g. `build_debug_symbols`, `sign_package`). ```toml build_debug_symbols = true sign_package = true +sign_with = "same" sign_key = "you@example.com" clean = false @@ -138,7 +153,7 @@ apt_mirror = "http:///ubuntu" ``` Config precedence (highest wins): `--config ` on the CLI > `/debian/debmagic.toml` > `$XDG_CONFIG_HOME/debmagic/config.toml`. -CLI flags like `--apt-mirror`/`--persistent`/`--sign`/`--clean` always override the matching config file value for that one invocation. +CLI flags like `--apt-mirror`/`--persistent`/`--sign`/`--no-sign`/`--clean`/`--no-clean` always override the matching config file value for that one invocation. ## What NOT to expect yet diff --git a/docs/usage/source.md b/docs/usage/source.md index 4b39bf0..a2aad90 100644 --- a/docs/usage/source.md +++ b/docs/usage/source.md @@ -16,21 +16,21 @@ debmagic build source --source-dir /path/to/parent/of/debian/dir --output-dir /p - Same `--source-dir`/`--output-dir`/`--distro` semantics as [`debmagic build`](build.md). - `--driver` defaults to `bare`: building a source package needs neither package build-dependencies nor a compiler, but the host must provide `dpkg-buildpackage` from `dpkg-dev`. Pass `--driver lxd`/`--driver incus`/`--driver docker` when the host does not provide a usable Debian build environment. - Container drivers install their base tooling, but package build-dependencies are installed only with `--clean yes`. + Container drivers install their base tooling, but package build-dependencies are installed only with `--clean`. Binary builds (`debmagic build binary`) still require `--driver` to be passed explicitly. (uploading-to-launchpad)= ## Uploading to Launchpad ```shell -debmagic build source --sign yes --sign-key you@example.com \ +debmagic build source --sign --sign-key you@example.com \ --source-dir . --output-dir /tmp/out dput ppa:your-lp-username/your-ppa /tmp/out/*_source.changes ``` -- `--sign yes` GPG-signs the `.dsc`/`.buildinfo`/`.changes` with `debsign` (from `devscripts`) after building. - This always runs on the host — never inside a driver's container — since it needs your own gpg keyring. -- `--sign-key` picks which key/uid to sign with (`debsign`'s `-k`); omit it to let `debsign` fall back to its own maintainer-address lookup. +- `--sign` GPG-signs the `.dsc`/`.buildinfo`/`.changes` with `debsign` (from `devscripts`) after building. + By default it runs on the host; with `--sign-with same` (or `auto` when `debsign` isn't installed on the host) it runs in a minimal same-distro container with your gpg-agent socket forwarded in — see [Signing and cleaning](build.md#signing-and-cleaning). +- `--sign-key` picks which key/uid to sign with (`debsign`'s `-k`); omit it to let `debsign` fall back to its own maintainer-address lookup (host signing only). - Both can be set as defaults in `debian/debmagic.toml`/`$XDG_CONFIG_HOME/debmagic/config.toml` instead of passing them every time: ```toml diff --git a/packages/debmagic/src/build/common.rs b/packages/debmagic/src/build/common.rs index aa7ffe0..e730cd7 100644 --- a/packages/debmagic/src/build/common.rs +++ b/packages/debmagic/src/build/common.rs @@ -134,6 +134,8 @@ pub struct BuildConfig { pub output_dir: PathBuf, pub distro: DistroVersion, pub sign_package: bool, + #[serde(default)] + pub sign_with: crate::build::signing::SignWith, /// GPG key ID/email to sign with (debsign's `-k` option). pub sign_key: Option, /// Build the automatic `-dbgsym` debug symbol package alongside the regular binaries. @@ -212,6 +214,16 @@ pub trait BuildDriver { fn reset_build_root(&self) -> std::io::Result<()>; + /// Sign `changes_file` (a path on the host) with `debsign` — on the host + /// for the bare driver, or inside a minimal same-distro container with + /// the host's gpg-agent socket forwarded in. `gpg` carries the agent + /// socket and key for container signing; bare ignores it. + fn sign_changes( + &self, + changes_file: &Path, + gpg: Option<&crate::build::signing::GpgForwarding>, + ) -> anyhow::Result<()>; + fn reused_environment(&self) -> bool { true } diff --git a/packages/debmagic/src/build/driver_bare.rs b/packages/debmagic/src/build/driver_bare.rs index b87d2f4..a163023 100644 --- a/packages/debmagic/src/build/driver_bare.rs +++ b/packages/debmagic/src/build/driver_bare.rs @@ -106,4 +106,13 @@ impl BuildDriver for DriverBare { } Ok(()) } + + fn sign_changes( + &self, + changes_file: &Path, + _gpg: Option<&crate::build::signing::GpgForwarding>, + ) -> anyhow::Result<()> { + crate::build::signing::check_host_debsign_available()?; + crate::build::signing::sign_on_host(changes_file, self.config.sign_key.as_deref()) + } } diff --git a/packages/debmagic/src/build/driver_docker.rs b/packages/debmagic/src/build/driver_docker.rs index 427b719..28d656f 100644 --- a/packages/debmagic/src/build/driver_docker.rs +++ b/packages/debmagic/src/build/driver_docker.rs @@ -5,7 +5,7 @@ use std::{ process::{Command, Stdio}, }; -use anyhow::anyhow; +use anyhow::{Context, anyhow}; use debmagic_common::distro::DistroVersion; use serde::{Deserialize, Serialize}; @@ -42,12 +42,8 @@ pub struct DriverDockerConfigOverrides { // Constants const ENVIRONMENT_LABEL: &str = "dev.debmagic.environment"; -fn bind_mount_arg(build_root: &Path) -> String { - format!( - "type=bind,src={},dst={}", - build_root.display(), - BUILD_DIR_IN_CONTAINER - ) +fn bind_mount_arg(src: &Path, dst: &str) -> String { + format!("type=bind,src={},dst={}", src.display(), dst) } const DOCKERFILE_TEMPLATE: &str = r#" FROM {base_image} @@ -76,6 +72,9 @@ fn shell_quote(s: &str) -> String { pub struct DriverDocker { config: BuildConfig, container_name: String, + /// Base image of the distro, used to spin up minimal one-shot containers + /// (e.g. for signing) that don't need the build environment's tooling. + base_image: String, reused_environment: bool, } @@ -267,6 +266,7 @@ impl DriverDocker { let mut driver = Self { config: config.clone(), container_name, + base_image: base_image.clone(), reused_environment: false, }; let environment_matches = container_environment_fingerprint(&driver.container_name)? @@ -309,7 +309,10 @@ impl DriverDocker { &format!("{ENVIRONMENT_LABEL}={desired_fingerprint}"), "--mount", ]) - .arg(bind_mount_arg(&config.build_root_dir)) + .arg(bind_mount_arg( + &config.build_root_dir, + BUILD_DIR_IN_CONTAINER, + )) .arg(&docker_image_name), "starting docker container", )?; @@ -333,12 +336,13 @@ impl DriverDocker { pub fn from_build_metadata( config: &BuildConfig, - _driver_config: &DriverConfig, + driver_config: &DriverConfig, build_metadata: &BuildMetadata, ) -> anyhow::Result { Ok(Self { config: config.clone(), container_name: container_name_from_metadata(build_metadata)?, + base_image: driver_config.docker.base_image_for_distro(&config.distro), reused_environment: true, }) } @@ -445,4 +449,56 @@ impl BuildDriver for DriverDocker { fn driver_type(&self) -> BuildDriverType { BuildDriverType::Docker } + + fn sign_changes( + &self, + changes_file: &Path, + gpg: Option<&crate::build::signing::GpgForwarding>, + ) -> anyhow::Result<()> { + use crate::build::signing; + + let gpg = gpg.context("docker container signing needs gpg forwarding info")?; + let output_dir = changes_file + .parent() + .context("changes file has no parent directory")?; + let staging_dir = self.config.build_temp_dir().join("sign"); + signing::stage_signing_material(&staging_dir, &gpg.sign_key)?; + let script = signing::sign_container_script( + signing::changes_filename(changes_file)?, + &gpg.sign_key, + // The sign container's root is not id-mapped; fix ownership of + // files it rewrites so the host user can manage them afterwards. + Some((unsafe { libc::geteuid() }, unsafe { libc::getegid() })), + ); + + println!( + "[docker] $ signing {} in a minimal {} container", + changes_file.display(), + self.config.distro.codename + ); + run_checked( + Command::new("docker") + .args(["run", "--rm", "--init"]) + .args([ + "--mount", + &format!( + "{},readonly", + bind_mount_arg(&staging_dir, signing::SIGN_STAGING_IN_CONTAINER) + ), + ]) + .arg(format!( + "--mount=type=bind,src={},dst={},readonly", + gpg.agent_extra_socket.display(), + signing::GPG_SOCKET_IN_CONTAINER + )) + .args([ + "--mount", + &bind_mount_arg(output_dir, signing::OUTPUT_DIR_IN_CONTAINER), + ]) + .arg(&self.base_image) + .args(["sh", "-ec", &script]), + "signing in docker container", + )?; + Ok(()) + } } diff --git a/packages/debmagic/src/build/driver_lxd.rs b/packages/debmagic/src/build/driver_lxd.rs index b129304..b571303 100644 --- a/packages/debmagic/src/build/driver_lxd.rs +++ b/packages/debmagic/src/build/driver_lxd.rs @@ -4,6 +4,7 @@ use std::{ process::{Command, Stdio}, }; +use anyhow::Context as _; use debmagic_common::distro::Distro; use serde::{Deserialize, Serialize}; @@ -111,6 +112,9 @@ pub struct DriverLxd { container_name: String, /// Resolved project name (None → omit `--project` flag). project: Option, + /// Base image of the distro, used to spin up minimal one-shot containers + /// (e.g. for signing) that don't need the build environment's tooling. + base_image: String, reused_environment: bool, } @@ -249,6 +253,7 @@ impl DriverLxd { config: config.clone(), container_name: container_name.clone(), project, + base_image: base_image.clone(), reused_environment: false, }; @@ -413,6 +418,7 @@ impl DriverLxd { pub fn from_build_metadata( variant: LxdVariant, config: &BuildConfig, + driver_config: &DriverConfig, build_metadata: &BuildMetadata, ) -> anyhow::Result { let project = build_metadata.driver_metadata.get("project").cloned(); @@ -422,6 +428,9 @@ impl DriverLxd { config: config.clone(), container_name: container_name_from_metadata(build_metadata)?, project, + base_image: driver_config + .lxd + .base_image_for_distro(variant, &config.distro), reused_environment: true, }) } @@ -578,4 +587,112 @@ impl BuildDriver for DriverLxd { LxdVariant::Incus => BuildDriverType::Incus, } } + + fn sign_changes( + &self, + changes_file: &Path, + gpg: Option<&crate::build::signing::GpgForwarding>, + ) -> anyhow::Result<()> { + use crate::build::signing; + + let gpg = gpg.context("container signing needs gpg forwarding info")?; + let output_dir = changes_file + .parent() + .context("changes file has no parent directory")?; + let staging_dir = self.config.build_temp_dir().join("sign"); + signing::stage_signing_material(&staging_dir, &gpg.sign_key)?; + // No chown needed: raw.idmap maps container root to the host user. + let script = signing::sign_container_script( + signing::changes_filename(changes_file)?, + &gpg.sign_key, + None, + ); + + let sign_container = resource_name( + "debmagic-sign", + &self.config.package_name, + &self.config.build_identifier(), + ); + let bin = self.variant.binary(); + // The sign container is ephemeral; it disappears when stopped. + let mut init = self.lxd_cmd("init"); + init.arg("--ephemeral"); + init.args([&self.base_image, sign_container.as_str()]); + run_checked(&mut init, &format!("initialising {bin} sign container"))?; + + let result = (|| -> anyhow::Result<()> { + let host_uid = unsafe { libc::geteuid() }; + let host_gid = unsafe { libc::getegid() }; + if host_uid != 0 { + let idmap = format!("uid {host_uid} 0\ngid {host_gid} 0"); + run_checked( + self.lxd_cmd("config") + .arg("set") + .arg(&sign_container) + .arg("raw.idmap") + .arg(&idmap), + "setting raw.idmap on sign container", + )?; + } + + run_checked( + self.lxd_cmd("config") + .arg("device") + .arg("add") + .arg(&sign_container) + .arg("debmagic-output") + .arg("disk") + .arg(format!("source={}", output_dir.display())) + .arg(format!("path={}", signing::OUTPUT_DIR_IN_CONTAINER)), + "mounting output directory into sign container", + )?; + run_checked( + self.lxd_cmd("config") + .arg("device") + .arg("add") + .arg(&sign_container) + .arg("debmagic-sign-staging") + .arg("disk") + .arg(format!("source={}", staging_dir.display())) + .arg(format!("path={}", signing::SIGN_STAGING_IN_CONTAINER)) + .arg("readonly=true"), + "mounting signing material into sign container", + )?; + // Forward the host gpg-agent's extra socket via a proxy device, + // like a manually configured unix proxy but scoped to signing. + run_checked( + self.lxd_cmd("config") + .arg("device") + .arg("add") + .arg(&sign_container) + .arg("debmagic-gpg-agent") + .arg("proxy") + .arg("bind=container") + .arg(format!("connect=unix:{}", gpg.agent_extra_socket.display())) + .arg(format!("listen=unix:{}", signing::GPG_SOCKET_IN_CONTAINER)) + .arg("uid=0") + .arg("gid=0"), + "forwarding gpg-agent socket into sign container", + )?; + + run_checked( + self.lxd_cmd("start").arg(&sign_container), + &format!("starting {bin} sign container"), + )?; + + let mut exec = self.lxd_cmd("exec"); + exec.arg(&sign_container); + exec.arg("--"); + exec.args(["sh", "-ec", &script]); + run_checked(&mut exec, "signing in container")?; + Ok(()) + })(); + + let mut stop = self.lxd_cmd("stop"); + let stop_result = run_checked( + stop.arg(&sign_container), + &format!("stopping {bin} sign container"), + ); + result.and(stop_result) + } } diff --git a/packages/debmagic/src/build/mod.rs b/packages/debmagic/src/build/mod.rs index 0476a6e..f7f92ae 100644 --- a/packages/debmagic/src/build/mod.rs +++ b/packages/debmagic/src/build/mod.rs @@ -11,7 +11,7 @@ use crate::build::config::DriverOverrides; use crate::build::source::{source_manifest_path, stage_source_tree}; use crate::{ build::{ - common::{BuildConfig, BuildDriver, BuildDriverType, BuildMetadata, run_checked}, + common::{BuildConfig, BuildDriver, BuildDriverType, BuildMetadata}, config::DriverConfig, driver_bare::DriverBare, driver_docker::DriverDocker, @@ -30,14 +30,75 @@ pub mod config; pub mod driver_bare; pub mod driver_docker; pub mod driver_lxd; +pub mod signing; pub mod source; struct Build { config: BuildConfig, pub driver: Box, + /// Prepared when signing happens inside a container: agent socket + + /// sign key, validated before the build starts. + gpg_forwarding: Option, attached: bool, } +/// Where debsign will actually run for this build. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +enum SignLocation { + Host, + Container, +} + +/// Resolve the effective sign location and validate everything signing will +/// need *before* the build starts, so a broken gpg setup doesn't waste a +/// whole build. +fn prepare_signing( + build_config: &BuildConfig, +) -> anyhow::Result<(SignLocation, Option)> { + let container_driver = build_config.driver != BuildDriverType::Bare; + let host_has_debsign = signing::check_host_debsign_available().is_ok(); + + let location = match build_config.sign_with { + signing::SignWith::Host => SignLocation::Host, + signing::SignWith::Same => { + if container_driver { + SignLocation::Container + } else { + // The bare driver's "build environment" is the host. + SignLocation::Host + } + } + signing::SignWith::Auto => { + if host_has_debsign || !container_driver { + SignLocation::Host + } else { + SignLocation::Container + } + } + }; + + match location { + SignLocation::Host => { + signing::check_host_debsign_available()?; + Ok((location, None)) + } + SignLocation::Container => { + let sign_key = build_config.sign_key.clone().ok_or_else(|| { + anyhow!( + "signing in a container requires sign_key to be set \ + (debsign's maintainer-based key lookup only works on the host)" + ) + })?; + let forwarding = signing::GpgForwarding { + agent_extra_socket: signing::gpg_agent_extra_socket()?, + sign_key: sign_key.clone(), + }; + signing::check_signing_key_available(&sign_key)?; + Ok((location, Some(forwarding))) + } + } +} + fn get_build_driver( config: &BuildConfig, driver_config: &DriverConfig, @@ -102,6 +163,7 @@ fn create_driver_from_metadata( Ok(Box::new(DriverLxd::from_build_metadata( variant, &metadata.config, + config, metadata, )?)) } @@ -117,9 +179,16 @@ impl Build { ) -> anyhow::Result { let driver = get_build_driver(config, driver_config, driver_overrides) .context(format!("failed to create {:?} build driver", config.driver))?; + let gpg_forwarding = if config.sign_package { + let (_location, forwarding) = prepare_signing(config)?; + forwarding + } else { + None + }; Ok(Self { config: config.clone(), driver, + gpg_forwarding, attached: false, }) } @@ -154,6 +223,7 @@ impl Build { let attached = send_socket_command(build_root, "attach").is_ok(); Ok(Self { + gpg_forwarding: None, config: metadata.config.clone(), driver, attached, @@ -256,6 +326,7 @@ fn prepare_build_env( build_root_dir: build_root.clone(), distro: distro_version.clone(), sign_package: config.sign_package, + sign_with: config.sign_with, sign_key: config.sign_key.clone(), build_debug_symbols: config.build_debug_symbols, clean: config.clean, @@ -400,7 +471,9 @@ fn run_build( &build.config.output_dir, )?; if build.config.sign_package { - sign_changes_file(&changes_file, build.config.sign_key.as_deref())?; + build + .driver + .sign_changes(&changes_file, build.gpg_forwarding.as_ref())?; } Ok(()) }); @@ -486,35 +559,6 @@ fn check_dpkg_buildpackage_available() -> anyhow::Result<()> { ) } -/// GPG-sign every `.changes` (and its referenced `.dsc`/`.buildinfo`) in -/// `output_dir` with `debsign`. Always runs on the host regardless of the -/// build driver, since signing needs the user's own gpg keyring, which an -/// ephemeral container doesn't have access to. -fn sign_changes_file(changes_file: &Path, sign_key: Option<&str>) -> anyhow::Result<()> { - check_command_available( - "debsign", - "It's part of devscripts; install it and set up a gpg signing key to use --sign.", - )?; - - let output_dir = changes_file.parent().ok_or_else(|| { - anyhow!( - "could not get output directory of {}", - changes_file.display() - ) - })?; - let filename = changes_file - .file_name() - .ok_or_else(|| anyhow!("could not get filename of {}", changes_file.display()))?; - - let mut cmd = Command::new("debsign"); - if let Some(key) = sign_key { - cmd.arg(format!("-k{key}")); - } - cmd.arg(filename).current_dir(output_dir); - run_checked(&mut cmd, &format!("signing {}", changes_file.display()))?; - Ok(()) -} - /// Build a `.dsc` + tarball + `.buildinfo` + `.changes` source package. /// /// If `config.clean` is set, build-dependencies are installed before diff --git a/packages/debmagic/src/build/signing.rs b/packages/debmagic/src/build/signing.rs new file mode 100644 index 0000000..7b07604 --- /dev/null +++ b/packages/debmagic/src/build/signing.rs @@ -0,0 +1,256 @@ +//! GPG signing of build artifacts (`.changes`/`.dsc`) via `debsign`. +//! +//! Signing can happen on the host (traditional, requires `devscripts` +//! locally) or inside a minimal same-distro container. In the container case +//! the host's gpg-agent *extra* socket is forwarded in, so private key +//! material never leaves the host — the agent on the host performs the +//! signing operations, and only the public key is imported into the +//! container's keyring. + +use std::{ + path::{Path, PathBuf}, + process::{Command, Stdio}, +}; + +use anyhow::{Context, anyhow}; +use serde::{Deserialize, Serialize}; + +use crate::build::common::run_checked; + +/// Where the forwarded agent socket is bind-mounted inside sign containers. +/// A fixed, always-existing path; the script symlinks it to gpg's lookup +/// locations so plain `debsign` works without extra flags or env vars. +pub const GPG_SOCKET_IN_CONTAINER: &str = "/tmp/debmagic-gpg/S.gpg-agent"; +/// Directory mounted read-only into sign containers, holding the exported +/// public key and ownertrust line produced on the host. +pub const SIGN_STAGING_IN_CONTAINER: &str = "/debmagic-sign"; +/// Mount point of the output directory inside sign containers. +pub const OUTPUT_DIR_IN_CONTAINER: &str = "/debmagic-output"; + +pub const PUBKEY_FILE: &str = "pubkey.asc"; +pub const OWNERTRUST_FILE: &str = "ownertrust.txt"; + +/// Selects where `debsign` runs. +#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, clap::ValueEnum, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SignWith { + /// Use the host if `debsign` is available there, otherwise a container + /// (requires a containerized build driver). + #[default] + Auto, + /// Always sign on the host with `debsign`. + Host, + /// Sign inside a minimal container of the same distro, forwarding the + /// host's gpg-agent socket. Requires `sign_key` to be set. + Same, +} + +/// Everything needed to GPG-sign inside a container: the host's gpg-agent +/// extra socket plus the public key to seed the container's keyring with. +pub struct GpgForwarding { + pub agent_extra_socket: PathBuf, + pub sign_key: String, +} + +/// Resolve the host's gpg-agent *extra* socket — the restricted variant +/// intended for forwarding into chroots/containers (signing works, key +/// export and management don't). +pub fn gpg_agent_extra_socket() -> anyhow::Result { + let output = Command::new("gpgconf") + .args(["--list-dirs", "agent-extra-socket"]) + .stdout(Stdio::piped()) + .output() + .context("failed to run gpgconf; is gpg installed?")?; + if !output.status.success() { + return Err(anyhow!( + "gpgconf --list-dirs agent-extra-socket failed; is gpg-agent set up?" + )); + } + let path = PathBuf::from(String::from_utf8_lossy(&output.stdout).trim().to_string()); + if !path.exists() { + return Err(anyhow!( + "gpg-agent extra socket {} does not exist; is gpg-agent running?", + path.display() + )); + } + Ok(path) +} + +/// Verify that the host gpg setup can sign with `sign_key` (secret key +/// available via the agent). Intended as a pre-flight check so builds don't +/// fail at the signing step after all the work is done. +pub fn check_signing_key_available(sign_key: &str) -> anyhow::Result<()> { + let output = Command::new("gpg") + .args(["--batch", "--list-secret-keys", sign_key]) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .output() + .context("failed to run gpg; is it installed?")?; + if !output.status.success() || output.stdout.is_empty() { + return Err(anyhow!( + "no secret key for '{sign_key}' available to gpg; \ + import it on the host or pick a different sign_key" + )); + } + Ok(()) +} + +/// Export the public key for `sign_key` from the host keyring. +pub fn export_public_key(sign_key: &str) -> anyhow::Result> { + let output = Command::new("gpg") + .args(["--batch", "--export", sign_key]) + .stdout(Stdio::piped()) + .output() + .context("failed to run gpg --export")?; + if !output.status.success() || output.stdout.is_empty() { + return Err(anyhow!( + "failed to export public key for '{sign_key}' from the host keyring" + )); + } + Ok(output.stdout) +} + +/// Fingerprint of the key `debsign` will use, for ownertrust seeding. +pub fn key_fingerprint(sign_key: &str) -> anyhow::Result { + let output = Command::new("gpg") + .args(["--batch", "--with-colons", "--list-secret-keys", sign_key]) + .stdout(Stdio::piped()) + .output() + .context("failed to run gpg --list-secret-keys")?; + if !output.status.success() { + return Err(anyhow!("failed to look up fingerprint for '{sign_key}'")); + } + for line in String::from_utf8_lossy(&output.stdout).lines() { + let fields: Vec<&str> = line.split(':').collect(); + if fields.first() == Some(&"fpr") + && let Some(fpr) = fields.get(9) + { + return Ok(fpr.to_string()); + } + } + Err(anyhow!("no fingerprint found for key '{sign_key}'")) +} + +/// Stage the files a sign container needs (exported public key + ownertrust) +/// into `staging_dir` on the host; the drivers mount it read-only at +/// [`SIGN_STAGING_IN_CONTAINER`]. +pub fn stage_signing_material(staging_dir: &Path, sign_key: &str) -> anyhow::Result<()> { + std::fs::create_dir_all(staging_dir)?; + std::fs::write(staging_dir.join(PUBKEY_FILE), export_public_key(sign_key)?)?; + // Ownertrust format: "::"; 6 = ultimate. The + // key is the user's own, freshly imported into a throwaway keyring. + let ownertrust = format!("{}:6:\n", key_fingerprint(sign_key)?); + std::fs::write(staging_dir.join(OWNERTRUST_FILE), ownertrust)?; + Ok(()) +} + +/// Shell script run inside a sign container: install debsign, link the +/// forwarded agent socket where gpg looks for it, seed the throwaway keyring +/// with the public key plus ownertrust, then sign. Runs as root; `chown_to` +/// fixes ownership of the bind-mounted output dir afterwards when the +/// container's root is not id-mapped to the host user (docker). +pub fn sign_container_script( + changes_filename: &str, + sign_key: &str, + chown_to: Option<(u32, u32)>, +) -> String { + let chown = match chown_to { + Some((uid, gid)) => format!( + " && chown -R {uid}:{gid} {out}", + out = OUTPUT_DIR_IN_CONTAINER + ), + None => String::new(), + }; + format!( + "set -e; \ + export GNUPGHOME=/root/.gnupg; \ + mkdir -p /run/user/0/gnupg \"$GNUPGHOME\"; \ + chmod 700 /run/user/0/gnupg \"$GNUPGHOME\"; \ + ln -sf {sock} /run/user/0/gnupg/S.gpg-agent; \ + ln -sf {sock} \"$GNUPGHOME/S.gpg-agent\"; \ + apt-get update -qq; \ + apt-get install -y -qq devscripts; \ + gpg --batch --import {staging}/{pubkey}; \ + gpg --batch --import-ownertrust {staging}/{ownertrust}; \ + cd {out} && debsign -k{key} {changes}{chown}", + sock = GPG_SOCKET_IN_CONTAINER, + staging = SIGN_STAGING_IN_CONTAINER, + pubkey = PUBKEY_FILE, + ownertrust = OWNERTRUST_FILE, + out = OUTPUT_DIR_IN_CONTAINER, + key = shell_single_quote(sign_key), + changes = shell_single_quote(changes_filename), + ) +} + +/// Sign `changes_file` on the host with `debsign`. +pub fn sign_on_host(changes_file: &Path, sign_key: Option<&str>) -> anyhow::Result<()> { + let output_dir = changes_file.parent().ok_or_else(|| { + anyhow!( + "could not get output directory of {}", + changes_file.display() + ) + })?; + let filename = changes_file + .file_name() + .ok_or_else(|| anyhow!("could not get filename of {}", changes_file.display()))?; + + let mut cmd = Command::new("debsign"); + if let Some(key) = sign_key { + cmd.arg(format!("-k{key}")); + } + cmd.arg(filename).current_dir(output_dir); + run_checked(&mut cmd, &format!("signing {}", changes_file.display()))?; + Ok(()) +} + +pub fn check_host_debsign_available() -> anyhow::Result<()> { + match Command::new("debsign") + .arg("--version") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + { + Ok(_) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Err(anyhow!( + "debsign not found on PATH. It's part of devscripts; install it and set up a \ + gpg signing key, or set sign_with to \"same\" with a container driver." + )), + Err(e) => Err(e).context("failed to check for debsign"), + } +} + +pub fn changes_filename(changes_file: &Path) -> anyhow::Result<&str> { + changes_file + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| anyhow!("could not get filename of {}", changes_file.display())) +} + +fn shell_single_quote(s: &str) -> String { + format!("'{}'", s.replace('\'', "'\\''")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sign_script_quotes_filename_and_key() { + let script = sign_container_script("pkg_1.0_amd64.changes", "me@example.com", None); + assert!(script.contains("debsign -k'me@example.com' 'pkg_1.0_amd64.changes'")); + assert!(script.contains("gpg --batch --import /debmagic-sign/pubkey.asc")); + assert!(!script.contains("chown")); + } + + #[test] + fn sign_script_chowns_output_when_requested() { + let script = sign_container_script("x.changes", "key", Some((1000, 100))); + assert!(script.contains("chown -R 1000:100 /debmagic-output")); + } + + #[test] + fn shell_quote_escapes_single_quotes() { + assert_eq!(shell_single_quote("a'b"), "'a'\\''b'"); + } +} diff --git a/packages/debmagic/src/build/source.rs b/packages/debmagic/src/build/source.rs index f6a7cb8..92f6be8 100644 --- a/packages/debmagic/src/build/source.rs +++ b/packages/debmagic/src/build/source.rs @@ -456,6 +456,7 @@ mod tests { output_dir: test_root.join("output"), distro: debmagic_common::distro::get_distro_version("trixie").unwrap(), sign_package: false, + sign_with: crate::build::signing::SignWith::default(), sign_key: None, build_debug_symbols: false, clean: false, diff --git a/packages/debmagic/src/cli.rs b/packages/debmagic/src/cli.rs index c7883e8..6551fb3 100644 --- a/packages/debmagic/src/cli.rs +++ b/packages/debmagic/src/cli.rs @@ -1,7 +1,7 @@ use std::path::PathBuf; use crate::build::common::{BuildDriverType, SourceSyncMode}; -use clap::{Args, Parser, Subcommand, builder::BoolishValueParser}; +use clap::{Args, Parser, Subcommand}; #[derive(Parser, Debug)] #[command(version, about, long_about = None)] @@ -109,14 +109,27 @@ pub struct CommonBuildArgs { #[arg( long, - value_parser = BoolishValueParser::new(), - help = "Sign the resulting .changes/.dsc with debsign after building (yes/no). Defaults to the 'sign_package' setting in the config file (false if unset). Always runs on the host, using your own gpg keyring, regardless of --driver." + action = clap::ArgAction::SetTrue, + help = "Sign the resulting .changes/.dsc with debsign after building. Defaults to the 'sign_package' setting in the config file (false if unset)." )] pub sign: Option, + #[arg( + long, + action = clap::ArgAction::SetFalse, + help = "Do not sign the resulting .changes/.dsc, overriding a 'sign_package = true' default in the config file." + )] + pub no_sign: Option, + + #[arg( + long = "sign-with", + help = "Where debsign runs: 'host' signs on the host (requires devscripts there), 'same' signs inside a minimal same-distro container with the host gpg-agent socket forwarded in (requires --sign-key), 'auto' (default) uses the host if debsign is available there, else a container. Defaults to the 'sign_with' setting in the config file." + )] + pub sign_with: Option, + #[arg( long = "sign-key", - help = "GPG key ID/email to sign with, passed to debsign's -k option. Defaults to the 'sign_key' setting in the config file, or debsign's own maintainer-based key lookup if unset." + help = "GPG key ID/email to sign with, passed to debsign's -k option. Defaults to the 'sign_key' setting in the config file, or debsign's own maintainer-based key lookup if unset. Required when signing in a container." )] pub sign_key: Option, diff --git a/packages/debmagic/src/config.rs b/packages/debmagic/src/config.rs index 84c671a..2d8c199 100644 --- a/packages/debmagic/src/config.rs +++ b/packages/debmagic/src/config.rs @@ -2,6 +2,7 @@ use std::path::PathBuf; use crate::build::common::SourceSyncMode; use crate::build::config::DriverConfig; +use crate::build::signing::SignWith; use anyhow::{Context, anyhow}; use config::{Config as ConfigBuilder, File}; use serde::Deserialize; @@ -18,8 +19,12 @@ pub struct Config { pub build_debug_symbols: bool, /// Sign the resulting `.changes`/`.dsc` with `debsign` after building. pub sign_package: bool, + /// Where `debsign` runs: on the host or inside a minimal same-distro + /// container with the host's gpg-agent socket forwarded in. + pub sign_with: SignWith, /// GPG key ID/email to sign with (debsign's `-k` option). `None` lets - /// debsign fall back to its own maintainer-based key lookup. + /// debsign fall back to its own maintainer-based key lookup, but + /// container signing requires an explicit key. pub sign_key: Option, /// Run `debian/rules clean` before building (like `dpkg-buildpackage` /// does unless passed `-nc`). Disabled by default because non-incremental @@ -37,6 +42,7 @@ impl Default for Config { source_sync_mode: SourceSyncMode::default(), build_debug_symbols: false, sign_package: false, + sign_with: SignWith::default(), sign_key: None, clean: false, } diff --git a/packages/debmagic/src/main.rs b/packages/debmagic/src/main.rs index a986798..99a4e03 100644 --- a/packages/debmagic/src/main.rs +++ b/packages/debmagic/src/main.rs @@ -98,6 +98,12 @@ fn main() -> anyhow::Result<()> { if let Some(sign) = build_args.sign { config.sign_package = sign; } + if let Some(no_sign) = build_args.no_sign { + config.sign_package = !no_sign; + } + if let Some(sign_with) = build_args.sign_with { + config.sign_with = sign_with; + } if let Some(sign_key) = build_args.sign_key.clone() { config.sign_key = Some(sign_key); } diff --git a/packages/debmagic/tests/signing.rs b/packages/debmagic/tests/signing.rs new file mode 100644 index 0000000..fda4269 --- /dev/null +++ b/packages/debmagic/tests/signing.rs @@ -0,0 +1,240 @@ +//! End-to-end test for container signing: builds a throwaway gpg key with a +//! loopback pinentry, crafts a minimal source package artifact set +//! (`.dsc` + `.changes`), then has the docker driver sign it in a minimal +//! container via the forwarded gpg-agent socket. +//! +//! Requires docker and gpg on the host. Ignored by default; run with: +//! +//! ```shell +//! cargo test --test signing -- --ignored --nocapture +//! ``` + +use std::{ + fs, + path::{Path, PathBuf}, + process::{Command, Stdio}, +}; + +/// Run `cmd`, panicking with stdout+stderr on failure. +fn run(cmd: &mut Command) -> String { + let output = cmd.output().expect("failed to spawn command"); + if !output.status.success() { + panic!( + "command failed: {:?}\nstdout: {}\nstderr: {}", + cmd, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + String::from_utf8_lossy(&output.stdout).into_owned() +} + +struct TestGpgHome { + dir: PathBuf, +} + +impl TestGpgHome { + /// Create an isolated GNUPGHOME with a throwaway signing key and an + /// agent that answers without pinentry. + fn create() -> Self { + let dir = std::env::temp_dir().join(format!("debmagic-sign-test-{}", uuid::Uuid::new_v4())); + fs::create_dir_all(&dir).unwrap(); + // gpg refuses to use a homedir others could access. + run(Command::new("chmod").args(["700"]).arg(&dir)); + + fs::write(dir.join("gpg-agent.conf"), "allow-loopback-pinentry\n").unwrap(); + fs::write(dir.join("gpg.conf"), "pinentry-mode loopback\n").unwrap(); + + run(Command::new("gpgconf") + .env("GNUPGHOME", &dir) + .args(["--launch", "gpg-agent"])); + + run(Command::new("gpg").env("GNUPGHOME", &dir).args([ + "--batch", + "--passphrase", + "", + "--quick-generate-key", + "debmagic sign test ", + "ed25519", + "sign", + "never", + ])); + + Self { dir } + } + + fn agent_extra_socket(&self) -> PathBuf { + let out = run(Command::new("gpgconf") + .env("GNUPGHOME", &self.dir) + .args(["--list-dirs", "agent-extra-socket"])); + PathBuf::from(out.trim()) + } + + fn fingerprint(&self) -> String { + let out = run(Command::new("gpg").env("GNUPGHOME", &self.dir).args([ + "--batch", + "--with-colons", + "--list-secret-keys", + "sign@example.invalid", + ])); + for line in out.lines() { + let fields: Vec<&str> = line.split(':').collect(); + if fields.first() == Some(&"fpr") { + return fields[9].to_string(); + } + } + panic!("no fingerprint found"); + } +} + +impl Drop for TestGpgHome { + fn drop(&mut self) { + let _ = Command::new("gpgconf") + .env("GNUPGHOME", &self.dir) + .args(["--kill", "all"]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + let _ = fs::remove_dir_all(&self.dir); + } +} + +/// Write a minimal artifact set (`hello.txt`, `.dsc`, `.changes`) with +/// consistent sizes and sha256 checksums, so `debsign` accepts it. +fn write_fake_artifacts(output_dir: &Path) -> PathBuf { + fs::create_dir_all(output_dir).unwrap(); + let payload = b"hello from debmagic sign test\n"; + fs::write(output_dir.join("hello.txt"), payload).unwrap(); + + let sha256 = |data: &[u8]| -> String { + let mut child = Command::new("sha256sum") + .arg("-") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .expect("failed to spawn sha256sum"); + use std::io::Write; + child + .stdin + .as_mut() + .unwrap() + .write_all(data) + .expect("failed to pipe to sha256sum"); + let out = child.wait_with_output().expect("sha256sum failed"); + String::from_utf8_lossy(&out.stdout) + .split_whitespace() + .next() + .unwrap() + .to_string() + }; + + let files_entry = + |name: &str, data: &[u8]| format!(" {} {} {}", sha256(data), data.len(), name); + + let dsc_content = format!( + "Format: 3.0 (native)\nSource: debmagic-sign-test\nBinary: debmagic-sign-test\nVersion: 1.0\nMaintainer: debmagic sign test \nArchitecture: all\nFiles:\n{}\n", + files_entry("hello.txt", payload) + ); + let dsc_name = "debmagic-sign-test_1.0.dsc"; + fs::write(output_dir.join(dsc_name), &dsc_content).unwrap(); + + let changes_content = format!( + "Format: 1.8\nSource: debmagic-sign-test\nBinary: debmagic-sign-test\nVersion: 1.0\nMaintainer: debmagic sign test \nArchitecture: source all\nDistribution: unstable\nFiles:\n{}\n{}\n", + files_entry(dsc_name, dsc_content.as_bytes()), + files_entry("hello.txt", payload) + ); + let changes_path = output_dir.join("debmagic-sign-test_1.0_amd64.changes"); + fs::write(&changes_path, &changes_content).unwrap(); + changes_path +} + +#[test] +#[ignore = "needs docker and gpg on the host"] +fn docker_signs_changes_with_forwarded_agent() { + // Skip early with a clear message if docker isn't usable. + if Command::new("docker") + .args(["info"]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .map(|s| !s.success()) + .unwrap_or(true) + { + eprintln!("docker not available, skipping"); + return; + } + + let gpg_home = TestGpgHome::create(); + let work_dir = std::env::temp_dir().join(format!("debmagic-sign-out-{}", uuid::Uuid::new_v4())); + let changes_path = write_fake_artifacts(&work_dir); + + let output_dir = changes_path.parent().unwrap(); + let staging_dir = work_dir.join("staging"); + fs::create_dir_all(&staging_dir).unwrap(); + + // Stage pubkey + ownertrust the same way the driver does. The export is + // binary OpenPGP data, so capture raw bytes rather than a String. + let pubkey = Command::new("gpg") + .env("GNUPGHOME", &gpg_home.dir) + .args(["--batch", "--export", "sign@example.invalid"]) + .output() + .expect("gpg export failed"); + assert!(pubkey.status.success()); + fs::write(staging_dir.join("pubkey.asc"), pubkey.stdout).unwrap(); + fs::write( + staging_dir.join("ownertrust.txt"), + format!("{}:6:\n", gpg_home.fingerprint()), + ) + .unwrap(); + + let socket = gpg_home.agent_extra_socket(); + let script = "set -e; \ + export GNUPGHOME=/root/.gnupg; \ + mkdir -p /run/user/0/gnupg \"$GNUPGHOME\"; \ + chmod 700 /run/user/0/gnupg \"$GNUPGHOME\"; \ + ln -sf /tmp/debmagic-gpg/S.gpg-agent /run/user/0/gnupg/S.gpg-agent; \ + ln -sf /tmp/debmagic-gpg/S.gpg-agent \"$GNUPGHOME/S.gpg-agent\"; \ + apt-get update -qq; \ + apt-get install -y -qq devscripts; \ + gpg --batch --import /debmagic-sign/pubkey.asc; \ + gpg --batch --import-ownertrust /debmagic-sign/ownertrust.txt; \ + cd /debmagic-output && debsign -k'sign@example.invalid' 'debmagic-sign-test_1.0_amd64.changes'"; + + run(Command::new("docker") + .args(["run", "--rm", "--init"]) + .args([ + "--mount", + &format!( + "type=bind,src={},dst=/debmagic-sign,readonly", + staging_dir.display() + ), + ]) + .arg(format!( + "--mount=type=bind,src={},dst=/tmp/debmagic-gpg/S.gpg-agent,readonly", + socket.display() + )) + .args([ + "--mount", + &format!( + "type=bind,src={},dst=/debmagic-output", + output_dir.display() + ), + ]) + .arg("docker.io/debian:trixie") + .args(["sh", "-ec", script])); + + let signed = fs::read_to_string(&changes_path).unwrap(); + assert!( + signed.contains("-----BEGIN PGP SIGNATURE-----"), + "changes file was not signed:\n{signed}" + ); + + // Verify the signature against the test keyring. The .changes file has + // the message and signature in one clearsigned document. + run(Command::new("gpg") + .env("GNUPGHOME", &gpg_home.dir) + .args(["--batch", "--verify"]) + .arg(&changes_path)); + + let _ = fs::remove_dir_all(&work_dir); +} From f9d2303a20717bbc564622a4d500cd684332b978 Mon Sep 17 00:00:00 2001 From: Jonas Jelten Date: Tue, 4 Aug 2026 17:10:25 +0200 Subject: [PATCH 6/7] docs(build): enhance --- docs/index.md | 1 + docs/usage/build.md | 139 ++++++++++++++++---------------- docs/usage/config.md | 70 ++++++++++++++++ docs/usage/packaging.md | 5 ++ packages/debmagic/src/config.rs | 1 + 5 files changed, 147 insertions(+), 69 deletions(-) create mode 100644 docs/usage/config.md create mode 100644 docs/usage/packaging.md diff --git a/docs/index.md b/docs/index.md index 9bca4fd..7a0b0dd 100644 --- a/docs/index.md +++ b/docs/index.md @@ -9,6 +9,7 @@ usage/getting-started.md usage/build.md usage/source.md +usage/config.md usage/modules/index.md ``` diff --git a/docs/usage/build.md b/docs/usage/build.md index 39b0a02..e25ddae 100644 --- a/docs/usage/build.md +++ b/docs/usage/build.md @@ -1,26 +1,39 @@ # Building packages Quick reference to build a Debian/Ubuntu package with `debmagic`. -It covers `debmagic build binary` — the generic entry point that works on *any* `debian/`-packaged source tree, including ones that don't use `debmagic-pkg` to write `debian/rules`. -For building just a `.dsc`/tarball without compiling anything, see [`debmagic build source`](source.md) instead. + ## TL;DR +- Entry point: `debmagic build binary` — build on *any* `debian/`-packaged source tree (including packages with [`debian/rules.py`](packaging.md)) +- Source package: [`debmagic build source`](source.md), creates a `.dsc` without compilation + ```shell +cd your-package debmagic build binary --driver lxd \ - --source-dir /path/to/parent/of/debian/dir \ - --output-dir /path/to/put/the/.deb/files \ - --apt-mirror http:///ubuntu + --output-dir /tmp/build-artifacts ``` -- `--source-dir` is the directory *containing* `debian/`, not `debian/` itself. - Defaults to the current directory. -- `--output-dir` is where the resulting `.deb`/`.udeb`/`.ddeb`, `.buildinfo` and `.changes` files end up. - It's created if missing. - Defaults to the current directory. -- `--driver` is required. - Use `lxd` or `incus` if available (isolated containers); fall back to `docker`, then `bare` (builds directly on the host with no isolation, so only use this if you already trust the host environment). -- Exit code is non-zero on failure; stderr/stdout carry the real `dpkg-buildpackage`/`apt-get` output, so grep that for the actual error instead of guessing from the exit code alone. +## Available options + +`debmagic build`: + +| Option | Description | +|---|---| +| `--driver <...>` | [Build environment driver to use](#picking-a-driver) | +| `--source-sync ` | [Which source files to include](#source-file-staging) | +| `--persistent` | Retain the build environment. This does not do incremental builds. | +| `--incremental` | Do incremental builds by syncing changed sources only; implies `persistent` | +| `--distro ` | [Select the target distro/release](#selecting-a-distrorelease) (e.g. `trixie`, `resolute`) | +| `--proposed` | Use [build dependencies from `proposed`](#proposed-dependencies) pocket | +| `--sign` | [GPG-sign the resulting `.changes`/`.dsc`/`.buildinfo`](#signing) with `debsign` | +| `--clean` | Run [`debian/rules clean` before building](#cleaning) | +| `--debug-symbols` | [Build the automatic `-dbgsym` debug symbol packages](#building-debug-symbol-packages) | +| `--apt-mirror ` | [Mirror URL](#mirror-selection) | +| `--source-dir ` | Directory containing the `debian/` package directory | +| `--output-dir ` | Directory to put the resulting build artifacts | + +[`debmagic shell`](#inspecting-a-failed-build) — attach an interactive shell to a build environment ## Picking a driver @@ -32,9 +45,24 @@ Check what's installed and use the first that applies, in this order: | `docker` | `docker info` | Full container isolation | | `bare` | none (no daemon) | None — build-deps install with `sudo apt-get` directly on the host; only use in a disposable/CI environment | -There's no auto-detection; pick one and pass it explicitly every time. +There's no auto-detection; pick one and pass it explicitly every time (or configure it in a [`debmagic.toml`](config.md) file). + +## Inspecting a failed build + +By default a failed build tears down the container, so nothing is left to inspect. +If a build might fail and you need to inspect it afterwards, pass `--persistent` up front, then once the run finishes: + +```shell +# if you're in the package still +debmagic shell +# from the outside: +debmagic shell --source-dir /path/to/parent/of/debian/dir +``` + +This attaches an interactive shell inside the still-running (or restartable) build environment, at the package's build directory. + -## Speeding up builds with a mirror +## Mirror selection Fresh containers install their base tooling plus every `Build-Depends`, so slow mirrors directly translate into slow builds. Pass `--apt-mirror` to use a faster mirror for build-dependency resolution after the base tooling is bootstrapped from the image's configured archives: @@ -43,18 +71,11 @@ Pass `--apt-mirror` to use a faster mirror for build-dependency resolution after debmagic build binary --driver lxd --apt-mirror http:///ubuntu ... ``` -Notes: +You can persistently set this flag in `.config/debmagic/config.toml`. -- Works for the `lxd`, `incus` and `docker` drivers. - It's a no-op for `bare`, which uses the host's own apt sources. -- The image, mirror, proposed-pocket setting and host user IDs form the build-environment identity. - Changing any of them automatically replaces an incompatible persistent container. -- Handles both the classic `sources.list` format and the deb822 `*.sources` format (Ubuntu 24.04+). -- To avoid repeating the flag, set it once in `$XDG_CONFIG_HOME/debmagic/config.toml` (see below) instead — a mirror is a property of your machine, not of a package, so it belongs in the global config rather than in the repo's `debian/debmagic.toml`. +## Source file staging -## Selecting which source files are staged - -Before building, debmagic stages the source tree into the build environment. +Before building, `debmagic` stages the source tree into the build environment. `--source-sync ` controls which files are staged, so you always know what ends up in the build and in a generated source package: | Mode | Stages | Notes | @@ -66,9 +87,13 @@ Before building, debmagic stages the source tree into the build environment. If the source directory is not a git worktree, `tracked` and `committed` fall back to `worktree` with a warning. Git submodules are skipped with a note, since their contents aren't tracked by the parent repository. -To persist a mode, set `source_sync_mode = "committed"` in `debian/debmagic.toml`. +To persist a mode, set `source_sync_mode = "committed"` in [`debmagic.toml`](config.md). + +## Repeating a build + +You can iterate on the same build for faster compile times. -## Iterating on a build (faster repeat runs) +### Persistent container Every `debmagic build binary` invocation creates a new container by default and tears it down afterwards. For repeated attempts against the same package and distro, add `--persistent` to retain and reuse the running environment while restaging the source tree for each build: @@ -78,37 +103,34 @@ debmagic build binary --driver lxd --persistent \ --source-dir . --output-dir /tmp/out ``` -Use `--incremental` to retain the environment and synchronize only source changes while preserving generated files and unchanged source inodes. -Incremental mode is binary-only, implies `--persistent`, and cannot be combined with `--clean yes`. +### Incremental builds -## Inspecting a failed build - -By default a failed build tears down the container, so nothing is left to inspect. -If a build might fail and you need to inspect it afterwards, pass `--persistent` up front, then once the run finishes: - -```shell -debmagic shell --source-dir /path/to/parent/of/debian/dir -``` +Use `--incremental` to retain the environment and synchronize only source changes while preserving generated files and unchanged source inodes. +This flag implies `--persistent`, and cannot be combined with `--clean yes`. -This attaches an interactive shell inside the still-running (or restartable) build environment, at the package's build directory. ## Selecting a distro/release Only needed when `debian/changelog`'s top entry doesn't unambiguously determine the target: pass `--distro ` (e.g. `--distro noble`, `--distro trixie`). If the changelog has a single unambiguous entry, omit it. +## Proposed dependencies + +If needed, build dependencies can be used from `-proposed`. +Pass `--proposed` to enable the proposed pocket in the build environment. + ## Building debug symbol packages By default `debmagic build binary` passes `DEB_BUILD_OPTIONS=noautodbgsym` to `dpkg-buildpackage`, which suppresses debhelper's automatic `-dbgsym` package (the detached debug info package debhelper otherwise builds by default from compat 9 onward). Pass `--debug-symbols` to build it for one invocation: ```shell -debmagic build binary --driver lxd --debug-symbols --source-dir . --output-dir /tmp/out +debmagic build binary --debug-symbols --output-dir /tmp/out ``` -Or set `build_debug_symbols = true` in `debian/debmagic.toml`/`$XDG_CONFIG_HOME/debmagic/config.toml` to always build it. +Or set `build_debug_symbols = true` in the [`debmagic.toml`](config.md). -## Signing and cleaning +## Signing `--sign` (plus optionally `--sign-key you@example.com`) GPG-signs the resulting `.changes`/`.dsc`/`.buildinfo` with `debsign` after building. This is mainly useful for [source builds destined for Launchpad](source.md#uploading-to-launchpad), but works for binary builds too. @@ -123,40 +145,19 @@ Where `debsign` runs is selected by `--sign-with` (config: `sign_with`): Container signing requires an explicit `--sign-key`, since debsign's maintainer-based key lookup only works on the host. Signing prerequisites (agent running, secret key available) are validated before the build starts, so a broken gpg setup fails fast instead of after the build. + +Defaults can be set in [`debmagic.toml`](config.md). + +## Cleaning + `--clean` runs `debian/rules clean` before building, like plain `dpkg-buildpackage` does unless passed `-nc`; `--no-clean` skips it even if the config file defaults to cleaning. Non-incremental builds already stage a clean source tree, while incremental builds preserve outputs intentionally. Enable cleaning only for packages whose `clean` target performs required setup or code generation. -Both default to the `sign_package`/`sign_with`/`sign_key`/`clean` settings in the config file (see below) if not passed on the CLI. - ## Persisting options in a config file -Instead of repeating CLI flags on every invocation, drop a config file. -There are two locations, with different scopes: - -- `$XDG_CONFIG_HOME/debmagic/config.toml` — your machine-wide defaults (mirror, signing key, persistent driver, ...). -- `/debian/debmagic.toml` — per-package defaults, committed next to `debian/rules` (e.g. `build_debug_symbols`, `sign_package`). - -```toml -build_debug_symbols = true -sign_package = true -sign_with = "same" -sign_key = "you@example.com" -clean = false - -[driver] -persistent = true -apt_mirror = "http:///ubuntu" - -[driver.lxd] -# project = "my-project" -``` - -Config precedence (highest wins): `--config ` on the CLI > `/debian/debmagic.toml` > `$XDG_CONFIG_HOME/debmagic/config.toml`. -CLI flags like `--apt-mirror`/`--persistent`/`--sign`/`--no-sign`/`--clean`/`--no-clean` always override the matching config file value for that one invocation. +Instead of repeating CLI flags on every invocation, drop a [config file](config.md). -## What NOT to expect yet +## Internals -- `debmagic test` and `debmagic check` are not implemented yet — don't rely on them for lintian/test output. - Rely on `debmagic build binary`'s own `dpkg-buildpackage` run (which already runs `dh_auto_test` unless the package's `debian/rules` disables it). - Container/device names are derived and sanitized internally (alphanumeric + hyphen, ≤63 chars for LXD/Incus) — don't try to predict or construct them yourself; use `debmagic shell` instead of `lxc`/`docker` commands directly. diff --git a/docs/usage/config.md b/docs/usage/config.md new file mode 100644 index 0000000..36d7510 --- /dev/null +++ b/docs/usage/config.md @@ -0,0 +1,70 @@ +# Configuration (`debmagic.toml`) + +Persistent build settings live in a `debmagic.toml` file. +Every option has a sensible default, so the file is optional — add only what you want to change. + +## Search order + +Config files are merged in order of increasing precedence, so later files override earlier ones: + +1. `~/.config/debmagic/config.toml` — your machine-wide defaults (e.g. a fast `apt_mirror`). +2. `/debian/debmagic.toml` — per-package settings, committed with the source. +3. An explicit `--config ` passed on the command line. + +Only files that exist are read; missing ones are skipped. +Command-line flags override whatever the merged config resolves to. + +## Options + +All keys are optional. + +| Key | Type | Default | Description | +|---|---|---|---| +| `driver.persistent` | bool | `false` | Keep and reuse the build environment across runs instead of tearing it down. | +| `driver.apt_mirror` | string | — | Mirror used for build-dependency resolution. Not used by the `bare` driver. | +| `driver.proposed` | bool | `false` | Also enable the `-proposed` pocket. Not used by the `bare` driver. | +| `driver.docker.base_images` | map | — | Base image per distro, keyed by `":"` (e.g. `"debian:trixie"`). Falls back to `docker.io/:`. | +| `driver.lxd.project` | string | — | LXD/Incus project to use. `None` uses the default project. | +| `driver.lxd.base_images` | map | — | Base image per distro, keyed by `":"`. Falls back to the driver's default remote image. | +| `temp_build_dir` | path | `/tmp/debmagic` | Where build trees are staged. | +| `incremental` | bool | `false` | Retain the environment and sync only source changes, preserving generated files. Binary-only; implies `persistent`; incompatible with `clean`. | +| `source_sync_mode` | enum | `tracked` | Which source files are staged (see below). | +| `build_debug_symbols` | bool | `false` | Build the automatic `-dbgsym` debug symbol package. | +| `sign_package` | bool | `false` | Sign the resulting `.changes`/`.dsc` with `debsign`. | +| `sign_with` | enum | `auto` | Where `debsign` runs (see below). | +| `sign_key` | string | — | GPG key ID/email for `debsign -k`. Required for container signing. | +| `clean` | bool | `false` | Run `debian/rules clean` before building. Disabled by default; incompatible with `incremental`. | + +### `source_sync_mode` + +| Value | Stages | +|---|---| +| `tracked` (default) | Git-tracked files, including uncommitted modifications. Untracked files are left out and reported as a warning. | +| `committed` | Same files as `tracked`, but fails if the worktree has uncommitted changes or untracked files. | +| `worktree` | Everything that isn't git-ignored, tracked or not. | + +### `sign_with` + +| Value | Behavior | +|---|---| +| `auto` (default) | Sign on the host if `debsign` is available there, otherwise in a container. | +| `host` | Always sign on the host with `debsign`. | +| `same` | Sign inside a minimal same-distro container, forwarding the host's gpg-agent socket. Requires `sign_key`. | + +## Example + + +```toml +build_debug_symbols = true +sign_package = true +sign_with = "same" +sign_key = "you@example.com or gpg key id" +clean = false + +[driver] +persistent = true +apt_mirror = "http:///ubuntu" + +[driver.lxd] +# project = "my=lxd-project-id" +``` diff --git a/docs/usage/packaging.md b/docs/usage/packaging.md new file mode 100644 index 0000000..d056dd0 --- /dev/null +++ b/docs/usage/packaging.md @@ -0,0 +1,5 @@ +# Packaging with debmagic + +`debmagic-pkg` allows you to write a package build recipe in Python. + +The documentation is not yet existent. \ No newline at end of file diff --git a/packages/debmagic/src/config.rs b/packages/debmagic/src/config.rs index 2d8c199..3c5c8b7 100644 --- a/packages/debmagic/src/config.rs +++ b/packages/debmagic/src/config.rs @@ -7,6 +7,7 @@ use anyhow::{Context, anyhow}; use config::{Config as ConfigBuilder, File}; use serde::Deserialize; +/// documented in docs/usage/config.md #[derive(Deserialize, Debug)] #[serde(default)] pub struct Config { From e2b914de248548c7835bcaee5aac1a072662d8b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Loipf=C3=BChrer?= Date: Tue, 4 Aug 2026 18:00:49 +0200 Subject: [PATCH 7/7] test: fix sample config initialization of BuildConfig --- packages/debmagic/src/build/common.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/debmagic/src/build/common.rs b/packages/debmagic/src/build/common.rs index e730cd7..796e708 100644 --- a/packages/debmagic/src/build/common.rs +++ b/packages/debmagic/src/build/common.rs @@ -274,6 +274,8 @@ mod tests { persistent: false, package_name: "debmagic".to_string(), sign_key: None, + sign_with: crate::build::signing::SignWith::Auto, + source_sync_mode: crate::build::common::SourceSyncMode::Tracked, } }