diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1e01757..af777a4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -155,7 +155,7 @@ jobs: - name: Install debmagic (cli) run: uv pip install packages/debmagic - name: Run Debmagic build on ourself - run: uv run debmagic build --driver=docker + run: uv run debmagic build binary --driver=docker # TODO: integration tests currently don't work in the CI since they require running apt source on debian trixie -> CI runs on ubuntu # integration-tests: diff --git a/Cargo.lock b/Cargo.lock index aab1ddb..34ea967 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -90,7 +90,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -193,7 +193,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -319,6 +319,29 @@ dependencies = [ "typenum", ] +[[package]] +name = "deb822-derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b6e5cafe61e77421a090e2a33b8a2e4e2ff1b106fd906ebade111307064d981" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "deb822-lossless" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812bb5c8052a89edc6d45d1bc3b3400e8186dd166e9b0a9520bfa5a2bd8477ee" +dependencies = [ + "deb822-derive", + "regex", + "rowan", + "serde", +] + [[package]] name = "debian-changelog" version = "0.2.14" @@ -334,6 +357,20 @@ dependencies = [ "whoami", ] +[[package]] +name = "debian-control" +version = "0.1.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3367f1dc94222cb9cf16e07c782e442b1928a26dd8dcee3aa4d2dbd32ca11d21" +dependencies = [ + "chrono", + "deb822-lossless", + "debversion", + "regex", + "rowan", + "url", +] + [[package]] name = "debmagic" version = "0.0.1-alpha.5" @@ -342,6 +379,7 @@ dependencies = [ "clap", "config", "debian-changelog", + "debian-control", "debmagic-common", "dirs", "glob", @@ -404,6 +442,17 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "dlv-list" version = "0.5.2" @@ -445,6 +494,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -551,6 +609,108 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "ignore" version = "0.4.25" @@ -620,7 +780,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn", + "syn 2.0.111", ] [[package]] @@ -640,6 +800,12 @@ dependencies = [ "redox_syscall", ] +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + [[package]] name = "log" version = "0.4.29" @@ -714,6 +880,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + [[package]] name = "pest" version = "2.8.4" @@ -744,7 +916,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -757,6 +929,15 @@ dependencies = [ "sha2", ] +[[package]] +name = "potential_utf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +dependencies = [ + "zerovec", +] + [[package]] name = "proc-macro2" version = "1.0.103" @@ -926,7 +1107,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -951,6 +1132,12 @@ dependencies = [ "serde_core", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.10.9" @@ -968,12 +1155,24 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + [[package]] name = "smawk" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c" +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + [[package]] name = "strsim" version = "0.11.1" @@ -991,6 +1190,28 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "test-case" version = "3.3.1" @@ -1009,7 +1230,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -1020,7 +1241,7 @@ checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", "test-case-core", ] @@ -1058,7 +1279,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -1070,6 +1291,16 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinystr" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "toml" version = "0.9.10+spec-1.1.0" @@ -1143,6 +1374,24 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -1157,6 +1406,7 @@ checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" dependencies = [ "getrandom 0.3.4", "js-sys", + "sha1_smol", "wasm-bindgen", ] @@ -1229,7 +1479,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.111", "wasm-bindgen-shared", ] @@ -1282,7 +1532,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -1293,7 +1543,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.111", ] [[package]] @@ -1344,6 +1594,12 @@ version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + [[package]] name = "yaml-rust2" version = "0.10.4" @@ -1355,6 +1611,83 @@ dependencies = [ "hashlink", ] +[[package]] +name = "yoke" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "zmij" version = "0.1.8" diff --git a/Cargo.toml b/Cargo.toml index 0a10d23..25c28d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,10 +20,11 @@ glob = ">=0.3.2" libc = ">=0.2.169" serde = { version = ">=1.0.217", features = ["derive"] } serde_json = ">=1.0.139" -uuid = { version = ">=1.10.0", features = ["v4"] } +uuid = { version = ">=1.10.0", features = ["v4", "v5"] } chrono = { version = ">=0.4.42" } regex = { version = ">=1.12.2" } debian-changelog = { version = ">=0.2.14" } +debian-control = { version = ">=0.1.39" } ignore = { version = ">=0.4.25" } # dev dependencies diff --git a/README.md b/README.md index ee1ca6a..502783b 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,8 @@ Included features: - debugging tools - `debmagic shell` - enter current running/finished package environment +> [!TIP] +> Want to use debmagic to build a package? See [docs/usage/build.md](docs/usage/build.md) for a quickstart. ## Debmagic packaging diff --git a/debian/control b/debian/control index 827b876..f06c3bf 100644 --- a/debian/control +++ b/debian/control @@ -26,6 +26,7 @@ Build-Depends: librust-test-case-dev (>=3.3.1), librust-pyo3-dev (>=0.27.2), librust-debian-changelog-dev (>=0.2.14), + librust-debian-control-dev (>= 0.1.39), librust-ignore-dev (>=0.4.25) Rules-Requires-Root: no X-Style: black diff --git a/debian/debmagic.1 b/debian/debmagic.1 index 4550f3e..f0d3cfe 100644 --- a/debian/debmagic.1 +++ b/debian/debmagic.1 @@ -1,43 +1,37 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.49.3. -.TH DEBMAGIC "1" "November 2025" "debmagic 0.1.0" "User Commands" +.TH DEBMAGIC "1" "July 2026" "debmagic 0.0.1-alpha1" "User Commands" .SH NAME -debmagic \- manual page for debmagic 0.1.0 +debmagic \- build Debian packages in isolated environments +.SH SYNOPSIS +.B debmagic +[\fI\,OPTIONS\/\fR] \fI\,\/\fR .SH DESCRIPTION -usage: debmagic [\-h] [\-\-version] {help,version,debuild,build} ... -.PP -Debmagic -.SS "positional arguments:" -.IP -{help,version,debuild,build} +.SS "Commands:" .TP -help -Show this help page and exit +build +Build a debian package: 'binary' (.deb) or 'source' (.dsc) packages +.TP +shell +Open an interactive shell to the currently active build environment +.TP +test +Run tests +.TP +check +Check the project .TP version -Print the version information and exit +Show version information .TP -debuild -Simply run debuild in the current working directory +help +Print this message or the help of the given subcommand(s) +.SH OPTIONS .TP -build -Buidl a debian package with the selected -containerization driver -.SS "options:" +\fB\-c\fR, \fB\-\-config\fR +Path to config file .TP \fB\-h\fR, \fB\-\-help\fR -show this help message and exit +Print help .TP -\fB\-\-version\fR -show program's version number and exit -.SH "SEE ALSO" -The full documentation for -.B debmagic -is maintained as a Texinfo manual. If the -.B info -and -.B debmagic -programs are properly installed at your site, the command -.IP -.B info debmagic -.PP -should give you access to the complete manual. +\fB\-V\fR, \fB\-\-version\fR +Print version diff --git a/docs/conf.py b/docs/conf.py index b002e1d..ffd3ac8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -30,7 +30,6 @@ # html settings html_theme = "sphinx_rtd_theme" -html_static_path = ["_static"] html_context = { "display_github": True, "github_user": "SFTtech", diff --git a/docs/index.md b/docs/index.md index ec902c9..7a0b0dd 100644 --- a/docs/index.md +++ b/docs/index.md @@ -7,6 +7,9 @@ :caption: Usage usage/getting-started.md +usage/build.md +usage/source.md +usage/config.md usage/modules/index.md ``` diff --git a/docs/usage/build.md b/docs/usage/build.md new file mode 100644 index 0000000..e25ddae --- /dev/null +++ b/docs/usage/build.md @@ -0,0 +1,163 @@ +# Building packages + +Quick reference to build a Debian/Ubuntu package with `debmagic`. + + +## TL;DR + +- Entry point: `debmagic build binary` — build on *any* `debian/`-packaged source tree (including packages with [`debian/rules.py`](packaging.md)) +- Source package: [`debmagic build source`](source.md), creates a `.dsc` without compilation + +```shell +cd your-package +debmagic build binary --driver lxd \ + --output-dir /tmp/build-artifacts +``` + +## Available options + +`debmagic build`: + +| Option | Description | +|---|---| +| `--driver <...>` | [Build environment driver to use](#picking-a-driver) | +| `--source-sync ` | [Which source files to include](#source-file-staging) | +| `--persistent` | Retain the build environment. This does not do incremental builds. | +| `--incremental` | Do incremental builds by syncing changed sources only; implies `persistent` | +| `--distro ` | [Select the target distro/release](#selecting-a-distrorelease) (e.g. `trixie`, `resolute`) | +| `--proposed` | Use [build dependencies from `proposed`](#proposed-dependencies) pocket | +| `--sign` | [GPG-sign the resulting `.changes`/`.dsc`/`.buildinfo`](#signing) with `debsign` | +| `--clean` | Run [`debian/rules clean` before building](#cleaning) | +| `--debug-symbols` | [Build the automatic `-dbgsym` debug symbol packages](#building-debug-symbol-packages) | +| `--apt-mirror ` | [Mirror URL](#mirror-selection) | +| `--source-dir ` | Directory containing the `debian/` package directory | +| `--output-dir ` | Directory to put the resulting build artifacts | + +[`debmagic shell`](#inspecting-a-failed-build) — attach an interactive shell to a build environment + +## Picking a driver + +Check what's installed and use the first that applies, in this order: + +| Driver | Check it's available | Isolation | +|---|---|---| +| `lxd` / `incus` | `lxc list` / `incus list` | Full container isolation | +| `docker` | `docker info` | Full container isolation | +| `bare` | none (no daemon) | None — build-deps install with `sudo apt-get` directly on the host; only use in a disposable/CI environment | + +There's no auto-detection; pick one and pass it explicitly every time (or configure it in a [`debmagic.toml`](config.md) file). + +## Inspecting a failed build + +By default a failed build tears down the container, so nothing is left to inspect. +If a build might fail and you need to inspect it afterwards, pass `--persistent` up front, then once the run finishes: + +```shell +# if you're in the package still +debmagic shell +# from the outside: +debmagic shell --source-dir /path/to/parent/of/debian/dir +``` + +This attaches an interactive shell inside the still-running (or restartable) build environment, at the package's build directory. + + +## Mirror selection + +Fresh containers install their base tooling plus every `Build-Depends`, so slow mirrors directly translate into slow builds. +Pass `--apt-mirror` to use a faster mirror for build-dependency resolution after the base tooling is bootstrapped from the image's configured archives: + +```shell +debmagic build binary --driver lxd --apt-mirror http:///ubuntu ... +``` + +You can persistently set this flag in `.config/debmagic/config.toml`. + +## Source file staging + +Before building, `debmagic` stages the source tree into the build environment. +`--source-sync ` controls which files are staged, so you always know what ends up in the build and in a generated source package: + +| Mode | Stages | Notes | +|---|---|---| +| `tracked` (default) | git-tracked files, including uncommitted modifications | Untracked files are left out and listed as a warning — `git add` them or switch modes to include them | +| `committed` | the same files as `tracked` | Fails if the worktree has uncommitted changes or untracked files; use for reproducible, reviewable source packages | +| `worktree` | everything that isn't git-ignored, tracked or not | | + +If the source directory is not a git worktree, `tracked` and `committed` fall back to `worktree` with a warning. +Git submodules are skipped with a note, since their contents aren't tracked by the parent repository. + +To persist a mode, set `source_sync_mode = "committed"` in [`debmagic.toml`](config.md). + +## Repeating a build + +You can iterate on the same build for faster compile times. + +### Persistent container + +Every `debmagic build binary` invocation creates a new container by default and tears it down afterwards. +For repeated attempts against the same package and distro, add `--persistent` to retain and reuse the running environment while restaging the source tree for each build: + +```shell +debmagic build binary --driver lxd --persistent \ + --source-dir . --output-dir /tmp/out +``` + +### Incremental builds + +Use `--incremental` to retain the environment and synchronize only source changes while preserving generated files and unchanged source inodes. +This flag implies `--persistent`, and cannot be combined with `--clean yes`. + + +## Selecting a distro/release + +Only needed when `debian/changelog`'s top entry doesn't unambiguously determine the target: pass `--distro ` (e.g. `--distro noble`, `--distro trixie`). +If the changelog has a single unambiguous entry, omit it. + +## Proposed dependencies + +If needed, build dependencies can be used from `-proposed`. +Pass `--proposed` to enable the proposed pocket in the build environment. + +## Building debug symbol packages + +By default `debmagic build binary` passes `DEB_BUILD_OPTIONS=noautodbgsym` to `dpkg-buildpackage`, which suppresses debhelper's automatic `-dbgsym` package (the detached debug info package debhelper otherwise builds by default from compat 9 onward). +Pass `--debug-symbols` to build it for one invocation: + +```shell +debmagic build binary --debug-symbols --output-dir /tmp/out +``` + +Or set `build_debug_symbols = true` in the [`debmagic.toml`](config.md). + +## Signing + +`--sign` (plus optionally `--sign-key you@example.com`) GPG-signs the resulting `.changes`/`.dsc`/`.buildinfo` with `debsign` after building. +This is mainly useful for [source builds destined for Launchpad](source.md#uploading-to-launchpad), but works for binary builds too. +If your config file defaults to signing, pass `--no-sign` to skip it for one invocation. + +Where `debsign` runs is selected by `--sign-with` (config: `sign_with`): + +- `auto` (default): sign on the host if `debsign` is installed there, otherwise in a container (requires a container driver). +- `host`: always sign on the host, using your own gpg keyring — requires `devscripts` installed locally. +- `same`: sign inside a minimal same-distro container, forwarding the host's gpg-agent socket (`gpgconf --list-dirs agent-extra-socket`) into it. + Only signing *operations* cross the socket; private key material never enters the container, and only the public key is imported into its throwaway keyring. + Container signing requires an explicit `--sign-key`, since debsign's maintainer-based key lookup only works on the host. + +Signing prerequisites (agent running, secret key available) are validated before the build starts, so a broken gpg setup fails fast instead of after the build. + +Defaults can be set in [`debmagic.toml`](config.md). + +## Cleaning + +`--clean` runs `debian/rules clean` before building, like plain `dpkg-buildpackage` does unless passed `-nc`; `--no-clean` skips it even if the config file defaults to cleaning. +Non-incremental builds already stage a clean source tree, while incremental builds preserve outputs intentionally. +Enable cleaning only for packages whose `clean` target performs required setup or code generation. + +## Persisting options in a config file + +Instead of repeating CLI flags on every invocation, drop a [config file](config.md). + +## Internals + +- Container/device names are derived and sanitized internally (alphanumeric + hyphen, ≤63 chars for LXD/Incus) — don't try to predict or construct them yourself; use `debmagic shell` instead of `lxc`/`docker` commands directly. diff --git a/docs/usage/config.md b/docs/usage/config.md new file mode 100644 index 0000000..36d7510 --- /dev/null +++ b/docs/usage/config.md @@ -0,0 +1,70 @@ +# Configuration (`debmagic.toml`) + +Persistent build settings live in a `debmagic.toml` file. +Every option has a sensible default, so the file is optional — add only what you want to change. + +## Search order + +Config files are merged in order of increasing precedence, so later files override earlier ones: + +1. `~/.config/debmagic/config.toml` — your machine-wide defaults (e.g. a fast `apt_mirror`). +2. `/debian/debmagic.toml` — per-package settings, committed with the source. +3. An explicit `--config ` passed on the command line. + +Only files that exist are read; missing ones are skipped. +Command-line flags override whatever the merged config resolves to. + +## Options + +All keys are optional. + +| Key | Type | Default | Description | +|---|---|---|---| +| `driver.persistent` | bool | `false` | Keep and reuse the build environment across runs instead of tearing it down. | +| `driver.apt_mirror` | string | — | Mirror used for build-dependency resolution. Not used by the `bare` driver. | +| `driver.proposed` | bool | `false` | Also enable the `-proposed` pocket. Not used by the `bare` driver. | +| `driver.docker.base_images` | map | — | Base image per distro, keyed by `":"` (e.g. `"debian:trixie"`). Falls back to `docker.io/:`. | +| `driver.lxd.project` | string | — | LXD/Incus project to use. `None` uses the default project. | +| `driver.lxd.base_images` | map | — | Base image per distro, keyed by `":"`. Falls back to the driver's default remote image. | +| `temp_build_dir` | path | `/tmp/debmagic` | Where build trees are staged. | +| `incremental` | bool | `false` | Retain the environment and sync only source changes, preserving generated files. Binary-only; implies `persistent`; incompatible with `clean`. | +| `source_sync_mode` | enum | `tracked` | Which source files are staged (see below). | +| `build_debug_symbols` | bool | `false` | Build the automatic `-dbgsym` debug symbol package. | +| `sign_package` | bool | `false` | Sign the resulting `.changes`/`.dsc` with `debsign`. | +| `sign_with` | enum | `auto` | Where `debsign` runs (see below). | +| `sign_key` | string | — | GPG key ID/email for `debsign -k`. Required for container signing. | +| `clean` | bool | `false` | Run `debian/rules clean` before building. Disabled by default; incompatible with `incremental`. | + +### `source_sync_mode` + +| Value | Stages | +|---|---| +| `tracked` (default) | Git-tracked files, including uncommitted modifications. Untracked files are left out and reported as a warning. | +| `committed` | Same files as `tracked`, but fails if the worktree has uncommitted changes or untracked files. | +| `worktree` | Everything that isn't git-ignored, tracked or not. | + +### `sign_with` + +| Value | Behavior | +|---|---| +| `auto` (default) | Sign on the host if `debsign` is available there, otherwise in a container. | +| `host` | Always sign on the host with `debsign`. | +| `same` | Sign inside a minimal same-distro container, forwarding the host's gpg-agent socket. Requires `sign_key`. | + +## Example + + +```toml +build_debug_symbols = true +sign_package = true +sign_with = "same" +sign_key = "you@example.com or gpg key id" +clean = false + +[driver] +persistent = true +apt_mirror = "http:///ubuntu" + +[driver.lxd] +# project = "my=lxd-project-id" +``` diff --git a/docs/usage/getting-started.md b/docs/usage/getting-started.md index 517dc5a..9a7d9e7 100644 --- a/docs/usage/getting-started.md +++ b/docs/usage/getting-started.md @@ -20,6 +20,12 @@ uvx debmagic apt install debmagic ``` +## Building an existing package (CLI) + +`debmagic build` builds *any* Debian-packaged source tree inside a throwaway build environment, driven by a build driver. + +To learn about `debmagic build`, see the [Building packages](build.md) page. + ## Example debian/rules.py Python `debian/rules.py` equivalent of [Ubuntu 24.04 htop](https://git.launchpad.net/ubuntu/+source/htop/tree/debian/rules?h=ubuntu/noble): diff --git a/docs/usage/packaging.md b/docs/usage/packaging.md new file mode 100644 index 0000000..d056dd0 --- /dev/null +++ b/docs/usage/packaging.md @@ -0,0 +1,5 @@ +# Packaging with debmagic + +`debmagic-pkg` allows you to write a package build recipe in Python. + +The documentation is not yet existent. \ No newline at end of file diff --git a/docs/usage/source.md b/docs/usage/source.md new file mode 100644 index 0000000..a2aad90 --- /dev/null +++ b/docs/usage/source.md @@ -0,0 +1,58 @@ +# Building source packages + +`debmagic build source` runs `dpkg-buildpackage -S -d -nc` to build a `.dsc`, tarball(s), `.buildinfo` and `.changes` without building binaries. +It's a target of the same [`debmagic build`](build.md) command that builds binary packages (`debmagic build binary`). + +## TL;DR + +```shell +# be in debian package directory, output to current directory: +debmagic build source + +# or outside the debian package dir: +debmagic build source --source-dir /path/to/parent/of/debian/dir --output-dir /path/to/put/the/artifacts +``` + +- Same `--source-dir`/`--output-dir`/`--distro` semantics as [`debmagic build`](build.md). +- `--driver` defaults to `bare`: building a source package needs neither package build-dependencies nor a compiler, but the host must provide `dpkg-buildpackage` from `dpkg-dev`. + Pass `--driver lxd`/`--driver incus`/`--driver docker` when the host does not provide a usable Debian build environment. + Container drivers install their base tooling, but package build-dependencies are installed only with `--clean`. + Binary builds (`debmagic build binary`) still require `--driver` to be passed explicitly. + +(uploading-to-launchpad)= +## Uploading to Launchpad + +```shell +debmagic build source --sign --sign-key you@example.com \ + --source-dir . --output-dir /tmp/out +dput ppa:your-lp-username/your-ppa /tmp/out/*_source.changes +``` + +- `--sign` GPG-signs the `.dsc`/`.buildinfo`/`.changes` with `debsign` (from `devscripts`) after building. + By default it runs on the host; with `--sign-with same` (or `auto` when `debsign` isn't installed on the host) it runs in a minimal same-distro container with your gpg-agent socket forwarded in — see [Signing and cleaning](build.md#signing-and-cleaning). +- `--sign-key` picks which key/uid to sign with (`debsign`'s `-k`); omit it to let `debsign` fall back to its own maintainer-address lookup (host signing only). +- Both can be set as defaults in `debian/debmagic.toml`/`$XDG_CONFIG_HOME/debmagic/config.toml` instead of passing them every time: + + ```toml + sign_package = true + sign_key = "you@example.com" + ``` + +## What ends up in the source package + +The same file selection as `debmagic build` uses to populate the build environment: everything under `--source-dir` except files matched by `.gitignore` (build artifacts, virtualenvs, ...). +Untracked-but-not-ignored files are included, so uncommitted work-in-progress changes are packaged too — useful while iterating locally. +`debian/source/options` (`tar-ignore`/`diff-ignore` patterns, etc.) is honored as usual, since it's `dpkg-source` itself that reads it. + +## What's NOT run + +By default, `debian/rules` is never invoked (neither `dpkg-source` nor `dpkg-genchanges` need it), so this also works for source trees whose build-dependencies aren't installed anywhere. +Pass `--clean` to opt into running `debian/rules clean` first (like plain `dpkg-buildpackage` does unless passed `-nc`) — useful if a package's `clean` target is (ab)used for setup/codegen that should end up in the source package. +This also installs build-dependencies first, since the `clean` target usually needs its own tooling; a source build without `--clean` (the default) needs none of that. + +## Known limitation: `--driver bare` on a non-Debian host + +`dpkg-genbuildinfo` reads the dpkg status database (`/var/lib/dpkg/status`) to record installed package versions, unlike plain `dpkg-source`. +On a host that has `dpkg-dev` installed but isn't itself Debian/Ubuntu-based (or otherwise lacks a real dpkg database), this step fails with an error like `cannot open /var/lib/dpkg/status`. +Use `--driver lxd`/`--driver incus` in that case to build inside a proper Debian-ish container instead. + diff --git a/packages/debmagic-common/src/debian/version.rs b/packages/debmagic-common/src/debian/version.rs index b8c8b26..a6e5400 100644 --- a/packages/debmagic-common/src/debian/version.rs +++ b/packages/debmagic-common/src/debian/version.rs @@ -48,6 +48,10 @@ impl PackageVersion { self.upstream.clone() } + pub fn upstream_version(&self) -> &str { + &self.upstream + } + /// upstream version plus packaging revision pub fn upstream_revision(&self) -> String { if let Some(revision) = &self.revision { diff --git a/packages/debmagic-common/src/distro.rs b/packages/debmagic-common/src/distro.rs index a66279c..e4f18f6 100644 --- a/packages/debmagic-common/src/distro.rs +++ b/packages/debmagic-common/src/distro.rs @@ -14,6 +14,25 @@ pub struct DistroVersion { pub codename: String, /// numeric or semver version, e.g. "24.04" for ubuntu or "12" for debian pub version: String, + /// true for unreleased development releases (affects image selection) + #[serde(default)] + pub is_devel: bool, +} + +impl DistroVersion { + fn new(distro: Distro, codename: &str, version: &str) -> Self { + Self { + distro, + codename: codename.to_string(), + version: version.to_string(), + is_devel: false, + } + } + + fn devel(mut self) -> Self { + self.is_devel = true; + self + } } impl Distro { @@ -32,161 +51,35 @@ impl std::fmt::Display for Distro { } static DISTRO_INFO_MAP: LazyLock> = LazyLock::new(|| { + use Distro::{Debian, Ubuntu}; HashMap::from([ // debian ( "experimental", - DistroVersion { - distro: Distro::Debian, - codename: "experimental".to_string(), - version: "".to_string(), - }, - ), - ( - "unstable", - DistroVersion { - distro: Distro::Debian, - codename: "unstable".to_string(), - version: "".to_string(), - }, - ), - ( - "sid", - DistroVersion { - distro: Distro::Debian, - codename: "sid".to_string(), - version: "".to_string(), - }, - ), - ( - "testing", - DistroVersion { - distro: Distro::Debian, - codename: "testing".to_string(), - version: "".to_string(), - }, - ), - ( - "duke", - DistroVersion { - distro: Distro::Debian, - codename: "duke".to_string(), - version: "15".to_string(), - }, - ), - ( - "forky", - DistroVersion { - distro: Distro::Debian, - codename: "forky".to_string(), - version: "14".to_string(), - }, - ), - ( - "trixie", - DistroVersion { - distro: Distro::Debian, - codename: "trixie".to_string(), - version: "13".to_string(), - }, - ), - ( - "bookworm", - DistroVersion { - distro: Distro::Debian, - codename: "bookworm".to_string(), - version: "12".to_string(), - }, - ), - ( - "bullseye", - DistroVersion { - distro: Distro::Debian, - codename: "bullseye".to_string(), - version: "11".to_string(), - }, - ), - ( - "buster", - DistroVersion { - distro: Distro::Debian, - codename: "buster".to_string(), - version: "10".to_string(), - }, - ), - ( - "stretch", - DistroVersion { - distro: Distro::Debian, - codename: "stretch".to_string(), - version: "9".to_string(), - }, - ), + DistroVersion::new(Debian, "experimental", ""), + ), + ("unstable", DistroVersion::new(Debian, "unstable", "")), + ("sid", DistroVersion::new(Debian, "sid", "")), + ("testing", DistroVersion::new(Debian, "testing", "")), + ("duke", DistroVersion::new(Debian, "duke", "15")), + ("forky", DistroVersion::new(Debian, "forky", "14")), + ("trixie", DistroVersion::new(Debian, "trixie", "13")), + ("bookworm", DistroVersion::new(Debian, "bookworm", "12")), + ("bullseye", DistroVersion::new(Debian, "bullseye", "11")), + ("buster", DistroVersion::new(Debian, "buster", "10")), + ("stretch", DistroVersion::new(Debian, "stretch", "9")), // ubuntu ( - "resolute", - DistroVersion { - distro: Distro::Ubuntu, - codename: "resolute".to_string(), - version: "26.04".to_string(), - }, - ), - ( - "questing", - DistroVersion { - distro: Distro::Ubuntu, - codename: "questing".to_string(), - version: "25.10".to_string(), - }, - ), - ( - "noble", - DistroVersion { - distro: Distro::Ubuntu, - codename: "noble".to_string(), - version: "24.04".to_string(), - }, - ), - ( - "jammy", - DistroVersion { - distro: Distro::Ubuntu, - codename: "jammy".to_string(), - version: "22.04".to_string(), - }, - ), - ( - "focal", - DistroVersion { - distro: Distro::Ubuntu, - codename: "focal".to_string(), - version: "20.04".to_string(), - }, - ), - ( - "bionic", - DistroVersion { - distro: Distro::Ubuntu, - codename: "bionic".to_string(), - version: "18.04".to_string(), - }, - ), - ( - "xenial", - DistroVersion { - distro: Distro::Ubuntu, - codename: "xenial".to_string(), - version: "16.04".to_string(), - }, - ), - ( - "trusty", - DistroVersion { - distro: Distro::Ubuntu, - codename: "trusty".to_string(), - version: "14.04".to_string(), - }, + "stonking", + DistroVersion::new(Ubuntu, "stonking", "26.10").devel(), ), + ("resolute", DistroVersion::new(Ubuntu, "resolute", "26.04")), + ("noble", DistroVersion::new(Ubuntu, "noble", "24.04")), + ("jammy", DistroVersion::new(Ubuntu, "jammy", "22.04")), + ("focal", DistroVersion::new(Ubuntu, "focal", "20.04")), + ("bionic", DistroVersion::new(Ubuntu, "bionic", "18.04")), + ("xenial", DistroVersion::new(Ubuntu, "xenial", "16.04")), + ("trusty", DistroVersion::new(Ubuntu, "trusty", "14.04")), ]) }); diff --git a/packages/debmagic/Cargo.toml b/packages/debmagic/Cargo.toml index 5281842..6be06a1 100644 --- a/packages/debmagic/Cargo.toml +++ b/packages/debmagic/Cargo.toml @@ -19,4 +19,5 @@ serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } uuid = { workspace = true, features = ["v4"] } debian-changelog = { workspace = true } +debian-control = { workspace = true } ignore = { workspace = true } diff --git a/packages/debmagic/src/build.rs b/packages/debmagic/src/build.rs deleted file mode 100644 index 7bd28d9..0000000 --- a/packages/debmagic/src/build.rs +++ /dev/null @@ -1,544 +0,0 @@ -use core::time; -use std::net::Shutdown; -use std::os::unix::net::{UnixListener, UnixStream}; -use std::sync::{Arc, Mutex}; -use std::{ - fs, - io::{self, BufReader, IsTerminal, Read, Write, stdout}, - path::{Path, PathBuf}, - thread, -}; - -use crate::build::config::DriverOverrides; -use crate::{ - build::{ - common::{BuildConfig, BuildDriver, BuildDriverType, BuildMetadata}, - config::DriverConfig, - driver_bare::DriverBare, - driver_docker::DriverDocker, - }, - config::Config, - package::PackageDescription, -}; -use anyhow::{Context, anyhow}; -use debmagic_common::distro::DistroVersion; -use glob::glob; - -pub mod common; -pub mod config; -pub mod driver_bare; -pub mod driver_docker; - -struct Build { - config: BuildConfig, - pub driver: Box, -} - -fn get_build_driver( - config: &BuildConfig, - driver_config: &DriverConfig, - driver_overrides: &DriverOverrides, -) -> anyhow::Result> { - match config.driver { - BuildDriverType::Docker => Ok(Box::new(DriverDocker::create( - config, - driver_config, - &driver_overrides.docker, - )?)), - BuildDriverType::Bare => Ok(Box::new(DriverBare::create( - config, - driver_config, - &driver_overrides.bare, - ))), - // BuildDriverType::Lxd => ... - } -} - -fn create_driver_from_metadata( - config: &DriverConfig, - metadata: &BuildMetadata, -) -> anyhow::Result> { - let driver: anyhow::Result> = match &metadata.config.driver { - BuildDriverType::Docker => Ok(Box::new(DriverDocker::from_build_metadata( - &metadata.config, - config, - metadata, - ))), - BuildDriverType::Bare => Ok(Box::new(DriverBare::from_build_metadata( - &metadata.config, - config, - metadata, - ))), - // BuildDriverType::Lxd => ... - }; - driver -} - -impl Build { - pub fn create( - config: &BuildConfig, - driver_config: &DriverConfig, - driver_overrides: &DriverOverrides, - ) -> anyhow::Result { - let driver = get_build_driver(config, driver_config, driver_overrides) - .context(format!("failed to create {:?} build driver", config.driver))?; - Ok(Self { - config: config.clone(), - driver, - }) - } - - pub fn from_build_root( - build_root: &Path, - driver_config: &DriverConfig, - ) -> anyhow::Result { - let build_metadata_path = build_root.join("build.json"); - if !build_metadata_path.is_file() { - return Err(anyhow!("No build.json found")); - } - // read metadata from file - let file = fs::OpenOptions::new() - .read(true) - .open(&build_metadata_path)?; - let metadata = || -> anyhow::Result { - let reader = BufReader::new(&file); - let metadata: BuildMetadata = serde_json::from_reader(reader).with_context(|| { - format!( - "Failed to read build metadata from {} - invalid json", - build_metadata_path.display() - ) - })?; - Ok(metadata) - }(); - - let metadata = metadata?; - - let driver = create_driver_from_metadata(driver_config, &metadata)?; - - // Try to signal the main debmagic build process that a shell attached - send_socket_command(build_root, "attach") - .context("No debmagic build is currently running for this source directory")?; - - Ok(Self { - config: metadata.config.clone(), - driver, - }) - } - - pub fn detach(&self) -> anyhow::Result<()> { - let build_root = &self.config.build_root_dir; - send_socket_command(build_root, "detach")?; - Ok(()) - } - - pub fn write_metadata(&self) -> anyhow::Result<()> { - let metadata = BuildMetadata { - config: self.config.clone(), - driver_metadata: self.driver.get_build_metadata(), - }; - let path = self.config.build_root_dir.join("build.json"); - let json = serde_json::to_string_pretty(&metadata) - .context("Failed to serialize build metadata")?; - fs::write(path, json)?; - Ok(()) - } -} - -fn copy_glob(src_dir: &Path, pattern: &str, dest_dir: &Path) -> anyhow::Result<()> { - let full_pattern = src_dir.join(pattern).to_string_lossy().into_owned(); - for entry in glob(&full_pattern)? { - let path = entry?; - if path.is_file() { - let filename = path.file_name().ok_or(anyhow!( - "Could not retrieve filename from {}", - path.display() - ))?; - fs::copy(&path, dest_dir.join(filename))?; - } - } - Ok(()) -} - -fn socket_path_for_build(build_root: &Path) -> PathBuf { - build_root.join("build.sock") -} - -fn start_socket_server( - build_root: &Path, - should_exit: Arc>, -) -> anyhow::Result> { - let sock = socket_path_for_build(build_root); - if sock.exists() { - // try to remove stale socket file - let _ = fs::remove_file(&sock); - } - - let listener = UnixListener::bind(&sock) - .with_context(|| format!("failed to bind unix socket {}", sock.display()))?; - - // Set non-blocking mode so we can check the exit flag - listener - .set_nonblocking(true) - .context("failed to set socket non-blocking")?; - - let handle = thread::spawn(move || { - let mut num_attached = 0u64; - loop { - // Check if we should exit - let exit_requested = *should_exit.lock().unwrap(); - if exit_requested && num_attached == 0 { - break; - } - - match listener.accept() { - Ok((mut s, _)) => { - let mut buf = String::new(); - if s.read_to_string(&mut buf).is_err() { - let _ = s.shutdown(Shutdown::Both); - continue; - } - let cmd = buf.trim(); - match cmd { - "attach" => { - num_attached += 1; - } - "detach" => { - num_attached = num_attached.saturating_sub(1); - } - _ => {} - } - let _ = s.shutdown(Shutdown::Both); - } - Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => { - // No connection available, sleep briefly to avoid busy-waiting - thread::sleep(time::Duration::from_millis(10)); - } - Err(_) => break, - } - } - let _ = fs::remove_file(&sock); - }); - - Ok(handle) -} - -fn send_socket_command(build_root: &Path, cmd: &str) -> anyhow::Result<()> { - let sock = socket_path_for_build(build_root); - let mut stream = UnixStream::connect(&sock) - .with_context(|| format!("failed to connect to socket {}", sock.display()))?; - stream - .write_all(cmd.as_bytes()) - .context("failed to send socket command")?; - stream.shutdown(Shutdown::Write).ok(); - Ok(()) -} - -fn copy_dir_all(src: impl AsRef, dst: impl AsRef) -> anyhow::Result<()> { - fs::create_dir_all(&dst)?; - - let walker = ignore::WalkBuilder::new(&src) - .standard_filters(true) - .hidden(false) - .filter_entry(|entry| !(entry.path().is_dir() && entry.path().ends_with(".git"))) - .build(); - - for entry in walker { - let entry = entry?; - let file_type = entry.file_type().ok_or(anyhow!( - "failed to get file type of {}", - entry.path().display() - ))?; - - // get path of entry relative to src - let relative_path = entry - .path() - .strip_prefix(src.as_ref()) - .context("failed to get relative path")?; - - if file_type.is_dir() { - fs::create_dir_all(dst.as_ref().join(relative_path))?; - } else if file_type.is_file() { - fs::copy(entry.path(), dst.as_ref().join(relative_path)) - .context(format!("failed to copy file: {}", entry.path().display()))?; - } - // handle hardlinks, symlinks and similar weird filetypes - } - Ok(()) -} - -fn get_build_root_and_identifier( - config: &Config, - package: &PackageDescription, -) -> (String, PathBuf) { - let package_identifier = format!("{}-{}", package.name, package.version); - let build_root = config.temp_build_dir.join(&package_identifier); - (package_identifier, build_root) -} - -/// Determine which distro version to use for the build. -/// -/// If only one distro version is specified in the changelog, it's used automatically. -/// If multiple distro versions are specified, an explicit --distro is required. -/// If --distro is provided, it's validated against the changelog versions. -fn resolve_distro_version( - changelog_distros: &[String], - explicit_distro: Option<&str>, -) -> anyhow::Result { - let resolved_codename = match (changelog_distros.len(), explicit_distro) { - (0, _) => Err(anyhow!("changelog contains no distributions")), - (1, None) => Ok(changelog_distros[0].clone()), - (1, Some(explicit)) => { - if explicit == changelog_distros[0] { - Ok(explicit.to_string()) - } else { - Err(anyhow!( - "explicit distro version '{}' conflicts with distribution specified in changelog '{}'", - explicit, - changelog_distros[0] - )) - } - } - (_, None) => Err(anyhow!( - "changelog contains multiple distributions ({}), please specify which one to build for with --distro", - changelog_distros.join(", ") - )), - (_, Some(explicit)) => { - if changelog_distros.contains(&explicit.to_string()) { - Ok(explicit.to_string()) - } else { - Err(anyhow!( - "explicit distro version '{}' not found in changelog distributions: {}", - explicit, - changelog_distros.join(", ") - )) - } - } - }?; - let resolved = debmagic_common::distro::get_distro_version(&resolved_codename) - .ok_or_else(|| anyhow!("unknown distro codename '{}'", resolved_codename))?; - Ok(resolved) -} - -fn prepare_build_env( - config: &Config, - driver_overrides: &DriverOverrides, - package: &PackageDescription, - driver_type: BuildDriverType, - output_dir: &Path, - explicit_distro_version: Option<&str>, -) -> anyhow::Result { - let (package_identifier, build_root) = get_build_root_and_identifier(config, package); - if build_root.exists() { - fs::remove_dir_all(&build_root)?; - } - - let distro_version = resolve_distro_version(&package.distro_versions, explicit_distro_version) - .context("failed to determine distro version")?; - - let build_config = BuildConfig { - driver: driver_type, - package_identifier, - source_dir: package.source_dir.clone(), - output_dir: output_dir.to_path_buf(), - build_root_dir: build_root, - distro: distro_version.clone(), - sign_package: false, - }; - - build_config - .create_dirs() - .context("failed to create build directories")?; - - copy_dir_all(&build_config.source_dir, build_config.build_source_dir()) - .context("failed to copy source tree to build directory")?; - - let build = Build::create(&build_config, &config.driver, driver_overrides)?; - Ok(build) -} - -pub fn get_shell_in_build(config: &Config, package: &PackageDescription) -> anyhow::Result<()> { - let (_package_identifier, build_root) = get_build_root_and_identifier(config, package); - let build = Build::from_build_root(&build_root, &config.driver)?; - let result = build - .driver - .interactive_shell(&build.config.build_source_dir()); - - build.detach()?; - - result?; - Ok(()) -} - -pub fn build_package( - config: &Config, - package: &PackageDescription, - driver_type: BuildDriverType, - driver_overrides: &DriverOverrides, - output_dir: &Path, - explicit_distro_version: Option<&str>, -) -> anyhow::Result<()> { - let build = prepare_build_env( - config, - driver_overrides, - package, - driver_type, - output_dir, - explicit_distro_version, - ) - .context("failed to prepare build environment")?; - build - .write_metadata() - .context("failed to write build metadata")?; - - let should_exit = Arc::new(Mutex::new(false)); - let socket_server_handle = - start_socket_server(&build.config.build_root_dir, should_exit.clone())?; - - let result = (|| -> anyhow::Result<()> { - build.driver.run_command( - &["apt-get", "-y", "build-dep", "."], - &build.config.build_source_dir(), - true, - )?; - build.driver.run_command( - &["dpkg-buildpackage", "-us", "-uc", "-ui", "-nc", "-b"], - &build.config.build_source_dir(), - false, - )?; - - if build.config.sign_package { - // SIGN .changes and .dsc files - // changes = *.changes / *.dsc - // driver.run_command(&["debsign", opts, changes], &build_config.build_source_dir(), false)?; - // driver.run_command(&["debrsign", opts, username, changes], &build_config.build_source_dir(), false)?; - } - - let parent_dir = build.config.build_source_dir().join(".."); - copy_glob(&parent_dir, "*.deb", &build.config.output_dir)?; - copy_glob(&parent_dir, "*.changes", &build.config.output_dir)?; - copy_glob(&parent_dir, "*.buildinfo", &build.config.output_dir)?; - copy_glob(&parent_dir, "*.dsc", &build.config.output_dir)?; - - Ok(()) - })(); - - if let Err(e) = result { - if stdout().is_terminal() { - eprintln!("Build failed: {e}. Dropping into shell..."); - let res = build - .driver - .interactive_shell(&build.config.build_source_dir()); - if let Err(shell_error) = res { - eprintln!("Dropping into shell failed: {shell_error}"); - } - } else { - eprintln!("Build failed: {e}"); - } - build.driver.cleanup(); - *should_exit.lock().unwrap() = true; - if !socket_server_handle.is_finished() { - println!("Waiting for all attached shells to exit..."); - } - socket_server_handle.join().ok(); - return Err(e); - } - - // Signal the socket server to exit and wait for it to complete - *should_exit.lock().unwrap() = true; - if !socket_server_handle.is_finished() { - println!("Waiting for all attached shells to exit..."); - } - socket_server_handle.join().ok(); - - build.driver.cleanup(); - Ok(()) -} - -#[cfg(test)] -mod tests { - use debmagic_common::distro::Distro; - - use super::*; - - #[test] - fn test_resolve_distro_version_single_distro_no_explicit() { - let distros = vec!["forky".to_string()]; - let result = resolve_distro_version(&distros, None); - assert!(result.is_ok()); - let distro_version = result.unwrap(); - assert_eq!(distro_version.codename, "forky"); - assert_eq!(distro_version.distro, Distro::Debian); - } - - #[test] - fn test_resolve_distro_version_single_distro_matching_explicit() { - let distros = vec!["forky".to_string()]; - let result = resolve_distro_version(&distros, Some("forky")); - assert!(result.is_ok()); - let distro_version = result.unwrap(); - assert_eq!(distro_version.codename, "forky"); - assert_eq!(distro_version.distro, Distro::Debian); - } - - #[test] - fn test_resolve_distro_version_single_distro_conflicting_explicit() { - let distros = vec!["forky".to_string()]; - let result = resolve_distro_version(&distros, Some("duke")); - assert!(result.is_err()); - assert!( - result - .unwrap_err() - .to_string() - .contains("conflicts with distribution specified in changelog") - ); - } - - #[test] - fn test_resolve_distro_version_multiple_distros_no_explicit() { - let distros = vec!["forky".to_string(), "duke".to_string()]; - let result = resolve_distro_version(&distros, None); - assert!(result.is_err()); - assert!( - result - .unwrap_err() - .to_string() - .contains("multiple distributions") - ); - } - - #[test] - fn test_resolve_distro_version_multiple_distros_explicit_valid() { - let distros = vec!["forky".to_string(), "duke".to_string()]; - let result = resolve_distro_version(&distros, Some("duke")); - assert!(result.is_ok()); - let distro_version = result.unwrap(); - assert_eq!(distro_version.codename, "duke"); - assert_eq!(distro_version.distro, Distro::Debian); - } - - #[test] - fn test_resolve_distro_version_multiple_distros_explicit_invalid() { - let distros = vec!["forky".to_string(), "duke".to_string()]; - let result = resolve_distro_version(&distros, Some("trixie")); - assert!(result.is_err()); - assert!( - result - .unwrap_err() - .to_string() - .contains("not found in changelog distributions") - ); - } - - #[test] - fn test_resolve_distro_version_empty_distros() { - let distros: Vec = vec![]; - let result = resolve_distro_version(&distros, None); - assert!(result.is_err()); - assert!( - result - .unwrap_err() - .to_string() - .contains("changelog contains no distributions") - ); - } -} diff --git a/packages/debmagic/src/build/artifacts.rs b/packages/debmagic/src/build/artifacts.rs new file mode 100644 index 0000000..d91c520 --- /dev/null +++ b/packages/debmagic/src/build/artifacts.rs @@ -0,0 +1,219 @@ +use std::{ + ffi::OsStr, + fs, + path::{Component, Path, PathBuf}, +}; + +use anyhow::{Context, anyhow, bail}; +use debian_control::lossless::changes::Changes; + +fn changes_file_in(build_dir: &Path) -> anyhow::Result { + let mut paths = fs::read_dir(build_dir) + .with_context(|| { + format!( + "failed to read build output directory {}", + build_dir.display() + ) + })? + .filter_map(|entry| entry.ok().map(|entry| entry.path())) + .filter(|path| path.extension() == Some(OsStr::new("changes"))); + + let path = paths + .next() + .ok_or_else(|| anyhow!("build produced no .changes file in {}", build_dir.display()))?; + if paths.next().is_some() { + bail!( + "build produced multiple .changes files in {}; refusing to guess which upload set to export", + build_dir.display() + ); + } + Ok(path) +} + +fn artifact_filename(filename: &str) -> anyhow::Result<&OsStr> { + let path = Path::new(filename); + let mut components = path.components(); + match (components.next(), components.next()) { + (Some(Component::Normal(filename)), None) => Ok(filename), + _ => bail!("invalid artifact filename in .changes: {filename:?}"), + } +} + +fn reject_destination_symlink(path: &Path) -> anyhow::Result<()> { + match fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_symlink() => { + bail!("refusing to overwrite artifact symlink {}", path.display()) + } + Ok(_) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + } +} + +pub fn export_build_artifacts(build_dir: &Path, output_dir: &Path) -> anyhow::Result { + fs::create_dir_all(output_dir) + .with_context(|| format!("failed to create output directory {}", output_dir.display()))?; + + let changes_path = changes_file_in(build_dir)?; + let changes_metadata = fs::symlink_metadata(&changes_path)?; + if !changes_metadata.file_type().is_file() { + bail!( + "changes file {} is not a regular file", + changes_path.display() + ); + } + let changes = Changes::from_file(&changes_path) + .with_context(|| format!("failed to parse {}", changes_path.display()))?; + let files = changes + .files() + .ok_or_else(|| anyhow!("{} has no Files field", changes_path.display()))?; + + for file in files { + let filename = artifact_filename(&file.filename)?; + let source = build_dir.join(filename); + let metadata = fs::symlink_metadata(&source).with_context(|| { + format!( + "artifact {} referenced by {} does not exist", + source.display(), + changes_path.display() + ) + })?; + if !metadata.file_type().is_file() { + bail!("build artifact {} is not a regular file", source.display()); + } + if metadata.len() != file.size as u64 { + bail!( + "artifact {} has size {}, but {} records {}", + source.display(), + metadata.len(), + changes_path.display(), + file.size + ); + } + let destination = output_dir.join(filename); + reject_destination_symlink(&destination)?; + fs::copy(&source, destination) + .with_context(|| format!("failed to copy build artifact {}", source.display()))?; + } + + let changes_filename = changes_path + .file_name() + .ok_or_else(|| anyhow!("invalid .changes path: {}", changes_path.display()))?; + let exported_changes = output_dir.join(changes_filename); + reject_destination_symlink(&exported_changes)?; + fs::copy(&changes_path, &exported_changes) + .with_context(|| format!("failed to copy {}", changes_path.display()))?; + Ok(exported_changes) +} + +#[cfg(test)] +mod tests { + use std::os::unix::fs::symlink; + + use super::*; + + fn test_dir(name: &str) -> PathBuf { + let path = std::env::temp_dir().join(format!("debmagic-{name}-{}", uuid::Uuid::new_v4())); + fs::create_dir(&path).unwrap(); + path + } + + fn write_changes(path: &Path, files: &[(&str, &str)]) { + let files = files + .iter() + .map(|(filename, contents)| { + fs::write(path.parent().unwrap().join(filename), contents).unwrap(); + format!( + " d41d8cd98f00b204e9800998ecf8427e {} misc optional {}", + contents.len(), + filename + ) + }) + .collect::>() + .join("\n"); + fs::write( + path, + format!("Format: 1.8\nSource: test\nFiles:\n{files}\n"), + ) + .unwrap(); + } + + #[test] + fn exports_every_file_listed_by_changes() { + let build_dir = test_dir("artifact-build"); + let output_dir = test_dir("artifact-output"); + let changes_path = build_dir.join("test_1_amd64.changes"); + let artifacts = [ + ("test_1_amd64.deb", "deb"), + ("test_1_amd64.udeb", "udeb"), + ("test-dbgsym_1_amd64.ddeb", "ddeb"), + ("test_1_amd64.buildinfo", "buildinfo"), + ]; + write_changes(&changes_path, &artifacts); + + let exported_changes = export_build_artifacts(&build_dir, &output_dir).unwrap(); + + assert_eq!(exported_changes, output_dir.join("test_1_amd64.changes")); + for (filename, contents) in artifacts { + assert_eq!( + fs::read_to_string(output_dir.join(filename)).unwrap(), + contents + ); + } + fs::remove_dir_all(build_dir).unwrap(); + fs::remove_dir_all(output_dir).unwrap(); + } + + #[test] + fn rejects_ambiguous_changes_files() { + let build_dir = test_dir("artifact-ambiguous"); + let output_dir = test_dir("artifact-ambiguous-output"); + fs::write(build_dir.join("one.changes"), "").unwrap(); + fs::write(build_dir.join("two.changes"), "").unwrap(); + + let error = export_build_artifacts(&build_dir, &output_dir).unwrap_err(); + + assert!(error.to_string().contains("multiple .changes files")); + fs::remove_dir_all(build_dir).unwrap(); + fs::remove_dir_all(output_dir).unwrap(); + } + + #[test] + fn rejects_artifact_symlinks() { + let build_dir = test_dir("artifact-symlink"); + let output_dir = test_dir("artifact-symlink-output"); + let changes_path = build_dir.join("test_1_amd64.changes"); + write_changes(&changes_path, &[("test_1_amd64.deb", "deb")]); + fs::remove_file(build_dir.join("test_1_amd64.deb")).unwrap(); + fs::write(build_dir.join("target"), "deb").unwrap(); + symlink("target", build_dir.join("test_1_amd64.deb")).unwrap(); + + let error = export_build_artifacts(&build_dir, &output_dir).unwrap_err(); + + assert!(error.to_string().contains("not a regular file")); + fs::remove_dir_all(build_dir).unwrap(); + fs::remove_dir_all(output_dir).unwrap(); + } + + #[test] + fn rejects_destination_symlinks() { + let build_dir = test_dir("artifact-destination-symlink"); + let output_dir = test_dir("artifact-destination-symlink-output"); + let changes_path = build_dir.join("test_1_amd64.changes"); + write_changes(&changes_path, &[("test_1_amd64.deb", "deb")]); + let target = output_dir.join("target"); + fs::write(&target, "unchanged").unwrap(); + symlink("target", output_dir.join("test_1_amd64.deb")).unwrap(); + + let error = export_build_artifacts(&build_dir, &output_dir).unwrap_err(); + + assert!( + error + .to_string() + .contains("refusing to overwrite artifact symlink") + ); + assert_eq!(fs::read_to_string(target).unwrap(), "unchanged"); + fs::remove_dir_all(build_dir).unwrap(); + fs::remove_dir_all(output_dir).unwrap(); + } +} diff --git a/packages/debmagic/src/build/attach.rs b/packages/debmagic/src/build/attach.rs new file mode 100644 index 0000000..15110d4 --- /dev/null +++ b/packages/debmagic/src/build/attach.rs @@ -0,0 +1,93 @@ +//! Unix-socket attach/detach tracking for build environments. +//! +//! While a build keeps its environment alive after a failure, a small +//! socket server in the build root lets concurrent `debmagic shell` +//! sessions register themselves, so the environment is only torn down +//! once the last attached shell detaches. + +use core::time; +use std::net::Shutdown; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::sync::{Arc, Mutex}; +use std::{ + fs, io, + io::{Read, Write}, + path::{Path, PathBuf}, + thread, +}; + +use anyhow::Context; + +fn socket_path_for_build(build_root: &Path) -> PathBuf { + build_root.join("build.sock") +} + +pub fn start_socket_server( + build_root: &Path, + should_exit: Arc>, +) -> anyhow::Result> { + let sock = socket_path_for_build(build_root); + if sock.exists() { + // try to remove stale socket file + let _ = fs::remove_file(&sock); + } + + let listener = UnixListener::bind(&sock) + .with_context(|| format!("failed to bind unix socket {}", sock.display()))?; + + // Set non-blocking mode so we can check the exit flag + listener + .set_nonblocking(true) + .context("failed to set socket non-blocking")?; + + let handle = thread::spawn(move || { + let mut num_attached = 0u64; + loop { + // Check if we should exit + let exit_requested = *should_exit.lock().unwrap(); + if exit_requested && num_attached == 0 { + break; + } + + match listener.accept() { + Ok((mut s, _)) => { + let mut buf = String::new(); + if s.read_to_string(&mut buf).is_err() { + let _ = s.shutdown(Shutdown::Both); + continue; + } + let cmd = buf.trim(); + match cmd { + "attach" => { + num_attached += 1; + } + "detach" => { + num_attached = num_attached.saturating_sub(1); + } + _ => {} + } + let _ = s.shutdown(Shutdown::Both); + } + Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => { + // No connection available, sleep briefly to avoid busy-waiting + thread::sleep(time::Duration::from_millis(10)); + } + Err(_) => break, + } + } + let _ = fs::remove_file(&sock); + }); + + Ok(handle) +} + +pub fn send_socket_command(build_root: &Path, cmd: &str) -> anyhow::Result<()> { + let sock = socket_path_for_build(build_root); + let mut stream = UnixStream::connect(&sock) + .with_context(|| format!("failed to connect to socket {}", sock.display()))?; + stream + .write_all(cmd.as_bytes()) + .context("failed to send socket command")?; + stream.shutdown(Shutdown::Write).ok(); + Ok(()) +} diff --git a/packages/debmagic/src/build/common.rs b/packages/debmagic/src/build/common.rs index e420f3f..796e708 100644 --- a/packages/debmagic/src/build/common.rs +++ b/packages/debmagic/src/build/common.rs @@ -3,17 +3,115 @@ use std::{ fmt::Debug, fs, path::{Path, PathBuf}, + process::Command, }; +use anyhow::Context; use clap::ValueEnum; use debmagic_common::distro::DistroVersion; use serde::{Deserialize, Serialize}; +/// Path at which the build root is bind-mounted inside container-based +/// drivers (Docker, LXD, Incus). +pub const BUILD_DIR_IN_CONTAINER: &str = "/debmagic"; + +/// Rewrite a path inside the host's build root to the equivalent path inside +/// a container that has it bind-mounted at [`BUILD_DIR_IN_CONTAINER`]. +pub fn translate_path_in_container( + build_root_dir: &Path, + path_in_source: &Path, +) -> std::io::Result { + path_in_source + .strip_prefix(build_root_dir) + .map(|rel| Path::new(BUILD_DIR_IN_CONTAINER).join(rel)) + .map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::NotFound, + "Path is not relative to build root".to_string(), + ) + }) +} + +/// Run `cmd`, failing with `context` (and, on a clean but unsuccessful exit, +/// its exit status) if it can't be spawned or exits unsuccessfully. +pub fn run_checked(cmd: &mut Command, context: &str) -> anyhow::Result<()> { + let status = cmd + .status() + .with_context(|| format!("Error running {context}"))?; + if !status.success() { + anyhow::bail!("{context} failed (exit status: {status})"); + } + Ok(()) +} + +pub fn resource_name(prefix: &str, label: &str, identifier: &str) -> String { + const MAX_LEN: usize = 63; + const HASH_LEN: usize = 16; + + let mut hash = uuid::Uuid::new_v5(&uuid::Uuid::NAMESPACE_OID, identifier.as_bytes()) + .simple() + .to_string(); + hash.truncate(HASH_LEN); + let max_label_len = MAX_LEN - prefix.len() - hash.len() - 2; + let label = label + .chars() + .map(|character| { + if character.is_ascii_alphanumeric() { + character.to_ascii_lowercase() + } else { + '-' + } + }) + .take(max_label_len) + .collect::(); + format!("{prefix}-{label}-{hash}") +} + +pub fn environment_fingerprint(parts: &[&str]) -> String { + uuid::Uuid::new_v5(&uuid::Uuid::NAMESPACE_OID, parts.join("\0").as_bytes()) + .simple() + .to_string() +} + +/// Metadata key under which container-based drivers store their container's +/// name for later reattachment via `from_build_metadata`. +const CONTAINER_NAME_KEY: &str = "container_name"; + +pub fn container_name_metadata(name: &str) -> DriverSpecificBuildMetadata { + DriverSpecificBuildMetadata::from([(CONTAINER_NAME_KEY.to_string(), name.to_string())]) +} + +pub fn container_name_from_metadata(build_metadata: &BuildMetadata) -> anyhow::Result { + build_metadata + .driver_metadata + .get(CONTAINER_NAME_KEY) + .cloned() + .context("build metadata has no container_name") +} + #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum, Serialize, Deserialize)] pub enum BuildDriverType { Docker, Bare, - // Lxd + Lxd, + Incus, +} + +/// Selects which files from the source directory are staged into the build tree. +#[derive( + Debug, Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum, Serialize, Deserialize, +)] +#[serde(rename_all = "snake_case")] +pub enum SourceSyncMode { + /// Git-tracked files, including uncommitted modifications. Untracked + /// files are not staged and reported as a warning. + #[default] + Tracked, + /// Like `tracked`, but the build fails if the worktree has uncommitted + /// changes or untracked files. + Committed, + /// All files except git-ignored ones, regardless of git tracking state. + Worktree, } pub type DriverSpecificBuildMetadata = HashMap; @@ -28,12 +126,33 @@ pub struct BuildMetadata { pub struct BuildConfig { pub driver: BuildDriverType, + #[serde(default)] + pub package_name: String, pub package_identifier: String, pub build_root_dir: PathBuf, pub source_dir: PathBuf, pub output_dir: PathBuf, pub distro: DistroVersion, pub sign_package: bool, + #[serde(default)] + pub sign_with: crate::build::signing::SignWith, + /// GPG key ID/email to sign with (debsign's `-k` option). + pub sign_key: Option, + /// Build the automatic `-dbgsym` debug symbol package alongside the regular binaries. + #[serde(default)] + pub build_debug_symbols: bool, + /// Run `debian/rules clean` before building. + #[serde(default)] + pub clean: bool, + /// Keep the build environment running after the build. + #[serde(default)] + pub persistent: bool, + /// Synchronize source inputs while preserving build-generated files. + #[serde(default)] + pub incremental: bool, + /// Which source files are staged into the build tree. + #[serde(default)] + pub source_sync_mode: SourceSyncMode, } impl BuildConfig { @@ -44,14 +163,6 @@ impl BuildConfig { ) } - /// Identifier safe for Docker image tags and container names. - /// - /// Debian versions may contain `~`, `+`, or `:` (e.g. `0.0.1~alpha2`), which - /// are invalid in Docker references. - pub fn docker_identifier(&self) -> String { - sanitize_docker_reference(&self.build_identifier()) - } - pub fn build_work_dir(&self) -> PathBuf { self.build_root_dir.join("work") } @@ -73,34 +184,76 @@ impl BuildConfig { } } -/// Replace characters that Docker image/container names disallow. -fn sanitize_docker_reference(name: &str) -> String { - name.chars() - .map(|c| match c { - 'a'..='z' | 'A'..='Z' | '0'..='9' | '.' | '_' | '-' => c, - _ => '-', - }) - .collect() -} +/// Source of a Python script that rewrites the default Debian/Ubuntu apt +/// sources to point at a mirror. Replaces the base image's default sources +/// file(s) outright with one debmagic owns, rather than parsing and +/// patching them in place, defaulting to the release/updates/security +/// pockets it detects from `/etc/os-release`. +pub const APT_MIRROR_SCRIPT: &str = include_str!("scripts/mirror.py"); pub trait BuildDriver { fn get_build_metadata(&self) -> DriverSpecificBuildMetadata; - fn run_command(&self, cmd: &[&str], cwd: &Path, requires_root: bool) -> std::io::Result<()>; + fn run_command_env( + &self, + cmd: &[&str], + cwd: &Path, + requires_root: bool, + env_add: &[(&str, &str)], + ) -> std::io::Result<()>; - fn cleanup(&self); + fn run_command(&self, cmd: &[&str], cwd: &Path, requires_root: bool) -> std::io::Result<()> { + self.run_command_env(cmd, cwd, requires_root, &[]) + } + + fn cleanup(&self) -> anyhow::Result<()>; fn interactive_shell(&self, cwd: &Path) -> std::io::Result<()>; fn driver_type(&self) -> BuildDriverType; + + fn reset_build_root(&self) -> std::io::Result<()>; + + /// Sign `changes_file` (a path on the host) with `debsign` — on the host + /// for the bare driver, or inside a minimal same-distro container with + /// the host's gpg-agent socket forwarded in. `gpg` carries the agent + /// socket and key for container signing; bare ignores it. + fn sign_changes( + &self, + changes_file: &Path, + gpg: Option<&crate::build::signing::GpgForwarding>, + ) -> anyhow::Result<()>; + + fn reused_environment(&self) -> bool { + true + } } #[cfg(test)] mod tests { use super::*; + use debmagic_common::distro::{Distro, DistroVersion}; use std::path::PathBuf; + #[test] + fn resource_names_are_valid_stable_and_distinct() { + let first = resource_name("debmagic", "package", "package-1.0~beta-1:2-debian-forky"); + let second = resource_name("debmagic", "package", "package-1.0-beta-1:2-debian-forky"); + + assert_eq!( + first, + resource_name("debmagic", "package", "package-1.0~beta-1:2-debian-forky") + ); + assert_ne!(first, second); + assert!(first.starts_with("debmagic-package-")); + assert_eq!(first.len(), "debmagic-package-".len() + 16); + assert!(first.len() <= 63); + assert!(first.chars().all(|character| character.is_ascii_lowercase() + || character.is_ascii_digit() + || character == '-')); + } + fn sample_config(package_identifier: &str) -> BuildConfig { BuildConfig { driver: BuildDriverType::Docker, @@ -112,8 +265,17 @@ mod tests { distro: Distro::Debian, codename: "forky".to_string(), version: "15".to_string(), + is_devel: false, }, sign_package: false, + incremental: false, + build_debug_symbols: false, + clean: false, + persistent: false, + package_name: "debmagic".to_string(), + sign_key: None, + sign_with: crate::build::signing::SignWith::Auto, + source_sync_mode: crate::build::common::SourceSyncMode::Tracked, } } @@ -124,23 +286,5 @@ mod tests { config.build_identifier(), "debmagic-0.0.1~alpha2-debian-forky" ); - assert_eq!( - config.docker_identifier(), - "debmagic-0.0.1-alpha2-debian-forky" - ); - } - - #[test] - fn docker_identifier_replaces_epoch_and_plus() { - let config = sample_config("pkg-1:2.0.0+dfsg1"); - assert_eq!(config.docker_identifier(), "pkg-1-2.0.0-dfsg1-debian-forky"); - } - - #[test] - fn sanitize_docker_reference_keeps_allowed_chars() { - assert_eq!( - sanitize_docker_reference("debmagic-0.0.1-alpha1_x86"), - "debmagic-0.0.1-alpha1_x86" - ); } } diff --git a/packages/debmagic/src/build/config.rs b/packages/debmagic/src/build/config.rs index 193d454..79a4aaf 100644 --- a/packages/debmagic/src/build/config.rs +++ b/packages/debmagic/src/build/config.rs @@ -2,17 +2,27 @@ use serde::Deserialize; use crate::build::driver_bare::{DriverBareConfig, DriverBareConfigOverrides}; use crate::build::driver_docker::{DriverDockerConfig, DriverDockerConfigOverrides}; +use crate::build::driver_lxd::{DriverLxdConfig, DriverLxdConfigOverrides}; #[derive(Deserialize, Debug, Clone, Default)] #[serde(default)] pub struct DriverConfig { pub persistent: bool, + /// Not used by the bare driver, which builds on the host's own sources. + pub apt_mirror: Option, + /// Also enable the `-proposed` pocket. Not used by the bare + /// driver, which builds on the host's own sources. + pub proposed: bool, pub docker: DriverDockerConfig, pub bare: DriverBareConfig, + pub lxd: DriverLxdConfig, } #[derive(Deserialize, Debug, Clone, Default)] pub struct DriverOverrides { + pub apt_mirror: Option, + pub proposed: Option, pub docker: DriverDockerConfigOverrides, pub bare: DriverBareConfigOverrides, + pub lxd: DriverLxdConfigOverrides, } diff --git a/packages/debmagic/src/build/driver_bare.rs b/packages/debmagic/src/build/driver_bare.rs index 647df21..a163023 100644 --- a/packages/debmagic/src/build/driver_bare.rs +++ b/packages/debmagic/src/build/driver_bare.rs @@ -50,7 +50,13 @@ impl BuildDriver for DriverBare { DriverSpecificBuildMetadata::from([]) } - fn run_command(&self, cmd: &[&str], cwd: &Path, requires_root: bool) -> std::io::Result<()> { + fn run_command_env( + &self, + cmd: &[&str], + cwd: &Path, + requires_root: bool, + env_add: &[(&str, &str)], + ) -> std::io::Result<()> { let mut full_cmd: Vec = Vec::new(); let is_root = unsafe { libc::geteuid() == 0 }; @@ -64,6 +70,7 @@ impl BuildDriver for DriverBare { command.args(&full_cmd[1..]); command.current_dir(cwd); + command.envs(env_add.iter().copied()); let status = command.status()?; @@ -77,8 +84,8 @@ impl BuildDriver for DriverBare { } } - fn cleanup(&self) { - // No-op for bare driver + fn cleanup(&self) -> anyhow::Result<()> { + Ok(()) } fn interactive_shell(&self, _cwd: &Path) -> std::io::Result<()> { @@ -92,4 +99,20 @@ impl BuildDriver for DriverBare { fn driver_type(&self) -> BuildDriverType { BuildDriverType::Bare } + + fn reset_build_root(&self) -> std::io::Result<()> { + if self.config.build_root_dir.exists() { + std::fs::remove_dir_all(&self.config.build_root_dir)?; + } + Ok(()) + } + + fn sign_changes( + &self, + changes_file: &Path, + _gpg: Option<&crate::build::signing::GpgForwarding>, + ) -> anyhow::Result<()> { + crate::build::signing::check_host_debsign_available()?; + crate::build::signing::sign_on_host(changes_file, self.config.sign_key.as_deref()) + } } diff --git a/packages/debmagic/src/build/driver_docker.rs b/packages/debmagic/src/build/driver_docker.rs index e80c6f7..28d656f 100644 --- a/packages/debmagic/src/build/driver_docker.rs +++ b/packages/debmagic/src/build/driver_docker.rs @@ -5,13 +5,16 @@ use std::{ process::{Command, Stdio}, }; -use anyhow::anyhow; +use anyhow::{Context, anyhow}; use debmagic_common::distro::DistroVersion; use serde::{Deserialize, Serialize}; use crate::build::{ common::{ - BuildConfig, BuildDriver, BuildDriverType, BuildMetadata, DriverSpecificBuildMetadata, + APT_MIRROR_SCRIPT, BUILD_DIR_IN_CONTAINER, BuildConfig, BuildDriver, BuildDriverType, + BuildMetadata, DriverSpecificBuildMetadata, container_name_from_metadata, + container_name_metadata, environment_fingerprint, resource_name, run_checked, + translate_path_in_container, }, config::DriverConfig, }; @@ -37,71 +40,98 @@ pub struct DriverDockerConfigOverrides { } // Constants -const BUILD_DIR_IN_CONTAINER: &str = "/debmagic"; -const DOCKER_USER: &str = "user"; +const ENVIRONMENT_LABEL: &str = "dev.debmagic.environment"; + +fn bind_mount_arg(src: &Path, dst: &str) -> String { + format!("type=bind,src={},dst={}", src.display(), dst) +} const DOCKERFILE_TEMPLATE: &str = r#" FROM {base_image} -ARG USERNAME={docker_user} ARG USER_UID=1000 ARG USER_GID=$USER_UID -RUN apt-get update && apt-get install -y sudo dpkg-dev python3 -RUN groupadd --gid $USER_GID $USERNAME \ - && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME \ - && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \ - && chmod 0440 /etc/sudoers.d/$USERNAME -RUN mkdir -p /build/package/debian -RUN --mount=type=bind,source=debian/control,target=/build/package/debian/control apt-get -y build-dep /build/package -RUN mkdir -p {build_dir} -RUN chown $USERNAME:$USERNAME {build_dir} -USER $USERNAME +RUN apt-get update && apt-get install -y dpkg-dev python3 +{apt_mirror_setup} +RUN set -e; \ + getent group "$USER_GID" >/dev/null || groupadd --gid "$USER_GID" debmagic; \ + getent passwd "$USER_UID" >/dev/null || useradd --uid "$USER_UID" --gid "$USER_GID" -m debmagic +RUN mkdir -p {build_dir} && chown $USER_UID:$USER_GID {build_dir} +USER $USER_UID:$USER_GID ENTRYPOINT ["sleep", "infinity"] "#; -#[derive(Debug, Serialize, Deserialize)] -pub struct DockerDriverBuildMetadata { - pub container_name: String, +// Ubuntu images ship python3 already; installed explicitly above for the +// others (e.g. Debian's slim images) before the mirror script needs it. +const APT_MIRROR_SCRIPT_FILENAME: &str = "mirror.py"; + +/// Quote `s` as a single POSIX shell word for embedding in a Dockerfile +/// `RUN` instruction (which is interpreted by `/bin/sh -c`). +fn shell_quote(s: &str) -> String { + format!("'{}'", s.replace('\'', "'\\''")) } pub struct DriverDocker { config: BuildConfig, - driver_config: DriverConfig, container_name: String, + /// Base image of the distro, used to spin up minimal one-shot containers + /// (e.g. for signing) that don't need the build environment's tooling. + base_image: String, + reused_environment: bool, +} + +/// Replace characters that Docker image/container names disallow. +fn sanitize_docker_reference(name: &str) -> String { + name.chars() + .map(|c| match c { + 'a'..='z' | 'A'..='Z' | '0'..='9' | '.' | '_' | '-' => c, + _ => '-', + }) + .collect() } fn build_build_image( config: &BuildConfig, - driver_config: &DriverConfig, - overrides: &DriverDockerConfigOverrides, -) -> anyhow::Result { - let base_image = overrides - .base_image - .clone() - .unwrap_or_else(|| driver_config.docker.base_image_for_distro(&config.distro)); + base_image: &str, + apt_mirror: Option<&str>, + proposed: bool, + image_name: &str, +) -> anyhow::Result<()> { + let apt_mirror_setup = if apt_mirror.is_some() || proposed { + fs::write( + config.build_temp_dir().join(APT_MIRROR_SCRIPT_FILENAME), + APT_MIRROR_SCRIPT, + ) + .map_err(|e| anyhow!("Failed to write apt mirror script: {e}"))?; + + let mut args = vec![ + "--codename".to_string(), + shell_quote(&config.distro.codename), + ]; + if let Some(mirror) = apt_mirror { + args.extend(["--mirror".to_string(), shell_quote(mirror)]); + } + if proposed { + args.push("--proposed".to_string()); + } - let debian_control_file_path = config.build_source_dir().join("debian").join("control"); + let setup = format!( + "COPY {script} /tmp/{script}\nRUN python3 /tmp/{script} {args} && rm /tmp/{script} && apt-get update\n", + script = APT_MIRROR_SCRIPT_FILENAME, + args = args.join(" "), + ); + setup + } else { + String::new() + }; let formatted_dockerfile = DOCKERFILE_TEMPLATE - .replace("{base_image}", &base_image) - .replace("{docker_user}", DOCKER_USER) - .replace("{build_dir}", BUILD_DIR_IN_CONTAINER) - .replace( - "{debian_control_file}", - &debian_control_file_path.to_string_lossy(), - ); + .replace("{base_image}", base_image) + .replace("{apt_mirror_setup}", &apt_mirror_setup) + .replace("{build_dir}", BUILD_DIR_IN_CONTAINER); let dockerfile_path = config.build_temp_dir().join("Dockerfile"); fs::write(&dockerfile_path, formatted_dockerfile) .map_err(|e| anyhow!("Failed to write Dockerfile, {e}"))?; - fs::create_dir_all(config.build_temp_dir().join("debian")) - .map_err(|e| anyhow!("Failed to create debian directory: {e}"))?; - fs::copy( - &debian_control_file_path, - config.build_temp_dir().join("debian").join("control"), - ) - .map_err(|e| anyhow!("Failed to copy debian control file: {e}"))?; - - let docker_image_name = format!("debmagic-{}", config.docker_identifier()); let mut build_args = Vec::new(); let uid = unsafe { libc::geteuid() }; @@ -117,150 +147,232 @@ fn build_build_image( build_cmd .args(["build"]) .args(&build_args) - .args(["--tag", &docker_image_name, "-f"]) + .args(["--tag", image_name, "-f"]) .arg(dockerfile_path) .arg(config.build_temp_dir()); - let status = build_cmd - .status() - .map_err(|e| anyhow!("Error running docker build: {}", e))?; - if !status.success() { - return Err(anyhow!("Error creating docker image")); - } + run_checked(&mut build_cmd, "building docker image")?; - Ok(docker_image_name) + Ok(()) } -fn does_container_exist(container_name: &str) -> anyhow::Result { - let mut ps_cmd = Command::new("docker"); - ps_cmd.args(["ps", "--all", "--format", "json"]); - ps_cmd.stdout(Stdio::piped()); +fn does_image_exist(image_name: &str) -> anyhow::Result { + let status = Command::new("docker") + .args(["image", "inspect", image_name]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .map_err(|error| anyhow!("Failed to inspect Docker image: {error}"))?; + Ok(status.success()) +} - let output = ps_cmd +/// The environment fingerprint stored in the container's labels, or `None` +/// if the container does not exist (or has no fingerprint label). +fn container_environment_fingerprint(container_name: &str) -> anyhow::Result> { + let output = Command::new("docker") + .args(["inspect", container_name]) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) .output() - .map_err(|_| anyhow!("Failed to read docker ps output"))?; - + .map_err(|error| anyhow!("failed to inspect Docker container: {error}"))?; if !output.status.success() { - return Err(anyhow!("failed to query running docker containers")); - } - let stdout = String::from_utf8_lossy(&output.stdout); - for line in stdout.lines() { - if let Ok(container) = serde_json::from_str::(line) - && let Some(names) = container.get("Names") - && names == container_name - { - return Ok(true); - } + // docker inspect exits non-zero when the container doesn't exist. + return Ok(None); } - Ok(false) + let containers: Vec = serde_json::from_slice(&output.stdout) + .map_err(|error| anyhow!("failed to parse Docker inspect output: {error}"))?; + Ok(containers + .first() + .and_then(|container| container.pointer("/Config/Labels")) + .and_then(|labels| labels.get(ENVIRONMENT_LABEL)) + .and_then(serde_json::Value::as_str) + .map(str::to_owned)) } impl DriverDocker { + fn container_start(&self) -> anyhow::Result<()> { + run_checked( + Command::new("docker").args(["start", &self.container_name]), + "starting docker container", + ) + } + + fn container_stop(&self) -> anyhow::Result<()> { + run_checked( + Command::new("docker").args(["stop", &self.container_name]), + "stopping docker container", + ) + } + + fn container_is_running(&self) -> anyhow::Result { + let output = Command::new("docker") + .args([ + "inspect", + "--format", + "{{.State.Running}}", + &self.container_name, + ]) + .output() + .map_err(|error| anyhow!("failed to inspect Docker container: {error}"))?; + if !output.status.success() { + return Err(anyhow!( + "Docker container {} not found", + self.container_name + )); + } + Ok(String::from_utf8_lossy(&output.stdout).trim() == "true") + } + + fn container_remove_force(&self) -> anyhow::Result<()> { + run_checked( + Command::new("docker").args(["rm", "-f", &self.container_name]), + "removing docker container", + ) + } + pub fn create( config: &BuildConfig, driver_config: &DriverConfig, overrides: &DriverDockerConfigOverrides, + apt_mirror: Option<&str>, + proposed: bool, ) -> anyhow::Result { - let container_name = format!("debmagic-{}", config.docker_identifier()); - let container_exists = does_container_exist(&container_name)?; - - if driver_config.persistent && container_exists { - let mut start_cmd = Command::new("docker"); - start_cmd.args(["start", &container_name]); - let status = start_cmd - .status() - .map_err(|e| anyhow!("Error running docker start: {}", e))?; - if !status.success() { - return Err(anyhow!("Error starting docker container")); + let base_image = overrides + .base_image + .clone() + .unwrap_or_else(|| driver_config.docker.base_image_for_distro(&config.distro)); + let uid = unsafe { libc::geteuid() }.to_string(); + let gid = unsafe { libc::getegid() }.to_string(); + let build_root = config.build_root_dir.to_string_lossy(); + let proposed_fingerprint = proposed.to_string(); + let image_fingerprint = environment_fingerprint(&[ + "docker", + DOCKERFILE_TEMPLATE, + APT_MIRROR_SCRIPT, + &base_image, + &config.distro.codename, + apt_mirror.unwrap_or(""), + &proposed_fingerprint, + &uid, + &gid, + ]); + let desired_fingerprint = + environment_fingerprint(&["docker-container", &image_fingerprint, build_root.as_ref()]); + let container_name = resource_name( + "debmagic", + &config.package_name, + &sanitize_docker_reference(&config.build_identifier()), + ); + let mut driver = Self { + config: config.clone(), + container_name, + base_image: base_image.clone(), + reused_environment: false, + }; + let environment_matches = container_environment_fingerprint(&driver.container_name)? + .as_deref() + == Some(&desired_fingerprint); + driver.reused_environment = config.persistent && environment_matches; + let created_container; + + if config.persistent && environment_matches { + created_container = false; + if !driver.container_is_running()? { + driver.container_start()?; } } else { - if container_exists { - let mut rm_cmd = Command::new("docker"); - rm_cmd.args(["rm", "-f", &container_name]); - let status = rm_cmd - .status() - .map_err(|e| anyhow!("Error running docker rm: {}", e))?; - if !status.success() { - return Err(anyhow!("Error removing existing docker container")); - } + // The container may not exist; removal errors don't matter here. + let _ = Command::new("docker") + .args(["rm", "-f", &driver.container_name]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + + let docker_image_name = format!("debmagic-env-{}", &image_fingerprint[..16]); + if !does_image_exist(&docker_image_name)? { + build_build_image( + config, + &base_image, + apt_mirror, + proposed, + &docker_image_name, + )?; } + run_checked( + Command::new("docker") + .args([ + "run", + "--detach", + "--name", + &driver.container_name, + "--label", + &format!("{ENVIRONMENT_LABEL}={desired_fingerprint}"), + "--mount", + ]) + .arg(bind_mount_arg( + &config.build_root_dir, + BUILD_DIR_IN_CONTAINER, + )) + .arg(&docker_image_name), + "starting docker container", + )?; + created_container = true; + } - let docker_image_name = build_build_image(config, driver_config, overrides)?; - let mut run_cmd = Command::new("docker"); - run_cmd.args([ - "run", - "--detach", - "--name", - &container_name, - "--mount", - &format!( - "type=bind,src={},dst={}", - config.build_root_dir.display(), - BUILD_DIR_IN_CONTAINER - ), - &docker_image_name, - ]); - - let status = run_cmd - .status() - .map_err(|e| anyhow!("Error running docker run: {}", e))?; - if !status.success() { - return Err(anyhow!("Error starting docker container")); + let update_result = driver + .run_command(&["apt-get", "update"], &config.build_source_dir(), true) + .map_err(|error| anyhow!("Error running apt-get update in container: {error}")); + if let Err(error) = update_result { + if created_container && let Err(cleanup_error) = driver.container_remove_force() { + return Err(error.context(format!( + "also failed to remove Docker container: {cleanup_error}" + ))); } + return Err(error); } - Ok(Self { - config: config.clone(), - driver_config: driver_config.clone(), - container_name, - }) + Ok(driver) } pub fn from_build_metadata( config: &BuildConfig, driver_config: &DriverConfig, build_metadata: &BuildMetadata, - ) -> Self { - let container_name = build_metadata - .driver_metadata - .get("container_name") - .cloned() - .expect("Missing container_name in metadata"); - - Self { + ) -> anyhow::Result { + Ok(Self { config: config.clone(), - driver_config: driver_config.clone(), - container_name, - } + container_name: container_name_from_metadata(build_metadata)?, + base_image: driver_config.docker.base_image_for_distro(&config.distro), + reused_environment: true, + }) } + fn translate_path_in_container( &self, path_in_source: &Path, ) -> Result { - path_in_source - .strip_prefix(&self.config.build_root_dir) - .map(|rel| Path::new(BUILD_DIR_IN_CONTAINER).join(rel)) - .map_err(|_| { - std::io::Error::new( - std::io::ErrorKind::NotFound, - "Path is not relative to build root".to_string(), - ) - }) + translate_path_in_container(&self.config.build_root_dir, path_in_source) } } impl BuildDriver for DriverDocker { fn get_build_metadata(&self) -> DriverSpecificBuildMetadata { - let mut meta = DriverSpecificBuildMetadata::new(); - meta.insert("container_name".to_string(), self.container_name.clone()); - meta + container_name_metadata(&self.container_name) } - fn run_command(&self, cmd: &[&str], cwd: &Path, requires_root: bool) -> std::io::Result<()> { + fn run_command_env( + &self, + cmd: &[&str], + cwd: &Path, + requires_root: bool, + env_add: &[(&str, &str)], + ) -> std::io::Result<()> { let container_path = self .translate_path_in_container(cwd) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?; + println!("[{}] $ {}", self.container_name, cmd.join(" ")); + let mut exec_cmd = Command::new("docker"); exec_cmd.args(["exec", "--workdir"]); exec_cmd.arg(container_path); @@ -269,6 +381,10 @@ impl BuildDriver for DriverDocker { exec_cmd.args(["--user", "root"]); } + for (key, value) in env_add { + exec_cmd.args(["--env", &format!("{key}={value}")]); + } + exec_cmd.arg(&self.container_name); exec_cmd.args(cmd); @@ -279,25 +395,53 @@ impl BuildDriver for DriverDocker { Ok(()) } - fn cleanup(&self) { - if self.driver_config.persistent { - let _ = Command::new("docker") - .args(["stop", &self.container_name]) - .status(); + fn cleanup(&self) -> anyhow::Result<()> { + if self.config.persistent { + Ok(()) } else { - let _ = Command::new("docker") - .args(["rm", "-f", &self.container_name]) - .status(); + self.container_remove_force() } } + fn reset_build_root(&self) -> std::io::Result<()> { + let find_cmd = ["find", BUILD_DIR_IN_CONTAINER, "-mindepth", "1", "-delete"]; + println!("[{}] $ {}", self.container_name, find_cmd.join(" ")); + let status = Command::new("docker") + .args(["exec", "--user", "root", &self.container_name]) + .args(find_cmd) + .status()?; + if !status.success() { + return Err(std::io::Error::other( + "failed to reset Docker build directory", + )); + } + Ok(()) + } + + fn reused_environment(&self) -> bool { + self.reused_environment + } + fn interactive_shell(&self, cwd: &Path) -> std::io::Result<()> { let workdir = self.translate_path_in_container(cwd)?; - let _ = Command::new("docker") - .args(["exec", "-it", "--workdir"]) - .arg(&workdir) + let was_running = self.container_is_running().map_err(std::io::Error::other)?; + if !was_running { + self.container_start().map_err(std::io::Error::other)?; + } + + let mut command = Command::new("docker"); + command + .args(["exec", "-it", "--user", "root", "--workdir"]) + .arg(&workdir); + let status = command .args([&self.container_name, "/usr/bin/env", "bash"]) - .status()?; + .status(); + if !was_running { + self.container_stop().map_err(std::io::Error::other)?; + } + if !status?.success() { + return Err(std::io::Error::other("Docker shell failed")); + } Ok(()) } @@ -305,4 +449,56 @@ impl BuildDriver for DriverDocker { fn driver_type(&self) -> BuildDriverType { BuildDriverType::Docker } + + fn sign_changes( + &self, + changes_file: &Path, + gpg: Option<&crate::build::signing::GpgForwarding>, + ) -> anyhow::Result<()> { + use crate::build::signing; + + let gpg = gpg.context("docker container signing needs gpg forwarding info")?; + let output_dir = changes_file + .parent() + .context("changes file has no parent directory")?; + let staging_dir = self.config.build_temp_dir().join("sign"); + signing::stage_signing_material(&staging_dir, &gpg.sign_key)?; + let script = signing::sign_container_script( + signing::changes_filename(changes_file)?, + &gpg.sign_key, + // The sign container's root is not id-mapped; fix ownership of + // files it rewrites so the host user can manage them afterwards. + Some((unsafe { libc::geteuid() }, unsafe { libc::getegid() })), + ); + + println!( + "[docker] $ signing {} in a minimal {} container", + changes_file.display(), + self.config.distro.codename + ); + run_checked( + Command::new("docker") + .args(["run", "--rm", "--init"]) + .args([ + "--mount", + &format!( + "{},readonly", + bind_mount_arg(&staging_dir, signing::SIGN_STAGING_IN_CONTAINER) + ), + ]) + .arg(format!( + "--mount=type=bind,src={},dst={},readonly", + gpg.agent_extra_socket.display(), + signing::GPG_SOCKET_IN_CONTAINER + )) + .args([ + "--mount", + &bind_mount_arg(output_dir, signing::OUTPUT_DIR_IN_CONTAINER), + ]) + .arg(&self.base_image) + .args(["sh", "-ec", &script]), + "signing in docker container", + )?; + Ok(()) + } } diff --git a/packages/debmagic/src/build/driver_lxd.rs b/packages/debmagic/src/build/driver_lxd.rs new file mode 100644 index 0000000..b571303 --- /dev/null +++ b/packages/debmagic/src/build/driver_lxd.rs @@ -0,0 +1,698 @@ +use std::{ + fs, + path::{Path, PathBuf}, + process::{Command, Stdio}, +}; + +use anyhow::Context as _; +use debmagic_common::distro::Distro; +use serde::{Deserialize, Serialize}; + +use crate::build::{ + common::{ + APT_MIRROR_SCRIPT, BUILD_DIR_IN_CONTAINER, BuildConfig, BuildDriver, BuildDriverType, + BuildMetadata, DriverSpecificBuildMetadata, container_name_from_metadata, + container_name_metadata, environment_fingerprint, resource_name, run_checked, + translate_path_in_container, + }, + config::DriverConfig, +}; + +// The binary name differs between LXD and Incus, but everything else is shared. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub enum LxdVariant { + Lxd, + Incus, +} + +impl LxdVariant { + pub fn binary(self) -> &'static str { + match self { + LxdVariant::Lxd => "lxc", + LxdVariant::Incus => "incus", + } + } +} + +// The host user is mapped to BUILD_USER_UID/GID via raw.idmap so that the +// bind-mounted source tree is writable from within the container. +const BUILD_USER_UID: u32 = 1000; +const BUILD_USER_GID: u32 = 1000; +const ENVIRONMENT_CONFIG_KEY: &str = "user.debmagic.environment"; +const ENVIRONMENT_SETUP_VERSION: &str = "dpkg-dev python3; build-user-v1; raw.idmap-v1"; + +// ── Config ──────────────────────────────────────────────────────────────────── + +/// Persistent configuration for both LXD and Incus drivers (from `debmagic.toml`). +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[serde(default)] +pub struct DriverLxdConfig { + /// LXD/Incus project to use. `None` means use the default project. + pub project: Option, + /// Override base image per distro, keyed by `":"`. + /// If absent, falls back to `"images:/"`. + pub base_images: std::collections::HashMap, +} + +impl DriverLxdConfig { + pub fn base_image_for_distro( + &self, + variant: LxdVariant, + distro: &debmagic_common::distro::DistroVersion, + ) -> String { + self.base_images + .get(&format!("{}:{}", distro.distro, distro.codename)) + .cloned() + .unwrap_or_else(|| default_base_image(variant, distro)) + } +} + +fn default_base_image( + variant: LxdVariant, + distro: &debmagic_common::distro::DistroVersion, +) -> String { + use debmagic_common::distro::Distro; + match (&distro.distro, variant, distro.is_devel) { + // LXD ships a dedicated ubuntu: remote; daily builds are on ubuntu-daily:. + (Distro::Ubuntu, LxdVariant::Lxd, false) => format!("ubuntu:{}", distro.version), + (Distro::Ubuntu, LxdVariant::Lxd, true) => format!("ubuntu-daily:{}", distro.version), + // Incus uses the images: remote for everything; daily via /daily variant. + (Distro::Ubuntu, LxdVariant::Incus, false) => format!("images:ubuntu/{}", distro.version), + (Distro::Ubuntu, LxdVariant::Incus, true) => { + format!("images:ubuntu/{}/daily", distro.codename) + } + // Debian images live on images: for both variants, released and devel alike. + (Distro::Debian, _, _) => { + format!("images:debian/{}", debian_image_codename(&distro.codename)) + } + } +} + +// The images remote uses real codenames; changelog aliases like "unstable" +// have no dedicated image and map to sid. +fn debian_image_codename(codename: &str) -> &str { + match codename { + "unstable" => "sid", + other => other, + } +} + +/// Per-invocation overrides (CLI flags). +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct DriverLxdConfigOverrides { + pub base_image: Option, + pub project: Option, +} + +// ── Shared implementation ───────────────────────────────────────────────────── + +pub struct DriverLxd { + variant: LxdVariant, + config: BuildConfig, + container_name: String, + /// Resolved project name (None → omit `--project` flag). + project: Option, + /// Base image of the distro, used to spin up minimal one-shot containers + /// (e.g. for signing) that don't need the build environment's tooling. + base_image: String, + reused_environment: bool, +} + +impl DriverLxd { + fn project_args(&self) -> Vec { + match &self.project { + Some(p) => vec!["--project".to_string(), p.clone()], + None => vec![], + } + } + + fn lxd_cmd(&self, subcommand: &str) -> Command { + let mut cmd = Command::new(self.variant.binary()); + cmd.args(self.project_args()); + cmd.arg(subcommand); + cmd + } + + /// Query the LXD/Incus `list` entry for this container, if it exists. + fn container_list_entry(&self) -> anyhow::Result> { + let mut list_cmd = self.lxd_cmd("list"); + list_cmd.args(["--format", "json"]); + list_cmd.stdout(Stdio::piped()); + + let output = list_cmd + .output() + .map_err(|e| anyhow::anyhow!("Failed to list containers: {e}"))?; + + if !output.status.success() { + return Err(anyhow::anyhow!( + "failed to query {} containers", + self.variant.binary() + )); + } + + let stdout = String::from_utf8_lossy(&output.stdout); + let containers: Vec = serde_json::from_str(&stdout) + .map_err(|e| anyhow::anyhow!("Failed to parse container list JSON: {e}"))?; + + Ok(containers.into_iter().find(|container| { + container.get("name").and_then(|v| v.as_str()) == Some(self.container_name.as_str()) + })) + } + + fn is_entry_running(entry: &serde_json::Value) -> bool { + entry + .get("status") + .and_then(|v| v.as_str()) + .map(|s| s == "Running") + .unwrap_or(false) + } + + fn container_start(&self) -> anyhow::Result<()> { + run_checked( + self.lxd_cmd("start").arg(&self.container_name), + &format!("starting {} container", self.variant.binary()), + ) + } + + fn container_stop(&self) -> anyhow::Result<()> { + run_checked( + self.lxd_cmd("stop").arg(&self.container_name), + &format!("stopping {} container", self.variant.binary()), + ) + } + + fn container_delete_force(&self) -> anyhow::Result<()> { + run_checked( + self.lxd_cmd("delete") + .args(["--force", &self.container_name]), + &format!("removing {} container", self.variant.binary()), + ) + } + + fn container_environment_fingerprint(&self) -> anyhow::Result> { + let output = self + .lxd_cmd("config") + .args(["get", &self.container_name, ENVIRONMENT_CONFIG_KEY]) + .output() + .map_err(|error| { + anyhow::anyhow!( + "failed to inspect {} container: {error}", + self.variant.binary() + ) + })?; + if !output.status.success() { + return Err(anyhow::anyhow!( + "failed to inspect {} container {}", + self.variant.binary(), + self.container_name + )); + } + let fingerprint = String::from_utf8_lossy(&output.stdout).trim().to_string(); + Ok((!fingerprint.is_empty()).then_some(fingerprint)) + } + + pub fn create( + variant: LxdVariant, + config: &BuildConfig, + driver_config: &DriverConfig, + overrides: &DriverLxdConfigOverrides, + apt_mirror: Option<&str>, + proposed: bool, + ) -> anyhow::Result { + let container_name = + resource_name("debmagic", &config.package_name, &config.build_identifier()); + + // Project: CLI override > config file value + let project = overrides + .project + .clone() + .or_else(|| driver_config.lxd.project.clone()); + let base_image = overrides.base_image.clone().unwrap_or_else(|| { + driver_config + .lxd + .base_image_for_distro(variant, &config.distro) + }); + let host_uid = unsafe { libc::geteuid() }.to_string(); + let host_gid = unsafe { libc::getegid() }.to_string(); + let build_root = config.build_root_dir.to_string_lossy(); + let proposed_fingerprint = proposed.to_string(); + let desired_fingerprint = environment_fingerprint(&[ + variant.binary(), + ENVIRONMENT_SETUP_VERSION, + &base_image, + &config.distro.codename, + apt_mirror.unwrap_or(""), + &proposed_fingerprint, + &host_uid, + &host_gid, + build_root.as_ref(), + ]); + + let mut base = Self { + variant, + config: config.clone(), + container_name: container_name.clone(), + project, + base_image: base_image.clone(), + reused_environment: false, + }; + + let container_entry = base.container_list_entry()?; + let environment_matches = container_entry.is_some() + && base.container_environment_fingerprint()?.as_deref() == Some(&desired_fingerprint); + let reusing_container = config.persistent && environment_matches; + base.reused_environment = reusing_container; + + let mut initialized_container = false; + let setup_result = (|| -> anyhow::Result<()> { + if reusing_container { + let already_running = container_entry + .as_ref() + .is_some_and(DriverLxd::is_entry_running); + if !already_running { + base.container_start()?; + } + } else { + if container_entry.is_some() { + base.container_delete_force()?; + } + + let mut init = base.lxd_cmd("init"); + if !config.persistent { + init.arg("--ephemeral"); + } + init.args([&base_image, &container_name]); + run_checked( + &mut init, + &format!("initialising {} container", variant.binary()), + )?; + initialized_container = true; + + // Map the host user's uid/gid to BUILD_USER_UID inside the container + // so that files in the bind-mounted build root are writable. + let host_uid = unsafe { libc::geteuid() }; + let host_gid = unsafe { libc::getegid() }; + if host_uid != 0 { + let idmap = format!( + "uid {} {}\ngid {} {}", + host_uid, BUILD_USER_UID, host_gid, BUILD_USER_GID + ); + run_checked( + base.lxd_cmd("config") + .arg("set") + .arg(&container_name) + .arg("raw.idmap") + .arg(&idmap), + "setting raw.idmap on container", + )?; + } + + let device_name = resource_name( + "debmagic-src", + &config.package_name, + &config.build_identifier(), + ); + run_checked( + base.lxd_cmd("config") + .arg("device") + .arg("add") + .arg(&container_name) + .arg(&device_name) + .arg("disk") + .arg(format!("source={}", config.build_root_dir.display())) + .arg(format!("path={}", BUILD_DIR_IN_CONTAINER)), + &format!("mounting build root into {} container", variant.binary()), + )?; + + run_checked( + base.lxd_cmd("start").arg(&container_name), + &format!("starting {} container", variant.binary()), + )?; + + if config.distro.distro == Distro::Ubuntu { + base.exec_in_container(&["cloud-init", "status", "--wait"], None, true, &[]) + .map_err(|e| { + anyhow::anyhow!("Error waiting for cloud-init to finish: {e}") + })?; + } + } + + // Re-run on every reuse of a persistent container too, so that a + // previous invocation that crashed before finishing this setup (or a + // long-lived incremental container with an aging package cache) + // doesn't leave `apt-get build-dep` unable to resolve anything. + base.exec_in_container(&["apt-get", "update"], None, true, &[]) + .map_err(|e| anyhow::anyhow!("Error running apt-get update in container: {e}"))?; + + if !reusing_container { + // Install the base tooling that stock images don't include. + // build-dep is intentionally omitted here: build.rs runs it for + // every driver against the real mounted source tree. + base.exec_in_container( + &["apt-get", "install", "-y", "dpkg-dev", "python3"], + None, + true, + &[], + ) + .map_err(|e| anyhow::anyhow!("Error installing base packages in container: {e}"))?; + + let ensure_build_user = format!( + "getent group {gid} >/dev/null || groupadd --gid {gid} debmagic; \ + getent passwd {uid} >/dev/null || useradd --uid {uid} --gid {gid} -m debmagic", + uid = BUILD_USER_UID, + gid = BUILD_USER_GID, + ); + base.exec_in_container(&["sh", "-ec", &ensure_build_user], None, true, &[]) + .map_err(|e| anyhow::anyhow!("Error creating build user in container: {e}"))?; + + if apt_mirror.is_some() || proposed { + let script_path = config.build_temp_dir().join("mirror.py"); + fs::write(&script_path, APT_MIRROR_SCRIPT)?; + let container_script_path = base.translate_path_in_container(&script_path)?; + let mut args = vec![ + "python3".to_string(), + container_script_path.to_string_lossy().into_owned(), + "--codename".to_string(), + config.distro.codename.clone(), + ]; + if let Some(mirror) = apt_mirror { + args.extend(["--mirror".to_string(), mirror.to_string()]); + } + if proposed { + args.push("--proposed".to_string()); + } + let args = args.iter().map(String::as_str).collect::>(); + base.exec_in_container(&args, None, true, &[]) + .map_err(|e| anyhow::anyhow!("Error configuring apt sources: {e}"))?; + base.exec_in_container(&["apt-get", "update"], None, true, &[]) + .map_err(|e| { + anyhow::anyhow!("Error updating configured apt sources: {e}") + })?; + } + + run_checked( + base.lxd_cmd("config") + .arg("set") + .arg(&container_name) + .arg(ENVIRONMENT_CONFIG_KEY) + .arg(&desired_fingerprint), + &format!("recording {} environment", variant.binary()), + )?; + } + Ok(()) + })(); + + if let Err(error) = setup_result { + if initialized_container && let Err(cleanup_error) = base.container_delete_force() { + return Err(error.context(format!( + "also failed to remove {} container: {cleanup_error}", + variant.binary() + ))); + } + return Err(error); + } + + Ok(base) + } + + pub fn from_build_metadata( + variant: LxdVariant, + config: &BuildConfig, + driver_config: &DriverConfig, + build_metadata: &BuildMetadata, + ) -> anyhow::Result { + let project = build_metadata.driver_metadata.get("project").cloned(); + + Ok(Self { + variant, + config: config.clone(), + container_name: container_name_from_metadata(build_metadata)?, + project, + base_image: driver_config + .lxd + .base_image_for_distro(variant, &config.distro), + reused_environment: true, + }) + } + + fn translate_path_in_container( + &self, + path_in_source: &Path, + ) -> Result { + translate_path_in_container(&self.config.build_root_dir, path_in_source) + } + + /// Run `action` with the container running, restoring a previously + /// stopped container to the stopped state afterwards. + fn with_running_container( + &self, + action: impl FnOnce(&Self) -> std::io::Result<()>, + ) -> std::io::Result<()> { + let entry = self + .container_list_entry() + .map_err(std::io::Error::other)? + .ok_or_else(|| { + std::io::Error::new(std::io::ErrorKind::NotFound, "container not found") + })?; + let was_running = Self::is_entry_running(&entry); + if !was_running { + self.container_start().map_err(std::io::Error::other)?; + } + let result = action(self); + let stop_result = if was_running { + Ok(()) + } else { + self.container_stop() + }; + match (result, stop_result) { + (Ok(()), Ok(())) => Ok(()), + (Err(e), Ok(())) => Err(e), + (Ok(()), Err(e)) => Err(std::io::Error::other(e)), + (Err(e), Err(stop_error)) => Err(std::io::Error::other(format!( + "{e}; also failed to stop {} container: {stop_error}", + self.variant.binary() + ))), + } + } + + fn exec_in_container( + &self, + cmd: &[&str], + workdir: Option<&Path>, + as_root: bool, + env_add: &[(&str, &str)], + ) -> std::io::Result<()> { + println!("[{}] $ {}", self.container_name, cmd.join(" ")); + + let mut exec_cmd = self.lxd_cmd("exec"); + exec_cmd.arg(&self.container_name); + + if let Some(wd) = workdir { + exec_cmd.args(["--cwd", &wd.to_string_lossy()]); + } + + if !as_root { + // Run as BUILD_USER_UID — the host user is mapped to this uid via + // raw.idmap, so it owns the bind-mounted source tree inside. + exec_cmd.args(["--user", &BUILD_USER_UID.to_string()]); + exec_cmd.args(["--group", &BUILD_USER_GID.to_string()]); + } + + for (key, value) in env_add { + exec_cmd.args(["--env", &format!("{key}={value}")]); + } + + exec_cmd.arg("--"); + exec_cmd.args(cmd); + + let status = exec_cmd.status()?; + if !status.success() { + return Err(std::io::Error::other(format!( + "{} exec failed", + self.variant.binary() + ))); + } + Ok(()) + } +} + +impl BuildDriver for DriverLxd { + fn get_build_metadata(&self) -> DriverSpecificBuildMetadata { + let mut meta = container_name_metadata(&self.container_name); + if let Some(ref p) = self.project { + meta.insert("project".to_string(), p.clone()); + } + meta + } + + fn run_command_env( + &self, + cmd: &[&str], + cwd: &Path, + requires_root: bool, + env_add: &[(&str, &str)], + ) -> std::io::Result<()> { + let container_path = self + .translate_path_in_container(cwd) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?; + + self.exec_in_container(cmd, Some(&container_path), requires_root, env_add) + } + + fn cleanup(&self) -> anyhow::Result<()> { + if self.config.persistent { + Ok(()) + } else { + // Ephemeral containers are auto-deleted after stopping. + self.container_stop() + } + } + + fn reset_build_root(&self) -> std::io::Result<()> { + self.with_running_container(|driver| { + driver.exec_in_container( + &["find", BUILD_DIR_IN_CONTAINER, "-mindepth", "1", "-delete"], + None, + true, + &[], + ) + }) + } + + fn reused_environment(&self) -> bool { + self.reused_environment + } + + fn interactive_shell(&self, cwd: &Path) -> std::io::Result<()> { + let workdir = self.translate_path_in_container(cwd)?; + self.with_running_container(|driver| { + let mut command = driver.lxd_cmd("exec"); + command + .arg(&driver.container_name) + .args(["--cwd", &workdir.to_string_lossy()]); + let status = command.arg("--").args(["/usr/bin/env", "bash"]).status()?; + if !status.success() { + return Err(std::io::Error::other(format!( + "{} shell failed", + driver.variant.binary() + ))); + } + Ok(()) + }) + } + + fn driver_type(&self) -> BuildDriverType { + match self.variant { + LxdVariant::Lxd => BuildDriverType::Lxd, + LxdVariant::Incus => BuildDriverType::Incus, + } + } + + fn sign_changes( + &self, + changes_file: &Path, + gpg: Option<&crate::build::signing::GpgForwarding>, + ) -> anyhow::Result<()> { + use crate::build::signing; + + let gpg = gpg.context("container signing needs gpg forwarding info")?; + let output_dir = changes_file + .parent() + .context("changes file has no parent directory")?; + let staging_dir = self.config.build_temp_dir().join("sign"); + signing::stage_signing_material(&staging_dir, &gpg.sign_key)?; + // No chown needed: raw.idmap maps container root to the host user. + let script = signing::sign_container_script( + signing::changes_filename(changes_file)?, + &gpg.sign_key, + None, + ); + + let sign_container = resource_name( + "debmagic-sign", + &self.config.package_name, + &self.config.build_identifier(), + ); + let bin = self.variant.binary(); + // The sign container is ephemeral; it disappears when stopped. + let mut init = self.lxd_cmd("init"); + init.arg("--ephemeral"); + init.args([&self.base_image, sign_container.as_str()]); + run_checked(&mut init, &format!("initialising {bin} sign container"))?; + + let result = (|| -> anyhow::Result<()> { + let host_uid = unsafe { libc::geteuid() }; + let host_gid = unsafe { libc::getegid() }; + if host_uid != 0 { + let idmap = format!("uid {host_uid} 0\ngid {host_gid} 0"); + run_checked( + self.lxd_cmd("config") + .arg("set") + .arg(&sign_container) + .arg("raw.idmap") + .arg(&idmap), + "setting raw.idmap on sign container", + )?; + } + + run_checked( + self.lxd_cmd("config") + .arg("device") + .arg("add") + .arg(&sign_container) + .arg("debmagic-output") + .arg("disk") + .arg(format!("source={}", output_dir.display())) + .arg(format!("path={}", signing::OUTPUT_DIR_IN_CONTAINER)), + "mounting output directory into sign container", + )?; + run_checked( + self.lxd_cmd("config") + .arg("device") + .arg("add") + .arg(&sign_container) + .arg("debmagic-sign-staging") + .arg("disk") + .arg(format!("source={}", staging_dir.display())) + .arg(format!("path={}", signing::SIGN_STAGING_IN_CONTAINER)) + .arg("readonly=true"), + "mounting signing material into sign container", + )?; + // Forward the host gpg-agent's extra socket via a proxy device, + // like a manually configured unix proxy but scoped to signing. + run_checked( + self.lxd_cmd("config") + .arg("device") + .arg("add") + .arg(&sign_container) + .arg("debmagic-gpg-agent") + .arg("proxy") + .arg("bind=container") + .arg(format!("connect=unix:{}", gpg.agent_extra_socket.display())) + .arg(format!("listen=unix:{}", signing::GPG_SOCKET_IN_CONTAINER)) + .arg("uid=0") + .arg("gid=0"), + "forwarding gpg-agent socket into sign container", + )?; + + run_checked( + self.lxd_cmd("start").arg(&sign_container), + &format!("starting {bin} sign container"), + )?; + + let mut exec = self.lxd_cmd("exec"); + exec.arg(&sign_container); + exec.arg("--"); + exec.args(["sh", "-ec", &script]); + run_checked(&mut exec, "signing in container")?; + Ok(()) + })(); + + let mut stop = self.lxd_cmd("stop"); + let stop_result = run_checked( + stop.arg(&sign_container), + &format!("stopping {bin} sign container"), + ); + result.and(stop_result) + } +} diff --git a/packages/debmagic/src/build/mod.rs b/packages/debmagic/src/build/mod.rs new file mode 100644 index 0000000..f7f92ae --- /dev/null +++ b/packages/debmagic/src/build/mod.rs @@ -0,0 +1,689 @@ +use std::sync::{Arc, Mutex}; +use std::{ + fs, io, + io::{BufReader, IsTerminal, stdout}, + path::{Path, PathBuf}, + process::{Command, Stdio}, +}; + +use crate::build::attach::{send_socket_command, start_socket_server}; +use crate::build::config::DriverOverrides; +use crate::build::source::{source_manifest_path, stage_source_tree}; +use crate::{ + build::{ + common::{BuildConfig, BuildDriver, BuildDriverType, BuildMetadata}, + config::DriverConfig, + driver_bare::DriverBare, + driver_docker::DriverDocker, + driver_lxd::{DriverLxd, LxdVariant}, + }, + config::Config, + package::PackageDescription, +}; +use anyhow::{Context, anyhow}; +use debmagic_common::distro::DistroVersion; + +pub mod artifacts; +pub mod attach; +pub mod common; +pub mod config; +pub mod driver_bare; +pub mod driver_docker; +pub mod driver_lxd; +pub mod signing; +pub mod source; + +struct Build { + config: BuildConfig, + pub driver: Box, + /// Prepared when signing happens inside a container: agent socket + + /// sign key, validated before the build starts. + gpg_forwarding: Option, + attached: bool, +} + +/// Where debsign will actually run for this build. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +enum SignLocation { + Host, + Container, +} + +/// Resolve the effective sign location and validate everything signing will +/// need *before* the build starts, so a broken gpg setup doesn't waste a +/// whole build. +fn prepare_signing( + build_config: &BuildConfig, +) -> anyhow::Result<(SignLocation, Option)> { + let container_driver = build_config.driver != BuildDriverType::Bare; + let host_has_debsign = signing::check_host_debsign_available().is_ok(); + + let location = match build_config.sign_with { + signing::SignWith::Host => SignLocation::Host, + signing::SignWith::Same => { + if container_driver { + SignLocation::Container + } else { + // The bare driver's "build environment" is the host. + SignLocation::Host + } + } + signing::SignWith::Auto => { + if host_has_debsign || !container_driver { + SignLocation::Host + } else { + SignLocation::Container + } + } + }; + + match location { + SignLocation::Host => { + signing::check_host_debsign_available()?; + Ok((location, None)) + } + SignLocation::Container => { + let sign_key = build_config.sign_key.clone().ok_or_else(|| { + anyhow!( + "signing in a container requires sign_key to be set \ + (debsign's maintainer-based key lookup only works on the host)" + ) + })?; + let forwarding = signing::GpgForwarding { + agent_extra_socket: signing::gpg_agent_extra_socket()?, + sign_key: sign_key.clone(), + }; + signing::check_signing_key_available(&sign_key)?; + Ok((location, Some(forwarding))) + } + } +} + +fn get_build_driver( + config: &BuildConfig, + driver_config: &DriverConfig, + driver_overrides: &DriverOverrides, +) -> anyhow::Result> { + let apt_mirror = driver_overrides + .apt_mirror + .as_deref() + .or(driver_config.apt_mirror.as_deref()); + let proposed = driver_overrides.proposed.unwrap_or(driver_config.proposed); + + match config.driver { + BuildDriverType::Docker => Ok(Box::new(DriverDocker::create( + config, + driver_config, + &driver_overrides.docker, + apt_mirror, + proposed, + )?)), + BuildDriverType::Bare => Ok(Box::new(DriverBare::create( + config, + driver_config, + &driver_overrides.bare, + ))), + BuildDriverType::Lxd | BuildDriverType::Incus => { + let variant = match config.driver { + BuildDriverType::Lxd => LxdVariant::Lxd, + _ => LxdVariant::Incus, + }; + Ok(Box::new(DriverLxd::create( + variant, + config, + driver_config, + &driver_overrides.lxd, + apt_mirror, + proposed, + )?)) + } + } +} + +fn create_driver_from_metadata( + config: &DriverConfig, + metadata: &BuildMetadata, +) -> anyhow::Result> { + let driver: anyhow::Result> = match &metadata.config.driver { + BuildDriverType::Docker => Ok(Box::new(DriverDocker::from_build_metadata( + &metadata.config, + config, + metadata, + )?)), + BuildDriverType::Bare => Ok(Box::new(DriverBare::from_build_metadata( + &metadata.config, + config, + metadata, + ))), + BuildDriverType::Lxd | BuildDriverType::Incus => { + let variant = match metadata.config.driver { + BuildDriverType::Lxd => LxdVariant::Lxd, + _ => LxdVariant::Incus, + }; + Ok(Box::new(DriverLxd::from_build_metadata( + variant, + &metadata.config, + config, + metadata, + )?)) + } + }; + driver +} + +impl Build { + pub fn create( + config: &BuildConfig, + driver_config: &DriverConfig, + driver_overrides: &DriverOverrides, + ) -> anyhow::Result { + let driver = get_build_driver(config, driver_config, driver_overrides) + .context(format!("failed to create {:?} build driver", config.driver))?; + let gpg_forwarding = if config.sign_package { + let (_location, forwarding) = prepare_signing(config)?; + forwarding + } else { + None + }; + Ok(Self { + config: config.clone(), + driver, + gpg_forwarding, + attached: false, + }) + } + + pub fn from_build_root( + build_root: &Path, + driver_config: &DriverConfig, + ) -> anyhow::Result { + let build_metadata_path = build_root.join("build.json"); + if !build_metadata_path.is_file() { + return Err(anyhow!("No build.json found")); + } + // read metadata from file + let file = fs::OpenOptions::new() + .read(true) + .open(&build_metadata_path)?; + let metadata = || -> anyhow::Result { + let reader = BufReader::new(&file); + let metadata: BuildMetadata = serde_json::from_reader(reader).with_context(|| { + format!( + "Failed to read build metadata from {} - invalid json", + build_metadata_path.display() + ) + })?; + Ok(metadata) + }(); + + let metadata = metadata?; + + let driver = create_driver_from_metadata(driver_config, &metadata)?; + + let attached = send_socket_command(build_root, "attach").is_ok(); + + Ok(Self { + gpg_forwarding: None, + config: metadata.config.clone(), + driver, + attached, + }) + } + + pub fn detach(&self) -> anyhow::Result<()> { + let build_root = &self.config.build_root_dir; + if self.attached { + send_socket_command(build_root, "detach")?; + } + Ok(()) + } + + pub fn write_metadata(&self) -> anyhow::Result<()> { + let metadata = BuildMetadata { + config: self.config.clone(), + driver_metadata: self.driver.get_build_metadata(), + }; + let path = self.config.build_root_dir.join("build.json"); + let json = serde_json::to_string_pretty(&metadata) + .context("Failed to serialize build metadata")?; + fs::write(path, json)?; + Ok(()) + } +} + +fn get_build_root_and_identifier( + config: &Config, + package: &PackageDescription, +) -> (String, PathBuf) { + let package_identifier = format!("{}-{}", package.name, package.version); + let build_root = config.temp_build_dir.join(&package_identifier); + (package_identifier, build_root) +} + +/// Determine which distro version to use for the build. +/// +/// If only one distro version is specified in the changelog, it's used automatically. +/// If multiple distro versions are specified, an explicit --distro is required. +/// If --distro is provided, it's validated against the changelog versions. +fn resolve_distro_version( + changelog_distros: &[String], + explicit_distro: Option<&str>, +) -> anyhow::Result { + let resolved_codename = match (changelog_distros.len(), explicit_distro) { + (0, _) => Err(anyhow!("changelog contains no distributions")), + (1, None) => Ok(changelog_distros[0].clone()), + (1, Some(explicit)) => { + if explicit == changelog_distros[0] { + Ok(explicit.to_string()) + } else { + Err(anyhow!( + "explicit distro version '{}' conflicts with distribution specified in changelog '{}'", + explicit, + changelog_distros[0] + )) + } + } + (_, None) => Err(anyhow!( + "changelog contains multiple distributions ({}), please specify which one to build for with --distro", + changelog_distros.join(", ") + )), + (_, Some(explicit)) => { + if changelog_distros.contains(&explicit.to_string()) { + Ok(explicit.to_string()) + } else { + Err(anyhow!( + "explicit distro version '{}' not found in changelog distributions: {}", + explicit, + changelog_distros.join(", ") + )) + } + } + }?; + let resolved = debmagic_common::distro::get_distro_version(&resolved_codename) + .ok_or_else(|| anyhow!("unknown distro codename '{}'", resolved_codename))?; + Ok(resolved) +} + +fn prepare_build_env( + config: &Config, + driver_overrides: &DriverOverrides, + package: &PackageDescription, + driver_type: BuildDriverType, + output_dir: &Path, + explicit_distro_version: Option<&str>, +) -> anyhow::Result { + let (package_identifier, build_root) = get_build_root_and_identifier(config, package); + + let distro_version = resolve_distro_version(&package.distro_versions, explicit_distro_version) + .context("failed to determine distro version")?; + + let build_config = BuildConfig { + driver: driver_type, + package_name: package.name.clone(), + package_identifier, + source_dir: package.source_dir.clone(), + output_dir: output_dir.to_path_buf(), + build_root_dir: build_root.clone(), + distro: distro_version.clone(), + sign_package: config.sign_package, + sign_with: config.sign_with, + sign_key: config.sign_key.clone(), + build_debug_symbols: config.build_debug_symbols, + clean: config.clean, + persistent: config.driver.persistent, + incremental: config.incremental, + source_sync_mode: config.source_sync_mode, + }; + + if config.driver.persistent && build_root.exists() { + // For persistent containers, starting first lets root inside delete + // container-owned files the host user can't remove. + let build = Build::create(&build_config, &config.driver, driver_overrides) + .context(format!("failed to create {:?} build driver", driver_type))?; + if !config.incremental + || !source_manifest_path(&build_config).is_file() + || !build.driver.reused_environment() + { + build + .driver + .reset_build_root() + .context("failed to reset persistent build directory")?; + } + build_config + .create_dirs() + .context("failed to create build directories")?; + stage_source_tree(&build_config, package)?; + return Ok(build); + } + + if build_root.exists() + && let Err(e) = fs::remove_dir_all(&build_root) + { + if e.kind() == io::ErrorKind::PermissionDenied { + // Some files were created by a privileged user inside a container + // and can't be deleted by the host user directly. Load the previous + // build's driver and ask it to clean up from inside. + let metadata_path = build_root.join("build.json"); + if metadata_path.is_file() + && let Ok(file) = fs::OpenOptions::new().read(true).open(&metadata_path) + && let Ok(metadata) = + serde_json::from_reader::<_, BuildMetadata>(BufReader::new(&file)) + && let Ok(driver) = create_driver_from_metadata(&config.driver, &metadata) + { + let _ = driver.reset_build_root(); + } + fs::remove_dir_all(&build_root).with_context(|| { + format!( + "failed to remove build root {}; try: sudo rm -rf {}", + build_root.display(), + build_root.display() + ) + })?; + } else { + return Err(e.into()); + } + } + + build_config + .create_dirs() + .context("failed to create build directories")?; + + stage_source_tree(&build_config, package)?; + + let build = Build::create(&build_config, &config.driver, driver_overrides)?; + Ok(build) +} + +pub fn get_shell_in_build(config: &Config, package: &PackageDescription) -> anyhow::Result<()> { + let (_package_identifier, build_root) = get_build_root_and_identifier(config, package); + let build = Build::from_build_root(&build_root, &config.driver)?; + let result = build + .driver + .interactive_shell(&build.config.build_source_dir()); + + build.detach()?; + + result?; + Ok(()) +} + +fn deb_build_options(existing: Option<&str>, build_debug_symbols: bool) -> String { + let mut options = existing + .unwrap_or_default() + .split_whitespace() + .filter(|option| *option != "noautodbgsym") + .collect::>(); + if !build_debug_symbols { + options.push("noautodbgsym"); + } + options.join(" ") +} + +/// Everything needed to run one package build, independent of whether the +/// build produces binary or source packages. +pub struct BuildRequest<'a> { + pub config: &'a Config, + pub package: &'a PackageDescription, + pub driver_type: BuildDriverType, + pub driver_overrides: &'a DriverOverrides, + pub output_dir: &'a Path, + pub explicit_distro_version: Option<&'a str>, +} + +/// Shared build orchestration: prepare the environment, run `build_commands` +/// in it, export the artifacts to the output dir, sign them if requested, and +/// clean up (dropping into a shell first on failure of an interactive binary +/// build). While `shell_on_failure` is set, a socket server lets concurrent +/// `debmagic shell` sessions attach to the environment. +fn run_build( + request: &BuildRequest, + shell_on_failure: bool, + build_commands: impl FnOnce(&Build) -> anyhow::Result<()>, +) -> anyhow::Result<()> { + let build = prepare_build_env( + request.config, + request.driver_overrides, + request.package, + request.driver_type, + request.output_dir, + request.explicit_distro_version, + ) + .context("failed to prepare build environment")?; + build + .write_metadata() + .context("failed to write build metadata")?; + + let should_exit = Arc::new(Mutex::new(false)); + let socket_server_handle = + start_socket_server(&build.config.build_root_dir, should_exit.clone())?; + + let stop_socket_server = || { + *should_exit.lock().unwrap() = true; + if !socket_server_handle.is_finished() { + println!("Waiting for all attached shells to exit..."); + } + socket_server_handle.join().ok(); + }; + + let result = build_commands(&build).and_then(|()| { + let changes_file = artifacts::export_build_artifacts( + &build.config.build_work_dir(), + &build.config.output_dir, + )?; + if build.config.sign_package { + build + .driver + .sign_changes(&changes_file, build.gpg_forwarding.as_ref())?; + } + Ok(()) + }); + + if let Err(error) = result { + if shell_on_failure && stdout().is_terminal() { + eprintln!("Build failed: {error}. Dropping into shell..."); + if let Err(shell_error) = build + .driver + .interactive_shell(&build.config.build_source_dir()) + { + eprintln!("Dropping into shell failed: {shell_error}"); + } + } else { + eprintln!("Build failed: {error}"); + } + if let Err(cleanup_error) = build.driver.cleanup() { + eprintln!("Failed to clean up build environment: {cleanup_error}"); + } + stop_socket_server(); + return Err(error); + } + + stop_socket_server(); + build + .driver + .cleanup() + .context("failed to clean up build environment")?; + Ok(()) +} + +pub fn build_package(request: &BuildRequest) -> anyhow::Result<()> { + run_build(request, true, |build| { + build.driver.run_command( + &["apt-get", "-y", "build-dep", "."], + &build.config.build_source_dir(), + true, + )?; + let inherited_options = std::env::var("DEB_BUILD_OPTIONS").ok(); + let options = deb_build_options( + inherited_options.as_deref(), + build.config.build_debug_symbols, + ); + let env_add = [("DEB_BUILD_OPTIONS", options.as_str())]; + let mut dpkg_buildpackage_args = vec!["dpkg-buildpackage", "-us", "-uc", "-ui"]; + if !build.config.clean { + // Non-incremental builds already stage a clean source tree, while + // incremental builds preserve their outputs intentionally. + dpkg_buildpackage_args.push("-nc"); + } + dpkg_buildpackage_args.push("-b"); + build.driver.run_command_env( + &dpkg_buildpackage_args, + &build.config.build_source_dir(), + false, + &env_add, + )?; + Ok(()) + }) +} + +/// Confirm `cmd` is on `PATH`, failing with an actionable message (rather +/// than a raw "command not found") if it isn't. +fn check_command_available(cmd: &str, install_hint: &str) -> anyhow::Result<()> { + match Command::new(cmd) + .arg("--version") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + { + Ok(_) => Ok(()), + Err(e) if e.kind() == io::ErrorKind::NotFound => { + Err(anyhow!("{cmd} not found on PATH. {install_hint}")) + } + Err(e) => Err(e).with_context(|| format!("failed to check for {cmd}")), + } +} + +fn check_dpkg_buildpackage_available() -> anyhow::Result<()> { + check_command_available( + "dpkg-buildpackage", + "It's part of dpkg-dev; install it, or pass --driver lxd/incus/docker to build inside a Debian-ish container instead.", + ) +} + +/// Build a `.dsc` + tarball + `.buildinfo` + `.changes` source package. +/// +/// If `config.clean` is set, build-dependencies are installed before +/// `dpkg-buildpackage` runs `debian/rules clean` once. +pub fn build_source_package(request: &BuildRequest) -> anyhow::Result<()> { + if request.driver_type == BuildDriverType::Bare { + check_dpkg_buildpackage_available()?; + } + + run_build(request, false, |build| { + let build_source_dir = build.config.build_source_dir(); + if build.config.clean { + build.driver.run_command( + &["apt-get", "-y", "build-dep", "."], + &build_source_dir, + true, + )?; + } + let mut args = vec!["dpkg-buildpackage", "-S", "-d", "-us", "-uc", "-ui"]; + if !build.config.clean { + args.push("-nc"); + } + build.driver.run_command(&args, &build_source_dir, false)?; + Ok(()) + }) + .context("failed to build source package") +} + +#[cfg(test)] +mod tests { + use debmagic_common::distro::Distro; + + use super::*; + + #[test] + fn debug_symbol_option_preserves_other_build_options() { + assert_eq!( + deb_build_options(Some("nocheck parallel=8"), false), + "nocheck parallel=8 noautodbgsym" + ); + assert_eq!( + deb_build_options(Some("nocheck noautodbgsym parallel=8"), true), + "nocheck parallel=8" + ); + } + + #[test] + fn test_resolve_distro_version_single_distro_no_explicit() { + let distros = vec!["forky".to_string()]; + let result = resolve_distro_version(&distros, None); + assert!(result.is_ok()); + let distro_version = result.unwrap(); + assert_eq!(distro_version.codename, "forky"); + assert_eq!(distro_version.distro, Distro::Debian); + } + + #[test] + fn test_resolve_distro_version_single_distro_matching_explicit() { + let distros = vec!["forky".to_string()]; + let result = resolve_distro_version(&distros, Some("forky")); + assert!(result.is_ok()); + let distro_version = result.unwrap(); + assert_eq!(distro_version.codename, "forky"); + assert_eq!(distro_version.distro, Distro::Debian); + } + + #[test] + fn test_resolve_distro_version_single_distro_conflicting_explicit() { + let distros = vec!["forky".to_string()]; + let result = resolve_distro_version(&distros, Some("duke")); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("conflicts with distribution specified in changelog") + ); + } + + #[test] + fn test_resolve_distro_version_multiple_distros_no_explicit() { + let distros = vec!["forky".to_string(), "duke".to_string()]; + let result = resolve_distro_version(&distros, None); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("multiple distributions") + ); + } + + #[test] + fn test_resolve_distro_version_multiple_distros_explicit_valid() { + let distros = vec!["forky".to_string(), "duke".to_string()]; + let result = resolve_distro_version(&distros, Some("duke")); + assert!(result.is_ok()); + let distro_version = result.unwrap(); + assert_eq!(distro_version.codename, "duke"); + assert_eq!(distro_version.distro, Distro::Debian); + } + + #[test] + fn test_resolve_distro_version_multiple_distros_explicit_invalid() { + let distros = vec!["forky".to_string(), "duke".to_string()]; + let result = resolve_distro_version(&distros, Some("trixie")); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("not found in changelog distributions") + ); + } + + #[test] + fn test_resolve_distro_version_empty_distros() { + let distros: Vec = vec![]; + let result = resolve_distro_version(&distros, None); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("changelog contains no distributions") + ); + } +} diff --git a/packages/debmagic/src/build/scripts/mirror.py b/packages/debmagic/src/build/scripts/mirror.py new file mode 100644 index 0000000..85239c0 --- /dev/null +++ b/packages/debmagic/src/build/scripts/mirror.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Point apt at a mirror by replacing the base image's default sources. + +debmagic picks the base image itself, so the default sources file it ships +(classic `sources.list` or deb822 `*.sources`) is a known quantity: rather +than parsing and patching it in place, it is simply removed and replaced +with a single deb822 file that debmagic owns and fully controls. Unless +overridden, the suites/components enabled default to the release, its +`-updates` and its `-security` pockets - the same ones any base image +ships out of the box. +""" + +from __future__ import annotations + +import argparse +import glob +import os + +OS_RELEASE_FILE = "/etc/os-release" + +# Default apt configuration shipped by the base images debmagic uses; +# removed unconditionally in favour of MANAGED_SOURCES_FILE below. +DEFAULT_SOURCE_FILES = [ + "/etc/apt/sources.list", + "/etc/apt/sources.list.d/ubuntu.sources", + "/etc/apt/sources.list.d/debian.sources", +] + +MANAGED_SOURCES_FILE = "/etc/apt/sources.list.d/debmagic.sources" + +# Debian codenames with no separate updates/security/backports pockets. +DEBIAN_ROLLING_CODENAMES = {"unstable", "sid", "testing", "experimental"} +DEBIAN_WITHOUT_PROPOSED = {"unstable", "sid", "experimental"} + + +def read_os_release(path: str = OS_RELEASE_FILE) -> dict[str, str]: + values: dict[str, str] = {} + with open(path, encoding="utf-8") as f: + for line in f: + stripped_line = line.strip() + if not stripped_line or stripped_line.startswith("#") or "=" not in stripped_line: + continue + key, _, value = stripped_line.partition("=") + values[key] = value.strip().strip('"') + return values + + +def default_suites(os_release: dict[str, str], codename: str) -> list[str]: + if os_release.get("ID") == "debian" and codename in DEBIAN_ROLLING_CODENAMES: + return [codename] + return [codename, f"{codename}-updates", f"{codename}-security"] + + +def default_components(os_release: dict[str, str]) -> list[str]: + if os_release.get("ID") == "ubuntu": + return ["main", "restricted", "universe", "multiverse"] + + major = int((os_release.get("VERSION_ID") or "0").split(".")[0] or 0) + if major >= 12: + return ["main", "contrib", "non-free", "non-free-firmware"] + return ["main", "contrib", "non-free"] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--mirror", + help="apt mirror base URI, e.g. http://archive.ubuntu.com/ubuntu/", + ) + parser.add_argument( + "--codename", + help="target distribution codename; defaults to VERSION_CODENAME from the base image", + ) + parser.add_argument( + "--suite", + dest="suites", + action="append", + help="suite/pocket to enable, e.g. noble or noble-security (repeatable); " + "defaults to the release plus its '-updates' and '-security' pockets", + ) + parser.add_argument( + "--component", + dest="components", + action="append", + help="component to enable, e.g. main or universe (repeatable); defaults to those enabled by the base image", + ) + parser.add_argument( + "--proposed", + action="store_true", + help="also enable the '-proposed' pocket", + ) + return parser.parse_args() + + +def iter_sources() -> list[tuple[str, list[str]]]: + """Yield (uri, suites) from deb822 *.sources files and legacy sources.list.""" + sources: list[tuple[str, list[str]]] = [] + for path in glob.glob("/etc/apt/sources.list.d/*.sources"): + uri: str | None = None + suites: list[str] = [] + enabled = True + with open(path, encoding="utf-8") as f: + for line in [*f, "\n"]: + stripped_line = line.strip() + if not stripped_line: + if uri and suites and enabled: + sources.append((uri, suites)) + uri, suites, enabled = None, [], True + continue + key, _, value = stripped_line.partition(":") + key, value = key.strip(), value.strip() + if key == "URIs": + uri = value.split()[0] + elif key == "Suites": + suites = value.split() + elif key == "Enabled": + enabled = value.lower() != "no" + for path in ["/etc/apt/sources.list", *glob.glob("/etc/apt/sources.list.d/*.list")]: + try: + f = open(path, encoding="utf-8") + except FileNotFoundError: + continue + with f: + for line in f: + fields = line.split() + if len(fields) >= 3 and fields[0] in {"deb", "deb-src"}: + fields = fields[1:] + if fields[0].startswith("["): + fields = fields[fields.index(next(f for f in fields if f.endswith("]"))) + 1 :] + if len(fields) >= 2: + sources.append((fields[0], fields[1:])) + return sources + + +def configured_mirror(codename: str) -> str: + for uri, suites in iter_sources(): + if codename in suites: + return uri + raise RuntimeError(f"could not determine the configured mirror for {codename}") + + +def proposed_suite(os_release: dict[str, str], codename: str) -> str: + if os_release.get("ID") == "debian": + if codename in DEBIAN_WITHOUT_PROPOSED: + raise ValueError(f"Debian {codename} has no proposed-updates pocket") + return f"{codename}-proposed-updates" + return f"{codename}-proposed" + + +def main() -> None: + args = parse_args() + os_release = read_os_release() + codename = args.codename or os_release["VERSION_CODENAME"] + + suites = args.suites or default_suites(os_release, codename) + components = args.components or default_components(os_release) + if args.proposed: + suites = [*suites, proposed_suite(os_release, codename)] + + if not args.mirror: + if not args.proposed: + raise ValueError("--mirror or --proposed is required") + suites = [proposed_suite(os_release, codename)] + args.mirror = configured_mirror(codename) + source_files = [] + source_groups = [(args.mirror, suites)] + else: + source_files = DEFAULT_SOURCE_FILES + if os_release.get("ID") == "debian" and codename not in DEBIAN_ROLLING_CODENAMES: + security_suite = f"{codename}-security" + source_groups = [ + (args.mirror, [suite for suite in suites if suite != security_suite]), + (configured_mirror(security_suite), [security_suite]), + ] + else: + source_groups = [(args.mirror, suites)] + + for path in source_files: + try: + os.remove(path) + except FileNotFoundError: + pass + + os.makedirs(os.path.dirname(MANAGED_SOURCES_FILE), exist_ok=True) + with open(MANAGED_SOURCES_FILE, "w", encoding="utf-8") as f: + for index, (mirror, group_suites) in enumerate(source_groups): + if index: + f.write("\n") + f.write("Types: deb deb-src\n") + f.write(f"URIs: {mirror}\n") + f.write(f"Suites: {' '.join(group_suites)}\n") + f.write(f"Components: {' '.join(components)}\n") + + +if __name__ == "__main__": + main() diff --git a/packages/debmagic/src/build/signing.rs b/packages/debmagic/src/build/signing.rs new file mode 100644 index 0000000..7b07604 --- /dev/null +++ b/packages/debmagic/src/build/signing.rs @@ -0,0 +1,256 @@ +//! GPG signing of build artifacts (`.changes`/`.dsc`) via `debsign`. +//! +//! Signing can happen on the host (traditional, requires `devscripts` +//! locally) or inside a minimal same-distro container. In the container case +//! the host's gpg-agent *extra* socket is forwarded in, so private key +//! material never leaves the host — the agent on the host performs the +//! signing operations, and only the public key is imported into the +//! container's keyring. + +use std::{ + path::{Path, PathBuf}, + process::{Command, Stdio}, +}; + +use anyhow::{Context, anyhow}; +use serde::{Deserialize, Serialize}; + +use crate::build::common::run_checked; + +/// Where the forwarded agent socket is bind-mounted inside sign containers. +/// A fixed, always-existing path; the script symlinks it to gpg's lookup +/// locations so plain `debsign` works without extra flags or env vars. +pub const GPG_SOCKET_IN_CONTAINER: &str = "/tmp/debmagic-gpg/S.gpg-agent"; +/// Directory mounted read-only into sign containers, holding the exported +/// public key and ownertrust line produced on the host. +pub const SIGN_STAGING_IN_CONTAINER: &str = "/debmagic-sign"; +/// Mount point of the output directory inside sign containers. +pub const OUTPUT_DIR_IN_CONTAINER: &str = "/debmagic-output"; + +pub const PUBKEY_FILE: &str = "pubkey.asc"; +pub const OWNERTRUST_FILE: &str = "ownertrust.txt"; + +/// Selects where `debsign` runs. +#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, clap::ValueEnum, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SignWith { + /// Use the host if `debsign` is available there, otherwise a container + /// (requires a containerized build driver). + #[default] + Auto, + /// Always sign on the host with `debsign`. + Host, + /// Sign inside a minimal container of the same distro, forwarding the + /// host's gpg-agent socket. Requires `sign_key` to be set. + Same, +} + +/// Everything needed to GPG-sign inside a container: the host's gpg-agent +/// extra socket plus the public key to seed the container's keyring with. +pub struct GpgForwarding { + pub agent_extra_socket: PathBuf, + pub sign_key: String, +} + +/// Resolve the host's gpg-agent *extra* socket — the restricted variant +/// intended for forwarding into chroots/containers (signing works, key +/// export and management don't). +pub fn gpg_agent_extra_socket() -> anyhow::Result { + let output = Command::new("gpgconf") + .args(["--list-dirs", "agent-extra-socket"]) + .stdout(Stdio::piped()) + .output() + .context("failed to run gpgconf; is gpg installed?")?; + if !output.status.success() { + return Err(anyhow!( + "gpgconf --list-dirs agent-extra-socket failed; is gpg-agent set up?" + )); + } + let path = PathBuf::from(String::from_utf8_lossy(&output.stdout).trim().to_string()); + if !path.exists() { + return Err(anyhow!( + "gpg-agent extra socket {} does not exist; is gpg-agent running?", + path.display() + )); + } + Ok(path) +} + +/// Verify that the host gpg setup can sign with `sign_key` (secret key +/// available via the agent). Intended as a pre-flight check so builds don't +/// fail at the signing step after all the work is done. +pub fn check_signing_key_available(sign_key: &str) -> anyhow::Result<()> { + let output = Command::new("gpg") + .args(["--batch", "--list-secret-keys", sign_key]) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .output() + .context("failed to run gpg; is it installed?")?; + if !output.status.success() || output.stdout.is_empty() { + return Err(anyhow!( + "no secret key for '{sign_key}' available to gpg; \ + import it on the host or pick a different sign_key" + )); + } + Ok(()) +} + +/// Export the public key for `sign_key` from the host keyring. +pub fn export_public_key(sign_key: &str) -> anyhow::Result> { + let output = Command::new("gpg") + .args(["--batch", "--export", sign_key]) + .stdout(Stdio::piped()) + .output() + .context("failed to run gpg --export")?; + if !output.status.success() || output.stdout.is_empty() { + return Err(anyhow!( + "failed to export public key for '{sign_key}' from the host keyring" + )); + } + Ok(output.stdout) +} + +/// Fingerprint of the key `debsign` will use, for ownertrust seeding. +pub fn key_fingerprint(sign_key: &str) -> anyhow::Result { + let output = Command::new("gpg") + .args(["--batch", "--with-colons", "--list-secret-keys", sign_key]) + .stdout(Stdio::piped()) + .output() + .context("failed to run gpg --list-secret-keys")?; + if !output.status.success() { + return Err(anyhow!("failed to look up fingerprint for '{sign_key}'")); + } + for line in String::from_utf8_lossy(&output.stdout).lines() { + let fields: Vec<&str> = line.split(':').collect(); + if fields.first() == Some(&"fpr") + && let Some(fpr) = fields.get(9) + { + return Ok(fpr.to_string()); + } + } + Err(anyhow!("no fingerprint found for key '{sign_key}'")) +} + +/// Stage the files a sign container needs (exported public key + ownertrust) +/// into `staging_dir` on the host; the drivers mount it read-only at +/// [`SIGN_STAGING_IN_CONTAINER`]. +pub fn stage_signing_material(staging_dir: &Path, sign_key: &str) -> anyhow::Result<()> { + std::fs::create_dir_all(staging_dir)?; + std::fs::write(staging_dir.join(PUBKEY_FILE), export_public_key(sign_key)?)?; + // Ownertrust format: "::"; 6 = ultimate. The + // key is the user's own, freshly imported into a throwaway keyring. + let ownertrust = format!("{}:6:\n", key_fingerprint(sign_key)?); + std::fs::write(staging_dir.join(OWNERTRUST_FILE), ownertrust)?; + Ok(()) +} + +/// Shell script run inside a sign container: install debsign, link the +/// forwarded agent socket where gpg looks for it, seed the throwaway keyring +/// with the public key plus ownertrust, then sign. Runs as root; `chown_to` +/// fixes ownership of the bind-mounted output dir afterwards when the +/// container's root is not id-mapped to the host user (docker). +pub fn sign_container_script( + changes_filename: &str, + sign_key: &str, + chown_to: Option<(u32, u32)>, +) -> String { + let chown = match chown_to { + Some((uid, gid)) => format!( + " && chown -R {uid}:{gid} {out}", + out = OUTPUT_DIR_IN_CONTAINER + ), + None => String::new(), + }; + format!( + "set -e; \ + export GNUPGHOME=/root/.gnupg; \ + mkdir -p /run/user/0/gnupg \"$GNUPGHOME\"; \ + chmod 700 /run/user/0/gnupg \"$GNUPGHOME\"; \ + ln -sf {sock} /run/user/0/gnupg/S.gpg-agent; \ + ln -sf {sock} \"$GNUPGHOME/S.gpg-agent\"; \ + apt-get update -qq; \ + apt-get install -y -qq devscripts; \ + gpg --batch --import {staging}/{pubkey}; \ + gpg --batch --import-ownertrust {staging}/{ownertrust}; \ + cd {out} && debsign -k{key} {changes}{chown}", + sock = GPG_SOCKET_IN_CONTAINER, + staging = SIGN_STAGING_IN_CONTAINER, + pubkey = PUBKEY_FILE, + ownertrust = OWNERTRUST_FILE, + out = OUTPUT_DIR_IN_CONTAINER, + key = shell_single_quote(sign_key), + changes = shell_single_quote(changes_filename), + ) +} + +/// Sign `changes_file` on the host with `debsign`. +pub fn sign_on_host(changes_file: &Path, sign_key: Option<&str>) -> anyhow::Result<()> { + let output_dir = changes_file.parent().ok_or_else(|| { + anyhow!( + "could not get output directory of {}", + changes_file.display() + ) + })?; + let filename = changes_file + .file_name() + .ok_or_else(|| anyhow!("could not get filename of {}", changes_file.display()))?; + + let mut cmd = Command::new("debsign"); + if let Some(key) = sign_key { + cmd.arg(format!("-k{key}")); + } + cmd.arg(filename).current_dir(output_dir); + run_checked(&mut cmd, &format!("signing {}", changes_file.display()))?; + Ok(()) +} + +pub fn check_host_debsign_available() -> anyhow::Result<()> { + match Command::new("debsign") + .arg("--version") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + { + Ok(_) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Err(anyhow!( + "debsign not found on PATH. It's part of devscripts; install it and set up a \ + gpg signing key, or set sign_with to \"same\" with a container driver." + )), + Err(e) => Err(e).context("failed to check for debsign"), + } +} + +pub fn changes_filename(changes_file: &Path) -> anyhow::Result<&str> { + changes_file + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| anyhow!("could not get filename of {}", changes_file.display())) +} + +fn shell_single_quote(s: &str) -> String { + format!("'{}'", s.replace('\'', "'\\''")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sign_script_quotes_filename_and_key() { + let script = sign_container_script("pkg_1.0_amd64.changes", "me@example.com", None); + assert!(script.contains("debsign -k'me@example.com' 'pkg_1.0_amd64.changes'")); + assert!(script.contains("gpg --batch --import /debmagic-sign/pubkey.asc")); + assert!(!script.contains("chown")); + } + + #[test] + fn sign_script_chowns_output_when_requested() { + let script = sign_container_script("x.changes", "key", Some((1000, 100))); + assert!(script.contains("chown -R 1000:100 /debmagic-output")); + } + + #[test] + fn shell_quote_escapes_single_quotes() { + assert_eq!(shell_single_quote("a'b"), "'a'\\''b'"); + } +} diff --git a/packages/debmagic/src/build/source.rs b/packages/debmagic/src/build/source.rs new file mode 100644 index 0000000..92f6be8 --- /dev/null +++ b/packages/debmagic/src/build/source.rs @@ -0,0 +1,606 @@ +//! Staging of the source tree into the build directory. +//! +//! Which files are staged is selected by `SourceSyncMode`: git-tracked +//! files (optionally requiring a clean worktree) or everything that isn't +//! git-ignored. Staging copies the selected entries, preserving symlinks +//! and keeping unchanged files untouched so incremental builds keep their +//! outputs; a manifest of staged entries lets incremental syncs remove +//! stale paths. + +use std::cmp::Reverse; +use std::io::{self, BufReader, Read}; +use std::os::unix::fs::{PermissionsExt, symlink}; +use std::path::{Component, Path, PathBuf}; +use std::{fs, process::Command}; + +use anyhow::{Context, anyhow, bail}; +use glob::glob; + +use crate::build::common::{BuildConfig, SourceSyncMode}; +use crate::package::PackageDescription; + +/// Paths of files tracked by git in `src`, as reported by `git ls-files`. +/// Returns `None` if `src` is not inside a git worktree. +fn git_tracked_paths(src: &Path) -> anyhow::Result>> { + let output = Command::new("git") + .args(["-C"]) + .arg(src) + .args(["ls-files", "-z"]) + .output() + .context("failed to run git ls-files")?; + if !output.status.success() { + return Ok(None); + } + let mut paths = Vec::new(); + for raw in output.stdout.split(|byte| *byte == 0) { + if raw.is_empty() { + continue; + } + let path = PathBuf::from(String::from_utf8(raw.to_vec()).with_context(|| { + format!("git-tracked path is not valid UTF-8 in {}", src.display()) + })?); + validate_source_path(&path)?; + paths.push(path); + } + Ok(Some(paths)) +} + +/// Paths of files git knows about but does not track (respecting ignore +/// rules), for warning about what a `tracked` sync leaves out. +fn git_untracked_paths(src: &Path) -> Vec { + let output = Command::new("git") + .args(["-C"]) + .arg(src) + .args(["ls-files", "-z", "--others", "--exclude-standard"]) + .output(); + match output { + Ok(output) if output.status.success() => output + .stdout + .split(|byte| *byte == 0) + .filter(|raw| !raw.is_empty()) + .filter_map(|raw| String::from_utf8(raw.to_vec()).ok()) + .map(PathBuf::from) + .collect(), + _ => Vec::new(), + } +} + +/// Ensure the git worktree in `src` has no uncommitted changes and no +/// untracked files, as required by `SourceSyncMode::Committed`. +fn git_ensure_clean_worktree(src: &Path) -> anyhow::Result<()> { + let output = Command::new("git") + .args(["-C"]) + .arg(src) + .args(["status", "--porcelain"]) + .output() + .context("failed to run git status")?; + if !output.status.success() { + // Not a git worktree; the caller falls back to worktree staging. + return Ok(()); + } + let stdout = String::from_utf8_lossy(&output.stdout); + let entries: Vec<&str> = stdout.lines().filter(|line| !line.is_empty()).collect(); + if entries.is_empty() { + return Ok(()); + } + let mut message = + String::from("source-sync mode 'committed' requires a clean git worktree, but found:\n"); + for entry in entries.iter().take(20) { + message.push_str(&format!(" {entry}\n")); + } + if entries.len() > 20 { + message.push_str(&format!(" ... and {} more\n", entries.len() - 20)); + } + message.push_str("commit the changes or use a different --source-sync mode"); + bail!(message) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "snake_case")] +enum SourcePathKind { + Directory, + File, + Symlink, +} + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +struct SourcePath { + path: PathBuf, + kind: SourcePathKind, +} + +fn validate_source_path(path: &Path) -> anyhow::Result<()> { + if path.as_os_str().is_empty() + || !path + .components() + .all(|component| matches!(component, Component::Normal(_))) + { + bail!("invalid source manifest path: {}", path.display()); + } + Ok(()) +} + +fn source_tree_entries(src: &Path, mode: SourceSyncMode) -> anyhow::Result> { + match mode { + SourceSyncMode::Worktree => worktree_entries(src), + SourceSyncMode::Tracked | SourceSyncMode::Committed => { + if mode == SourceSyncMode::Committed { + git_ensure_clean_worktree(src)?; + } + match git_tracked_paths(src)? { + Some(paths) => tracked_entries(src, &paths), + None => { + eprintln!( + "debmagic: warning: {} is not a git worktree, falling back to 'worktree' source sync", + src.display() + ); + worktree_entries(src) + } + } + } + } +} + +fn entry_kind(path: &Path) -> anyhow::Result { + let metadata = fs::symlink_metadata(path) + .with_context(|| format!("failed to stat source path {}", path.display()))?; + if metadata.is_dir() { + Ok(SourcePathKind::Directory) + } else if metadata.is_file() { + Ok(SourcePathKind::File) + } else if metadata.is_symlink() { + Ok(SourcePathKind::Symlink) + } else { + Err(anyhow!( + "unsupported file type in source tree: {}", + path.display() + )) + } +} + +/// Build the entry list from git-tracked paths: tracked files (plus their +/// parent directories), with submodule gitlinks skipped. +fn tracked_entries(src: &Path, paths: &[PathBuf]) -> anyhow::Result> { + let mut seen = std::collections::HashSet::new(); + let mut entries = Vec::new(); + for path in paths { + // Add parent directories first. + let mut ancestors: Vec<&Path> = path.ancestors().skip(1).collect(); + ancestors.pop(); // drop the empty "" ancestor + ancestors.reverse(); + for ancestor in ancestors { + if seen.insert(ancestor.to_path_buf()) { + entries.push(SourcePath { + path: ancestor.to_path_buf(), + kind: SourcePathKind::Directory, + }); + } + } + let full_path = src.join(path); + // Submodule gitlinks are directories; their contents are not staged + // since git does not track them as part of this repository. + if full_path.is_dir() { + eprintln!( + "debmagic: skipping git submodule {}; its contents are not staged", + path.display() + ); + continue; + } + entries.push(SourcePath { + path: path.clone(), + kind: entry_kind(&full_path)?, + }); + } + entries.sort_by_key(|entry| entry.path.components().count()); + Ok(entries) +} + +fn worktree_entries(src: &Path) -> anyhow::Result> { + let walker = ignore::WalkBuilder::new(src) + .standard_filters(true) + .hidden(false) + .filter_entry(|entry| !(entry.path().is_dir() && entry.path().ends_with(".git"))) + .build(); + + let mut entries = Vec::new(); + for entry in walker { + let entry = entry?; + let relative_path = entry + .path() + .strip_prefix(src) + .context("failed to get relative path")?; + if relative_path.as_os_str().is_empty() { + continue; + } + entries.push(SourcePath { + path: relative_path.to_path_buf(), + kind: entry_kind(entry.path())?, + }); + } + entries.sort_by_key(|entry| entry.path.components().count()); + Ok(entries) +} + +fn remove_path(path: &Path) -> std::io::Result<()> { + match fs::symlink_metadata(path) { + Ok(metadata) if metadata.is_dir() => fs::remove_dir_all(path), + Ok(_) => fs::remove_file(path), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error), + } +} + +fn files_match(source: &Path, destination: &Path) -> std::io::Result { + let source_metadata = fs::metadata(source)?; + let destination_metadata = match fs::symlink_metadata(destination) { + Ok(metadata) if metadata.is_file() => metadata, + Ok(_) => return Ok(false), + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error), + }; + if source_metadata.len() != destination_metadata.len() + || source_metadata.permissions().mode() != destination_metadata.permissions().mode() + { + return Ok(false); + } + + let mut source = BufReader::new(fs::File::open(source)?); + let mut destination = BufReader::new(fs::File::open(destination)?); + let mut source_buffer = [0; 8192]; + let mut destination_buffer = [0; 8192]; + loop { + let source_len = source.read(&mut source_buffer)?; + let destination_len = destination.read(&mut destination_buffer)?; + if source_len != destination_len + || source_buffer[..source_len] != destination_buffer[..destination_len] + { + return Ok(false); + } + if source_len == 0 { + return Ok(true); + } + } +} + +fn copy_source_entries(src: &Path, dst: &Path, entries: &[SourcePath]) -> anyhow::Result<()> { + fs::create_dir_all(dst)?; + for entry in entries { + let source = src.join(&entry.path); + let destination = dst.join(&entry.path); + match entry.kind { + SourcePathKind::Directory => match fs::symlink_metadata(&destination) { + Ok(metadata) if metadata.is_dir() => {} + Ok(_) => { + remove_path(&destination)?; + fs::create_dir_all(&destination)?; + } + Err(error) if error.kind() == io::ErrorKind::NotFound => { + fs::create_dir_all(&destination)?; + } + Err(error) => return Err(error.into()), + }, + SourcePathKind::File => { + if !files_match(&source, &destination)? { + remove_path(&destination)?; + fs::copy(&source, &destination) + .with_context(|| format!("failed to copy file: {}", source.display()))?; + } + } + SourcePathKind::Symlink => { + let target = fs::read_link(&source)?; + if fs::read_link(&destination).ok().as_deref() != Some(target.as_path()) { + remove_path(&destination)?; + symlink(target, &destination)?; + } + } + } + } + Ok(()) +} + +fn copy_glob(src_dir: &Path, pattern: &str, dest_dir: &Path) -> anyhow::Result<()> { + let full_pattern = src_dir.join(pattern).to_string_lossy().into_owned(); + for entry in glob(&full_pattern)? { + let path = entry?; + if path.is_file() { + let filename = path.file_name().ok_or(anyhow!( + "Could not retrieve filename from {}", + path.display() + ))?; + fs::copy(&path, dest_dir.join(filename))?; + } + } + Ok(()) +} + +pub fn source_manifest_path(build_config: &BuildConfig) -> PathBuf { + build_config.build_root_dir.join("source-manifest.json") +} + +fn write_source_manifest(build_config: &BuildConfig, entries: &[SourcePath]) -> anyhow::Result<()> { + let manifest_path = source_manifest_path(build_config); + let temporary_path = manifest_path.with_extension("json.tmp"); + fs::write(&temporary_path, serde_json::to_vec_pretty(entries)?)?; + fs::rename(temporary_path, manifest_path)?; + Ok(()) +} + +fn sync_source_tree(build_config: &BuildConfig) -> anyhow::Result<()> { + let manifest_path = source_manifest_path(build_config); + let previous: Vec = serde_json::from_reader(BufReader::new( + fs::File::open(&manifest_path) + .with_context(|| format!("failed to open {}", manifest_path.display()))?, + )) + .with_context(|| format!("failed to read {}", manifest_path.display()))?; + for entry in &previous { + validate_source_path(&entry.path)?; + } + let current = source_tree_entries(&build_config.source_dir, build_config.source_sync_mode)?; + + let current_kinds = current + .iter() + .map(|entry| (entry.path.as_path(), entry.kind)) + .collect::>(); + let mut stale = previous + .iter() + .filter(|entry| current_kinds.get(entry.path.as_path()) != Some(&entry.kind)) + .collect::>(); + stale.sort_by_key(|entry| Reverse(entry.path.components().count())); + for entry in stale { + let destination = build_config.build_source_dir().join(&entry.path); + if entry.kind == SourcePathKind::Directory + && !current_kinds.contains_key(entry.path.as_path()) + { + match fs::remove_dir(&destination) { + Ok(()) => {} + Err(error) + if matches!( + error.kind(), + io::ErrorKind::NotFound | io::ErrorKind::DirectoryNotEmpty + ) => {} + Err(error) => return Err(error.into()), + } + } else { + remove_path(&destination)?; + } + } + + copy_source_entries( + &build_config.source_dir, + &build_config.build_source_dir(), + ¤t, + )?; + write_source_manifest(build_config, ¤t) +} + +pub fn stage_source_tree( + build_config: &BuildConfig, + package: &PackageDescription, +) -> anyhow::Result<()> { + if build_config.source_sync_mode == SourceSyncMode::Tracked { + let untracked = git_untracked_paths(&build_config.source_dir); + if !untracked.is_empty() { + eprintln!( + "debmagic: warning: {} untracked file(s) not staged into the build tree:", + untracked.len() + ); + for path in untracked.iter().take(20) { + eprintln!(" {}", path.display()); + } + if untracked.len() > 20 { + eprintln!(" ... and {} more", untracked.len() - 20); + } + eprintln!(" git add them or use --source-sync worktree to include them"); + } + } + if build_config.incremental && source_manifest_path(build_config).is_file() { + sync_source_tree(build_config).context("failed to synchronize source tree")?; + } else { + let entries = source_tree_entries(&build_config.source_dir, build_config.source_sync_mode)?; + copy_source_entries( + &build_config.source_dir, + &build_config.build_source_dir(), + &entries, + ) + .context("failed to copy source tree to build directory")?; + write_source_manifest(build_config, &entries)?; + } + + let source_parent = build_config + .source_dir + .parent() + .ok_or_else(|| anyhow!("source directory has no parent"))?; + let prefix = format!("{}_{}", package.name, package.version.upstream_version()); + copy_glob( + source_parent, + &format!("{prefix}.orig.tar.*"), + &build_config.build_work_dir(), + )?; + copy_glob( + source_parent, + &format!("{prefix}.orig-*.tar.*"), + &build_config.build_work_dir(), + )?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::os::unix::fs::MetadataExt; + + use crate::build::common::BuildDriverType; + + use super::*; + + #[test] + fn incremental_sync_updates_sources_and_preserves_build_outputs() -> anyhow::Result<()> { + let test_root = std::env::temp_dir().join(format!( + "debmagic-incremental-test-{}", + uuid::Uuid::new_v4() + )); + let source_dir = test_root.join("source"); + let build_root_dir = test_root.join("build"); + fs::create_dir_all(source_dir.join("cache"))?; + fs::write(source_dir.join("changed.txt"), "before")?; + fs::write(source_dir.join("unchanged.txt"), "unchanged")?; + fs::write(source_dir.join("removed.txt"), "remove me")?; + fs::write(source_dir.join("cache/input.c"), "source")?; + symlink("changed.txt", source_dir.join("link"))?; + + let build_config = BuildConfig { + driver: BuildDriverType::Bare, + package_name: "example".to_string(), + package_identifier: "example-1.0".to_string(), + build_root_dir: build_root_dir.clone(), + source_dir: source_dir.clone(), + output_dir: test_root.join("output"), + distro: debmagic_common::distro::get_distro_version("trixie").unwrap(), + sign_package: false, + sign_with: crate::build::signing::SignWith::default(), + sign_key: None, + build_debug_symbols: false, + clean: false, + persistent: true, + incremental: true, + source_sync_mode: SourceSyncMode::Worktree, + }; + build_config.create_dirs()?; + let initial_entries = source_tree_entries(&source_dir, SourceSyncMode::Worktree)?; + copy_source_entries( + &source_dir, + &build_config.build_source_dir(), + &initial_entries, + )?; + write_source_manifest(&build_config, &initial_entries)?; + let unchanged_inode = + fs::metadata(build_config.build_source_dir().join("unchanged.txt"))?.ino(); + fs::write( + build_config.build_source_dir().join("cache/output.o"), + "compiled", + )?; + + fs::write(source_dir.join("changed.txt"), "after")?; + fs::remove_file(source_dir.join("removed.txt"))?; + fs::remove_file(source_dir.join("cache/input.c"))?; + fs::remove_dir(source_dir.join("cache"))?; + fs::remove_file(source_dir.join("link"))?; + symlink("added.txt", source_dir.join("link"))?; + fs::write(source_dir.join("added.txt"), "new")?; + + sync_source_tree(&build_config)?; + + let staged = build_config.build_source_dir(); + assert_eq!(fs::read_to_string(staged.join("changed.txt"))?, "after"); + assert_eq!(fs::read_to_string(staged.join("added.txt"))?, "new"); + assert_eq!( + fs::metadata(staged.join("unchanged.txt"))?.ino(), + unchanged_inode + ); + assert_eq!(fs::read_link(staged.join("link"))?, Path::new("added.txt")); + assert!(!staged.join("removed.txt").exists()); + assert!(!staged.join("cache/input.c").exists()); + assert_eq!( + fs::read_to_string(staged.join("cache/output.o"))?, + "compiled" + ); + + fs::remove_dir_all(test_root)?; + Ok(()) + } + + #[test] + fn source_manifest_paths_must_be_relative_and_normal() { + assert!(validate_source_path(Path::new("debian/control")).is_ok()); + for path in ["", ".", "../outside", "debian/../outside", "/tmp/outside"] { + assert!(validate_source_path(Path::new(path)).is_err(), "{path}"); + } + } + + /// Create a git repo with one committed file in a fresh temp dir. + fn git_test_repo() -> anyhow::Result { + let repo = std::env::temp_dir().join(format!("debmagic-git-test-{}", uuid::Uuid::new_v4())); + fs::create_dir_all(repo.join("debian"))?; + fs::write(repo.join("debian/control"), "Source: example")?; + let run = |args: &[&str]| -> anyhow::Result<()> { + let status = Command::new("git") + .arg("-C") + .arg(&repo) + .args(args) + .status()?; + if status.success() { + Ok(()) + } else { + Err(anyhow!("git {:?} failed", args)) + } + }; + run(&["init", "-q"])?; + run(&["add", "debian/control"])?; + run(&[ + "-c", + "user.email=t@t", + "-c", + "user.name=t", + "commit", + "-qm", + "init", + ])?; + Ok(repo) + } + + #[test] + fn tracked_sync_stages_only_git_tracked_files() -> anyhow::Result<()> { + let repo = git_test_repo()?; + fs::write(repo.join("untracked.txt"), "not staged")?; + fs::write(repo.join("dirty.txt"), "uncommitted but tracked? no")?; + // A tracked file with uncommitted modifications is staged with its + // worktree content. + fs::write(repo.join("debian/rules"), "new content")?; + Command::new("git") + .arg("-C") + .arg(&repo) + .args(["add", "debian/rules"]) + .status()?; + + let entries = source_tree_entries(&repo, SourceSyncMode::Tracked)?; + let paths: Vec<&Path> = entries.iter().map(|e| e.path.as_path()).collect(); + assert!(paths.contains(&Path::new("debian"))); + assert!(paths.contains(&Path::new("debian/control"))); + assert!(paths.contains(&Path::new("debian/rules"))); + assert!(!paths.contains(&Path::new("untracked.txt"))); + assert!(!paths.contains(&Path::new("dirty.txt"))); + assert_eq!(git_untracked_paths(&repo).len(), 2); + + fs::remove_dir_all(repo)?; + Ok(()) + } + + #[test] + fn committed_sync_requires_clean_worktree() -> anyhow::Result<()> { + let repo = git_test_repo()?; + assert!(source_tree_entries(&repo, SourceSyncMode::Committed).is_ok()); + + fs::write(repo.join("untracked.txt"), "dirty")?; + let result = source_tree_entries(&repo, SourceSyncMode::Committed); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("requires a clean git worktree") + ); + + fs::remove_dir_all(repo)?; + Ok(()) + } + + #[test] + fn tracked_sync_falls_back_outside_git_worktree() -> anyhow::Result<()> { + let dir = std::env::temp_dir().join(format!("debmagic-nogit-{}", uuid::Uuid::new_v4())); + fs::create_dir_all(&dir)?; + fs::write(dir.join("file.txt"), "content")?; + let entries = source_tree_entries(&dir, SourceSyncMode::Tracked)?; + assert!(entries.iter().any(|e| e.path == Path::new("file.txt"))); + fs::remove_dir_all(dir)?; + Ok(()) + } +} diff --git a/packages/debmagic/src/cli.rs b/packages/debmagic/src/cli.rs index 22eac71..6551fb3 100644 --- a/packages/debmagic/src/cli.rs +++ b/packages/debmagic/src/cli.rs @@ -1,6 +1,6 @@ use std::path::PathBuf; -use crate::build::common::BuildDriverType; +use crate::build::common::{BuildDriverType, SourceSyncMode}; use clap::{Args, Parser, Subcommand}; #[derive(Parser, Debug)] @@ -15,8 +15,8 @@ pub struct Cli { #[derive(Subcommand, Debug)] pub enum Commands { - #[command(about = "Build a debian package")] - Build(BuildSubcommandArgs), + #[command(about = "Build a debian package: 'binary' (.deb) or 'source' (.dsc) packages")] + Build(Box), #[command(about = "Open an interactive shell to the currently active build environment")] Shell(ShellSubcommandArgs), #[command(about = "Run tests")] @@ -40,6 +40,7 @@ pub struct CommonCli { #[derive(Args, Debug)] pub struct DockerArgs { #[arg( + id = "docker-base-image", long = "driver-docker-base-image", help = "If passed will override the base image for the current build" )] @@ -47,25 +48,105 @@ pub struct DockerArgs { } #[derive(Args, Debug)] -pub struct BuildSubcommandArgs { - #[arg(short, long, help = "Build driver type")] - pub driver: BuildDriverType, +pub struct LxdArgs { + #[arg( + id = "lxd-base-image", + long = "driver-lxd-base-image", + help = "Override the base image (image alias) for the LXD/Incus container" + )] + pub base_image: Option, + + #[arg( + long = "driver-lxd-project", + help = "LXD/Incus project to register the container in" + )] + pub project: Option, +} + +/// Flags shared between `debmagic build binary` and `debmagic build source`. +#[derive(Args, Debug)] +pub struct CommonBuildArgs { + #[arg( + short, + long, + help = "Build driver type. Required for binary builds; source-only builds default to 'bare', since those need no build-deps or compilation." + )] + pub driver: Option, - #[arg(long, action = clap::ArgAction::SetTrue, help = "Persist the build environment after the build finished")] - pub persist_driver: Option, + #[arg(long, action = clap::ArgAction::SetTrue, help = "Keep the build environment for reuse after the build finishes")] + pub persistent: Option, + + #[arg( + long = "source-sync", + help = "Which source files are staged into the build tree: 'tracked' stages git-tracked files including uncommitted changes and warns about untracked files (default), 'committed' additionally fails if the worktree is dirty, 'worktree' stages everything that is not git-ignored. Defaults to the 'source_sync_mode' setting in the config file." + )] + pub source_sync: Option, #[command(flatten)] pub docker: DockerArgs, - #[arg(short, long, action = clap::ArgAction::SetTrue, help = "Enable incremental builds. This implies --persist-driver")] - pub incremental: Option, + #[command(flatten)] + pub lxd: LxdArgs, + + #[arg( + long = "apt-mirror", + help = "Apt mirror URL to use inside the build environment instead of the default archive.ubuntu.com/security.ubuntu.com/deb.debian.org, e.g. http://my-mirror.example/ubuntu. Ignored by the bare driver." + )] + pub apt_mirror: Option, #[arg( long, - help = "Select the target distribution version, only required in the debian changelog specifies multiple versions" + action = clap::ArgAction::SetTrue, + help = "Also enable the '-proposed' pocket in the build environment. Ignored by the bare driver." + )] + pub proposed: Option, + + #[arg( + long, + help = "Select the target distribution version, only required if the debian changelog specifies multiple versions" )] pub distro: Option, + #[arg( + long, + action = clap::ArgAction::SetTrue, + help = "Sign the resulting .changes/.dsc with debsign after building. Defaults to the 'sign_package' setting in the config file (false if unset)." + )] + pub sign: Option, + + #[arg( + long, + action = clap::ArgAction::SetFalse, + help = "Do not sign the resulting .changes/.dsc, overriding a 'sign_package = true' default in the config file." + )] + pub no_sign: Option, + + #[arg( + long = "sign-with", + help = "Where debsign runs: 'host' signs on the host (requires devscripts there), 'same' signs inside a minimal same-distro container with the host gpg-agent socket forwarded in (requires --sign-key), 'auto' (default) uses the host if debsign is available there, else a container. Defaults to the 'sign_with' setting in the config file." + )] + pub sign_with: Option, + + #[arg( + long = "sign-key", + help = "GPG key ID/email to sign with, passed to debsign's -k option. Defaults to the 'sign_key' setting in the config file, or debsign's own maintainer-based key lookup if unset. Required when signing in a container." + )] + pub sign_key: Option, + + #[arg( + long, + action = clap::ArgAction::SetTrue, + help = "Run 'debian/rules clean' before building, like plain dpkg-buildpackage does unless passed -nc. Defaults to the 'clean' setting in the config file (false if unset); non-incremental builds already stage a clean source tree, while incremental builds preserve outputs by design. For source builds this also installs build-dependencies first, since a clean target usually needs its own tooling." + )] + pub clean: Option, + + #[arg( + long, + action = clap::ArgAction::SetFalse, + help = "Do not run 'debian/rules clean' before building, overriding a 'clean = true' default in the config file." + )] + pub no_clean: Option, + #[command(flatten)] pub common: CommonCli, @@ -73,6 +154,44 @@ pub struct BuildSubcommandArgs { pub output_dir: Option, } +#[derive(Args, Debug)] +pub struct BuildSubcommandArgs { + #[command(subcommand)] + pub target: BuildTarget, +} + +#[derive(Subcommand, Debug)] +pub enum BuildTarget { + #[command(about = "Build binary .deb packages")] + Binary(BinaryTargetArgs), + #[command( + about = "Build a source package only (.dsc + tarball, plus .buildinfo/.changes), no build-deps or compilation required" + )] + Source(SourceTargetArgs), +} + +#[derive(Args, Debug)] +pub struct BinaryTargetArgs { + #[command(flatten)] + pub build: CommonBuildArgs, + + #[arg(short, long, action = clap::ArgAction::SetTrue, help = "Synchronize changed source inputs while preserving build outputs. Implies --persistent")] + pub incremental: Option, + + #[arg( + long = "debug-symbols", + action = clap::ArgAction::SetTrue, + help = "Also build the automatic '-dbgsym' debug symbol package" + )] + pub debug_symbols: Option, +} + +#[derive(Args, Debug)] +pub struct SourceTargetArgs { + #[command(flatten)] + pub build: CommonBuildArgs, +} + #[derive(Args, Debug)] pub struct ShellSubcommandArgs { #[command(flatten)] diff --git a/packages/debmagic/src/config.rs b/packages/debmagic/src/config.rs index 9ab1084..3c5c8b7 100644 --- a/packages/debmagic/src/config.rs +++ b/packages/debmagic/src/config.rs @@ -1,16 +1,37 @@ use std::path::PathBuf; +use crate::build::common::SourceSyncMode; use crate::build::config::DriverConfig; +use crate::build::signing::SignWith; use anyhow::{Context, anyhow}; use config::{Config as ConfigBuilder, File}; use serde::Deserialize; +/// documented in docs/usage/config.md #[derive(Deserialize, Debug)] #[serde(default)] pub struct Config { pub driver: DriverConfig, pub temp_build_dir: PathBuf, pub incremental: bool, + /// Which source files are staged into the build tree. + pub source_sync_mode: SourceSyncMode, + /// Always build the automatic `-dbgsym` debug symbol package. + pub build_debug_symbols: bool, + /// Sign the resulting `.changes`/`.dsc` with `debsign` after building. + pub sign_package: bool, + /// Where `debsign` runs: on the host or inside a minimal same-distro + /// container with the host's gpg-agent socket forwarded in. + pub sign_with: SignWith, + /// GPG key ID/email to sign with (debsign's `-k` option). `None` lets + /// debsign fall back to its own maintainer-based key lookup, but + /// container signing requires an explicit key. + pub sign_key: Option, + /// Run `debian/rules clean` before building (like `dpkg-buildpackage` + /// does unless passed `-nc`). Disabled by default because non-incremental + /// builds already stage a clean source tree and incremental builds preserve + /// outputs intentionally. + pub clean: bool, } impl Default for Config { @@ -19,6 +40,12 @@ impl Default for Config { driver: DriverConfig::default(), temp_build_dir: PathBuf::from("/tmp/debmagic"), incremental: false, + source_sync_mode: SourceSyncMode::default(), + build_debug_symbols: false, + sign_package: false, + sign_with: SignWith::default(), + sign_key: None, + clean: false, } } } diff --git a/packages/debmagic/src/main.rs b/packages/debmagic/src/main.rs index 81f8781..99a4e03 100644 --- a/packages/debmagic/src/main.rs +++ b/packages/debmagic/src/main.rs @@ -8,10 +8,12 @@ use clap::{CommandFactory, Parser}; use crate::{ build::{ - build_package, config::DriverOverrides, driver_bare::DriverBareConfigOverrides, - driver_docker::DriverDockerConfigOverrides, get_shell_in_build, + BuildRequest, build_package, build_source_package, common::BuildDriverType, + config::DriverOverrides, driver_bare::DriverBareConfigOverrides, + driver_docker::DriverDockerConfigOverrides, driver_lxd::DriverLxdConfigOverrides, + get_shell_in_build, }, - cli::{Cli, Commands}, + cli::{BuildTarget, Cli, Commands, CommonBuildArgs}, config::Config, package::PackageDescription, }; @@ -54,44 +56,110 @@ fn main() -> anyhow::Result<()> { let current_dir = env::current_dir()?; match &cli.command { Commands::Build(args) => { - let source_dir = args.common.source_dir.as_deref().unwrap_or(¤t_dir); + let (build_args, debug_symbols, incremental, is_source): ( + &CommonBuildArgs, + Option, + Option, + bool, + ) = match &args.target { + BuildTarget::Binary(binary_args) => ( + &binary_args.build, + binary_args.debug_symbols, + binary_args.incremental, + false, + ), + BuildTarget::Source(source_args) => (&source_args.build, None, None, true), + }; + + let source_dir = build_args + .common + .source_dir + .as_deref() + .unwrap_or(¤t_dir); let mut config = get_config(&cli, &Some(source_dir.to_path_buf()))?; // TODO: figure out a better way to override config from CLI args - maybe more generic, if that is even possible since // we want a nice cli which somewhat matches the config structure // but some config options only make sense in some cli subcommands -> these flags don't make sense in all commands // and should only be used in some - if let Some(persist_driver) = args.persist_driver { - config.driver.persistent = persist_driver; + if let Some(persistent) = build_args.persistent { + config.driver.persistent = persistent; } - if let Some(incremental) = args.incremental { + if is_source { + config.incremental = false; + } else if let Some(incremental) = incremental { config.incremental = incremental; } + + if let Some(debug_symbols) = debug_symbols { + config.build_debug_symbols = debug_symbols; + } + if let Some(sign) = build_args.sign { + config.sign_package = sign; + } + if let Some(no_sign) = build_args.no_sign { + config.sign_package = !no_sign; + } + if let Some(sign_with) = build_args.sign_with { + config.sign_with = sign_with; + } + if let Some(sign_key) = build_args.sign_key.clone() { + config.sign_key = Some(sign_key); + } + if let Some(clean) = build_args.clean { + config.clean = clean; + } + if let Some(no_clean) = build_args.no_clean { + config.clean = !no_clean; + } + if let Some(source_sync) = build_args.source_sync { + config.source_sync_mode = source_sync; + } if config.incremental { - // TODO: investigate if this is actually needed + if config.clean { + anyhow::bail!("incremental builds are incompatible with clean builds"); + } config.driver.persistent = true; } let driver_overrides = DriverOverrides { + apt_mirror: build_args.apt_mirror.clone(), + proposed: build_args.proposed, docker: DriverDockerConfigOverrides { - base_image: args.docker.base_image.clone(), + base_image: build_args.docker.base_image.clone(), }, bare: DriverBareConfigOverrides {}, + lxd: DriverLxdConfigOverrides { + base_image: build_args.lxd.base_image.clone(), + project: build_args.lxd.project.clone(), + }, }; let package = PackageDescription::from_dir( &path::absolute(source_dir).context("resolving source dir failed")?, )?; - let output_dir = args.output_dir.as_deref().unwrap_or(¤t_dir); - build_package( - &config, - &package, - args.driver, - &driver_overrides, - &path::absolute(output_dir).context("resolving output dir failed")?, - args.distro.as_deref(), - ) - .context("Building the package failed")?; + let output_dir = build_args.output_dir.as_deref().unwrap_or(¤t_dir); + let output_dir = path::absolute(output_dir).context("resolving output dir failed")?; + + let request = BuildRequest { + config: &config, + package: &package, + driver_type: if is_source { + build_args.driver.unwrap_or(BuildDriverType::Bare) + } else { + build_args.driver.context( + "--driver is required for binary builds (docker, bare, lxd or incus)", + )? + }, + driver_overrides: &driver_overrides, + output_dir: &output_dir, + explicit_distro_version: build_args.distro.as_deref(), + }; + if is_source { + build_source_package(&request).context("Building the source package failed")?; + } else { + build_package(&request).context("Building the package failed")?; + } } Commands::Shell(args) => { let source_dir = args.common.source_dir.as_deref().unwrap_or(¤t_dir); diff --git a/packages/debmagic/tests/signing.rs b/packages/debmagic/tests/signing.rs new file mode 100644 index 0000000..fda4269 --- /dev/null +++ b/packages/debmagic/tests/signing.rs @@ -0,0 +1,240 @@ +//! End-to-end test for container signing: builds a throwaway gpg key with a +//! loopback pinentry, crafts a minimal source package artifact set +//! (`.dsc` + `.changes`), then has the docker driver sign it in a minimal +//! container via the forwarded gpg-agent socket. +//! +//! Requires docker and gpg on the host. Ignored by default; run with: +//! +//! ```shell +//! cargo test --test signing -- --ignored --nocapture +//! ``` + +use std::{ + fs, + path::{Path, PathBuf}, + process::{Command, Stdio}, +}; + +/// Run `cmd`, panicking with stdout+stderr on failure. +fn run(cmd: &mut Command) -> String { + let output = cmd.output().expect("failed to spawn command"); + if !output.status.success() { + panic!( + "command failed: {:?}\nstdout: {}\nstderr: {}", + cmd, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + String::from_utf8_lossy(&output.stdout).into_owned() +} + +struct TestGpgHome { + dir: PathBuf, +} + +impl TestGpgHome { + /// Create an isolated GNUPGHOME with a throwaway signing key and an + /// agent that answers without pinentry. + fn create() -> Self { + let dir = std::env::temp_dir().join(format!("debmagic-sign-test-{}", uuid::Uuid::new_v4())); + fs::create_dir_all(&dir).unwrap(); + // gpg refuses to use a homedir others could access. + run(Command::new("chmod").args(["700"]).arg(&dir)); + + fs::write(dir.join("gpg-agent.conf"), "allow-loopback-pinentry\n").unwrap(); + fs::write(dir.join("gpg.conf"), "pinentry-mode loopback\n").unwrap(); + + run(Command::new("gpgconf") + .env("GNUPGHOME", &dir) + .args(["--launch", "gpg-agent"])); + + run(Command::new("gpg").env("GNUPGHOME", &dir).args([ + "--batch", + "--passphrase", + "", + "--quick-generate-key", + "debmagic sign test ", + "ed25519", + "sign", + "never", + ])); + + Self { dir } + } + + fn agent_extra_socket(&self) -> PathBuf { + let out = run(Command::new("gpgconf") + .env("GNUPGHOME", &self.dir) + .args(["--list-dirs", "agent-extra-socket"])); + PathBuf::from(out.trim()) + } + + fn fingerprint(&self) -> String { + let out = run(Command::new("gpg").env("GNUPGHOME", &self.dir).args([ + "--batch", + "--with-colons", + "--list-secret-keys", + "sign@example.invalid", + ])); + for line in out.lines() { + let fields: Vec<&str> = line.split(':').collect(); + if fields.first() == Some(&"fpr") { + return fields[9].to_string(); + } + } + panic!("no fingerprint found"); + } +} + +impl Drop for TestGpgHome { + fn drop(&mut self) { + let _ = Command::new("gpgconf") + .env("GNUPGHOME", &self.dir) + .args(["--kill", "all"]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + let _ = fs::remove_dir_all(&self.dir); + } +} + +/// Write a minimal artifact set (`hello.txt`, `.dsc`, `.changes`) with +/// consistent sizes and sha256 checksums, so `debsign` accepts it. +fn write_fake_artifacts(output_dir: &Path) -> PathBuf { + fs::create_dir_all(output_dir).unwrap(); + let payload = b"hello from debmagic sign test\n"; + fs::write(output_dir.join("hello.txt"), payload).unwrap(); + + let sha256 = |data: &[u8]| -> String { + let mut child = Command::new("sha256sum") + .arg("-") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .expect("failed to spawn sha256sum"); + use std::io::Write; + child + .stdin + .as_mut() + .unwrap() + .write_all(data) + .expect("failed to pipe to sha256sum"); + let out = child.wait_with_output().expect("sha256sum failed"); + String::from_utf8_lossy(&out.stdout) + .split_whitespace() + .next() + .unwrap() + .to_string() + }; + + let files_entry = + |name: &str, data: &[u8]| format!(" {} {} {}", sha256(data), data.len(), name); + + let dsc_content = format!( + "Format: 3.0 (native)\nSource: debmagic-sign-test\nBinary: debmagic-sign-test\nVersion: 1.0\nMaintainer: debmagic sign test \nArchitecture: all\nFiles:\n{}\n", + files_entry("hello.txt", payload) + ); + let dsc_name = "debmagic-sign-test_1.0.dsc"; + fs::write(output_dir.join(dsc_name), &dsc_content).unwrap(); + + let changes_content = format!( + "Format: 1.8\nSource: debmagic-sign-test\nBinary: debmagic-sign-test\nVersion: 1.0\nMaintainer: debmagic sign test \nArchitecture: source all\nDistribution: unstable\nFiles:\n{}\n{}\n", + files_entry(dsc_name, dsc_content.as_bytes()), + files_entry("hello.txt", payload) + ); + let changes_path = output_dir.join("debmagic-sign-test_1.0_amd64.changes"); + fs::write(&changes_path, &changes_content).unwrap(); + changes_path +} + +#[test] +#[ignore = "needs docker and gpg on the host"] +fn docker_signs_changes_with_forwarded_agent() { + // Skip early with a clear message if docker isn't usable. + if Command::new("docker") + .args(["info"]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .map(|s| !s.success()) + .unwrap_or(true) + { + eprintln!("docker not available, skipping"); + return; + } + + let gpg_home = TestGpgHome::create(); + let work_dir = std::env::temp_dir().join(format!("debmagic-sign-out-{}", uuid::Uuid::new_v4())); + let changes_path = write_fake_artifacts(&work_dir); + + let output_dir = changes_path.parent().unwrap(); + let staging_dir = work_dir.join("staging"); + fs::create_dir_all(&staging_dir).unwrap(); + + // Stage pubkey + ownertrust the same way the driver does. The export is + // binary OpenPGP data, so capture raw bytes rather than a String. + let pubkey = Command::new("gpg") + .env("GNUPGHOME", &gpg_home.dir) + .args(["--batch", "--export", "sign@example.invalid"]) + .output() + .expect("gpg export failed"); + assert!(pubkey.status.success()); + fs::write(staging_dir.join("pubkey.asc"), pubkey.stdout).unwrap(); + fs::write( + staging_dir.join("ownertrust.txt"), + format!("{}:6:\n", gpg_home.fingerprint()), + ) + .unwrap(); + + let socket = gpg_home.agent_extra_socket(); + let script = "set -e; \ + export GNUPGHOME=/root/.gnupg; \ + mkdir -p /run/user/0/gnupg \"$GNUPGHOME\"; \ + chmod 700 /run/user/0/gnupg \"$GNUPGHOME\"; \ + ln -sf /tmp/debmagic-gpg/S.gpg-agent /run/user/0/gnupg/S.gpg-agent; \ + ln -sf /tmp/debmagic-gpg/S.gpg-agent \"$GNUPGHOME/S.gpg-agent\"; \ + apt-get update -qq; \ + apt-get install -y -qq devscripts; \ + gpg --batch --import /debmagic-sign/pubkey.asc; \ + gpg --batch --import-ownertrust /debmagic-sign/ownertrust.txt; \ + cd /debmagic-output && debsign -k'sign@example.invalid' 'debmagic-sign-test_1.0_amd64.changes'"; + + run(Command::new("docker") + .args(["run", "--rm", "--init"]) + .args([ + "--mount", + &format!( + "type=bind,src={},dst=/debmagic-sign,readonly", + staging_dir.display() + ), + ]) + .arg(format!( + "--mount=type=bind,src={},dst=/tmp/debmagic-gpg/S.gpg-agent,readonly", + socket.display() + )) + .args([ + "--mount", + &format!( + "type=bind,src={},dst=/debmagic-output", + output_dir.display() + ), + ]) + .arg("docker.io/debian:trixie") + .args(["sh", "-ec", script])); + + let signed = fs::read_to_string(&changes_path).unwrap(); + assert!( + signed.contains("-----BEGIN PGP SIGNATURE-----"), + "changes file was not signed:\n{signed}" + ); + + // Verify the signature against the test keyring. The .changes file has + // the message and signature in one clearsigned document. + run(Command::new("gpg") + .env("GNUPGHOME", &gpg_home.dir) + .args(["--batch", "--verify"]) + .arg(&changes_path)); + + let _ = fs::remove_dir_all(&work_dir); +} diff --git a/tests/integration/test_packages.py b/tests/integration/test_packages.py index ea133c8..107de33 100644 --- a/tests/integration/test_packages.py +++ b/tests/integration/test_packages.py @@ -96,6 +96,7 @@ def test_build_package(test_env: Environment, package: str, version: str): "debmagic", "--", "build", + "binary", "--driver", "docker", "--driver-docker-base-image",