From c995ac4bf834969c4240678b69440cdf749f8175 Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Thu, 19 Mar 2026 15:32:48 +0100 Subject: [PATCH 1/9] Added container-based CFEngine package builder Introduced build-in-container, a Python/Docker-based build system that builds CFEngine packages inside containers using the existing build scripts. Ticket: ENT-13777 Signed-off-by: Lars Erik Wik --- build-in-container-inner.sh | 113 +++++++++++++ build-in-container.md | 117 +++++++++++++ build-in-container.py | 318 ++++++++++++++++++++++++++++++++++++ container/Dockerfile.debian | 42 +++++ 4 files changed, 590 insertions(+) create mode 100755 build-in-container-inner.sh create mode 100644 build-in-container.md create mode 100755 build-in-container.py create mode 100644 container/Dockerfile.debian diff --git a/build-in-container-inner.sh b/build-in-container-inner.sh new file mode 100755 index 000000000..3a6b39054 --- /dev/null +++ b/build-in-container-inner.sh @@ -0,0 +1,113 @@ +#!/bin/bash +set -e + +# Configuration via environment variables: +# PROJECT, BUILD_TYPE, EXPLICIT_ROLE, BUILD_NUMBER, EXPLICIT_VERSION + +BASEDIR=/home/builder/build +export BASEDIR +export AUTOBUILD_PATH="$BASEDIR/buildscripts" + +mkdir -p "$BASEDIR" + +# Bind-mounted directories may be owned by the host user's UID. +# Fix ownership so builder can write to them. +sudo chown -R "$(id -u):$(id -g)" "$HOME/.cache" /output + +# Prevent git "dubious ownership" errors +git config --global --add safe.directory '*' + +# === Sync source repos === +repos="buildscripts core masterfiles" +if [ "$PROJECT" = "nova" ]; then + repos="$repos enterprise nova mission-portal" +fi + +for repo in $repos; do + src="/srv/source/$repo" + # Use rsync -aL to follow symlinks during copy. + # The source dir may use symlinks (e.g., core -> cfengine/core/). + # -L resolves them at copy time, so the destination gets real files + # regardless of the host directory layout. + # Exclude acceptance test workdirs — they contain broken symlinks left + # over from previous test runs and are not needed for building. + if [ -d "$src" ] || [ -L "$src" ]; then + echo "Syncing $repo..." + sudo rsync -aL --exclude='config.cache' --exclude='workdir' --chown="$(id -u):$(id -g)" "$src/" "$BASEDIR/$repo/" + else + echo "ERROR: Required repository $repo not found" >&2 + exit 1 + fi +done + +install_mission_portal_deps() ( + set -e + + if [ -f "$BASEDIR/mission-portal/public/scripts/package.json" ]; then + echo "Installing npm dependencies..." + npm ci --prefix "$BASEDIR/mission-portal/public/scripts/" + echo "Building react components..." + npm run build --prefix "$BASEDIR/mission-portal/public/scripts/" + rm -rf "$BASEDIR/mission-portal/public/scripts/node_modules" + fi + + if [ -f "$BASEDIR/mission-portal/composer.json" ]; then + echo "Installing Mission Portal PHP dependencies..." + (cd "$BASEDIR/mission-portal" && php /usr/bin/composer.phar install --no-dev --ignore-platform-reqs) + fi + + if [ -f "$BASEDIR/nova/api/http/composer.json" ]; then + echo "Installing Nova API PHP dependencies..." + (cd "$BASEDIR/nova/api/http" && php /usr/bin/composer.phar install --no-dev --ignore-platform-reqs) + fi + + if [ -f "$BASEDIR/mission-portal/public/themes/default/bootstrap/cfengine_theme.less" ]; then + echo "Compiling Mission Portal styles..." + mkdir -p "$BASEDIR/mission-portal/public/themes/default/bootstrap/compiled/css" + (cd "$BASEDIR/mission-portal/public/themes/default/bootstrap" && + lessc --compress ./cfengine_theme.less ./compiled/css/cfengine.less.css) + fi + + if [ -f "$BASEDIR/mission-portal/ldap/composer.json" ]; then + echo "Installing LDAP API PHP dependencies..." + (cd "$BASEDIR/mission-portal/ldap" && php /usr/bin/composer.phar install --no-dev --ignore-platform-reqs) + fi +) + +# === Step runner with failure reporting === +# Disable set -e so we can capture exit codes and report which step failed. +set +e +run_step() { + local name="$1" + shift + echo "=== Running $name ===" + "$@" + local rc=$? + if [ $rc -ne 0 ]; then + echo "" + echo "=== FAILED: $name (exit code $rc) ===" + exit $rc + fi +} + +# === Build steps === +run_step "01-autogen" "$BASEDIR/buildscripts/build-scripts/autogen" +run_step "02-install-dependencies" "$BASEDIR/buildscripts/build-scripts/install-dependencies" +if [ "$EXPLICIT_ROLE" = "hub" ]; then + run_step "03-mission-portal-deps" install_mission_portal_deps +fi +run_step "04-configure" "$BASEDIR/buildscripts/build-scripts/configure" +run_step "05-compile" "$BASEDIR/buildscripts/build-scripts/compile" +run_step "06-package" "$BASEDIR/buildscripts/build-scripts/package" + +# === Copy output packages === +# Packages are created under $BASEDIR// by dpkg-buildpackage / rpmbuild. +# Exclude deps-packaging to avoid copying dependency packages. +find "$BASEDIR" -maxdepth 4 \ + -path "$BASEDIR/buildscripts/deps-packaging" -prune -o \ + \( -name '*.deb' -o -name '*.rpm' -o -name '*.pkg.tar.gz' \) -print \ + -exec cp {} /output/ \; + +echo "" +echo "=== Build complete ===" +ls -lh /output/ diff --git a/build-in-container.md b/build-in-container.md new file mode 100644 index 000000000..f47b48a74 --- /dev/null +++ b/build-in-container.md @@ -0,0 +1,117 @@ +# build-in-container + +Build CFEngine packages inside Docker containers using build scripts. Requires +only Docker and Python 3 on the host. + +## Quick start + +```bash +# Build a community agent .deb for Ubuntu 22 +./build-in-container.py --platform ubuntu-22 --project community --role agent --build-type DEBUG + +# Build a nova hub release package for Debian 12 +./build-in-container.py --platform debian-12 --project nova --role hub --build-type RELEASE +``` + +In the examples above, we run the script from inside `buildscripts/` (with +`buildscripts` as our current working directory). This is not required — if not +specified, defaults will: + +- Look for sources relative to the script (parent directory of + `build-in-container.py`). +- Place cache files in the user's home directory + (`~/.cache/cfengine/buildscripts`). +- Use the current working directory for output packages (`./output/`). + +## Usage + +``` +./build-in-container.py --platform PLATFORM --project PROJECT --role ROLE --build-type TYPE [OPTIONS] +``` + +### Required arguments + +| Option | Description | +| -------------- | ----------------------------------------------- | +| `--platform` | Target platform (e.g. `ubuntu-22`, `debian-12`) | +| `--project` | `community` or `nova` | +| `--role` | `agent` or `hub` | +| `--build-type` | `DEBUG` or `RELEASE` | + +### Optional arguments + +| Option | Default | Description | +| ------------------ | -------------------------------- | ----------------------------------------------------------- | +| `--output-dir` | `./output` | Where to write output packages | +| `--cache-dir` | `~/.cache/cfengine/buildscripts` | Dependency cache directory | +| `--build-number` | `1` | Build number for package versioning | +| `--version` | auto | Override version string | +| `--rebuild-image` | | Force rebuild of Docker image (bypasses Docker layer cache) | +| `--shell` | | Drop into a bash shell inside the container for debugging | +| `--list-platforms` | | List available platforms and exit | +| `--source-dir` | parent of `buildscripts/` | Root directory containing repos | + +## Supported platforms + +| Name | Base image | +| ----------- | -------------- | +| `ubuntu-20` | `ubuntu:20.04` | +| `ubuntu-22` | `ubuntu:22.04` | +| `ubuntu-24` | `ubuntu:24.04` | +| `debian-11` | `debian:11` | +| `debian-12` | `debian:12` | + +Adding a new Debian/Ubuntu platform requires only a new entry in the `PLATFORMS` +dict in `build-in-container.py`. Adding a non-debian based platform (e.g., +RHEL/CentOS) requires a new `container/Dockerfile.rhel` plus platform entries. + +## How it works + +The system has three components: + +1. **`build-in-container.py`** (Python) -- the orchestrator that runs on the host. + Parses arguments, builds the Docker image, and launches the container with + the correct mounts and environment variables. + +2. **`build-in-container-inner.sh`** (Bash) -- runs inside the container. Copies + source repos from the read-only mount, then calls the existing build scripts + in order. + +3. **`container/Dockerfile.debian`** -- parameterized Dockerfile shared by all + Debian/Ubuntu platforms via a `BASE_IMAGE` build arg. + +### Container mounts + +| Host path | Container path | Mode | Purpose | +| ---------------------------------------- | ----------------------------------------- | ---------- | ------------------------------------- | +| Source repos (parent of `buildscripts/`) | `/srv/source` | read-only | Protects host repos from modification | +| `~/.cache/cfengine/buildscripts/` | `/home/builder/.cache/buildscripts_cache` | read-write | Dependency cache shared across builds | +| `./output/` | `/output` | read-write | Output packages copied here | + +### Build steps + +The inner script runs these steps in order: + +1. **autogen** -- runs `autogen.sh` in each repo +2. **install-dependencies** -- builds and installs bundled dependencies +3. **mission-portal-deps** -- (hub only) installs PHP/npm/LESS assets +4. **configure** -- runs `./configure` with platform-appropriate flags +5. **compile** -- compiles and installs to the dist tree +6. **package** -- creates `.deb` or `.rpm` packages + +## Docker image management + +The Docker image is tagged `cfengine-builder-{platform}` and rebuilt +automatically when the Dockerfile changes (tracked via a content hash stored as +an image label). Use `--rebuild-image` to force a full rebuild bypassing the +Docker layer cache (useful when upstream packages change). + +## Debugging + +```bash +# Drop into a shell inside the container +./build-in-container.py --platform ubuntu-22 --project community --role agent --build-type DEBUG --shell +``` + +The shell session has the same mounts and environment as a build run. The +container is ephemeral (`--rm`), so any changes are lost on exit. diff --git a/build-in-container.py b/build-in-container.py new file mode 100755 index 000000000..bca788971 --- /dev/null +++ b/build-in-container.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +"""Container-based CFEngine package builder. + +Builds CFEngine packages inside Docker containers using the existing build +scripts. Each build runs in a fresh ephemeral container. +""" + +import argparse +import hashlib +import logging +import subprocess +import sys +from pathlib import Path + +log = logging.getLogger("build-in-container") + +PLATFORMS = { + "ubuntu-20": { + "base_image": "ubuntu:20.04", + "dockerfile": "Dockerfile.debian", + "extra_build_args": {"NCURSES_PKGS": "libncurses5 libncurses5-dev"}, + }, + "ubuntu-22": { + "base_image": "ubuntu:22.04", + "dockerfile": "Dockerfile.debian", + "extra_build_args": {}, + }, + "ubuntu-24": { + "base_image": "ubuntu:24.04", + "dockerfile": "Dockerfile.debian", + "extra_build_args": {}, + }, + "debian-11": { + "base_image": "debian:11", + "dockerfile": "Dockerfile.debian", + "extra_build_args": {}, + }, + "debian-12": { + "base_image": "debian:12", + "dockerfile": "Dockerfile.debian", + "extra_build_args": {}, + }, +} + +def detect_source_dir(): + """Find the root directory containing all repos (parent of buildscripts/).""" + script_dir = Path(__file__).resolve().parent + # The script lives in buildscripts/, so the source dir is one level up + source_dir = script_dir.parent + if not (source_dir / "buildscripts").is_dir(): + log.error(f"Cannot find buildscripts/ in {source_dir}") + sys.exit(1) + return source_dir + + +def dockerfile_hash(dockerfile_path): + """Compute SHA256 hash of a Dockerfile.""" + return hashlib.sha256(dockerfile_path.read_bytes()).hexdigest() + + +def image_needs_rebuild(image_tag, current_hash): + """Check if the Docker image needs rebuilding based on Dockerfile hash.""" + result = subprocess.run( + [ + "docker", + "inspect", + "--format", + '{{index .Config.Labels "dockerfile-hash"}}', + image_tag, + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + return True # Image doesn't exist + stored_hash = result.stdout.strip() + return stored_hash != current_hash + + +def build_image(platform_name, platform_config, script_dir, rebuild=False): + """Build the Docker image for the given platform.""" + image_tag = f"cfengine-builder-{platform_name}" + dockerfile_name = platform_config["dockerfile"] + dockerfile_path = script_dir / "container" / dockerfile_name + current_hash = dockerfile_hash(dockerfile_path) + + if not rebuild and not image_needs_rebuild(image_tag, current_hash): + log.info(f"Docker image {image_tag} is up to date.") + return image_tag + + log.info(f"Building Docker image {image_tag}...") + cmd = [ + "docker", + "build", + "-f", + str(dockerfile_path), + "--build-arg", + f"BASE_IMAGE={platform_config['base_image']}", + "--label", + f"dockerfile-hash={current_hash}", + "-t", + image_tag, + ] + + for key, value in platform_config.get("extra_build_args", {}).items(): + cmd.extend(["--build-arg", f"{key}={value}"]) + + if rebuild: + cmd.append("--no-cache") + + cmd.extend(["--network", "host"]) + + # Build context is the container/ directory + cmd.append(str(script_dir / "container")) + + result = subprocess.run(cmd) + if result.returncode != 0: + log.error("Docker image build failed.") + sys.exit(1) + + return image_tag + + +def run_container(args, image_tag, source_dir, script_dir): + """Run the build inside a Docker container.""" + output_dir = Path(args.output_dir).resolve() + cache_dir = Path(args.cache_dir).resolve() + + # Pre-create host directories so Docker doesn't create them as root + output_dir.mkdir(parents=True, exist_ok=True) + cache_dir.mkdir(parents=True, exist_ok=True) + + cmd = ["docker", "run", "--rm", "--network", "host"] + + if args.shell: + cmd.extend(["-it"]) + + # Mounts + cmd.extend( + [ + "-v", + f"{source_dir}:/srv/source:ro", + "-v", + f"{cache_dir}:/home/builder/.cache/buildscripts_cache", + "-v", + f"{output_dir}:/output", + ] + ) + + # Environment variables + # JOB_BASE_NAME is used by deps-packaging/pkg-cache to derive the cache + # label. Format: "label=". Without it, all platforms share NO_LABEL. + cache_label = f"label=container_{args.platform}" + cmd.extend( + [ + "-e", + f"PROJECT={args.project}", + "-e", + f"BUILD_TYPE={args.build_type}", + "-e", + f"EXPLICIT_ROLE={args.role}", + "-e", + f"BUILD_NUMBER={args.build_number}", + "-e", + f"JOB_BASE_NAME={cache_label}", + "-e", + "CACHE_IS_ONLY_LOCAL=yes", + ] + ) + + if args.version: + cmd.extend(["-e", f"EXPLICIT_VERSION={args.version}"]) + + cmd.append(image_tag) + + if args.shell: + cmd.append("/bin/bash") + else: + cmd.append(str(Path("/srv/source/buildscripts/build-in-container-inner.sh"))) + + result = subprocess.run(cmd) + return result.returncode + + +# Custom action so --list-platforms can exit early without triggering +# argparse's required-argument validation (same mechanism as --help). +class ListPlatformsAction(argparse.Action): + + def __call__(self, parser, namespace, values, option_string=None): + print("Available platforms:") + for name, config in PLATFORMS.items(): + print(f" {name:15s} ({config['base_image']})") + parser.exit() + + +def main(): + parser = argparse.ArgumentParser( + description="Build CFEngine packages in Docker containers." + ) + parser.add_argument( + "--platform", + required=True, + choices=list(PLATFORMS.keys()), + help="Target platform", + ) + parser.add_argument( + "--project", + required=True, + choices=["community", "nova"], + help="CFEngine edition", + ) + parser.add_argument( + "--role", + required=True, + choices=["agent", "hub"], + help="Component to build", + ) + parser.add_argument( + "--build-type", + dest="build_type", + required=True, + choices=["DEBUG", "RELEASE"], + help="Build type", + ) + parser.add_argument( + "--source-dir", + help="Root directory containing repos (default: parent of buildscripts/)", + ) + parser.add_argument( + "--output-dir", + default="./output", + help="Output directory for packages (default: ./output)", + ) + parser.add_argument( + "--cache-dir", + default=str(Path.home() / ".cache" / "cfengine" / "buildscripts"), + help="Dependency cache directory", + ) + parser.add_argument( + "--rebuild-image", + action="store_true", + help="Force rebuild of Docker image (--no-cache)", + ) + parser.add_argument( + "--shell", + action="store_true", + help="Drop into container shell for debugging", + ) + parser.add_argument( + "--list-platforms", + action=ListPlatformsAction, + nargs=0, + help="List available platforms and exit", + ) + parser.add_argument( + "--build-number", + default="1", + help="Build number for package versioning (default: 1)", + ) + parser.add_argument( + "--version", + help="Override version string", + ) + args = parser.parse_args() + + logging.basicConfig( + level=logging.INFO, + format="%(message)s", + ) + + # Detect source directory + if args.source_dir: + source_dir = Path(args.source_dir).resolve() + else: + source_dir = detect_source_dir() + + script_dir = source_dir / "buildscripts" + + if args.platform not in PLATFORMS: + log.error(f"Unknown platform '{args.platform}'") + sys.exit(1) + + platform_config = PLATFORMS[args.platform] + + # Build Docker image + image_tag = build_image( + args.platform, platform_config, script_dir, rebuild=args.rebuild_image + ) + + if not args.shell: + log.info( + f"Building {args.project} {args.role} for {args.platform} ({args.build_type})..." + ) + + # Run the container + rc = run_container(args, image_tag, source_dir, script_dir) + + if rc != 0: + log.error(f"Build failed (exit code {rc}).") + sys.exit(rc) + + if not args.shell: + output_dir = Path(args.output_dir).resolve() + packages = ( + list(output_dir.glob("*.deb")) + + list(output_dir.glob("*.rpm")) + + list(output_dir.glob("*.pkg.tar.gz")) + ) + if packages: + log.info("Output packages:") + for p in sorted(packages): + log.info(f" {p}") + else: + log.warning("No packages found in output directory.") + + +if __name__ == "__main__": + main() diff --git a/container/Dockerfile.debian b/container/Dockerfile.debian new file mode 100644 index 000000000..5b2abdcf3 --- /dev/null +++ b/container/Dockerfile.debian @@ -0,0 +1,42 @@ +ARG BASE_IMAGE=ubuntu:20.04 +FROM ${BASE_IMAGE} + +ENV DEBIAN_FRONTEND=noninteractive + +# Build tools extracted from ci/cfengine-build-host-setup.cf (debian|ubuntu section) +RUN apt-get update && apt-get install -y \ + autoconf automake binutils bison build-essential curl debhelper \ + dpkg-dev expat fakeroot flex gdb git libexpat1-dev \ + libmodule-load-conditional-perl libpam0g-dev libtool \ + pkg-config psmisc python3-pip rsync sudo systemd-coredump wget \ + && rm -rf /var/lib/apt/lists/* + +# ncurses: ubuntu-20 uses libncurses5/libncurses5-dev, newer use libncurses6/libncurses-dev +ARG NCURSES_PKGS="libncurses6 libncurses-dev" +RUN apt-get update && apt-get install -y ${NCURSES_PKGS} \ + && rm -rf /var/lib/apt/lists/* + +# Hub build tools: Node.js 20 LTS (system nodejs is too old for modern npm +# packages that use the node: protocol), PHP, and Composer +RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \ + && apt-get install -y nodejs php-cli \ + && rm -rf /var/lib/apt/lists/* +RUN npm install -g less +RUN curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/bin --filename=composer.phar + +# Remove system dev libraries that conflict with bundled deps +# (from cfengine-build-host-setup.cf lines 53-59) +RUN apt-get purge -y \ + libattr1-dev libssl-dev libpcre2-dev libacl1-dev \ + libyaml-dev libxml2-dev librsync-dev 2>/dev/null || true + +# Create build user with passwordless sudo (needed by install-dependencies, package, etc.) +RUN useradd -m -s /bin/bash builder \ + && echo "builder ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/builder + +USER builder +WORKDIR /home/builder + +# Pre-create the build directory so that when a named volume is mounted here, +# Docker initializes it with the correct ownership (builder:builder). +RUN mkdir -p /home/builder/build From 554c212de1c997d7513684f0c6e8330c4fce8a6a Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Wed, 1 Apr 2026 14:55:05 +0200 Subject: [PATCH 2/9] Made build args conditionally required in build-in-container.py Replaced the ListPlatformsAction custom argparse action with a simple --list-platforms store_true flag, and moved argument validation from argparse's required=True to manual post-parse checks. This allows future flags (e.g. --push-image) to bypass build-specific arguments like --project, --role, and --build-type. Signed-off-by: Lars Erik Wik --- build-in-container.py | 49 +++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/build-in-container.py b/build-in-container.py index bca788971..bf200225e 100755 --- a/build-in-container.py +++ b/build-in-container.py @@ -182,46 +182,36 @@ def run_container(args, image_tag, source_dir, script_dir): return result.returncode -# Custom action so --list-platforms can exit early without triggering -# argparse's required-argument validation (same mechanism as --help). -class ListPlatformsAction(argparse.Action): - - def __call__(self, parser, namespace, values, option_string=None): - print("Available platforms:") - for name, config in PLATFORMS.items(): - print(f" {name:15s} ({config['base_image']})") - parser.exit() - - def main(): parser = argparse.ArgumentParser( description="Build CFEngine packages in Docker containers." ) parser.add_argument( "--platform", - required=True, choices=list(PLATFORMS.keys()), help="Target platform", ) parser.add_argument( "--project", - required=True, choices=["community", "nova"], help="CFEngine edition", ) parser.add_argument( "--role", - required=True, choices=["agent", "hub"], help="Component to build", ) parser.add_argument( "--build-type", dest="build_type", - required=True, choices=["DEBUG", "RELEASE"], help="Build type", ) + parser.add_argument( + "--list-platforms", + action="store_true", + help="List available platforms and exit", + ) parser.add_argument( "--source-dir", help="Root directory containing repos (default: parent of buildscripts/)", @@ -246,12 +236,6 @@ def main(): action="store_true", help="Drop into container shell for debugging", ) - parser.add_argument( - "--list-platforms", - action=ListPlatformsAction, - nargs=0, - help="List available platforms and exit", - ) parser.add_argument( "--build-number", default="1", @@ -268,6 +252,25 @@ def main(): format="%(message)s", ) + if args.list_platforms: + print("Available platforms:") + for name, config in PLATFORMS.items(): + print(f" {name:15s} ({config['base_image']})") + sys.exit(0) + + # Validate required arguments for build mode + missing = [] + if not args.platform: + missing.append("--platform") + if not args.project: + missing.append("--project") + if not args.role: + missing.append("--role") + if not args.build_type: + missing.append("--build-type") + if missing: + parser.error(f"the following arguments are required: {', '.join(missing)}") + # Detect source directory if args.source_dir: source_dir = Path(args.source_dir).resolve() @@ -276,10 +279,6 @@ def main(): script_dir = source_dir / "buildscripts" - if args.platform not in PLATFORMS: - log.error(f"Unknown platform '{args.platform}'") - sys.exit(1) - platform_config = PLATFORMS[args.platform] # Build Docker image From 57bf999d8d532634e3821804f6c4f022d7130b2e Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Wed, 1 Apr 2026 14:57:26 +0200 Subject: [PATCH 3/9] Extracted argument parsing into parse_args() function Signed-off-by: Lars Erik Wik --- build-in-container.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/build-in-container.py b/build-in-container.py index bf200225e..6b6330457 100755 --- a/build-in-container.py +++ b/build-in-container.py @@ -182,7 +182,8 @@ def run_container(args, image_tag, source_dir, script_dir): return result.returncode -def main(): +def parse_args(): + """Parse and validate command-line arguments.""" parser = argparse.ArgumentParser( description="Build CFEngine packages in Docker containers." ) @@ -247,11 +248,6 @@ def main(): ) args = parser.parse_args() - logging.basicConfig( - level=logging.INFO, - format="%(message)s", - ) - if args.list_platforms: print("Available platforms:") for name, config in PLATFORMS.items(): @@ -271,6 +267,17 @@ def main(): if missing: parser.error(f"the following arguments are required: {', '.join(missing)}") + return args + + +def main(): + args = parse_args() + + logging.basicConfig( + level=logging.INFO, + format="%(message)s", + ) + # Detect source directory if args.source_dir: source_dir = Path(args.source_dir).resolve() From fd1ebb4573dce04970d817323030d4af98a8fa3b Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Wed, 1 Apr 2026 16:07:32 +0200 Subject: [PATCH 4/9] Added IMAGE_REGISTRY, IMAGE_VERSION constants and image_tag to PLATFORMS Each platform entry now includes a versioned image_tag field composed from the platform name and IMAGE_VERSION. The build_image() function uses this tag instead of constructing it locally. This prepares for registry-based image management where the tag includes a version for cache busting. Ticket: ENT-13784 Signed-off-by: Lars Erik Wik --- build-in-container.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/build-in-container.py b/build-in-container.py index 6b6330457..1eb034eef 100755 --- a/build-in-container.py +++ b/build-in-container.py @@ -14,28 +14,36 @@ log = logging.getLogger("build-in-container") +IMAGE_REGISTRY = "ghcr.io/cfengine" +IMAGE_VERSION = "1" + PLATFORMS = { "ubuntu-20": { + "image_tag": f"cfengine-builder-ubuntu-20:{IMAGE_VERSION}", "base_image": "ubuntu:20.04", "dockerfile": "Dockerfile.debian", "extra_build_args": {"NCURSES_PKGS": "libncurses5 libncurses5-dev"}, }, "ubuntu-22": { + "image_tag": f"cfengine-builder-ubuntu-22:{IMAGE_VERSION}", "base_image": "ubuntu:22.04", "dockerfile": "Dockerfile.debian", "extra_build_args": {}, }, "ubuntu-24": { + "image_tag": f"cfengine-builder-ubuntu-24:{IMAGE_VERSION}", "base_image": "ubuntu:24.04", "dockerfile": "Dockerfile.debian", "extra_build_args": {}, }, "debian-11": { + "image_tag": f"cfengine-builder-debian-11:{IMAGE_VERSION}", "base_image": "debian:11", "dockerfile": "Dockerfile.debian", "extra_build_args": {}, }, "debian-12": { + "image_tag": f"cfengine-builder-debian-12:{IMAGE_VERSION}", "base_image": "debian:12", "dockerfile": "Dockerfile.debian", "extra_build_args": {}, @@ -79,7 +87,7 @@ def image_needs_rebuild(image_tag, current_hash): def build_image(platform_name, platform_config, script_dir, rebuild=False): """Build the Docker image for the given platform.""" - image_tag = f"cfengine-builder-{platform_name}" + image_tag = platform_config["image_tag"] dockerfile_name = platform_config["dockerfile"] dockerfile_path = script_dir / "container" / dockerfile_name current_hash = dockerfile_hash(dockerfile_path) From 9b9b7cc26f2c40cb90da358524759af53b8aae71 Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Wed, 1 Apr 2026 16:16:20 +0200 Subject: [PATCH 5/9] Added --push-image CLI argument The --push-image flag is a standalone image management operation that builds and pushes a container image to the registry. It only requires --platform (not --project/--role/--build-type). Ticket: ENT-13784 Signed-off-by: Lars Erik Wik --- build-in-container.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/build-in-container.py b/build-in-container.py index 1eb034eef..4fdc80715 100755 --- a/build-in-container.py +++ b/build-in-container.py @@ -50,6 +50,7 @@ }, } + def detect_source_dir(): """Find the root directory containing all repos (parent of buildscripts/).""" script_dir = Path(__file__).resolve().parent @@ -240,6 +241,11 @@ def parse_args(): action="store_true", help="Force rebuild of Docker image (--no-cache)", ) + parser.add_argument( + "--push-image", + action="store_true", + help="Build image and push to registry, then exit", + ) parser.add_argument( "--shell", action="store_true", @@ -262,18 +268,21 @@ def parse_args(): print(f" {name:15s} ({config['base_image']})") sys.exit(0) - # Validate required arguments for build mode - missing = [] + # --platform is always required (except --list-platforms handled above) if not args.platform: - missing.append("--platform") + parser.error("missing required argument --platform") + + if args.push_image: + # No other arguments are required for --push-image + return args + + # Validate remaining required arguments for build mode if not args.project: - missing.append("--project") + parser.error("missing required argument --project") if not args.role: - missing.append("--role") + parser.error("missing required argument --role") if not args.build_type: - missing.append("--build-type") - if missing: - parser.error(f"the following arguments are required: {', '.join(missing)}") + parser.error("missing required argument --build-type") return args From 038a31623112b93dd15d76561afa4596079cbd9c Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Wed, 1 Apr 2026 16:17:38 +0200 Subject: [PATCH 6/9] Added registry_image_ref, pull_image, and push_image functions - registry_image_ref() returns the fully-qualified ghcr.io reference - pull_image() pulls from the registry, returns None on failure - push_image() tags a local image and pushes it to the registry Ticket: ENT-13784 Signed-off-by: Lars Erik Wik --- build-in-container.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/build-in-container.py b/build-in-container.py index 4fdc80715..9741c1626 100755 --- a/build-in-container.py +++ b/build-in-container.py @@ -130,6 +130,44 @@ def build_image(platform_name, platform_config, script_dir, rebuild=False): return image_tag +def registry_image_ref(platform_name): + """Return the fully-qualified registry image reference for a platform.""" + return f"{IMAGE_REGISTRY}/{PLATFORMS[platform_name]['image_tag']}" + + +def pull_image(platform_name): + """Pull a pre-built image from the registry. + + Returns the image reference on success or None on failure. + """ + ref = registry_image_ref(platform_name) + log.info(f"Pulling image {ref}...") + result = subprocess.run( + ["docker", "pull", ref], + capture_output=True, + text=True, + ) + if result.returncode != 0: + return None + return ref + + +def push_image(platform_name, local_tag): + """Tag a local image with the registry reference and push it.""" + ref = registry_image_ref(platform_name) + log.info(f"Tagging {local_tag} as {ref}...") + result = subprocess.run(["docker", "tag", local_tag, ref]) + if result.returncode != 0: + log.error("Docker tag failed.") + sys.exit(1) + + log.info(f"Pushing {ref}...") + result = subprocess.run(["docker", "push", ref]) + if result.returncode != 0: + log.error("Docker push failed.") + sys.exit(1) + + def run_container(args, image_tag, source_dir, script_dir): """Run the build inside a Docker container.""" output_dir = Path(args.output_dir).resolve() From 25a5a88da294f16e4ec114ac2229252c3555e834 Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Wed, 1 Apr 2026 16:19:08 +0200 Subject: [PATCH 7/9] Implemented image resolution flow with registry pull/push - --push-image: builds with --no-cache and pushes to registry, then exits - --rebuild-image: builds locally, skips registry - Default: pulls from registry, falls back to local build on failure Ticket: ENT-13784 Signed-off-by: Lars Erik Wik --- build-in-container.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/build-in-container.py b/build-in-container.py index 9741c1626..81828e62a 100755 --- a/build-in-container.py +++ b/build-in-container.py @@ -343,10 +343,23 @@ def main(): platform_config = PLATFORMS[args.platform] - # Build Docker image - image_tag = build_image( - args.platform, platform_config, script_dir, rebuild=args.rebuild_image - ) + if args.push_image: + image_tag = build_image( + args.platform, platform_config, script_dir, rebuild=True + ) + push_image(args.platform, image_tag) + return + + # Resolve image: pull from registry, fall back to local build + if args.rebuild_image: + image_tag = build_image( + args.platform, platform_config, script_dir, rebuild=True + ) + else: + image_tag = pull_image(args.platform) + if image_tag is None: + log.warning("Registry pull failed, building image locally...") + image_tag = build_image(args.platform, platform_config, script_dir) if not args.shell: log.info( From 1859e0c29900426ad422fbe095f144ca57ceff12 Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Wed, 1 Apr 2026 16:21:28 +0200 Subject: [PATCH 8/9] Updated documentation for container registry support Added --push-image to argument tables, documented the registry pull/push workflow, fallback behavior, and toolchain update process. Ticket: ENT-13784 Signed-off-by: Lars Erik Wik --- build-in-container.md | 45 +++++++++++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/build-in-container.md b/build-in-container.md index f47b48a74..e087fb3db 100644 --- a/build-in-container.md +++ b/build-in-container.md @@ -31,12 +31,12 @@ specified, defaults will: ### Required arguments -| Option | Description | -| -------------- | ----------------------------------------------- | -| `--platform` | Target platform (e.g. `ubuntu-22`, `debian-12`) | -| `--project` | `community` or `nova` | -| `--role` | `agent` or `hub` | -| `--build-type` | `DEBUG` or `RELEASE` | +| Option | Description | +| -------------- | ------------------------------------------------------- | +| `--platform` | Target platform (e.g. `ubuntu-22`, `debian-12`) | +| `--project` | `community` or `nova` (not required for `--push-image`) | +| `--role` | `agent` or `hub` (not required for `--push-image`) | +| `--build-type` | `DEBUG` or `RELEASE` (not required for `--push-image`) | ### Optional arguments @@ -47,6 +47,7 @@ specified, defaults will: | `--build-number` | `1` | Build number for package versioning | | `--version` | auto | Override version string | | `--rebuild-image` | | Force rebuild of Docker image (bypasses Docker layer cache) | +| `--push-image` | | Build image and push to registry, then exit | | `--shell` | | Drop into a bash shell inside the container for debugging | | `--list-platforms` | | List available platforms and exit | | `--source-dir` | parent of `buildscripts/` | Root directory containing repos | @@ -101,10 +102,34 @@ The inner script runs these steps in order: ## Docker image management -The Docker image is tagged `cfengine-builder-{platform}` and rebuilt -automatically when the Dockerfile changes (tracked via a content hash stored as -an image label). Use `--rebuild-image` to force a full rebuild bypassing the -Docker layer cache (useful when upstream packages change). +By default, the script pulls a pre-built image from the container registry +(`ghcr.io/cfengine`). If the pull fails (e.g. no network, image not yet +published), it falls back to building the image locally. + +Use `--rebuild-image` to skip the registry and force a local rebuild — useful +when iterating on the Dockerfile. The local build tracks the Dockerfile content +hash and skips rebuilding when nothing has changed. + +### Container registry + +Images are hosted at `ghcr.io/cfengine` and versioned via `IMAGE_VERSION` in +`build-in-container.py`. To push a new image: + +```bash +# Build and push a single platform +./build-in-container.py --platform ubuntu-22 --push-image +``` + +`--push-image` always builds with `--no-cache` to pick up the latest upstream +packages, then pushes to the registry. + +### Updating the toolchain + +1. Edit `container/Dockerfile.debian` as needed +2. Test locally with `--rebuild-image` +3. Bump `IMAGE_VERSION` in `build-in-container.py` +4. Commit the Dockerfile change + version bump +5. Push new images with `--push-image` (or trigger the GitHub Actions workflow) ## Debugging From 3a5b5f4f132cd2721c4fbb85dee4b0840ab24cdb Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Wed, 1 Apr 2026 16:22:33 +0200 Subject: [PATCH 9/9] Added GitHub Actions workflow for building base images Manually triggered workflow that builds and pushes container images to ghcr.io. Builds all platforms in parallel via a matrix strategy. Ticket: ENT-13784 Signed-off-by: Lars Erik Wik --- .github/workflows/build-base-images.yml | 31 +++++++++++++++++++++++++ build-in-container.py | 16 +++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 .github/workflows/build-base-images.yml diff --git a/.github/workflows/build-base-images.yml b/.github/workflows/build-base-images.yml new file mode 100644 index 000000000..a7aeac4dd --- /dev/null +++ b/.github/workflows/build-base-images.yml @@ -0,0 +1,31 @@ +name: Build base images + +on: + workflow_dispatch: + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + packages: write + strategy: + matrix: + platform: + - ubuntu-20 + - ubuntu-22 + - ubuntu-24 + - debian-11 + - debian-12 + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Log in to ghcr.io + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push image + run: ./build-in-container.py --platform ${{ matrix.platform }} --push-image diff --git a/build-in-container.py b/build-in-container.py index 81828e62a..fe04d49ea 100755 --- a/build-in-container.py +++ b/build-in-container.py @@ -152,9 +152,25 @@ def pull_image(platform_name): return ref +def image_exists_in_registry(platform_name): + """Check if an image tag already exists in the registry.""" + ref = registry_image_ref(platform_name) + result = subprocess.run( + ["docker", "manifest", "inspect", ref], + capture_output=True, + text=True, + ) + return result.returncode == 0 + + def push_image(platform_name, local_tag): """Tag a local image with the registry reference and push it.""" ref = registry_image_ref(platform_name) + + if image_exists_in_registry(platform_name): + log.error(f"Image {ref} already exists. Bump IMAGE_VERSION.") + sys.exit(1) + log.info(f"Tagging {local_tag} as {ref}...") result = subprocess.run(["docker", "tag", local_tag, ref]) if result.returncode != 0: