From 898e84ce36e8505aeb78165b15f7222dc5b82f85 Mon Sep 17 00:00:00 2001 From: Daniel Rossier Date: Fri, 14 Aug 2026 11:59:58 +0200 Subject: [PATCH] build: serve the TEZI feed from the tree instead of /var/www/html MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Propagated from the edgem1 tree (77707b54, "[env]: Add local deploy into the IB tree", David Truan) so both trees deploy the Verdin the same way. A verdin `http` deploy used to rsync into /var/www/html/verdin-imx8mp-prod with sudo, which meant an nginx serving that document root, a root-owned directory outside the tree, a machine-specific absolute path in a TRACKED local.conf, and — with a snap-packaged Docker, which cannot bind-mount outside $HOME — no way to publish from inside the build container at all. IB_HTTP_DEPLOY_PATH now defaults to ${IB_DIR}/build/deploy/tezi/${IB_PLATFORM}, which is already bind-mounted and owned by the calling user, so publishing needs no privilege; the sudo rsync stays only as a fallback for a document root that really is root-owned. It sits under build/deploy/ rather than build/tmp/ so a clean does not wipe a feed a board may be pulling from. do_platform_deploy also writes image_list.json next to image.json, making the feed directory self-contained: whoever serves it can use it as-is as its document root, and the board URL is always /image_list.json. Serving it is the new scripts/tezi-feed-serve.sh — a detached `python3 -m http.server` on IB_HTTP_FEED_PORT (8080, unprivileged) with --ensure/--status/--stop. deploy.sh calls --ensure so a deploy leaves a working feed behind; dbuild.sh calls it on the HOST after the container exits, since --rm would take a server started inside down with it. It is a no-op under $CI and inside the container. Per-machine deviations go in the new build/conf/site.conf, included by bitbake.conf after local.conf and untracked. Two related .gitignore fixes: site.conf must be ignored explicitly because !/build/conf/ re-includes the whole directory, and !/build/meta/ un-ignores the core layer, which "meta-*" never matched — build/meta/conf/bitbake.conf was tracked only because it predates the rule, so this very commit could not have added a file there. Verdin-specific parts of the edgem1 commit do not apply here: this tree has no .gitlab-ci.yml and no scripts/tezi-custom/, so the feed URL is handed to the board by hand with `tezictl feed-add http://:8080/image_list.json` (tezi-feed-serve.sh --status prints that line). --- .gitignore | 11 + build/conf/local.conf | 19 +- .../bsp/files/bsp_verdin-imx8mp.inc | 26 +- build/meta/conf/bitbake.conf | 7 + scripts/dbuild.sh | 42 ++- scripts/deploy.sh | 7 + scripts/tezi-feed-serve.sh | 243 ++++++++++++++++++ 7 files changed, 349 insertions(+), 6 deletions(-) create mode 100755 scripts/tezi-feed-serve.sh diff --git a/.gitignore b/.gitignore index 946af4f90a..1811775e79 100755 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,17 @@ /build/* !/build/conf/ !/build/meta-*/ +# The core layer has NO dash, so "meta-*" above does not match it: its classes +# (base, patch, utils, logging, terminal) and conf/ were tracked only because +# they predate the rule, and any file added there since would have been ignored +# in silence — a new class simply never reaching the repository. +!/build/meta/ + +# Per-machine bitbake overrides (see build/meta/conf/bitbake.conf). The rule +# just above re-includes all of build/conf/, so this one file needs its own. +# The TEZI feed published by deploy.sh lives in build/deploy/ (along with the +# feed server's pid/log) and is already covered by the /build/* rule. +/build/conf/site.conf # The rpi4 BSP deploys Broadcom's prebuilt bcm27xx device trees onto the boot # partition, alongside start4.elf & co. Unlike every other .dtb in the tree they diff --git a/build/conf/local.conf b/build/conf/local.conf index 85ec3eeca4..cfd7b76b9c 100644 --- a/build/conf/local.conf +++ b/build/conf/local.conf @@ -249,8 +249,23 @@ IB_STORAGE_MODE:verdin-imx8mp ?= "http" # IB_STORAGE_DEVICE:rpi4_64 = "mmcblk0" # IB_STORAGE_DEVICE:verdin-imx8mp = "sda" -# HTTP deploy path: TEZI files are placed here for TEZI network auto-install -IB_HTTP_DEPLOY_PATH:verdin-imx8mp = "/var/www/html/verdin-imx8mp-prod" +# HTTP deploy path: TEZI files are placed here for TEZI network auto-install, +# and scripts/tezi-feed-serve.sh serves this directory over HTTP. +# +# Tree-relative on purpose: this file is tracked, so it must not carry a +# machine-specific absolute path. Keeping the feed inside the tree also means +# it is already visible to the build container (dbuild.sh bind-mounts the +# tree) and writable by the calling user, so publishing needs no sudo. +# Under build/deploy/ rather than build/tmp/ so a clean does not wipe a feed +# a board may be about to pull from. +# +# To publish elsewhere on one machine, override this in build/conf/site.conf +# (untracked, loaded after this file). +IB_HTTP_DEPLOY_PATH ?= "${IB_DIR}/build/deploy/tezi/${IB_PLATFORM}" + +# Port scripts/tezi-feed-serve.sh listens on. Unprivileged so the feed server +# runs as your user; override in site.conf when two trees share a machine. +IB_HTTP_FEED_PORT ?= "8080" # Size of the rootfs partition IB_ROOTFS_SIZE ?= "2G" diff --git a/build/meta-bsp/recipes-bsp/bsp/files/bsp_verdin-imx8mp.inc b/build/meta-bsp/recipes-bsp/bsp/files/bsp_verdin-imx8mp.inc index 2cb2363452..cdee95a279 100644 --- a/build/meta-bsp/recipes-bsp/bsp/files/bsp_verdin-imx8mp.inc +++ b/build/meta-bsp/recipes-bsp/bsp/files/bsp_verdin-imx8mp.inc @@ -110,6 +110,7 @@ def __fetch_torizon_tezi_image(d, dest_dir): def __do_platform_deploy(d): """Stage a flashable TEZI image: our imx-boot (flash.bin) + boot.scr + the AVZ+SO3 ITB, with image.json patched to auto-install them.""" + import json import os import subprocess @@ -169,8 +170,29 @@ def __do_platform_deploy(d): if not final_dest: bb.fatal("verdin: no deploy destination (set IB_HTTP_DEPLOY_PATH or use the default)") - utils_sudo(["mkdir", "-p", final_dest], check=True) - utils_sudo(["rsync", "-a", "--delete", f"{staging}/", f"{final_dest}/"], check=True) + # TEZI feed index. Written next to image.json so the feed directory is + # self-contained: whatever serves it (scripts/tezi-feed-serve.sh, or a + # web server on another machine) can use it as its document root, and + # the board's feed URL is always /image_list.json. Kept relative + # for the same reason — no absolute path, no directory prefix. + with open(f"{staging}/image_list.json", "w") as f: + json.dump({"config_format": 1, + "autoinstall": "image.json", + "images": ["image.json"]}, f, indent=4) + + # Publish. The default IB_HTTP_DEPLOY_PATH lives inside the tree and is + # owned by the calling user, so no privilege is needed; a site.conf that + # points somewhere root-owned (e.g. a web server document root) still + # works through the sudo fallback. + try: + os.makedirs(final_dest, exist_ok=True) + subprocess.run(["rsync", "-a", "--delete", + f"{staging}/", f"{final_dest}/"], check=True) + except (PermissionError, subprocess.CalledProcessError) as e: + bb.note(f"publishing to {final_dest} unprivileged failed ({e}); retrying with sudo") + utils_sudo(["mkdir", "-p", final_dest], check=True) + utils_sudo(["rsync", "-a", "--delete", + f"{staging}/", f"{final_dest}/"], check=True) bb.plain("verdin: TEZI image staged at %s (imx-boot + boot.scr + ITB(s) for %s)" % (final_dest, IB_TARGET_ITS)) diff --git a/build/meta/conf/bitbake.conf b/build/meta/conf/bitbake.conf index a2b1a9dbac..e440ecd0b3 100644 --- a/build/meta/conf/bitbake.conf +++ b/build/meta/conf/bitbake.conf @@ -21,6 +21,13 @@ # Necessary to retrieve global variables from there include conf/local.conf +# Optional per-machine overrides, loaded after local.conf so they win. +# Untracked (see .gitignore): this is where a machine deviates from the +# tree defaults — e.g. publishing the TEZI feed somewhere else than +# ${IB_HTTP_DEPLOY_PATH}. Absent on most machines; `include` is silent +# when the file does not exist. +include conf/site.conf + # its own in staging ASSUME_PROVIDED = "\ patch-native \ diff --git a/scripts/dbuild.sh b/scripts/dbuild.sh index 068fb6dba3..42fb549cc2 100755 --- a/scripts/dbuild.sh +++ b/scripts/dbuild.sh @@ -127,6 +127,11 @@ set -- -e IB_TREE="$IB_ROOT" -e IB_CWD="$cwd" "$@" # bind-mounted and the container is privileged (it writes the HOST's # device, so double-check IB_STORAGE_DEVICE). # +# The DEFAULT feed lives inside the tree (IB_HTTP_DEPLOY_PATH in +# local.conf), which is already bind-mounted, so this loop does nothing +# for it. It only matters for a build/conf/site.conf that redirects the +# feed out of the tree. +# # A snap-packaged Docker cannot do this: the confined daemon only reaches # $HOME (and a few allowed paths), so bind-mounting e.g. /var/www/html # fails with "mkdir /var/www: read-only file system". Skip the mount @@ -137,8 +142,27 @@ case "$(command -v docker)" in /snap/*) _snap_docker=1 ;; esac +# The client path is not proof either way: the docker snap also ships a +# /usr/bin/docker wrapper, so a confined daemon can hide behind an +# ordinary-looking binary. Ask the daemon itself — a snap daemon keeps +# its root under /var/snap. Only worth doing when the path check missed. + +if [ "$_snap_docker" = "0" ]; then + case "$(docker info --format '{{.DockerRootDir}}' 2>/dev/null)" in + /var/snap/*|/snap/*) _snap_docker=1 ;; + esac +fi + for _feed in $(sed -n 's/^[[:space:]]*IB_HTTP_DEPLOY_PATH[^=]*=[[:space:]]*"\([^"]*\)".*/\1/p' \ - "$IB_ROOT/build/conf/local.conf" 2>/dev/null | sort -u); do + "$IB_ROOT/build/conf/local.conf" "$IB_ROOT/build/conf/site.conf" \ + 2>/dev/null | sort -u); do + # Tree-relative defaults are already inside the bind-mounted tree, and + # ${...} references are not expanded here — skip both. + case "$_feed" in + *'${'*) continue ;; + "$IB_ROOT"|"$IB_ROOT"/*) continue ;; + esac + [ -d "$_feed" ] || continue case "$_feed" in @@ -217,4 +241,18 @@ set -- --rm \ -e TERM="${TERM:-xterm}" \ "$@" -exec docker run "$@" +docker run "$@" +_rc=$? + +# The TEZI feed server has to run on the HOST: the container is --rm, so a +# server started inside it dies with the command that started it. It shares +# the host network namespace (--network host above), but not its lifetime. +# +# Run this after the container rather than before, so the very first +# `dbuild.sh deploy.sh ...` — which creates the feed inside the container — +# leaves a serving feed behind too. Idempotent, and a no-op when nothing has +# been published yet or a server is already up. + +"$IB_ROOT/scripts/tezi-feed-serve.sh" --ensure + +exit $_rc diff --git a/scripts/deploy.sh b/scripts/deploy.sh index aac547ed40..7d4d55445d 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -144,4 +144,11 @@ then then exit 1 fi + + # Make the freshly published TEZI feed reachable without a manual + # step. Idempotent and quiet when a server is already up; a no-op + # when the deploy did not publish an HTTP feed, in CI, and inside + # the build container (dbuild.sh does it on the host instead — + # a server started in the container would die with it). + ./scripts/tezi-feed-serve.sh --ensure fi diff --git a/scripts/tezi-feed-serve.sh b/scripts/tezi-feed-serve.sh new file mode 100755 index 0000000000..3a4354487a --- /dev/null +++ b/scripts/tezi-feed-serve.sh @@ -0,0 +1,243 @@ +#!/bin/sh + +# Serve the TEZI HTTP feed produced by `deploy.sh` so a board can pull it +# during a TEZI network auto-install. +# +# The feed directory (IB_HTTP_DEPLOY_PATH) lives inside the tree and is +# self-contained: deploy.sh writes image_list.json next to image.json, so +# this script only has to expose that directory as the document root. The +# board's feed URL is therefore always: +# +# http://:/image_list.json +# +# The server runs as the calling user on an unprivileged port — no root, no +# web server configuration, no document-root permissions to arrange, and +# nothing written outside the tree. +# +# Copyright (c) 2025-2026 EDGEMTech SA + +# Usage: +# tezi-feed-serve.sh Serve in the foreground (Ctrl-C to stop) +# tezi-feed-serve.sh --ensure Start detached if not already running +# tezi-feed-serve.sh --status Report whether it runs, and on which URL +# tezi-feed-serve.sh --stop Stop a detached server +# +# --ensure is what build.sh/deploy.sh/dbuild.sh call, so a normal deploy +# leaves a working feed behind with no manual step. It is idempotent and +# silent when the server is already up. + +progname=$(basename "$0") + +IB_ROOT=$(cd "$(dirname "$(command -v -- "$0")")/.." && pwd -P) + +pr_usage() +{ + printf "Serve the TEZI HTTP feed for network auto-install\n\n" + printf "Usage: %s [--ensure|--status|--stop|-h]\n" "$progname" +} + +pr_help() +{ + printf "\nAvailable options:\n" + printf " -h, --help Print this help\n" + printf " --ensure Start detached unless already running\n" + printf " --status Report state and feed URL\n" + printf " --stop Stop a detached server\n" + printf "\nWith no option the server runs in the foreground.\n" + printf "\nThe feed directory and port come from build/conf/local.conf,\n" + printf "overridable per machine in build/conf/site.conf:\n" + printf " IB_HTTP_DEPLOY_PATH, IB_HTTP_FEED_PORT\n" +} + +# --- configuration ---------------------------------------------------------- +# +# Read a bitbake variable the way the front-end scripts do: last matching +# assignment wins, site.conf after local.conf so a per-machine override +# takes precedence. Only plain `=`/`?=`/`:=` assignments are understood, +# which is all these two variables ever use. + +conf_get() +{ + _var=$1 + _val="" + for _conf in "$IB_ROOT/build/conf/local.conf" "$IB_ROOT/build/conf/site.conf"; do + [ -f "$_conf" ] || continue + _hit=$(sed -n "s/^[[:space:]]*${_var}[[:space:]]*[?:]*=[[:space:]]*\"\([^\"]*\)\".*/\1/p" \ + "$_conf" | tail -1) + [ -n "$_hit" ] && _val=$_hit + done + printf '%s' "$_val" +} + +# IB_PLATFORM is scoped-override aware in check-env.sh; here the bare +# assignment is enough because IB_HTTP_DEPLOY_PATH interpolates it. +platform=$(sed -n 's/^[[:space:]]*IB_PLATFORM[[:space:]]*??*=[[:space:]]*"\([^"]*\)".*/\1/p' \ + "$IB_ROOT/build/conf/local.conf" 2>/dev/null | tail -1) + +feed=$(conf_get IB_HTTP_DEPLOY_PATH) +port=$(conf_get IB_HTTP_FEED_PORT) +[ -n "$port" ] || port=8080 + +# Expand the two bitbake variables the default value uses. Anything else +# left unexpanded means the override is beyond what this parser handles. +feed=$(printf '%s' "$feed" | sed -e "s|\${IB_DIR}|$IB_ROOT|g" -e "s|\${IB_PLATFORM}|$platform|g") + +case "$feed" in + "") printf "%s: IB_HTTP_DEPLOY_PATH not found in build/conf/*.conf\n" "$progname" >&2; exit 1 ;; + *'${'*) printf "%s: cannot expand IB_HTTP_DEPLOY_PATH (%s)\n" "$progname" "$feed" >&2; exit 1 ;; +esac + +runtime_dir="$IB_ROOT/build/deploy" +pidfile="$runtime_dir/tezi-feed.pid" +logfile="$runtime_dir/tezi-feed.log" + +# --- helpers ---------------------------------------------------------------- + +# Host IP as the board sees it: first default route, skipping tunnel-like +# interfaces so an active VPN does not capture the flashing address. This is +# the address printed below and handed to the board with `tezictl feed-add`, +# so a wrong pick sends the board to a host it cannot reach. +host_ip() +{ + ip -4 route show default 2>/dev/null \ + | awk '{for (i = 1; i <= NF; i++) if ($i == "dev") print $(i + 1)}' \ + | while IFS= read -r dev; do + case "$dev" in vpn*|tun*|tap*|wg*|ppp*) continue ;; esac + ip -4 -o addr show dev "$dev" 2>/dev/null \ + | awk 'NR == 1 {sub(/\/.*/, "", $4); print $4; exit}' + break + done +} + +feed_url() +{ + _ip=$(host_ip) + [ -n "$_ip" ] || _ip="" + printf 'http://%s:%s/image_list.json' "$_ip" "$port" +} + +# A live pidfile means our server; a stale one is cleaned up so --ensure +# can restart after a reboot or a kill without manual intervention. +running_pid() +{ + [ -f "$pidfile" ] || return 1 + _pid=$(cat "$pidfile" 2>/dev/null) + case "$_pid" in ''|*[!0-9]*) rm -f "$pidfile"; return 1 ;; esac + if kill -0 "$_pid" 2>/dev/null; then + printf '%s' "$_pid" + return 0 + fi + rm -f "$pidfile" + return 1 +} + +# --- actions ---------------------------------------------------------------- + +do_status() +{ + if _pid=$(running_pid); then + printf "running (pid %s) %s\n" "$_pid" "$(feed_url)" + printf " feed: %s\n" "$feed" + printf " log : %s\n" "$logfile" + return 0 + fi + printf "not running (feed: %s)\n" "$feed" + return 1 +} + +do_stop() +{ + if _pid=$(running_pid); then + kill "$_pid" 2>/dev/null + rm -f "$pidfile" + printf "%s: stopped (pid %s)\n" "$progname" "$_pid" + return 0 + fi + printf "%s: not running\n" "$progname" + return 0 +} + +# Serving an empty or missing directory would answer 404 to the board and +# look like a network fault, so say plainly that a deploy has to run first. +check_feed() +{ + if [ ! -d "$feed" ]; then + printf "%s: feed directory does not exist yet: %s\n" "$progname" "$feed" >&2 + printf "%s: run a deploy first (e.g. deploy.sh bsp-linux)\n" "$progname" >&2 + return 1 + fi + if [ ! -f "$feed/image_list.json" ]; then + printf "%s: %s has no image_list.json — deploy has not published here yet\n" \ + "$progname" "$feed" >&2 + return 1 + fi + return 0 +} + +do_serve_foreground() +{ + check_feed || exit 1 + printf "%s: serving %s\n" "$progname" "$feed" + printf "%s: feed URL %s\n" "$progname" "$(feed_url)" + printf "%s: board cmd tezictl feed-add %s\n" "$progname" "$(feed_url)" + exec python3 -m http.server "$port" --directory "$feed" +} + +do_ensure() +{ + # Never leave a daemon behind on a CI runner: CI builds and deploys to + # verify, it never flashes a board from its own feed. + if [ -n "${CI:-}" ] || [ -n "${GITLAB_CI:-}" ]; then + return 0 + fi + + # Inside the build container this is a no-op: the container is --rm, so + # a server started here would die with it. dbuild.sh starts one on the + # host before the container runs (it shares the host network namespace). + if [ -f /.dockerenv ] || [ -n "${container:-}" ]; then + return 0 + fi + + running_pid >/dev/null && return 0 + check_feed || return 0 + + command -v python3 >/dev/null 2>&1 || { + printf "%s: python3 not found — cannot serve the feed\n" "$progname" >&2 + return 0 + } + + mkdir -p "$runtime_dir" + + # Detached and immune to the parent shell going away, so the feed + # outlives the deploy that started it. + nohup python3 -m http.server "$port" --directory "$feed" \ + >>"$logfile" 2>&1 & + _pid=$! + printf '%s\n' "$_pid" > "$pidfile" + + # Give it a moment to fail (port already taken by something else) so the + # user hears about it now rather than when a board fails to flash. + sleep 1 + if ! kill -0 "$_pid" 2>/dev/null; then + rm -f "$pidfile" + printf "%s: feed server failed to start on port %s — see %s\n" \ + "$progname" "$port" "$logfile" >&2 + printf "%s: another server on that port? override IB_HTTP_FEED_PORT in build/conf/site.conf\n" \ + "$progname" >&2 + return 0 + fi + + printf "[tezi-feed] serving %s on %s (pid %s)\n" "$feed" "$(feed_url)" "$_pid" + return 0 +} + +# --- main ------------------------------------------------------------------- + +case "${1:-}" in + -h|--help) pr_usage; pr_help; exit 0 ;; + --ensure) do_ensure; exit 0 ;; + --status) do_status; exit $? ;; + --stop) do_stop; exit $? ;; + "") do_serve_foreground ;; + *) pr_usage >&2; exit 1 ;; +esac