Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 121 additions & 0 deletions deploy.sh
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,21 @@ ENABLED_EXTENSION_MANIFESTS="$LOCAL_BOOTSTRAP_MANIFEST"
# USE_AF_XDP=1 to make and rejects a cached AF_PACKET-only binary.
ANYSCAN_USE_AF_XDP="${ANYSCAN_USE_AF_XDP:-0}"

# Opt-in kernel backport upgrade. Mirrors install-external-deps.sh —
# see PR 65 issuecomment-4336192354 / anygpt-42 / anygpt-44. Default 0
# leaves the running kernel untouched (existing AMIs unchanged). 1
# installs the Debian bookworm-backports kernel image so the host can
# run kernel 6.16+ with the in-flight ena_xdp_zc patches that AF_XDP
# zerocopy on ENA needs. Never auto-reboots; the operator schedules
# the reboot. After install probes /sys/module/ena/version + dmesg
# for ena_xdp_zc support.
ANYSCAN_INSTALL_KERNEL_BACKPORT="${ANYSCAN_INSTALL_KERNEL_BACKPORT:-0}"
ANYSCAN_KERNEL_BACKPORT_MIN_VERSION="${ANYSCAN_KERNEL_BACKPORT_MIN_VERSION:-6.16}"
ANYSCAN_KERNEL_BACKPORT_PACKAGE="${ANYSCAN_KERNEL_BACKPORT_PACKAGE:-linux-image-cloud-amd64}"
ANYSCAN_KERNEL_BACKPORT_SUITE="${ANYSCAN_KERNEL_BACKPORT_SUITE:-bookworm-backports}"
ANYSCAN_KERNEL_BACKPORT_SOURCES_LIST="${ANYSCAN_KERNEL_BACKPORT_SOURCES_LIST:-/etc/apt/sources.list.d/anyscan-bookworm-backports.list}"
ANYSCAN_KERNEL_BACKPORT_MIRROR="${ANYSCAN_KERNEL_BACKPORT_MIRROR:-http://deb.debian.org/debian}"

print_banner() {
printf '═══════════════════════════════════════════════════════════\n'
printf ' AnyScan Rust Deploy Script \n'
Expand Down Expand Up @@ -105,6 +120,110 @@ binary_has_afxdp_linkage() {
return 1
}

# Mirrors install-external-deps.sh::kernel_version_at_least.
kernel_version_at_least() {
local have="$1" need="$2"
local have_major have_minor need_major need_minor
have_major="${have%%.*}"
have_minor="${have#*.}"
have_minor="${have_minor%%.*}"
have_minor="${have_minor%%[!0-9]*}"
need_major="${need%%.*}"
need_minor="${need#*.}"
need_minor="${need_minor%%.*}"
need_minor="${need_minor%%[!0-9]*}"
have_major="${have_major:-0}"
have_minor="${have_minor:-0}"
need_major="${need_major:-0}"
need_minor="${need_minor:-0}"
if [ "$have_major" -gt "$need_major" ]; then
return 0
fi
if [ "$have_major" -lt "$need_major" ]; then
return 1
fi
if [ "$have_minor" -ge "$need_minor" ]; then
return 0
fi
return 1
}

# Mirrors install-external-deps.sh::probe_ena_xdp_zc.
probe_ena_xdp_zc() {
if [ ! -e /sys/module/ena/version ]; then
printf '[!] ena driver not loaded on running kernel — cannot confirm AF_XDP zerocopy support. Reboot into the backport kernel and re-run this probe.\n' >&2
return 1
fi
local ena_ver
ena_ver="$(cat /sys/module/ena/version 2>/dev/null || true)"
printf '[*] ena driver version on running kernel: %s\n' "${ena_ver:-unknown}"
if command -v dmesg >/dev/null 2>&1 \
&& dmesg 2>/dev/null | grep -qiE 'ena_xdp_zc|ena.*xdp.*zerocopy|ena.*xdp_zc'; then
printf '[*] ena_xdp_zc indicator detected in dmesg — AF_XDP zerocopy should be available.\n'
return 0
fi
printf '[!] ena_xdp_zc indicator NOT found in dmesg on running kernel %s. AF_XDP zerocopy may not be available; the scanner will fall back to drv+copy mode. Reboot into kernel %s+ and re-run if you just installed the backport image.\n' \
"$(uname -r 2>/dev/null || echo unknown)" "$ANYSCAN_KERNEL_BACKPORT_MIN_VERSION" >&2
return 1
}

# Mirrors install-external-deps.sh::install_kernel_backport_if_requested.
# deploy.sh runs as root (the script enforces this above), so the
# function takes a slightly simpler path than the install-external-deps.sh
# variant — no sudo branch is needed here.
install_kernel_backport_if_requested() {
if [ "${ANYSCAN_INSTALL_KERNEL_BACKPORT:-0}" != "1" ]; then
return 0
fi
local current_kernel current_kernel_ver
current_kernel="$(uname -r 2>/dev/null || echo unknown)"
current_kernel_ver="${current_kernel%%-*}"
printf '[*] ANYSCAN_INSTALL_KERNEL_BACKPORT=1 — current kernel %s (need >= %s for ena_xdp_zc).\n' \
"$current_kernel" "$ANYSCAN_KERNEL_BACKPORT_MIN_VERSION"
if kernel_version_at_least "$current_kernel_ver" "$ANYSCAN_KERNEL_BACKPORT_MIN_VERSION"; then
printf '[*] Running kernel already meets %s+; backport image install skipped.\n' \
"$ANYSCAN_KERNEL_BACKPORT_MIN_VERSION"
probe_ena_xdp_zc || true
return 0
fi
if ! command -v apt-get >/dev/null 2>&1; then
printf '[*] Skipping kernel backport: apt-get not on PATH (this knob targets Debian-family hosts).\n'
return 0
fi
if [ ! -f "$ANYSCAN_KERNEL_BACKPORT_SOURCES_LIST" ]; then
printf '[*] Writing apt source for %s to %s...\n' \
"$ANYSCAN_KERNEL_BACKPORT_SUITE" "$ANYSCAN_KERNEL_BACKPORT_SOURCES_LIST"
if ! printf 'deb %s %s main\n' \
"$ANYSCAN_KERNEL_BACKPORT_MIRROR" \
"$ANYSCAN_KERNEL_BACKPORT_SUITE" \
>"$ANYSCAN_KERNEL_BACKPORT_SOURCES_LIST"; then
printf '[!] Failed to write %s; cannot install backport kernel image. Skipping.\n' \
"$ANYSCAN_KERNEL_BACKPORT_SOURCES_LIST" >&2
return 0
fi
else
printf '[*] Reusing existing apt source list at %s.\n' "$ANYSCAN_KERNEL_BACKPORT_SOURCES_LIST"
fi
printf '[*] Refreshing apt indexes for %s...\n' "$ANYSCAN_KERNEL_BACKPORT_SUITE"
if ! apt-get update >/dev/null 2>&1; then
printf '[!] apt-get update failed; cannot install backport kernel image. Skipping.\n' >&2
return 0
fi
printf '[*] Installing %s from %s (no auto-reboot)...\n' \
"$ANYSCAN_KERNEL_BACKPORT_PACKAGE" "$ANYSCAN_KERNEL_BACKPORT_SUITE"
if ! apt-get install -y --no-install-recommends \
-t "$ANYSCAN_KERNEL_BACKPORT_SUITE" \
"$ANYSCAN_KERNEL_BACKPORT_PACKAGE" >/dev/null 2>&1; then
printf '[!] Failed to install %s from %s; existing kernel unchanged.\n' \
"$ANYSCAN_KERNEL_BACKPORT_PACKAGE" "$ANYSCAN_KERNEL_BACKPORT_SUITE" >&2
return 0
fi
printf '[*] REBOOT REQUIRED: backport kernel image %s staged on disk. This script does NOT auto-reboot — schedule a maintenance window and reboot to activate kernel %s+.\n' \
"$ANYSCAN_KERNEL_BACKPORT_PACKAGE" "$ANYSCAN_KERNEL_BACKPORT_MIN_VERSION"
probe_ena_xdp_zc || true
return 0
}

install_vulnscanner_binary() {
local source_bin=""
local make_args=()
Expand Down Expand Up @@ -167,6 +286,8 @@ if [ "$EUID" -ne 0 ]; then
exit 1
fi

install_kernel_backport_if_requested

if ! command -v cargo >/dev/null 2>&1; then
printf '[!] cargo was not found in PATH. Install the Rust toolchain before deploying.\n' >&2
exit 1
Expand Down
163 changes: 163 additions & 0 deletions install-external-deps.sh
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,28 @@ VULNSCANNER_INSTALLED_BIN="/opt/anyscan/bin/scanner"
# to. See plans/2026-04-27-portscan-afxdp-plan-v1.md §3.6.
ANYSCAN_USE_AF_XDP="${ANYSCAN_USE_AF_XDP:-0}"

# Opt-in kernel backport upgrade. Default 0 leaves the running kernel
# untouched (existing AMIs unchanged). Setting 1 installs the Debian
# bookworm-backports kernel image so the host can run kernel 6.16+
# with the in-flight `ena_xdp_zc` ENA driver patches that AF_XDP
# zerocopy on ENA needs. The PR 65 §10 / anygpt-42 live bench showed
# ENA on kernel 6.12.74 caps the c6in.metal 8-NIC cap=4 throughput
# at ~22M pps in drv+copy mode, vs the 30-50M projection driver-mode
# zerocopy was supposed to deliver. See PR 65 issuecomment-4336192354
# for the constraint trace.
#
# Never auto-reboots. The new kernel is staged on disk and the operator
# has to schedule the reboot themselves. After install the script
# probes `/sys/module/ena/version` + dmesg for `ena_xdp_zc` support
# and warns if absent so the operator knows whether the
# CURRENTLY-RUNNING kernel will deliver zerocopy.
ANYSCAN_INSTALL_KERNEL_BACKPORT="${ANYSCAN_INSTALL_KERNEL_BACKPORT:-0}"
ANYSCAN_KERNEL_BACKPORT_MIN_VERSION="${ANYSCAN_KERNEL_BACKPORT_MIN_VERSION:-6.16}"
ANYSCAN_KERNEL_BACKPORT_PACKAGE="${ANYSCAN_KERNEL_BACKPORT_PACKAGE:-linux-image-cloud-amd64}"
ANYSCAN_KERNEL_BACKPORT_SUITE="${ANYSCAN_KERNEL_BACKPORT_SUITE:-bookworm-backports}"
ANYSCAN_KERNEL_BACKPORT_SOURCES_LIST="${ANYSCAN_KERNEL_BACKPORT_SOURCES_LIST:-/etc/apt/sources.list.d/anyscan-bookworm-backports.list}"
ANYSCAN_KERNEL_BACKPORT_MIRROR="${ANYSCAN_KERNEL_BACKPORT_MIRROR:-http://deb.debian.org/debian}"

# True when the existing scanner binary was linked against libxdp at build
# time. The AF_XDP build path (USE_AF_XDP=1 in the engine Makefile) adds
# `-lxdp -lbpf -lelf -lz` so libxdp.so shows up as a dynamic dependency;
Expand Down Expand Up @@ -57,6 +79,146 @@ vulnscanner_make_args() {
fi
}

# Lexicographic numeric compare of two `<major>.<minor>` version strings.
# Returns 0 (true) when $1 >= $2. Tolerant of missing fields and
# non-numeric trailing tokens (`6.16.0-rc1` / `6.16+deb13` etc).
kernel_version_at_least() {
local have="$1" need="$2"
local have_major have_minor need_major need_minor
have_major="${have%%.*}"
have_minor="${have#*.}"
have_minor="${have_minor%%.*}"
have_minor="${have_minor%%[!0-9]*}"
need_major="${need%%.*}"
need_minor="${need#*.}"
need_minor="${need_minor%%.*}"
need_minor="${need_minor%%[!0-9]*}"
have_major="${have_major:-0}"
have_minor="${have_minor:-0}"
need_major="${need_major:-0}"
need_minor="${need_minor:-0}"
if [ "$have_major" -gt "$need_major" ]; then
return 0
fi
if [ "$have_major" -lt "$need_major" ]; then
return 1
fi
if [ "$have_minor" -ge "$need_minor" ]; then
return 0
fi
return 1
}

# Probe the ena driver for AF_XDP zerocopy capability. ena_xdp_zc is
# the upstream patch series (in-flight against kernel 6.16+) that lets
# ENA advertise XDP_ZC; without it the scanner's AF_XDP path falls
# back to drv+copy and caps c6in.metal 8-NIC cap=4 throughput at
# ~22M pps (anygpt-42 live bench). Best-effort: a kernel with no
# `/sys/module/ena/version` (ena module not loaded) or no
# ena_xdp_zc indicator in dmesg just emits a warning so the operator
# knows zerocopy is unavailable on the CURRENTLY-RUNNING kernel
# (most useful right after a reboot into the backport kernel).
probe_ena_xdp_zc() {
if [ ! -e /sys/module/ena/version ]; then
printf '[!] ena driver not loaded on running kernel — cannot confirm AF_XDP zerocopy support. Reboot into the backport kernel and re-run this probe.\n' >&2
return 1
fi
local ena_ver
ena_ver="$(cat /sys/module/ena/version 2>/dev/null || true)"
printf '[*] ena driver version on running kernel: %s\n' "${ena_ver:-unknown}"
if command -v dmesg >/dev/null 2>&1 \
&& dmesg 2>/dev/null | grep -qiE 'ena_xdp_zc|ena.*xdp.*zerocopy|ena.*xdp_zc'; then
printf '[*] ena_xdp_zc indicator detected in dmesg — AF_XDP zerocopy should be available.\n'
return 0
fi
printf '[!] ena_xdp_zc indicator NOT found in dmesg on running kernel %s. AF_XDP zerocopy may not be available; the scanner will fall back to drv+copy mode. Reboot into kernel %s+ and re-run if you just installed the backport image.\n' \
"$(uname -r 2>/dev/null || echo unknown)" "$ANYSCAN_KERNEL_BACKPORT_MIN_VERSION" >&2
return 1
}

# Opt-in path that installs a backport kernel image (default
# linux-image-cloud-amd64 from Debian bookworm-backports) on hosts
# whose stock kernel is older than 6.16. Default OFF — existing
# AMIs are unchanged. Never auto-reboots: installing a kernel image
# only stages it on disk, the operator has to schedule the reboot
# themselves. After the install (or if the kernel is already new
# enough) probes /sys/module/ena/version + dmesg for ena_xdp_zc
# support and warns if absent.
install_kernel_backport_if_requested() {
if [ "${ANYSCAN_INSTALL_KERNEL_BACKPORT:-0}" != "1" ]; then
return 0
fi
local current_kernel current_kernel_ver
current_kernel="$(uname -r 2>/dev/null || echo unknown)"
current_kernel_ver="${current_kernel%%-*}"
printf '[*] ANYSCAN_INSTALL_KERNEL_BACKPORT=1 — current kernel %s (need >= %s for ena_xdp_zc).\n' \
"$current_kernel" "$ANYSCAN_KERNEL_BACKPORT_MIN_VERSION"
if kernel_version_at_least "$current_kernel_ver" "$ANYSCAN_KERNEL_BACKPORT_MIN_VERSION"; then
printf '[*] Running kernel already meets %s+; backport image install skipped.\n' \
"$ANYSCAN_KERNEL_BACKPORT_MIN_VERSION"
probe_ena_xdp_zc || true
return 0
fi
if ! command -v apt-get >/dev/null 2>&1; then
printf '[*] Skipping kernel backport: apt-get not on PATH (this knob targets Debian-family hosts).\n'
return 0
fi
local apt_cmd=() tee_cmd=()
if [ "$(id -u 2>/dev/null || echo 1)" = "0" ]; then
apt_cmd=(apt-get)
tee_cmd=(tee)
elif command -v sudo >/dev/null 2>&1; then
if ! sudo -n true >/dev/null 2>&1; then
printf '[*] Skipping kernel backport: sudo would prompt for a password.\n'
printf ' Install manually if you want kernel %s+:\n' "$ANYSCAN_KERNEL_BACKPORT_MIN_VERSION"
printf ' echo "deb %s %s main" | sudo tee %s\n' \
"$ANYSCAN_KERNEL_BACKPORT_MIRROR" \
"$ANYSCAN_KERNEL_BACKPORT_SUITE" \
"$ANYSCAN_KERNEL_BACKPORT_SOURCES_LIST"
printf ' sudo apt-get update && sudo apt-get install -y -t %s %s\n' \
"$ANYSCAN_KERNEL_BACKPORT_SUITE" "$ANYSCAN_KERNEL_BACKPORT_PACKAGE"
return 0
fi
apt_cmd=(sudo -n apt-get)
tee_cmd=(sudo -n tee)
else
printf '[*] Skipping kernel backport: not root and sudo is not available.\n'
return 0
fi
if [ ! -f "$ANYSCAN_KERNEL_BACKPORT_SOURCES_LIST" ]; then
printf '[*] Writing apt source for %s to %s...\n' \
"$ANYSCAN_KERNEL_BACKPORT_SUITE" "$ANYSCAN_KERNEL_BACKPORT_SOURCES_LIST"
if ! printf 'deb %s %s main\n' \
"$ANYSCAN_KERNEL_BACKPORT_MIRROR" \
"$ANYSCAN_KERNEL_BACKPORT_SUITE" \
| "${tee_cmd[@]}" "$ANYSCAN_KERNEL_BACKPORT_SOURCES_LIST" >/dev/null; then
printf '[!] Failed to write %s; cannot install backport kernel image. Skipping.\n' \
"$ANYSCAN_KERNEL_BACKPORT_SOURCES_LIST" >&2
return 0
fi
else
printf '[*] Reusing existing apt source list at %s.\n' "$ANYSCAN_KERNEL_BACKPORT_SOURCES_LIST"
fi
printf '[*] Refreshing apt indexes for %s...\n' "$ANYSCAN_KERNEL_BACKPORT_SUITE"
if ! "${apt_cmd[@]}" update >/dev/null 2>&1; then
printf '[!] apt-get update failed; cannot install backport kernel image. Skipping.\n' >&2
return 0
fi
printf '[*] Installing %s from %s (no auto-reboot)...\n' \
"$ANYSCAN_KERNEL_BACKPORT_PACKAGE" "$ANYSCAN_KERNEL_BACKPORT_SUITE"
if ! "${apt_cmd[@]}" install -y --no-install-recommends \
-t "$ANYSCAN_KERNEL_BACKPORT_SUITE" \
"$ANYSCAN_KERNEL_BACKPORT_PACKAGE" >/dev/null 2>&1; then
printf '[!] Failed to install %s from %s; existing kernel unchanged.\n' \
"$ANYSCAN_KERNEL_BACKPORT_PACKAGE" "$ANYSCAN_KERNEL_BACKPORT_SUITE" >&2
return 0
fi
printf '[*] REBOOT REQUIRED: backport kernel image %s staged on disk. This script does NOT auto-reboot — schedule a maintenance window and reboot to activate kernel %s+.\n' \
"$ANYSCAN_KERNEL_BACKPORT_PACKAGE" "$ANYSCAN_KERNEL_BACKPORT_MIN_VERSION"
probe_ena_xdp_zc || true
return 0
}

EXTENSION_MANIFESTS="$SCRIPT_DIR/local-bootstrap-provisioner.json,$SCRIPT_DIR/vulnscanner-zmap-adapter.json"
ANYGPT_API_ENV_FILE_DEFAULT="$REPO_ROOT/apps/api/.env"

Expand Down Expand Up @@ -144,6 +306,7 @@ if ! command -v git >/dev/null 2>&1; then
fi

install_afxdp_build_deps
install_kernel_backport_if_requested

if [ -d "$VULNSCANNER_REPO_DIR/.git" ]; then
printf '[*] Updating external repository in %s...\n' "$VULNSCANNER_REPO_DIR"
Expand Down
12 changes: 12 additions & 0 deletions package-worker-bundle.sh
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,17 @@ ANYSCAN_USE_AF_XDP="${ANYSCAN_USE_AF_XDP:-0}"
ANYSCAN_VULNSCANNER_REPO_DIR_DEFAULT="$SCRIPT_DIR/../../anyscan-engine-c"
ANYSCAN_VULNSCANNER_REPO_DIR="${ANYSCAN_VULNSCANNER_REPO_DIR:-$ANYSCAN_VULNSCANNER_REPO_DIR_DEFAULT}"

# Opt-in kernel backport flag (mirrors install-external-deps.sh /
# deploy.sh). Bundles do not contain a kernel image — the actual
# install happens at deploy time via install-external-deps.sh or
# deploy.sh on the target host. The bundle README records the flag
# so a downstream operator knows the producer's intent (e.g. "this
# bundle was built expecting kernel 6.16+ on the target"). Default
# 0; existing AMIs unchanged. See PR 65 issuecomment-4336192354 for
# the ENA / ena_xdp_zc constraint trace and anygpt-44 for this
# wire-up.
ANYSCAN_INSTALL_KERNEL_BACKPORT="${ANYSCAN_INSTALL_KERNEL_BACKPORT:-0}"

print_banner() {
printf '═══════════════════════════════════════════════════════════\n'
printf ' Remote Agent Packager \n'
Expand Down Expand Up @@ -744,6 +755,7 @@ Bundle control route:
Bundle scanner build:
scanner_included: ${include_scanner}
use_af_xdp: ${ANYSCAN_USE_AF_XDP}
install_kernel_backport: ${ANYSCAN_INSTALL_KERNEL_BACKPORT}
EOF

bundle_path="$DIST_DIR/${bundle_name}.tar.gz"
Expand Down
29 changes: 29 additions & 0 deletions runtime.worker.env.template
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,35 @@ POLL_INTERVAL_SECONDS=15
# plans/2026-04-27-portscan-afxdp-plan-v1.md §3.6.
# ANYSCAN_USE_AF_XDP=0

# Opt-in kernel backport upgrade (read by install-external-deps.sh,
# package-worker-bundle.sh, deploy.sh — NOT by the agentd runtime).
# Set 1 to install the Debian bookworm-backports kernel image
# (linux-image-cloud-amd64 by default) on a Debian-family host whose
# stock kernel is older than 6.16. Kernel 6.16+ carries the in-flight
# `ena_xdp_zc` ENA driver patches that AF_XDP zerocopy on ENA needs;
# without them the scanner's AF_XDP path falls back to drv+copy and
# caps c6in.metal 8-NIC cap=4 throughput at ~22M pps (anygpt-42 live
# bench, vs the 30-50M projection in
# plans/2026-04-27-portscan-afxdp-plan-v1.md §10).
#
# Default 0 — existing AMIs are unchanged. The scripts NEVER
# auto-reboot; the kernel image is staged on disk and the operator
# schedules the reboot. After install the scripts probe
# /sys/module/ena/version + dmesg for `ena_xdp_zc` support and warn
# if absent so the operator knows whether the CURRENTLY-RUNNING
# kernel will deliver zerocopy. Out of scope here: AMI rebuild,
# auto-reboot, the ena driver patches themselves.
#
# Override the package / suite / source list / mirror with the
# matching ANYSCAN_KERNEL_BACKPORT_* variables if you carry a
# different backport channel (e.g. an internal Debian mirror).
# ANYSCAN_INSTALL_KERNEL_BACKPORT=0
# ANYSCAN_KERNEL_BACKPORT_MIN_VERSION=6.16
# ANYSCAN_KERNEL_BACKPORT_PACKAGE=linux-image-cloud-amd64
# ANYSCAN_KERNEL_BACKPORT_SUITE=bookworm-backports
# ANYSCAN_KERNEL_BACKPORT_SOURCES_LIST=/etc/apt/sources.list.d/anyscan-bookworm-backports.list
# ANYSCAN_KERNEL_BACKPORT_MIRROR=http://deb.debian.org/debian

# Installed bundle asset locations
EXTENSION_MANIFEST_PATHS=/opt/agentd/extensions/bootstrap-provisioner.json,/opt/agentd/extensions/portscan-adapter.json
ARTIFACT_DIR=/var/lib/agentd/artifacts
Expand Down
Loading