diff --git a/.github/workflows/test-ova.yml b/.github/workflows/test-ova.yml index ee99979..5f90291 100644 --- a/.github/workflows/test-ova.yml +++ b/.github/workflows/test-ova.yml @@ -22,13 +22,13 @@ jobs: - name: Checkout uses: actions/checkout@v7 - - name: Install bats and jq + - name: Install bats, jq and zstd run: | sudo apt-get update -qq - sudo apt-get install -y --no-install-recommends bats jq + sudo apt-get install -y --no-install-recommends bats jq zstd - name: Run logic and config tests - run: bats ova/tests/generate-env.bats ova/tests/generate-compose.bats ova/tests/firewall.bats + run: bats ova/tests/generate-env.bats ova/tests/generate-compose.bats ova/tests/firewall.bats ova/tests/dg-ctl.bats integration: name: Real bring-up @@ -37,10 +37,10 @@ jobs: - name: Checkout uses: actions/checkout@v7 - - name: Install bats and jq + - name: Install bats, jq and zstd run: | sudo apt-get update -qq - sudo apt-get install -y --no-install-recommends bats jq + sudo apt-get install -y --no-install-recommends bats jq zstd - name: Login to GitHub container registry uses: docker/login-action@v4 diff --git a/ova/defguard.pkr.hcl b/ova/defguard.pkr.hcl index 46fb548..7c5e7bd 100644 --- a/ova/defguard.pkr.hcl +++ b/ova/defguard.pkr.hcl @@ -92,6 +92,21 @@ build { destination = "/tmp/defguard-init.service" } + provisioner "file" { + source = "files/dg-ctl" + destination = "/tmp/dg-ctl" + } + + provisioner "file" { + source = "files/install-dg-ctl.sh" + destination = "/tmp/install-dg-ctl.sh" + } + + provisioner "file" { + source = "manifest.json" + destination = "/tmp/manifest.json" + } + provisioner "file" { source = "files/defguard-firewall.sh" destination = "/tmp/defguard-firewall.sh" @@ -126,6 +141,13 @@ build { "echo 'DEFGUARD_GATEWAY_TAG=${var.gateway_tag}' | sudo tee -a /opt/stacks/defguard/init/.image-tags > /dev/null", "sudo mv /tmp/99-defguard.cfg /etc/cloud/cloud.cfg.d/99-defguard.cfg", "sudo mv /tmp/defguard-init.service /etc/systemd/system/defguard-init.service", + "sudo mkdir -p /opt/defguard/backups", + "sudo install -m 750 -o root -g root /tmp/dg-ctl /opt/defguard/dg-ctl", + "sudo install -m 750 -o root -g root /tmp/install-dg-ctl.sh /opt/defguard/install-dg-ctl.sh", + "sudo ln -sf /opt/defguard/dg-ctl /usr/local/bin/dg-ctl", + "jq -n --arg v \"$(jq -r .ova_version /tmp/manifest.json)\" --arg ref \"$(jq -r .template_ref /tmp/manifest.json)\" --arg core '${var.core_tag}' --arg proxy '${var.proxy_tag}' --arg gateway '${var.gateway_tag}' '{ova_version: $v, template_ref: $ref, tags: {core: $core, proxy: $proxy, gateway: $gateway}}' | sudo tee /opt/defguard/state.json > /dev/null", + "sudo chmod 600 /opt/defguard/state.json", + "rm -f /tmp/manifest.json", "sudo mv /tmp/defguard-firewall.sh /opt/stacks/defguard/defguard-firewall.sh", "sudo chmod +x /opt/stacks/defguard/defguard-firewall.sh", "sudo mv /tmp/defguard-firewall.service /etc/systemd/system/defguard-firewall.service", diff --git a/ova/files/dg-ctl b/ova/files/dg-ctl new file mode 100644 index 0000000..5342780 --- /dev/null +++ b/ova/files/dg-ctl @@ -0,0 +1,1478 @@ +#!/bin/bash +# defguard OVA maintenance CLI, installed as /opt/defguard/dg-ctl. +set -euo pipefail + +DG_CTL_VERSION="2.1.0" + +STACK_DIR="${DEFGUARD_STACK_DIR:-/opt/stacks/defguard}" +OVA_DIR="${DEFGUARD_OVA_DIR:-/opt/defguard}" +BACKUPS_DIR="$OVA_DIR/backups" +STATE_FILE="$OVA_DIR/state.json" + +COMPOSE_FILE="$STACK_DIR/docker-compose.yml" +ENV_FILE="$STACK_DIR/.env" +INIT_DIR="$STACK_DIR/init" +APPLIED_PROFILES_FILE="$INIT_DIR/.applied-profiles" +LEGACY_LAYOUT=0 +LEGACY_COMPOSE_MODE="" +DEPLOYMENT_MODE="auto" +INIT_SERVICE_FILE="${DEFGUARD_INIT_SERVICE_FILE:-/etc/systemd/system/defguard-init.service}" + +OVA_REPO="${DEFGUARD_OVA_REPO:-DefGuard/deployment}" +MANIFEST_URL="${DEFGUARD_OVA_MANIFEST_URL:-https://raw.githubusercontent.com/$OVA_REPO/main/ova/manifest.json}" +SOURCE_DIR="${DEFGUARD_OVA_SOURCE_DIR:-}" + +KEEP_BACKUPS="${KEEP_BACKUPS:-3}" +HEALTH_TIMEOUT="${HEALTH_TIMEOUT:-180}" +# Window used to catch a service that starts and then crash-loops. +HEALTH_SETTLE="${HEALTH_SETTLE:-15}" +FETCH_CONNECT_TIMEOUT="${FETCH_CONNECT_TIMEOUT:-10}" +FETCH_TIMEOUT="${FETCH_TIMEOUT:-60}" + +WORK_DIR="" +LOCK_FILE="$OVA_DIR/.dg-ctl.lock" +LOCK_FD="" +LOCK_DIR="$OVA_DIR/.dg-ctl.lock.d" +LOCK_MODE="" +ACTIVE_BACKUP_DIR="" +BACKUP_ID_RESULT="" +STACK_WAS_RUNNING=0 +STACK_QUIESCED=0 +LEAVE_STACK_STOPPED=0 +RECOVERY_BACKUP_ID="" +RECOVERY_ACTIVE=0 +RECOVERY_IN_PROGRESS=0 +RESTORE_START_STACK=1 + +cleanup() { + local status=$? + trap - EXIT + + if [ "$status" -ne 0 ] && [ "$STACK_QUIESCED" = 1 ] && [ "$RECOVERY_IN_PROGRESS" = 0 ]; then + RECOVERY_IN_PROGRESS=1 + if [ "$RECOVERY_ACTIVE" = 1 ] && [ -n "$RECOVERY_BACKUP_ID" ] && [ "$RECOVERY_BACKUP_ID" != "none" ]; then + warn "operation failed; attempting automatic rollback to $RECOVERY_BACKUP_ID" + RESTORE_START_STACK="$STACK_WAS_RUNNING" + if ( restore_backup "$RECOVERY_BACKUP_ID" ); then + STACK_QUIESCED=0 + else + warn "automatic rollback failed; the stack may be stopped. Inspect Docker and the backup before retrying." + fi + elif [ "$STACK_WAS_RUNNING" = 1 ]; then + warn "operation failed; attempting to restart the previously running stack" + if ! compose up -d --remove-orphans; then + warn "could not restart the stack after the failed operation" + else + STACK_QUIESCED=0 + fi + fi + fi + + [ -n "$WORK_DIR" ] && rm -rf "$WORK_DIR" + [ "$LOCK_MODE" = mkdir ] && rm -rf "$LOCK_DIR" + if [ "$status" -ne 0 ] && [ -n "$ACTIVE_BACKUP_DIR" ] && [ ! -f "$ACTIVE_BACKUP_DIR/meta.json" ]; then + rm -rf "$ACTIVE_BACKUP_DIR" + fi + return "$status" +} +trap cleanup EXIT + +log() { echo "[dg-ctl] $*"; } +warn() { echo "[dg-ctl] WARNING: $*" >&2; } +die() { echo "[dg-ctl] ERROR: $*" >&2; exit 1; } + +legacy_active_profiles_csv() { + [ -f "$STACK_DIR/active-profiles" ] || return 0 + tr '[:space:]' '\n' < "$STACK_DIR/active-profiles" \ + | sed '/^$/d' | tr '\n' ',' | sed 's/,$//' +} + +legacy_compose_profiles_csv() { + local profiles + profiles="$(legacy_active_profiles_csv)" + if [ -f "$STACK_DIR/enable-docker-management" ]; then + [ -n "$profiles" ] && profiles+=',' + profiles+=dockge + fi + printf '%s\n' "$profiles" +} + +detect_stack_layout() { + COMPOSE_FILE="$STACK_DIR/docker-compose.yml" + INIT_DIR="$STACK_DIR/init" + APPLIED_PROFILES_FILE="$INIT_DIR/.applied-profiles" + LEGACY_LAYOUT=0 + LEGACY_COMPOSE_MODE="" + DEPLOYMENT_MODE="auto" + + if [ -f "$COMPOSE_FILE" ]; then + if [ -f "$INIT_DIR/.deployment-mode" ]; then + DEPLOYMENT_MODE="$(tr -d '[:space:]' < "$INIT_DIR/.deployment-mode")" + case "$DEPLOYMENT_MODE" in + full|segmented) ;; + *) die "invalid deployment mode in $INIT_DIR/.deployment-mode" ;; + esac + fi + return 0 + fi + + if [ -f "$STACK_DIR/docker-compose.yaml" ]; then + LEGACY_LAYOUT=1 + COMPOSE_FILE="$STACK_DIR/docker-compose.yaml" + if [ -n "$(legacy_active_profiles_csv)" ]; then + [ -f "$STACK_DIR/docker-compose.standalone.yaml" ] \ + || die "legacy standalone Compose file is missing" + LEGACY_COMPOSE_MODE=standalone + DEPLOYMENT_MODE=segmented + else + LEGACY_COMPOSE_MODE=full + DEPLOYMENT_MODE=full + fi + return 0 + fi +} + +compose() { + if [ "$LEGACY_LAYOUT" = 1 ]; then + local legacy_compose_file="$COMPOSE_FILE" + [ "$LEGACY_COMPOSE_MODE" = standalone ] \ + && legacy_compose_file="$STACK_DIR/docker-compose.standalone.yaml" + COMPOSE_PROFILES="$(legacy_compose_profiles_csv)" \ + docker compose -f "$legacy_compose_file" \ + --project-directory "$STACK_DIR" "$@" + else + docker compose -f "$COMPOSE_FILE" --project-directory "$STACK_DIR" "$@" + fi +} + +need_root() { + [ "${DEFGUARD_OVA_ALLOW_NONROOT:-0}" = "1" ] && return 0 + [ "$(id -u)" = "0" ] || die "must run as root (try: sudo $0 $*)" +} + +acquire_lock() { + mkdir -p "$OVA_DIR" + if command -v flock >/dev/null 2>&1; then + exec {LOCK_FD}>"$LOCK_FILE" + flock -n "$LOCK_FD" || die "another dg-ctl operation is already running" + LOCK_MODE=flock + else + if ! mkdir "$LOCK_DIR" 2>/dev/null; then + local owner="" + [ -f "$LOCK_DIR/pid" ] && owner="$(cat "$LOCK_DIR/pid" 2>/dev/null || true)" + if [[ "$owner" =~ ^[0-9]+$ ]] && ! kill -0 "$owner" 2>/dev/null; then + rm -rf "$LOCK_DIR" + mkdir "$LOCK_DIR" 2>/dev/null \ + || die "another dg-ctl operation is already running" + else + die "another dg-ctl operation is already running" + fi + fi + LOCK_MODE="mkdir" + printf '%s\n' "$$" >"$LOCK_DIR/pid" + fi +} + +canonicalize_path() { + local suffix="" current="$1" parent base resolved + while [ "$current" != "/" ] && [[ "$current" == */ ]]; do current="${current%/}"; done + [ -n "$current" ] || current="/" + while [ ! -e "$current" ]; do + parent="$(dirname "$current")" + base="$(basename "$current")" + [ "$parent" != "$current" ] || return 1 + suffix="/$base$suffix" + current="$parent" + done + [ -d "$current" ] || return 1 + resolved="$(cd -P "$current" && pwd -P)" || return 1 + printf '%s%s\n' "$resolved" "$suffix" +} + +validate_safe_paths() { + [ ! -L "$STACK_DIR" ] || die "DEFGUARD_STACK_DIR may not be a symlink" + [ ! -L "$OVA_DIR" ] || die "DEFGUARD_OVA_DIR may not be a symlink" + + local stack_real ova_real + stack_real="$(canonicalize_path "$STACK_DIR")" \ + || die "DEFGUARD_STACK_DIR does not resolve to an existing directory" + ova_real="$(canonicalize_path "$OVA_DIR")" \ + || die "DEFGUARD_OVA_DIR is not a valid path" + [ -d "$stack_real" ] || die "DEFGUARD_STACK_DIR is not a directory" + [ ! -e "$OVA_DIR" ] || [ -d "$ova_real" ] \ + || die "DEFGUARD_OVA_DIR is not a directory" + [ "$stack_real" != "/" ] || die "DEFGUARD_STACK_DIR may not resolve to /" + [ "$ova_real" != "/" ] || die "DEFGUARD_OVA_DIR may not resolve to /" + case "$stack_real/" in + "$ova_real/"*) die "DEFGUARD_STACK_DIR and DEFGUARD_OVA_DIR may not overlap" ;; + esac + case "$ova_real/" in + "$stack_real/"*) die "DEFGUARD_STACK_DIR and DEFGUARD_OVA_DIR may not overlap" ;; + esac + + STACK_DIR="$stack_real" + OVA_DIR="$ova_real" + BACKUPS_DIR="$OVA_DIR/backups" + STATE_FILE="$OVA_DIR/state.json" + COMPOSE_FILE="$STACK_DIR/docker-compose.yml" + ENV_FILE="$STACK_DIR/.env" + INIT_DIR="$STACK_DIR/init" + APPLIED_PROFILES_FILE="$INIT_DIR/.applied-profiles" + LOCK_FILE="$OVA_DIR/.dg-ctl.lock" + LOCK_DIR="$OVA_DIR/.dg-ctl.lock.d" + detect_stack_layout +} + +need_stack() { + detect_stack_layout + [ -f "$COMPOSE_FILE" ] || die "$COMPOSE_FILE not found; this does not look like a defguard OVA host" + [ -f "$ENV_FILE" ] || die "$ENV_FILE not found" + docker compose version >/dev/null 2>&1 || die "docker compose v2 is not available" + command -v jq >/dev/null 2>&1 || die "jq is required" + command -v curl >/dev/null 2>&1 || die "curl is required" +} + +need_zstd() { + command -v zstd >/dev/null 2>&1 || die "zstd is required for backups (apt-get install zstd)" +} + +env_value() { + [ -f "$ENV_FILE" ] || return 0 + sed -n "s/^$1=//p" "$ENV_FILE" | tail -n1 +} + +fetch() { + local src="$1" dest="$2" + case "$src" in + file://*) cp "${src#file://}" "$dest" ;; + /*) cp "$src" "$dest" ;; + *) curl -fsSL --retry 3 --connect-timeout "$FETCH_CONNECT_TIMEOUT" \ + --max-time "$FETCH_TIMEOUT" -o "$dest" "$src" ;; + esac +} + +fetch_ova_file() { + local ref="$1" name="$2" dest="$3" + if [ -n "$SOURCE_DIR" ]; then + cp "$SOURCE_DIR/ova/files/$name" "$dest" + else + fetch "https://raw.githubusercontent.com/$OVA_REPO/$ref/ova/files/$name" "$dest" + fi +} + +read_manifest() { + local tmp + tmp="$(mktemp)" + fetch "$MANIFEST_URL" "$tmp" || { rm -f "$tmp"; die "could not fetch manifest from $MANIFEST_URL"; } + jq -e . "$tmp" || { rm -f "$tmp"; die "manifest at $MANIFEST_URL is not valid JSON"; } + rm -f "$tmp" +} + +state_value() { + [ -f "$STATE_FILE" ] || { echo "unknown"; return 0; } + jq -r --arg k "$1" '.[$k] // "unknown"' "$STATE_FILE" 2>/dev/null || echo "unknown" +} + +valid_version() { + [[ "$1" =~ ^[0-9]+\.[0-9]+\.[0-9]+([-.][0-9A-Za-z.-]+)?$ ]] +} + +valid_tag() { + [[ "$1" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]] +} + +valid_ref() { + local ref="$1" + [[ "$ref" =~ ^[A-Za-z0-9._/@+-]+$ ]] || return 1 + [[ "$ref" != /* && "$ref" != */ ]] || return 1 + [[ "$ref" != *..* && "$ref" != *//* && "$ref" != *"@{"* ]] || return 1 + [[ "$ref" != .* && "$ref" != */.* && "$ref" != *.lock ]] || return 1 +} + +valid_backup_id() { + [[ "$1" =~ ^[0-9]{8}-[0-9]{6}Z(-[A-Za-z0-9._-]+)*$ ]] +} + +validate_manifest() { + local manifest="$1" v previous="" duplicate + jq -e ' + type == "object" and + (.ova_version | type == "string") and + (.ova_version | test("^[0-9]+\\.[0-9]+\\.[0-9]+([-.][0-9A-Za-z.-]+)?$")) and + (.template_ref | type == "string") and + ((.migrations // []) | type == "array") and + all((.migrations // [])[]; type == "string" and test("^[0-9]+\\.[0-9]+\\.[0-9]+([-.][0-9A-Za-z.-]+)?$")) + ' <<<"$manifest" >/dev/null || die "manifest has an invalid schema or version" + + while IFS= read -r v; do + [ -n "$v" ] || continue + valid_version "$v" || die "manifest contains invalid migration version: $v" + [ -z "$previous" ] || version_le "$previous" "$v" \ + || die "manifest migrations are not in ascending order" + duplicate="$(jq -r --arg v "$v" '[.migrations[] | select(. == $v)] | length' <<<"$manifest")" + [ "$duplicate" = 1 ] || die "manifest contains duplicate migration: $v" + previous="$v" + done < <(jq -r '.migrations[]? // empty' <<<"$manifest") + + local ref + ref="$(jq -r '.template_ref' <<<"$manifest")" + valid_ref "$ref" || die "manifest contains an invalid template ref" +} + +resolve_template_ref() { + local ref="$1" response + valid_ref "$ref" || die "invalid template ref: $ref" + [[ "$OVA_REPO" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] \ + || die "invalid GitHub repository: $OVA_REPO" + if [ -n "$SOURCE_DIR" ] || [[ "$ref" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "$ref" + return 0 + fi + response="$(curl -fsSL --retry 3 --connect-timeout "$FETCH_CONNECT_TIMEOUT" \ + --max-time "$FETCH_TIMEOUT" -H 'Accept: application/vnd.github+json' \ + "https://api.github.com/repos/$OVA_REPO/commits/$ref")" \ + || die "could not resolve template ref $ref to an immutable commit" + jq -er '.sha | select(test("^[0-9a-fA-F]{40}$"))' <<<"$response" \ + || die "GitHub returned no immutable commit for template ref $ref" +} + +# "unknown" / "2.0-unknown" (state.json seeded before this field existed) sorts +# as the oldest possible version, so every migration is considered pending. +normalize_version() { + case "$1" in + ""|unknown|*-unknown) echo "0.0.0" ;; + *) echo "$1" ;; + esac +} + +numeric_cmp() { + local left="$1" right="$2" + local LC_ALL=C + + while [ "$left" != "0" ] && [ "${left#0}" != "$left" ]; do left="${left#0}"; done + while [ "$right" != "0" ] && [ "${right#0}" != "$right" ]; do right="${right#0}"; done + if [ "${#left}" -lt "${#right}" ]; then + echo -1 + elif [ "${#left}" -gt "${#right}" ]; then + echo 1 + elif [[ "$left" < "$right" ]]; then + echo -1 + elif [[ "$left" > "$right" ]]; then + echo 1 + else + echo 0 + fi +} + +version_cmp() { + local left="$1" right="$2" left_base right_base left_pre right_pre + local i max li ri cmp + local -a lparts=() rparts=() + + left_base="${left%%-*}" + right_base="${right%%-*}" + left_pre="" + right_pre="" + [[ "$left" == *-* ]] && left_pre="${left#*-}" + [[ "$right" == *-* ]] && right_pre="${right#*-}" + + IFS=. read -r -a lparts <<<"$left_base" + IFS=. read -r -a rparts <<<"$right_base" + for i in 0 1 2; do + cmp="$(numeric_cmp "${lparts[$i]}" "${rparts[$i]}")" + if [ "$cmp" -lt 0 ]; then + echo -1 + return 0 + elif [ "$cmp" -gt 0 ]; then + echo 1 + return 0 + fi + done + + # A release is newer than any pre-release of the same core version. + if [ -z "$left_pre" ] && [ -n "$right_pre" ]; then echo 1; return 0; fi + if [ -n "$left_pre" ] && [ -z "$right_pre" ]; then echo -1; return 0; fi + [ -z "$left_pre" ] && { echo 0; return 0; } + + IFS=. read -r -a lparts <<<"$left_pre" + IFS=. read -r -a rparts <<<"$right_pre" + max="${#lparts[@]}" + [ "${#rparts[@]}" -gt "$max" ] && max="${#rparts[@]}" + for ((i = 0; i < max; i++)); do + [ "$i" -lt "${#lparts[@]}" ] || { echo -1; return 0; } + [ "$i" -lt "${#rparts[@]}" ] || { echo 1; return 0; } + li="${lparts[$i]}" + ri="${rparts[$i]}" + if [[ "$li" =~ ^[0-9]+$ && "$ri" =~ ^[0-9]+$ ]]; then + cmp="$(numeric_cmp "$li" "$ri")" + if [ "$cmp" -lt 0 ]; then echo -1; return 0; fi + if [ "$cmp" -gt 0 ]; then echo 1; return 0; fi + elif [[ "$li" =~ ^[0-9]+$ ]]; then + echo -1 + return 0 + elif [[ "$ri" =~ ^[0-9]+$ ]]; then + echo 1 + return 0 + elif [[ "$li" < "$ri" ]]; then + echo -1 + return 0 + elif [[ "$li" > "$ri" ]]; then + echo 1 + return 0 + fi + done + echo 0 +} + +version_le() { + [ "$(version_cmp "$1" "$2")" -le 0 ] +} + +# resolve_pending_migrations FROM TO MANIFEST - prints, one per line, the +# migration versions in MANIFEST's "migrations" array with FROM < v <= TO. +resolve_pending_migrations() { + local from to manifest v + from="$(normalize_version "$1")" + to="$(normalize_version "$2")" + manifest="$3" + while IFS= read -r v; do + [ -n "$v" ] || continue + if ! version_le "$v" "$from" && version_le "$v" "$to"; then + echo "$v" + fi + done < <(jq -r '.migrations[]? // empty' <<<"$manifest") +} + +# Prints the profiles this stack was generated with, one per line. The +# generated compose file has no profiles: key left in it, so this has to come +# from what generate-compose.sh recorded. +resolve_applied_profiles() { + local -a profiles=() words=() unique=() + local line header services service p + + if [ "$LEGACY_LAYOUT" = 1 ]; then + if [ "$LEGACY_COMPOSE_MODE" = full ]; then + profiles+=(core edge gateway) + else + while IFS= read -r line; do + [ -n "${line//[[:space:]]/}" ] || continue + read -r -a words <<<"$line" + profiles+=("${words[@]}") + done < "$STACK_DIR/active-profiles" + fi + [ -f "$STACK_DIR/enable-docker-management" ] && profiles+=(dockge) + elif [ -s "$APPLIED_PROFILES_FILE" ]; then + while IFS= read -r line; do + [[ -n "${line//[[:space:]]/}" ]] || continue + read -r -a words <<<"$line" + profiles+=("${words[@]}") + done < "$APPLIED_PROFILES_FILE" + else + # OVAs built before .applied-profiles existed: the generator's header comment. + header="$(sed -n 's/^# Selected profiles: //p' "$COMPOSE_FILE" | head -n1)" + if [ -n "$header" ]; then + read -r -a profiles <<<"$header" + else + # Last resort: infer from the service list. + services="$(compose config --services 2>/dev/null)" \ + || die "could not inspect the Compose service list" + for service in $services; do + case "$service" in + core|db) profiles+=(core) ;; + edge) profiles+=(edge) ;; + gateway) profiles+=(gateway) ;; + dockge) profiles+=(dockge) ;; + esac + done + fi + fi + + [ "${#profiles[@]}" -gt 0 ] \ + || die "cannot determine which profiles this stack runs; create $APPLIED_PROFILES_FILE (one profile per line) and retry" + + local -A seen=() + for p in "${profiles[@]}"; do + case "$p" in + core|edge|gateway|dockge) ;; + *) die "unknown applied profile '$p'; expected core, edge, gateway, or dockge" ;; + esac + if [ -z "${seen[$p]+x}" ]; then + unique+=("$p") + seen[$p]=1 + fi + done + printf '%s\n' "${unique[@]}" +} + +has_profile() { + local want="$1" p + shift + for p in "$@"; do [ "$p" = "$want" ] && return 0; done + return 1 +} + +backup_id_now() { + local label="${1:-}" + local id + [ -z "$label" ] || [[ "$label" =~ ^[A-Za-z0-9._-]+$ ]] \ + || die "backup label may contain only letters, numbers, '.', '_' and '-'" + id="$(date -u +%Y%m%d-%H%M%SZ)" + [ -n "$label" ] && id="$id-$label" + echo "$id" +} + +stack_has_running() { + local output running + output="$(compose ps --format json 2>/dev/null)" || return 2 + [ -n "$output" ] || return 1 + running="$(jq -rs 'flatten | any(.[]?; .State == "running")' <<<"$output")" \ + || return 2 + [ "$running" = true ] +} + +assert_stack_stopped() { + local output running + output="$(compose ps --format json 2>/dev/null)" \ + || die "could not verify that the Compose stack stopped" + [ -z "$output" ] && return 0 + running="$(jq -rs 'flatten | any(.[]?; .State == "running")' <<<"$output")" \ + || die "could not parse the Compose container state" + [ "$running" != true ] \ + || die "Compose still has running containers; refusing to replace volumes" +} + +quiesce_stack() { + STACK_WAS_RUNNING=0 + if stack_has_running; then + STACK_WAS_RUNNING=1 + else + case "$?" in + 1) ;; + *) die "could not inspect the running Compose stack" ;; + esac + fi + STACK_QUIESCED=1 + compose stop >&2 + assert_stack_stopped +} + +atomic_install() { + local src="$1" dest="$2" mode="$3" owner="${4:-}" group="${5:-}" tmp + local -a install_args=(-m "$mode") + tmp="$(mktemp "${dest}.tmp.XXXXXX")" + [ -z "$owner" ] || install_args+=(-o "$owner") + [ -z "$group" ] || install_args+=(-g "$group") + if ! install "${install_args[@]}" "$src" "$tmp"; then + rm -f "$tmp" + return 1 + fi + mv -f "$tmp" "$dest" +} + +do_backup() { + local label="${1:-}" id dir profiles digests + local volumes_sha stack_sha ova_sha init_service_sha free_kb need_kb total_kb backup_kb + need_zstd + id="$(backup_id_now "$label")" + dir="$BACKUPS_DIR/$id" + # Same-second backups (e.g. two upgrades run back to back) would otherwise + # collide on the timestamp-only id; disambiguate instead of failing. + if [ -e "$dir" ]; then + local n=2 + while [ -e "$BACKUPS_DIR/$id-$n" ]; do n=$((n + 1)); done + id="$id-$n" + dir="$BACKUPS_DIR/$id" + fi + mkdir -p "$BACKUPS_DIR" + + free_kb="$(df -Pk "$OVA_DIR" | awk 'NR==2 {print $4}')" + total_kb="$(du -sk "$STACK_DIR" "$OVA_DIR" 2>/dev/null | awk '{sum += $1} END {print sum}')" + backup_kb="$(du -sk "$BACKUPS_DIR" 2>/dev/null | awk '{sum += $1} END {print sum}')" + [[ "$free_kb" =~ ^[0-9]+$ ]] || die "could not determine free space in $OVA_DIR" + [[ "$total_kb" =~ ^[0-9]+$ ]] || die "could not estimate backup size" + [[ "$backup_kb" =~ ^[0-9]+$ ]] || die "could not estimate existing backup size" + need_kb=$((total_kb - backup_kb + 1048576)) + [ "$need_kb" -gt 0 ] || need_kb=1048576 + if [ "$free_kb" -lt "$need_kb" ]; then + die "not enough free space in $OVA_DIR: ${free_kb}K free, ${need_kb}K of volumes to archive" + fi + + mkdir -p "$dir" + ACTIVE_BACKUP_DIR="$dir" + profiles="$(resolve_applied_profiles)" + + # Digests first: they are only readable while the containers/images are known + # to compose, and they cost nothing. + digests="$(compose images --format json 2>/dev/null || echo '[]')" + echo "$digests" > "$dir/image-digests" + + log "stopping stack for a consistent backup" >&2 + quiesce_stack + + log "archiving volumes" >&2 + if [ -d "$STACK_DIR/.volumes" ]; then + tar --zstd -cf "$dir/volumes.tar.zst" -C "$STACK_DIR" .volumes + else + warn ".volumes does not exist; archiving an empty volume set" + tar --zstd -cf "$dir/volumes.tar.zst" -C "$STACK_DIR" --files-from /dev/null + fi + log "archiving stack structure" >&2 + tar --zstd -cf "$dir/stack-structure.tar.zst" \ + --exclude='./.volumes' -C "$STACK_DIR" . + log "archiving OVA structure" >&2 + tar --zstd -cf "$dir/ova-structure.tar.zst" \ + --exclude='./backups' \ + --exclude='./.dg-ctl.lock' \ + --exclude='./.dg-ctl.lock.d' \ + --exclude='./dg-ctl' \ + --exclude='./install-dg-ctl.sh' \ + -C "$OVA_DIR" . + volumes_sha="$(sha256sum "$dir/volumes.tar.zst" | awk '{print $1}')" + stack_sha="$(sha256sum "$dir/stack-structure.tar.zst" | awk '{print $1}')" + ova_sha="$(sha256sum "$dir/ova-structure.tar.zst" | awk '{print $1}')" + + cp "$ENV_FILE" "$dir/env" + chmod 600 "$dir/env" + cp "$COMPOSE_FILE" "$dir/docker-compose.yml" + printf '%s\n' "$profiles" > "$dir/applied-profiles" + init_service_sha="" + if [ -f "$INIT_SERVICE_FILE" ]; then + cp "$INIT_SERVICE_FILE" "$dir/defguard-init.service" + init_service_sha="$(sha256sum "$dir/defguard-init.service" | awk '{print $1}')" + fi + + jq -n \ + --argjson format_version 3 \ + --arg id "$id" \ + --arg label "$label" \ + --arg created "$(date -u +%FT%TZ)" \ + --arg host "$(hostname)" \ + --arg ova_version "$(state_value ova_version)" \ + --arg core "$(env_value DEFGUARD_CORE_TAG)" \ + --arg proxy "$(env_value DEFGUARD_PROXY_TAG)" \ + --arg gateway "$(env_value DEFGUARD_GATEWAY_TAG)" \ + --arg volumes_sha "$volumes_sha" \ + --arg stack_sha "$stack_sha" \ + --arg ova_sha "$ova_sha" \ + --arg init_service_sha "$init_service_sha" \ + '{format_version: $format_version, id: $id, label: $label, created: $created, host: $host, ova_version: $ova_version, + tags: {core: $core, proxy: $proxy, gateway: $gateway}, + volumes_sha256: $volumes_sha, stack_sha256: $stack_sha, ova_sha256: $ova_sha, + init_service_sha256: $init_service_sha}' > "$dir/meta.json" + + if [ "$LEAVE_STACK_STOPPED" != 1 ]; then + if [ "$STACK_WAS_RUNNING" = 1 ]; then + compose up -d --remove-orphans + fi + STACK_QUIESCED=0 + fi + log "backup written to $dir" >&2 + BACKUP_ID_RESULT="$id" + ACTIVE_BACKUP_DIR="" + echo "$id" +} + +prune_backups() { + local keep="$KEEP_BACKUPS" all old b + [ "$keep" -gt 0 ] 2>/dev/null || return 0 + local listing + listing="$(find "$BACKUPS_DIR" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; \ + | LC_ALL=C sort)" || die "could not enumerate backups" + all=() + while IFS= read -r b; do + [ -n "$b" ] || continue + valid_backup_id "$b" || { warn "ignoring invalid backup directory $b"; continue; } + all+=("$b") + done <<<"$listing" + [ "${#all[@]}" -gt "$keep" ] || return 0 + old=("${all[@]:0:${#all[@]}-keep}") + for b in "${old[@]}"; do + log "pruning old backup $b (KEEP_BACKUPS=$keep)" + rm -rf "${BACKUPS_DIR:?}/${b:?}" + done +} + +do_list_backups() { + [ -d "$BACKUPS_DIR" ] || { log "no backups yet"; return 0; } + local b any=0 listing + listing="$(find "$BACKUPS_DIR" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; \ + | LC_ALL=C sort)" || die "could not enumerate backups" + printf '%-28s %-22s %-12s %s\n' ID CREATED OVA_VERSION SIZE + while IFS= read -r b; do + [ -n "$b" ] || continue + valid_backup_id "$b" || { warn "ignoring invalid backup directory $b"; continue; } + any=1 + printf '%-28s %-22s %-12s %s\n' \ + "$b" \ + "$(jq -r '.created // "?"' "$BACKUPS_DIR/$b/meta.json" 2>/dev/null || echo '?')" \ + "$(jq -r '.ova_version // "?"' "$BACKUPS_DIR/$b/meta.json" 2>/dev/null || echo '?')" \ + "$(du -sh "$BACKUPS_DIR/$b" | awk '{print $1}')" + done <<<"$listing" + [ "$any" = 1 ] || log "no backups yet" +} + +backup_format() { + local dir="$1" + jq -r '.format_version // 1' "$dir/meta.json" 2>/dev/null \ + || die "backup metadata is not valid JSON" +} + +verify_backup() { + local id="$1" dir format want have + dir="$BACKUPS_DIR/$id" + if [ -z "$id" ] || ! valid_backup_id "$id"; then + die "invalid backup id: $id" + fi + [ -d "$dir" ] || die "no such backup: $id" + for f in volumes.tar.zst env docker-compose.yml image-digests meta.json; do + [ -f "$dir/$f" ] || die "backup $id is incomplete: missing $f" + done + + format="$(backup_format "$dir")" + [[ "$format" =~ ^[123]$ ]] || die "backup $id has an unsupported format" + want="$(jq -r '.volumes_sha256 // ""' "$dir/meta.json")" + have="$(sha256sum "$dir/volumes.tar.zst" | awk '{print $1}')" + [ -n "$want" ] || die "backup $id has no volume checksum; refusing to restore" + [ "$want" = "$have" ] \ + || die "backup $id is corrupt: volumes.tar.zst checksum mismatch" + + if [ "$format" -ge 2 ]; then + local archive key + for archive in stack-structure.tar.zst ova-structure.tar.zst; do + [ -f "$dir/$archive" ] || die "backup $id is incomplete: missing $archive" + case "$archive" in + stack-structure.tar.zst) key=stack_sha256 ;; + ova-structure.tar.zst) key=ova_sha256 ;; + esac + want="$(jq -r --arg k "$key" '.[$k] // ""' "$dir/meta.json")" + have="$(sha256sum "$dir/$archive" | awk '{print $1}')" + [ -n "$want" ] || die "backup $id has no checksum for $archive" + [ "$want" = "$have" ] \ + || die "backup $id is corrupt: $archive checksum mismatch" + done + fi + + if [ "$format" -ge 3 ]; then + local init_service_sha + init_service_sha="$(jq -r '.init_service_sha256 // ""' "$dir/meta.json")" + if [ -n "$init_service_sha" ]; then + [ -f "$dir/defguard-init.service" ] \ + || die "backup $id is incomplete: missing defguard-init.service" + have="$(sha256sum "$dir/defguard-init.service" | awk '{print $1}')" + [ "$init_service_sha" = "$have" ] \ + || die "backup $id is corrupt: defguard-init.service checksum mismatch" + fi + fi +} + +install_init_service() { + local src="$1" + mkdir -p "$(dirname "$INIT_SERVICE_FILE")" + if [ "$INIT_SERVICE_FILE" = "/etc/systemd/system/defguard-init.service" ]; then + atomic_install "$src" "$INIT_SERVICE_FILE" 644 root root \ + || die "could not install $INIT_SERVICE_FILE" + else + atomic_install "$src" "$INIT_SERVICE_FILE" 644 \ + || die "could not install $INIT_SERVICE_FILE" + fi +} + +reload_init_service() { + [ "$INIT_SERVICE_FILE" = "/etc/systemd/system/defguard-init.service" ] || return 0 + command -v systemctl >/dev/null 2>&1 || die "systemctl is required to reload defguard-init.service" + systemctl daemon-reload || die "could not reload systemd after changing defguard-init.service" +} + +restore_image_digests() { + local file="$1" rows row repository tag digest image_id source + [ -f "$file" ] || { + warn "backup is missing image-digests; image identity cannot be verified" + return 0 + } + + rows="$(jq -c 'if type == "array" then .[] else . end' "$file" 2>/dev/null)" \ + || die "backup image-digests is not valid JSON" + while IFS= read -r row; do + [ -n "$row" ] || continue + repository="$(jq -r '.Repository // empty' <<<"$row")" + tag="$(jq -r '.Tag // empty' <<<"$row")" + digest="$(jq -r '.Digest // empty' <<<"$row")" + image_id="$(jq -r '.ID // empty' <<<"$row")" + [ -n "$repository" ] && [ "$repository" != "" ] || continue + [ -n "$tag" ] && [ "$tag" != "" ] || continue + + if [[ "$digest" =~ ^sha256:[0-9a-fA-F]{64}$ ]]; then + source="$repository@$digest" + if ! docker image inspect "$source" >/dev/null 2>&1; then + docker pull "$source" >/dev/null \ + || die "could not retrieve image $source required by the backup" + fi + elif [[ "$image_id" =~ ^sha256:[0-9a-fA-F]{64}$ ]]; then + source="$image_id" + docker image inspect "$source" >/dev/null 2>&1 \ + || die "image $source required by the backup is not available locally" + else + die "backup has no restorable image digest for $repository:$tag" + fi + docker image tag "$source" "$repository:$tag" >/dev/null \ + || die "could not restore image tag $repository:$tag from the backup" + done <<<"$rows" +} + +restore_backup() { + local id="$1" dir="$BACKUPS_DIR/$1" format + need_zstd + verify_backup "$id" + format="$(backup_format "$dir")" + + log "restoring backup $id" + STACK_QUIESCED=1 + compose stop || die "could not stop the Compose stack for rollback" + assert_stack_stopped + compose down --remove-orphans \ + || die "could not remove the current Compose stack for rollback" + + if [ "$format" -ge 2 ]; then + [ "$STACK_DIR" != "/" ] && [ "$OVA_DIR" != "/" ] \ + || die "refusing to restore into /" + + # Structural archives are complete snapshots of the migration-owned + # directories. Removing current non-volume contents also removes files + # introduced by a failed migration that did not exist in the backup. + find "$STACK_DIR" -mindepth 1 -maxdepth 1 ! -name .volumes \ + -exec rm -rf -- {} + \ + || die "could not clear the current stack structure for rollback" + rm -rf "$STACK_DIR/.volumes" \ + || die "could not clear the current stack volumes for rollback" + tar --zstd -xf "$dir/stack-structure.tar.zst" -C "$STACK_DIR" \ + || die "could not restore the stack structure from backup" + tar --zstd -xf "$dir/volumes.tar.zst" -C "$STACK_DIR" \ + || die "could not restore the volumes from backup" + + find "$OVA_DIR" -mindepth 1 -maxdepth 1 \ + ! -name backups ! -name .dg-ctl.lock ! -name .dg-ctl.lock.d ! -name dg-ctl \ + ! -name install-dg-ctl.sh -exec rm -rf -- {} + \ + || die "could not clear the current OVA structure for rollback" + tar --zstd -xf "$dir/ova-structure.tar.zst" -C "$OVA_DIR" \ + || die "could not restore the OVA structure from backup" + [ ! -f "$STATE_FILE" ] || chmod 600 "$STATE_FILE" \ + || die "could not secure restored state.json" + else + warn "backup $id uses the legacy format; structural migration files and state cannot be restored" + rm -rf "$STACK_DIR/.volumes" \ + || die "could not clear the current stack volumes for rollback" + tar --zstd -xf "$dir/volumes.tar.zst" -C "$STACK_DIR" \ + || die "could not restore the volumes from backup" + atomic_install "$dir/env" "$ENV_FILE" 600 \ + || die "could not restore the stack environment from backup" + atomic_install "$dir/docker-compose.yml" "$COMPOSE_FILE" 640 \ + || die "could not restore the Compose file from backup" + if [ -f "$dir/applied-profiles" ]; then + mkdir -p "$INIT_DIR" + atomic_install "$dir/applied-profiles" "$APPLIED_PROFILES_FILE" 640 \ + || die "could not restore applied profiles from backup" + fi + fi + + if [ "$format" -ge 3 ]; then + local init_service_sha + init_service_sha="$(jq -r '.init_service_sha256 // ""' "$dir/meta.json")" + if [ -n "$init_service_sha" ]; then + install_init_service "$dir/defguard-init.service" + reload_init_service + fi + fi + + # A structural rollback can switch between the new .yml layout and the old + # .yaml/.standalone layout. Re-select the restored Compose file before start. + detect_stack_layout + + restore_image_digests "$dir/image-digests" + if [ "$RESTORE_START_STACK" = 1 ]; then + compose up -d --remove-orphans \ + || die "could not start the restored Compose stack" + STACK_QUIESCED=0 + else + log "backup restored; leaving the stack stopped because it was stopped before the failed operation" + fi +} + +do_rollback() { + local id="" skip_tests=0 + while [ $# -gt 0 ]; do + case "$1" in + --skip-tests) skip_tests=1; shift ;; + -*) die "unknown option for rollback: $1" ;; + *) [ -z "$id" ] || die "rollback accepts exactly one backup id"; id="$1"; shift ;; + esac + done + [ -n "$id" ] || die "usage: dg-ctl rollback [--skip-tests] (see: dg-ctl list-backups)" + + restore_backup "$id" + if [ "$skip_tests" = 1 ]; then + log "rollback to $id complete; run '$0 test' when the stack has settled" + elif do_test; then + log "rollback to $id complete and healthy" + else + die "rollback to $id restored the stack but health checks failed; inspect with: journalctl -t defguard-core" + fi +} + +tcp_open() { + local host="$1" port="$2" + timeout 3 bash -c "exec 3<>/dev/tcp/$host/$port" 2>/dev/null +} + +check() { + local name="$1" + shift + if "$@"; then + echo " PASS $name" + return 0 + fi + echo " FAIL $name" + return 1 +} + +retry_until() { + [[ "$HEALTH_TIMEOUT" =~ ^[0-9]+$ ]] || return 1 + local deadline=$(( $(date +%s) + HEALTH_TIMEOUT )) + while :; do + "$@" && return 0 + [ "$(date +%s)" -ge "$deadline" ] && return 1 + sleep 3 + done +} + +service_running() { + local svc="$1" state + state="$(compose ps --format json "$svc" 2>/dev/null \ + | jq -rs 'flatten | .[0].State // "missing"' 2>/dev/null || echo missing)" + [ "$state" = "running" ] +} + +restart_count() { + local svc="$1" cid + cid="$(compose ps -q "$svc" 2>/dev/null | head -n1)" + [ -n "$cid" ] || return 1 + local count + count="$(docker inspect -f '{{.RestartCount}}' "$cid" 2>/dev/null)" || return 1 + [[ "$count" =~ ^[0-9]+$ ]] || return 1 + printf '%s\n' "$count" +} + +core_health() { curl -fsS http://localhost:8000/api/v1/health >/dev/null 2>&1; } + +db_ready() { + local user + user="$(env_value POSTGRES_USER)" + compose exec -T db pg_isready -U "${user:-defguard}" >/dev/null 2>&1 +} + +# 50051 is only published in segmented deployments. +edge_grpc_published() { compose port edge 50051 >/dev/null 2>&1; } + +no_fatal_logs() { + local tag="$1" since="$2" logs + command -v journalctl >/dev/null 2>&1 || return 0 + logs="$(journalctl -t "$tag" --since "$since" --no-pager 2>/dev/null)" || return 1 + ! grep -Eq 'FATAL|panicked at|thread .* panicked' <<<"$logs" +} + +# do_test [since] - since is a journalctl timestamp bounding the log scan. +do_test() { + local since="${1:--5 min}" + local -a profiles=() services=() + local profile_text service_text line + + profile_text="$(resolve_applied_profiles)" || { + log "health checks could not resolve applied profiles" + return 1 + } + service_text="$(compose config --services)" || { + log "health checks could not inspect Compose services" + return 1 + } + while IFS= read -r line; do + [ -n "$line" ] && profiles+=("$line") + done <<<"$profile_text" + while IFS= read -r line; do + [ -n "$line" ] && services+=("$line") + done <<<"$service_text" + [ "${#profiles[@]}" -gt 0 ] || { log "health checks resolved no profiles"; return 1; } + [ "${#services[@]}" -gt 0 ] || { log "health checks resolved no Compose services"; return 1; } + [[ "$HEALTH_SETTLE" =~ ^[0-9]+$ ]] || { log "HEALTH_SETTLE must be a non-negative integer"; return 1; } + + log "health checks (profiles: ${profiles[*]})" + local failed=0 svc + + local -A before=() + for svc in "${services[@]}"; do + check "$svc is running" retry_until service_running "$svc" || failed=1 + if ! before[$svc]="$(restart_count "$svc")"; then + echo " FAIL $svc restart count is readable" + failed=1 + fi + done + + if has_profile core "${profiles[@]}"; then + check "core /api/v1/health responds" retry_until core_health || failed=1 + check "db accepts connections" retry_until db_ready || failed=1 + fi + if has_profile edge "${profiles[@]}"; then + check "edge :8080 accepts connections" retry_until tcp_open localhost 8080 || failed=1 + check "edge :443 accepts connections" retry_until tcp_open localhost 443 || failed=1 + if edge_grpc_published; then + check "edge :50051 accepts connections" retry_until tcp_open localhost 50051 || failed=1 + fi + fi + if has_profile gateway "${profiles[@]}"; then + check "gateway health port :55003 accepts connections" retry_until tcp_open localhost 55003 || failed=1 + fi + + # Catch crash loops: a container can report running and die seconds later. + log "watching ${HEALTH_SETTLE}s for restarts" + sleep "$HEALTH_SETTLE" + for svc in "${services[@]}"; do + local now + if now="$(restart_count "$svc")"; then + check "$svc did not restart" [ "$now" = "${before[$svc]}" ] || failed=1 + else + echo " FAIL $svc restart count is readable" + failed=1 + fi + done + + for svc in "${services[@]}"; do + check "$svc logged no fatal errors" no_fatal_logs "defguard-$svc" "$since" || failed=1 + done + + [ "$failed" = 0 ] || return 1 + log "all health checks passed" +} + +set_env_tag() { + local file="$1" key="$2" value="$3" tmp found=0 line + valid_tag "$value" || die "invalid Docker image tag for $key: $value" + tmp="$(mktemp "${file}.tmp.XXXXXX")" + while IFS= read -r line || [ -n "$line" ]; do + if [[ "$line" == "$key="* ]]; then + printf '%s=%s\n' "$key" "$value" >>"$tmp" + found=1 + else + printf '%s\n' "$line" >>"$tmp" + fi + done <"$file" + if [ "$found" = 0 ]; then + printf '%s=%s\n' "$key" "$value" >>"$tmp" + fi + chmod 600 "$tmp" + mv -f "$tmp" "$file" +} + +set_env_value() { + local file="$1" key="$2" value="$3" tmp found=0 line + tmp="$(mktemp "${file}.tmp.XXXXXX")" + while IFS= read -r line || [ -n "$line" ]; do + if [[ "$line" == "$key="* ]]; then + printf '%s=%s\n' "$key" "$value" >>"$tmp" + found=1 + else + printf '%s\n' "$line" >>"$tmp" + fi + done <"$file" + if [ "$found" = 0 ]; then + printf '%s=%s\n' "$key" "$value" >>"$tmp" + fi + chmod 600 "$tmp" + mv -f "$tmp" "$file" +} + +do_upgrade() { + local ref="" core_tag="" proxy_tag="" gateway_tag="" + local assume_yes=0 skip_backup=0 skip_tests=0 + + while [ $# -gt 0 ]; do + case "$1" in + --ref) ref="${2:?--ref needs a value}"; shift 2 ;; + --core-tag) core_tag="${2:?--core-tag needs a value}"; shift 2 ;; + --proxy-tag) proxy_tag="${2:?--proxy-tag needs a value}"; shift 2 ;; + --gateway-tag) gateway_tag="${2:?--gateway-tag needs a value}"; shift 2 ;; + -y|--yes) assume_yes=1; shift ;; + --no-backup) skip_backup=1; shift ;; + --skip-tests) skip_tests=1; shift ;; + *) die "unknown option for upgrade: $1" ;; + esac + done + + [ "$skip_backup" = 1 ] || need_zstd + + local manifest ova_version resolved_ref from_version + manifest="$(read_manifest)" + validate_manifest "$manifest" + ova_version="$(jq -r '.ova_version' <<<"$manifest")" + : "${ref:=$(jq -r '.template_ref // "main"' <<<"$manifest")}" + : "${core_tag:=$(jq -r '.core_tag // empty' <<<"$manifest")}" + : "${proxy_tag:=$(jq -r '.proxy_tag // empty' <<<"$manifest")}" + : "${gateway_tag:=$(jq -r '.gateway_tag // empty' <<<"$manifest")}" + valid_ref "$ref" || die "invalid template ref: $ref" + valid_tag "$core_tag" || die "invalid Docker image tag for DEFGUARD_CORE_TAG: $core_tag" + valid_tag "$proxy_tag" || die "invalid Docker image tag for DEFGUARD_PROXY_TAG: $proxy_tag" + valid_tag "$gateway_tag" || die "invalid Docker image tag for DEFGUARD_GATEWAY_TAG: $gateway_tag" + + if [ -f "$STATE_FILE" ]; then + jq -e 'type == "object" and (.ova_version? | type == "string")' "$STATE_FILE" \ + >/dev/null || die "$STATE_FILE is corrupt; refusing to guess the installed version" + fi + from_version="$(state_value ova_version)" + if [[ "$from_version" != "unknown" && "$from_version" != *-unknown* ]]; then + valid_version "$from_version" \ + || die "installed OVA version is invalid: $from_version" + if version_le "$ova_version" "$from_version" && [ "$ova_version" != "$from_version" ]; then + die "target OVA version $ova_version is older than installed version $from_version" + fi + fi + + resolved_ref="$(resolve_template_ref "$ref")" + + local -a profiles=() + local profile_text profile_line + profile_text="$(resolve_applied_profiles)" || die "could not resolve applied profiles" + while IFS= read -r profile_line; do + [ -n "$profile_line" ] && profiles+=("$profile_line") + done <<<"$profile_text" + [ "${#profiles[@]}" -gt 0 ] || die "no applied profiles were resolved" + + local was_legacy="$LEGACY_LAYOUT" deployment_mode="$DEPLOYMENT_MODE" layout="current" + [ "$was_legacy" = 1 ] && layout="legacy -> current (migration)" + if [ "$deployment_mode" = auto ]; then + if has_profile core "${profiles[@]}" \ + && has_profile edge "${profiles[@]}" \ + && has_profile gateway "${profiles[@]}"; then + deployment_mode=full + else + deployment_mode=segmented + fi + fi + if [ "$was_legacy" = 1 ]; then + [ -f "$INIT_SERVICE_FILE" ] \ + || die "legacy OVA is missing $INIT_SERVICE_FILE; refusing an upgrade without a rollbackable startup unit" + fi + + cat < $core_tag + proxy: $(env_value DEFGUARD_PROXY_TAG) -> $proxy_tag + gateway: $(env_value DEFGUARD_GATEWAY_TAG) -> $gateway_tag + version: $(state_value ova_version) -> $ova_version +EOF + if [ "$assume_yes" != 1 ]; then + read -r -p "[dg-ctl] proceed? [y/N] " answer + case "$answer" in y|Y|yes|YES) ;; *) die "aborted" ;; esac + fi + + WORK_DIR="$(mktemp -d)" + local work="$WORK_DIR" + local new_init="$work/init" + local staged_stack="$work/staged-stack" + local new_env="$work/.env" + local staged_compose="$staged_stack/docker-compose.yml" + local new_service="$work/defguard-init.service" + mkdir -p "$new_init" + printf '%s\n' "$deployment_mode" > "$new_init/.deployment-mode" + + log "fetching compose template and generator at $resolved_ref" + local f + for f in docker-compose.template.yaml lib.sh generate-compose.sh; do + fetch_ova_file "$resolved_ref" "$f" "$new_init/$f" \ + || die "could not fetch ova/files/$f at ref $resolved_ref" + done + for f in lib.sh generate-compose.sh; do + bash -n "$new_init/$f" || die "fetched ova/files/$f does not parse" + done + if [ "$was_legacy" = 1 ]; then + fetch_ova_file "$resolved_ref" generate-env.sh "$new_init/generate-env.sh" \ + || die "could not fetch ova/files/generate-env.sh at ref $resolved_ref" + fetch_ova_file "$resolved_ref" defguard-init.service "$new_service" \ + || die "could not fetch ova/files/defguard-init.service at ref $resolved_ref" + bash -n "$new_init/generate-env.sh" \ + || die "fetched ova/files/generate-env.sh does not parse" + grep -Fq '/opt/stacks/defguard/init/generate-compose.sh' "$new_service" \ + || die "fetched defguard-init.service does not use the simplified init layout" + fi + + local -a pending_migrations + mapfile -t pending_migrations < <(resolve_pending_migrations "$from_version" "$ova_version" "$manifest") + if [ "${#pending_migrations[@]}" -gt 0 ]; then + log "fetching migrations: ${pending_migrations[*]}" + mkdir -p "$work/migrations" + local m + for m in "${pending_migrations[@]}"; do + fetch_ova_file "$resolved_ref" "migrations/$m.sh" "$work/migrations/$m.sh" \ + || die "could not fetch migration $m at ref $resolved_ref" + bash -n "$work/migrations/$m.sh" || die "migration $m does not parse; nothing was changed" + done + fi + + cp "$ENV_FILE" "$new_env" + set_env_tag "$new_env" DEFGUARD_CORE_TAG "$core_tag" + set_env_tag "$new_env" DEFGUARD_PROXY_TAG "$proxy_tag" + set_env_tag "$new_env" DEFGUARD_GATEWAY_TAG "$gateway_tag" + if [ "$was_legacy" = 1 ] && [ "$deployment_mode" = full ]; then + # The old all-in-one Compose file hard-coded these values; they were not + # present in its .env and would otherwise become empty in the new template. + set_env_value "$new_env" DEFGUARD_ADOPT_EDGE edge:50051 + set_env_value "$new_env" DEFGUARD_ADOPT_GATEWAY host.docker.internal:50066 + fi + chmod 600 "$new_env" + + # Validate both the raw template and the generated, flattened compose file + # before touching the live stack. + local -a profile_args=() + for f in "${profiles[@]}"; do profile_args+=(--profile "$f"); done + docker compose -f "$new_init/docker-compose.template.yaml" \ + --project-directory "$STACK_DIR" --env-file "$new_env" "${profile_args[@]}" \ + config -q >/dev/null 2>&1 \ + || die "fetched template at ref $resolved_ref does not parse; nothing was changed" + + mkdir -p "$staged_stack" + cp "$new_env" "$staged_stack/.env" + local -a stack_profiles=() + for f in "${profiles[@]}"; do + [ "$f" = "dockge" ] && continue + stack_profiles+=("$f") + done + printf '%s\n' "${stack_profiles[@]}" > "$staged_stack/active-profiles" + if has_profile dockge "${profiles[@]}"; then + touch "$staged_stack/enable-docker-management" + fi + DEFGUARD_DEPLOYMENT_MODE="$deployment_mode" \ + DEFGUARD_STACK_DIR="$staged_stack" DEFGUARD_INIT_DIR="$new_init" \ + bash "$new_init/generate-compose.sh" \ + || die "compose generation failed for ref $resolved_ref; nothing was changed" + [ -f "$staged_compose" ] || die "compose generator produced no docker-compose.yml" + docker compose -f "$staged_compose" --project-directory "$staged_stack" \ + --env-file "$staged_stack/.env" config -q >/dev/null 2>&1 \ + || die "generated compose at ref $resolved_ref does not parse; nothing was changed" + + local backup_id="none" + if [ "$skip_backup" = 1 ]; then + warn "--no-backup given: no rollback point will exist; the stack will still be stopped for migration safety" + quiesce_stack + else + LEAVE_STACK_STOPPED=1 + do_backup upgrade + backup_id="$BACKUP_ID_RESULT" + RECOVERY_ACTIVE=1 + RECOVERY_BACKUP_ID="$backup_id" + fi + + if [ "${#pending_migrations[@]}" -gt 0 ]; then + local m + for m in "${pending_migrations[@]}"; do + log "running migration $m" + if ! STACK_DIR="$STACK_DIR" OVA_DIR="$OVA_DIR" ENV_FILE="$ENV_FILE" \ + FROM_VERSION="$from_version" TO_VERSION="$ova_version" \ + bash "$work/migrations/$m.sh"; then + die "migration $m failed; stack left as-is. Restore data with: $0 rollback $backup_id" + fi + done + + # Migrations are allowed to add or change environment entries. Refresh the + # staged copy after they finish so the atomic install does not discard + # those changes; the target image tags remain authoritative. + log "refreshing staged environment after migrations" + cp "$ENV_FILE" "$new_env" + set_env_tag "$new_env" DEFGUARD_CORE_TAG "$core_tag" + set_env_tag "$new_env" DEFGUARD_PROXY_TAG "$proxy_tag" + set_env_tag "$new_env" DEFGUARD_GATEWAY_TAG "$gateway_tag" + if [ "$was_legacy" = 1 ] && [ "$deployment_mode" = full ]; then + set_env_value "$new_env" DEFGUARD_ADOPT_EDGE edge:50051 + set_env_value "$new_env" DEFGUARD_ADOPT_GATEWAY host.docker.internal:50066 + fi + chmod 600 "$new_env" + cp "$new_env" "$staged_stack/.env" + chmod 600 "$staged_stack/.env" + + # Migrations may alter interpolation inputs. Rebuild the staged Compose + # file after refreshing .env, restoring the profile flags consumed by the + # first generation pass. + rm -f "$staged_compose" + printf '%s\n' "${stack_profiles[@]}" > "$staged_stack/active-profiles" + if has_profile dockge "${profiles[@]}"; then + touch "$staged_stack/enable-docker-management" + fi + DEFGUARD_DEPLOYMENT_MODE="$deployment_mode" \ + DEFGUARD_STACK_DIR="$staged_stack" DEFGUARD_INIT_DIR="$new_init" \ + bash "$new_init/generate-compose.sh" \ + || die "compose regeneration after migrations failed; nothing was changed" + docker compose -f "$staged_compose" --project-directory "$staged_stack" \ + --env-file "$staged_stack/.env" config -q >/dev/null 2>&1 \ + || die "generated compose after migrations does not parse; nothing was changed" + fi + + log "installing staged environment and compose files" + atomic_install "$new_env" "$ENV_FILE" 600 + # From this point onward the live stack uses the new canonical filename. + COMPOSE_FILE="$STACK_DIR/docker-compose.yml" + LEGACY_LAYOUT=0 + LEGACY_COMPOSE_MODE="" + DEPLOYMENT_MODE="$deployment_mode" + atomic_install "$staged_compose" "$COMPOSE_FILE" 640 + mkdir -p "$INIT_DIR" + printf '%s\n' "${profiles[@]}" > "$work/applied-profiles" + atomic_install "$work/applied-profiles" "$APPLIED_PROFILES_FILE" 640 + rm -f "$STACK_DIR/active-profiles" "$STACK_DIR/enable-docker-management" + + log "pulling images" + compose pull + local started_at + started_at="$(date -u +'%Y-%m-%d %H:%M:%S')" + log "starting stack" + compose up -d --remove-orphans + + if [ "$skip_tests" = 1 ]; then + log "--skip-tests given; run '$0 test' when the stack has settled" + elif ! do_test "$started_at"; then + echo >&2 + warn "the upgraded stack did not pass health checks; automatic rollback will be attempted" + exit 1 + fi + + log "installing upgraded init assets" + local image_tags_tmp="$WORK_DIR/image-tags" + printf 'DEFGUARD_CORE_TAG=%s\nDEFGUARD_PROXY_TAG=%s\nDEFGUARD_GATEWAY_TAG=%s\n' \ + "$core_tag" "$proxy_tag" "$gateway_tag" > "$image_tags_tmp" + chmod 640 "$image_tags_tmp" + atomic_install "$new_init/docker-compose.template.yaml" "$INIT_DIR/docker-compose.template.yaml" 640 + atomic_install "$new_init/lib.sh" "$INIT_DIR/lib.sh" 640 + atomic_install "$new_init/generate-compose.sh" "$INIT_DIR/generate-compose.sh" 750 + atomic_install "$image_tags_tmp" "$INIT_DIR/.image-tags" 640 + atomic_install "$new_init/.deployment-mode" "$INIT_DIR/.deployment-mode" 640 + if [ "$was_legacy" = 1 ]; then + atomic_install "$new_init/generate-env.sh" "$INIT_DIR/generate-env.sh" 750 + install_init_service "$new_service" + reload_init_service + rm -f \ + "$STACK_DIR/docker-compose.yaml" \ + "$STACK_DIR/docker-compose.standalone.yaml" \ + "$STACK_DIR/start.sh" \ + "$STACK_DIR/generate-env.sh" \ + "$STACK_DIR/.image-tags" + fi + + local migrations_json state_tmp + migrations_json="$(printf '%s\n' "${pending_migrations[@]:-}" \ + | jq -R 'select(length > 0)' | jq -s .)" + state_tmp="$(mktemp "$WORK_DIR/state.json.XXXXXX")" + jq -n \ + --arg ova_version "$ova_version" \ + --arg ref "$resolved_ref" \ + --arg core "$core_tag" --arg proxy "$proxy_tag" --arg gateway "$gateway_tag" \ + --arg upgraded_at "$(date -u +%FT%TZ)" \ + --arg backup "$backup_id" \ + --arg cli "$DG_CTL_VERSION" \ + --argjson migrations "$migrations_json" \ + '{ova_version: $ova_version, template_ref: $ref, cli_version: $cli, + tags: {core: $core, proxy: $proxy, gateway: $gateway}, + upgraded_at: $upgraded_at, backup_id: $backup, migrations_applied: $migrations}' \ + >"$state_tmp" + atomic_install "$state_tmp" "$STATE_FILE" 600 + + RECOVERY_ACTIVE=0 + RECOVERY_BACKUP_ID="" + LEAVE_STACK_STOPPED=0 + STACK_QUIESCED=0 + + prune_backups + log "upgrade to $ova_version complete" +} + +do_version() { + echo "dg-ctl cli: $DG_CTL_VERSION" + echo "ova version: $(state_value ova_version)" + echo "template ref: $(state_value template_ref)" + echo "core tag: $(env_value DEFGUARD_CORE_TAG)" + echo "proxy tag: $(env_value DEFGUARD_PROXY_TAG)" + echo "gateway tag: $(env_value DEFGUARD_GATEWAY_TAG)" + local applied + applied="$(jq -r '.migrations_applied // [] | join(", ")' "$STATE_FILE" 2>/dev/null)" || applied="" + [ -n "$applied" ] && echo "migrations: $applied" + local manifest + if manifest="$(read_manifest 2>/dev/null)"; then + echo "available: $(jq -r '.ova_version // "unknown"' <<<"$manifest") (core $(jq -r '.core_tag' <<<"$manifest"), proxy $(jq -r '.proxy_tag' <<<"$manifest"), gateway $(jq -r '.gateway_tag' <<<"$manifest"))" + else + echo "available: could not reach $MANIFEST_URL" + fi +} + +do_self_update() { + local manifest ref resolved_ref tmp + manifest="$(read_manifest)" + validate_manifest "$manifest" + ref="$(jq -r '.template_ref // "main"' <<<"$manifest")" + resolved_ref="$(resolve_template_ref "$ref")" + tmp="$(mktemp)" + fetch_ova_file "$resolved_ref" "dg-ctl" "$tmp" \ + || { rm -f "$tmp"; die "could not fetch dg-ctl at ref $resolved_ref"; } + bash -n "$tmp" || { rm -f "$tmp"; die "fetched dg-ctl does not parse; keeping the current one"; } + atomic_install "$tmp" "$OVA_DIR/dg-ctl" 750 root root \ + || { rm -f "$tmp"; die "could not atomically install the updated CLI"; } + rm -f "$tmp" + log "CLI updated from ref $resolved_ref: $("$OVA_DIR/dg-ctl" --version 2>/dev/null || true)" +} + +usage() { + cat < [options] + + upgrade [--ref REF] [--core-tag T] [--proxy-tag T] [--gateway-tag T] + [--yes] [--no-backup] [--skip-tests] + stop safely, back up, run pending migrations, refresh + the stack from the release manifest, then health-test it + backup [--label L] cold full backup of volumes, structure, state and config + list-backups show local backups + rollback [--skip-tests] + restore a backup and bring that stack back up + test run the health checks against the running stack + version show installed and available versions + self-update replace this CLI with the published one + help this text + +environment: + DEFGUARD_OVA_MANIFEST_URL release manifest (default: $OVA_REPO@main) + DEFGUARD_OVA_SOURCE_DIR take ova/files/* from a local checkout + KEEP_BACKUPS backups to keep after an upgrade (default 3) + FETCH_CONNECT_TIMEOUT network connection timeout in seconds (default 10) + FETCH_TIMEOUT network transfer timeout in seconds (default 60) +EOF +} + +main() { + local cmd="${1:-help}" + [ $# -gt 0 ] && shift || true + case "$cmd" in + upgrade) need_root "$cmd"; need_stack; validate_safe_paths; acquire_lock; do_upgrade "$@" ;; + backup) + need_root "$cmd"; need_stack; validate_safe_paths; acquire_lock + local label="" + [ "${1:-}" = "--label" ] && { label="${2:?--label needs a value}"; shift 2; } + [ "$#" -eq 0 ] || die "unknown option for backup: $1" + mkdir -p "$BACKUPS_DIR" + do_backup "$label" + ;; + list-backups) do_list_backups ;; + rollback) need_root "$cmd"; need_stack; validate_safe_paths; acquire_lock; do_rollback "$@" ;; + test) need_stack; do_test "$@" ;; + version|--version|-v) do_version ;; + self-update) need_root "$cmd"; validate_safe_paths; acquire_lock; do_self_update ;; + help|--help|-h) usage ;; + *) usage >&2; die "unknown command: $cmd" ;; + esac +} + +main "$@" diff --git a/ova/files/docker-setup.sh b/ova/files/docker-setup.sh index 40eb776..cb64c9c 100644 --- a/ova/files/docker-setup.sh +++ b/ova/files/docker-setup.sh @@ -5,7 +5,8 @@ apt-get update apt-get full-upgrade -y # open-vm-tools: graceful shutdown, guest IP reporting, and time sync under VMware. # jq: strips profiles from generate-compose.sh's flattened docker-compose.yml -apt-get install -y ca-certificates curl open-vm-tools jq +# zstd: compresses stack backups taken by /opt/defguard/dg-ctl +apt-get install -y ca-certificates curl open-vm-tools jq zstd install -m 0755 -d /etc/apt/keyrings curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc chmod a+r /etc/apt/keyrings/docker.asc diff --git a/ova/files/generate-compose.sh b/ova/files/generate-compose.sh index 189f8da..741d64d 100755 --- a/ova/files/generate-compose.sh +++ b/ova/files/generate-compose.sh @@ -11,6 +11,7 @@ TEMPLATE="$INIT_DIR/docker-compose.template.yaml" COMPOSE_FILE="$STACK_DIR/docker-compose.yml" PROFILES_FILE="$STACK_DIR/active-profiles" ENABLE_DOCKER_MGMT_FILE="$STACK_DIR/enable-docker-management" +DEPLOYMENT_MODE_FILE="$INIT_DIR/.deployment-mode" if [ -f "$COMPOSE_FILE" ]; then echo "DefGuard: docker-compose.yml already exists, skipping generation." @@ -22,10 +23,35 @@ source "$INIT_DIR/lib.sh" mapfile -t _profiles < <(resolve_profiles "$STACK_DIR") -FULL_STACK=false -if is_full_stack "${_profiles[@]}"; then - FULL_STACK=true +PERSISTED_MODE="$(deployment_mode "$INIT_DIR")" \ + || { echo "DefGuard: invalid deployment mode in $DEPLOYMENT_MODE_FILE" >&2; exit 1; } +if [ -n "${DEFGUARD_DEPLOYMENT_MODE:-}" ]; then + MODE="$DEFGUARD_DEPLOYMENT_MODE" +elif [ "$PERSISTED_MODE" = segmented ]; then + # A legacy standalone deployment may legitimately select all three + # profiles; retain its segmented network semantics across regeneration. + MODE=segmented +elif [ -f "$PROFILES_FILE" ]; then + # An explicit mode-selection file is a fresh operator/cloud-init input and + # takes precedence over a marker left by an earlier generated file. + MODE=auto +else + MODE="$PERSISTED_MODE" fi +case "$MODE" in + auto|full|segmented) ;; + *) echo "DefGuard: invalid deployment mode '$MODE'" >&2; exit 1 ;; +esac +FULL_STACK=false +case "$MODE" in + full) FULL_STACK=true ;; + segmented) ;; + auto) + if is_full_stack "${_profiles[@]}"; then + FULL_STACK=true + fi + ;; +esac _profile_args=() for p in "${_profiles[@]}"; do _profile_args+=(--profile "$p"); done @@ -36,14 +62,17 @@ for p in "${_profiles[@]}"; do _profile_args+=(--profile "$p"); done # `docker compose up -d` with no COMPOSE_PROFILES set. jq strips that key and # injects the conditional depends_on/port tweaks; the result is then piped # back through `docker compose config` (stdin) to render real YAML rather -# than leaving the file as JSON-flavored-as-YAML. +# than leaving the file as JSON-flavored-as-YAML. Keep paths unexpanded because +# upgrades generate this file in a temporary staging directory before copying +# it into the permanent stack directory; running from STACK_DIR lets Compose +# still validate the relative .env file. TMP_COMPOSE_FILE="$COMPOSE_FILE.tmp" -trap 'rm -f "$TMP_COMPOSE_FILE"' EXIT +trap 'rm -f "$TMP_COMPOSE_FILE" "$DEPLOYMENT_MODE_FILE.tmp"' EXIT { echo "# Generated by defguard OVA first-boot init on $(date -u +%FT%TZ)." echo "# Selected profiles: ${_profiles[*]}" - docker compose -f "$TEMPLATE" -p defguard --project-directory "$STACK_DIR" --env-file "$STACK_DIR/.env" "${_profile_args[@]}" \ - config --format json \ + (cd "$STACK_DIR" && docker compose -f "$TEMPLATE" -p defguard --project-directory "$STACK_DIR" --env-file "$STACK_DIR/.env" "${_profile_args[@]}" \ + config --no-path-resolution --format json) \ | jq --argjson full "$FULL_STACK" ' del(.services[].profiles) | del(.name) | if $full then @@ -52,16 +81,28 @@ trap 'rm -f "$TMP_COMPOSE_FILE"' EXIT (.services.edge // empty) |= (.ports += ["50051:50051"]) end ' \ - | docker compose -f - -p defguard --project-directory "$STACK_DIR" config + | (cd "$STACK_DIR" && docker compose -f - -p defguard --project-directory "$STACK_DIR" config --no-path-resolution) } > "$TMP_COMPOSE_FILE" -sed -i \ +sed -i.bak \ -e "s/__DEFGUARD_CORE_TAG__/\${DEFGUARD_CORE_TAG}/" \ -e "s/__DEFGUARD_PROXY_TAG__/\${DEFGUARD_PROXY_TAG}/" \ -e "s/__DEFGUARD_GATEWAY_TAG__/\${DEFGUARD_GATEWAY_TAG}/" \ - "$TMP_COMPOSE_FILE" + "$TMP_COMPOSE_FILE" && rm -f "$TMP_COMPOSE_FILE.bak" mv "$TMP_COMPOSE_FILE" "$COMPOSE_FILE" +if [ "$FULL_STACK" = true ]; then + printf '%s\n' full > "$DEPLOYMENT_MODE_FILE.tmp" +else + printf '%s\n' segmented > "$DEPLOYMENT_MODE_FILE.tmp" +fi +mv "$DEPLOYMENT_MODE_FILE.tmp" "$DEPLOYMENT_MODE_FILE" + +# The generated file carries no profiles: keys, so record the selection for +# later regeneration (see /opt/defguard/dg-ctl upgrade). +mkdir -p "$INIT_DIR" +printf '%s\n' "${_profiles[@]}" > "$INIT_DIR/.applied-profiles" + rm -f "$PROFILES_FILE" "$ENABLE_DOCKER_MGMT_FILE" echo "DefGuard: generated $COMPOSE_FILE for profiles: ${_profiles[*]}" diff --git a/ova/files/generate-env.sh b/ova/files/generate-env.sh index 581fe30..dfeb97a 100644 --- a/ova/files/generate-env.sh +++ b/ova/files/generate-env.sh @@ -6,6 +6,8 @@ set -euo pipefail STACK_DIR="${DEFGUARD_STACK_DIR:-/opt/stacks/defguard}" INIT_DIR="${DEFGUARD_INIT_DIR:-$STACK_DIR/init}" ENV_FILE="$STACK_DIR/.env" +PROFILES_FILE="$STACK_DIR/active-profiles" +DEPLOYMENT_MODE_FILE="$INIT_DIR/.deployment-mode" if [ -f "$ENV_FILE" ]; then echo "DefGuard: .env already exists, skipping generation." @@ -31,9 +33,24 @@ fi # co-located on this host; segmented deployments must fill these in manually # since edge/gateway live on other VMs entirely. mapfile -t _profiles < <(resolve_profiles "$STACK_DIR") +PERSISTED_MODE="$(deployment_mode "$INIT_DIR")" \ + || { echo "DefGuard: invalid deployment mode in $DEPLOYMENT_MODE_FILE" >&2; exit 1; } +if [ -n "${DEFGUARD_DEPLOYMENT_MODE:-}" ]; then + MODE="$DEFGUARD_DEPLOYMENT_MODE" +elif [ "$PERSISTED_MODE" = segmented ]; then + MODE=segmented +elif [ -f "$PROFILES_FILE" ]; then + MODE=auto +else + MODE="$PERSISTED_MODE" +fi +case "$MODE" in + auto|full|segmented) ;; + *) echo "DefGuard: invalid deployment mode '$MODE'" >&2; exit 1 ;; +esac ADOPT_EDGE="" ADOPT_GATEWAY="" -if is_full_stack "${_profiles[@]}"; then +if [ "$MODE" = full ] || { [ "$MODE" = auto ] && is_full_stack "${_profiles[@]}"; }; then ADOPT_EDGE="edge:50051" ADOPT_GATEWAY="host.docker.internal:50066" fi diff --git a/ova/files/install-dg-ctl.sh b/ova/files/install-dg-ctl.sh new file mode 100644 index 0000000..0f45e34 --- /dev/null +++ b/ova/files/install-dg-ctl.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# Installs the defguard OVA maintenance CLI onto an already-deployed OVA VM, +# which shipped without it. Idempotent - re-running just refreshes the CLI. +# +# curl -fsSL https://raw.githubusercontent.com/DefGuard/deployment/main/ova/files/install-dg-ctl.sh | sudo bash +set -euo pipefail + +STACK_DIR="${DEFGUARD_STACK_DIR:-/opt/stacks/defguard}" +OVA_DIR="${DEFGUARD_OVA_DIR:-/opt/defguard}" +OVA_REPO="${DEFGUARD_OVA_REPO:-DefGuard/deployment}" +MANIFEST_URL="${DEFGUARD_OVA_MANIFEST_URL:-https://raw.githubusercontent.com/$OVA_REPO/main/ova/manifest.json}" +SOURCE_DIR="${DEFGUARD_OVA_SOURCE_DIR:-}" + +log() { echo "[dg-ctl-install] $*"; } +die() { echo "[dg-ctl-install] ERROR: $*" >&2; exit 1; } + +[ "$(id -u)" = "0" ] || die "run as root (pipe into 'sudo bash')" +if [ ! -f "$STACK_DIR/docker-compose.yml" ] \ + && [ ! -f "$STACK_DIR/docker-compose.yaml" ]; then + die "$STACK_DIR/docker-compose.yml or $STACK_DIR/docker-compose.yaml not found; this does not look like a defguard OVA host" +fi + +# jq and zstd are baked into OVA 2.1 images but not into earlier ones. +missing=() +command -v jq >/dev/null 2>&1 || missing+=(jq) +command -v zstd >/dev/null 2>&1 || missing+=(zstd) +command -v curl >/dev/null 2>&1 || missing+=(curl) +if [ "${#missing[@]}" -gt 0 ]; then + log "installing missing dependencies: ${missing[*]}" + apt-get update -qq + apt-get install -y --no-install-recommends "${missing[@]}" +fi + +fetch() { + local src="$1" dest="$2" + case "$src" in + file://*) cp "${src#file://}" "$dest" ;; + /*) cp "$src" "$dest" ;; + *) curl -fsSL --retry 3 -o "$dest" "$src" ;; + esac +} + +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +fetch "$MANIFEST_URL" "$work/manifest.json" || die "could not fetch manifest from $MANIFEST_URL" +REF="$(jq -r '.template_ref // "main"' "$work/manifest.json")" + +if [ -n "$SOURCE_DIR" ]; then + cp "$SOURCE_DIR/ova/files/dg-ctl" "$work/dg-ctl" +else + fetch "https://raw.githubusercontent.com/$OVA_REPO/$REF/ova/files/dg-ctl" "$work/dg-ctl" \ + || die "could not fetch dg-ctl at ref $REF" +fi +bash -n "$work/dg-ctl" || die "downloaded dg-ctl does not parse; aborting" + +mkdir -p "$OVA_DIR/backups" +install -m 750 -o root -g root "$work/dg-ctl" "$OVA_DIR/dg-ctl" +ln -sf "$OVA_DIR/dg-ctl" /usr/local/bin/dg-ctl + +# Seed state from what is actually deployed. ova_version is unknown for images +# built before the manifest existed; the tags in .env are the truth. +if [ ! -f "$OVA_DIR/state.json" ]; then + env_value() { sed -n "s/^$1=//p" "$STACK_DIR/.env" | tail -n1; } + jq -n \ + --arg core "$(env_value DEFGUARD_CORE_TAG)" \ + --arg proxy "$(env_value DEFGUARD_PROXY_TAG)" \ + --arg gateway "$(env_value DEFGUARD_GATEWAY_TAG)" \ + '{ova_version: "2.0-unknown", template_ref: "unknown", + tags: {core: $core, proxy: $proxy, gateway: $gateway}}' > "$OVA_DIR/state.json" + chmod 600 "$OVA_DIR/state.json" +fi + +log "installed $OVA_DIR/dg-ctl (ref $REF)" +log "next: sudo $OVA_DIR/dg-ctl upgrade" diff --git a/ova/files/lib.sh b/ova/files/lib.sh index fc0c339..088d04f 100644 --- a/ova/files/lib.sh +++ b/ova/files/lib.sh @@ -44,3 +44,17 @@ is_full_stack() { done [ "$has_core" = 1 ] && [ "$has_edge" = 1 ] && [ "$has_gateway" = 1 ] } + +# Reads the persisted deployment mode used by generate-compose.sh. Older +# layouts have no marker, so callers fall back to profile-based inference. +deployment_mode() { + local init_dir="$1" mode="" + if [ -f "$init_dir/.deployment-mode" ]; then + mode="$(tr -d '[:space:]' < "$init_dir/.deployment-mode")" + fi + case "$mode" in + "") echo auto ;; + full|segmented) echo "$mode" ;; + *) return 1 ;; + esac +} diff --git a/ova/files/migrations/README.md b/ova/files/migrations/README.md new file mode 100644 index 0000000..c57a5d2 --- /dev/null +++ b/ova/files/migrations/README.md @@ -0,0 +1,35 @@ +# OVA structural migrations + +`dg-ctl upgrade` runs these when it needs to change the layout of +`/opt/stacks/defguard` or `/opt/defguard` in a way the compose template +regeneration alone can't express (moving/renaming a volume directory, seeding +a new required `.env` var, dropping a new file under `init/`, etc). The +upgrader also recognizes the pre-simplification OVA layout from commit +`8192f5a` and migrates its static Compose files and startup unit first. + +## Adding one + +1. Add a script here named after the OVA version that introduces the change, + e.g. `2.2.0.sh`. +2. List that version in `migrations` in `ova/manifest.json`, in ascending + order. +3. The script receives, as environment variables: + - `STACK_DIR` - the stack directory (`/opt/stacks/defguard`) + - `OVA_DIR` - the OVA CLI directory (`/opt/defguard`) + - `ENV_FILE` - `$STACK_DIR/.env` + - `FROM_VERSION` / `TO_VERSION` - the OVA versions being upgraded between +4. Exit non-zero to abort the upgrade. The stack is stopped before migrations + run. The pre-upgrade backup includes the stack structure, OVA state, volumes, + and the `defguard-init.service` unit when present, so `dg-ctl rollback ` + can restore side effects from a failed migration. +5. Scripts must be idempotent: `dg-ctl upgrade` can be re-run after a partial + failure, and `resolve_pending_migrations` decides what's pending from the + *target* OVA version, not from what actually ran last time. + +`--no-backup` still stops the stack before migrations, but disables automatic +rollback. Use it only when an external backup and recovery plan already exist. + +`dg-ctl` only runs migrations with `FROM_VERSION < version <= TO_VERSION` +(string `FROM_VERSION`/`TO_VERSION` of `unknown` or `*-unknown`, from OVAs +that predate `state.json`, are treated as version `0.0.0` - every migration +runs). diff --git a/ova/manifest.json b/ova/manifest.json new file mode 100644 index 0000000..269f906 --- /dev/null +++ b/ova/manifest.json @@ -0,0 +1,8 @@ +{ + "ova_version": "2.1.0", + "core_tag": "2", + "proxy_tag": "2", + "gateway_tag": "2", + "template_ref": "main", + "migrations": [] +} diff --git a/ova/test/test-deployment-modes.sh b/ova/test/test-deployment-modes.sh index 66ba006..51204d0 100755 --- a/ova/test/test-deployment-modes.sh +++ b/ova/test/test-deployment-modes.sh @@ -125,6 +125,131 @@ verify_mode() { || { log "$mode: /opt/stacks/defguard contains '$actual', expected only docker-compose.yml, .env, .volumes, defguard-firewall.sh, init"; return 1; } reprovision_mode "$mode" "$ip" || return 1 + upgrade_mode "$mode" "$ip" || return 1 + + return 0 +} + +upgrade_mode() { + local mode="$1" ip="$2" names probe backup_id expected_mode + [ "${SKIP_UPGRADE_TEST:-0}" = "1" ] && { log "$mode: skipping upgrade test"; return 0; } + + log "$mode: exercising /opt/defguard/dg-ctl upgrade" + + local has_db=0 + case " ${EXPECT[$mode]} " in *" db "*) has_db=1 ;; esac + if [ "$has_db" = 1 ]; then + vm_ssh "$ip" "sudo docker compose -f /opt/stacks/defguard/docker-compose.yml exec -T db \ + psql -qtAX -U defguard -d defguard -c \"CREATE TABLE IF NOT EXISTS ova_upgrade_probe (v text); \ + INSERT INTO ova_upgrade_probe VALUES ('survived');\"" \ + || { log "$mode: could not seed the upgrade probe row"; return 1; } + fi + + vm_ssh "$ip" "sudo ${UPGRADE_ENV:-} /opt/defguard/dg-ctl upgrade --yes" \ + || { log "$mode: dg-ctl upgrade failed"; return 1; } + + vm_ssh "$ip" "sudo test -f /opt/defguard/state.json" \ + || { log "$mode: upgrade did not record /opt/defguard/state.json"; return 1; } + backup_id="$(vm_ssh "$ip" "sudo jq -r '.backup_id // \"none\"' /opt/defguard/state.json")" \ + || { log "$mode: could not read the upgrade backup id"; return 1; } + [ "$backup_id" != none ] \ + || { log "$mode: upgrade did not record a backup id"; return 1; } + vm_ssh "$ip" "for f in volumes.tar.zst stack-structure.tar.zst ova-structure.tar.zst env docker-compose.yml image-digests meta.json; do sudo test -f /opt/defguard/backups/$backup_id/\$f || exit 1; done" \ + || { log "$mode: upgrade backup is missing a required artifact"; return 1; } + + expected_mode=segmented + [ "$mode" = full ] && expected_mode=full + vm_ssh "$ip" "sudo grep -qx '$expected_mode' /opt/stacks/defguard/init/.deployment-mode" \ + || { log "$mode: upgrade did not preserve the deployment mode"; return 1; } + local applied_profile + if [ "$mode" = full ]; then + for applied_profile in core edge gateway; do + vm_ssh "$ip" "sudo grep -qx '$applied_profile' /opt/stacks/defguard/init/.applied-profiles" \ + || { log "$mode: upgrade did not preserve the $applied_profile profile"; return 1; } + done + else + vm_ssh "$ip" "sudo grep -qx '$mode' /opt/stacks/defguard/init/.applied-profiles" \ + || { log "$mode: upgrade did not preserve the applied profile"; return 1; } + fi + + names="$(wait_services "$ip" "${EXPECT[$mode]}")" \ + || { log "$mode: upgrade: expected services did not come back; running: $(tr '\n' ' ' <<<"$names")"; return 1; } + + local svc + for svc in ${FORBID[$mode]}; do + has_service "$names" "$svc" \ + && { log "$mode: upgrade: unexpected service '$svc' is running"; return 1; } + done + + if [ "$has_db" = 1 ]; then + probe="$(vm_ssh "$ip" "sudo docker compose -f /opt/stacks/defguard/docker-compose.yml exec -T db \ + psql -qtAX -U defguard -d defguard -c 'SELECT v FROM ova_upgrade_probe;'" | tr -d '[:space:]')" + [ "$probe" = "survived" ] \ + || { log "$mode: upgrade lost database state (probe returned '$probe')"; return 1; } + fi + + vm_ssh "$ip" "sudo /opt/defguard/dg-ctl test" \ + || { log "$mode: post-upgrade health checks failed"; return 1; } + + [ "$mode" = full ] || return 0 + legacy_upgrade_mode "$mode" "$ip" +} + +legacy_upgrade_mode() { + local mode="$1" ip="$2" output backup_id names probe applied_profile + [ "${SKIP_UPGRADE_TEST:-0}" = "1" ] && return 0 + + log "$mode: exercising legacy-layout migration through /opt/defguard/dg-ctl upgrade" + + # Turn the already-running full deployment into the pre-simplification + # layout. The containers stay up while the upgrader discovers the legacy + # files, then the upgrade itself performs the cold backup and migration. + vm_ssh "$ip" "sudo cp /opt/stacks/defguard/docker-compose.yml /opt/stacks/defguard/docker-compose.yaml && \ + sudo rm -f /opt/stacks/defguard/docker-compose.yml /opt/stacks/defguard/docker-compose.standalone.yaml \ + /opt/stacks/defguard/active-profiles /opt/stacks/defguard/enable-docker-management && \ + sudo rm -rf /opt/stacks/defguard/init && \ + sudo touch /opt/stacks/defguard/start.sh && \ + sudo sed -i 's#/opt/stacks/defguard/init/generate-compose.sh#/opt/stacks/defguard/start.sh#' \ + /etc/systemd/system/defguard-init.service" \ + || { log "$mode: could not prepare the legacy layout"; return 1; } + + output="$(vm_ssh "$ip" "sudo ${UPGRADE_ENV:-} /opt/defguard/dg-ctl upgrade --yes")" \ + || { log "$mode: legacy-layout upgrade failed"; return 1; } + [[ "$output" == *"layout: legacy -> current (migration)"* ]] \ + || { log "$mode: upgrade did not report the legacy-layout migration"; return 1; } + + vm_ssh "$ip" "sudo test -f /opt/stacks/defguard/docker-compose.yml && \ + sudo test ! -e /opt/stacks/defguard/docker-compose.yaml && \ + sudo test ! -e /opt/stacks/defguard/docker-compose.standalone.yaml && \ + sudo test ! -e /opt/stacks/defguard/start.sh && \ + sudo test -f /opt/stacks/defguard/init/generate-env.sh && \ + sudo grep -Fq '/opt/stacks/defguard/init/generate-compose.sh' \ + /etc/systemd/system/defguard-init.service" \ + || { log "$mode: legacy-layout artifacts were not migrated"; return 1; } + for applied_profile in core edge gateway; do + vm_ssh "$ip" "sudo grep -qx '$applied_profile' /opt/stacks/defguard/init/.applied-profiles" \ + || { log "$mode: legacy migration lost the $applied_profile profile"; return 1; } + done + + backup_id="$(vm_ssh "$ip" "sudo jq -r '.backup_id // \"none\"' /opt/defguard/state.json")" \ + || { log "$mode: could not read the legacy migration backup id"; return 1; } + vm_ssh "$ip" "sudo test -f /opt/defguard/backups/$backup_id/defguard-init.service" \ + || { log "$mode: legacy migration backup did not capture defguard-init.service"; return 1; } + + names="$(wait_services "$ip" "${EXPECT[$mode]}")" \ + || { log "$mode: legacy migration services did not come back; running: $(tr '\n' ' ' <<<"$names")"; return 1; } + local svc + for svc in ${FORBID[$mode]}; do + has_service "$names" "$svc" \ + && { log "$mode: legacy migration started unexpected service '$svc'"; return 1; } + done + + probe="$(vm_ssh "$ip" "sudo docker compose -f /opt/stacks/defguard/docker-compose.yml exec -T db \ + psql -qtAX -U defguard -d defguard -c 'SELECT v FROM ova_upgrade_probe;'" | tr -d '[:space:]')" + [ "$probe" = survived ] \ + || { log "$mode: legacy migration lost database state (probe returned '$probe')"; return 1; } + vm_ssh "$ip" "sudo /opt/defguard/dg-ctl test" \ + || { log "$mode: post-migration health checks failed"; return 1; } return 0 } diff --git a/ova/tests/dg-ctl.bats b/ova/tests/dg-ctl.bats new file mode 100644 index 0000000..88cbba5 --- /dev/null +++ b/ova/tests/dg-ctl.bats @@ -0,0 +1,506 @@ +#!/usr/bin/env bats +# Exercises /opt/defguard/dg-ctl against a temp stack: real `docker compose config` +# (offline), everything daemon-facing faked by tests/stub/docker-stub. + +load helpers + +CLI="$FILES_DIR/dg-ctl" + +setup() { + command -v docker >/dev/null 2>&1 || skip "docker not installed" + docker compose version >/dev/null 2>&1 || skip "docker compose v2 not available" + command -v jq >/dev/null 2>&1 || skip "jq not installed" + command -v zstd >/dev/null 2>&1 || skip "zstd not installed" + command -v sha256sum >/dev/null 2>&1 || skip "sha256sum not available" + make_stack + make_ova_home + stub_docker + # Health checks must not sit in retry loops in unit tests. + export HEALTH_TIMEOUT=1 HEALTH_SETTLE=0 +} + +teardown() { + teardown_stub_docker + teardown_ova_home + teardown_stack + teardown_migration_source +} + +@test "backup captures volumes, env, compose file and a matching checksum" { + seed_stack core + run bash "$CLI" backup --label unit + [ "$status" -eq 0 ] + id="${lines[-1]}" + dir="$OVA_HOME/backups/$id" + + for f in volumes.tar.zst stack-structure.tar.zst ova-structure.tar.zst env docker-compose.yml applied-profiles image-digests meta.json; do + [ -f "$dir/$f" ] + done + [ "$(jq -r '.volumes_sha256' "$dir/meta.json")" = "$(sha256sum "$dir/volumes.tar.zst" | awk '{print $1}')" ] + [ "$(jq -r '.format_version' "$dir/meta.json")" = "3" ] + [ "$(jq -r '.stack_sha256' "$dir/meta.json")" = "$(sha256sum "$dir/stack-structure.tar.zst" | awk '{print $1}')" ] + [ "$(jq -r '.ova_sha256' "$dir/meta.json")" = "$(sha256sum "$dir/ova-structure.tar.zst" | awk '{print $1}')" ] + [ "$(jq -r '.tags.core' "$dir/meta.json")" = "2" ] + [ "$(cat "$dir/applied-profiles")" = "core" ] + [ "$(file_mode "$dir/env")" = "600" ] +} + +@test "list-backups shows a created backup" { + seed_stack core + id="$(bash "$CLI" backup 2>/dev/null | tail -n1)" + run bash "$CLI" list-backups + [ "$status" -eq 0 ] + [[ "$output" == *"$id"* ]] +} + +@test "backups beyond KEEP_BACKUPS are pruned on upgrade" { + seed_stack core + write_manifest + for n in 1 2 3 4; do mkdir -p "$OVA_HOME/backups/2000010$n-000000Z"; done + KEEP_BACKUPS=2 run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -eq 0 ] + # 2 kept: the freshly taken one sorts last, so only it plus one placeholder. + [ "$(ls -1 "$OVA_HOME/backups" | wc -l)" -eq 2 ] +} + +@test "rollback restores volumes and refuses a corrupt archive" { + seed_stack core + id="$(bash "$CLI" backup 2>/dev/null | tail -n1)" + + echo "damaged-after-backup" > "$STACK_DIR/.volumes/db/marker" + run bash "$CLI" rollback --skip-tests "$id" + [ "$status" -eq 0 ] + [ "$(cat "$STACK_DIR/.volumes/db/marker")" = "seeded-db-file" ] + + printf 'corrupt' >> "$OVA_HOME/backups/$id/volumes.tar.zst" + echo "damaged-again" > "$STACK_DIR/.volumes/db/marker" + run bash "$CLI" rollback "$id" + [ "$status" -ne 0 ] + [[ "$output" == *"checksum mismatch"* ]] + # Refusal happens before anything is deleted. + [ "$(cat "$STACK_DIR/.volumes/db/marker")" = "damaged-again" ] +} + +@test "rollback rejects an unknown backup id" { + seed_stack core + run bash "$CLI" rollback 19700101-000000Z + [ "$status" -ne 0 ] + [[ "$output" == *"no such backup"* ]] +} + +@test "profile resolution prefers init/.applied-profiles" { + seed_stack core + printf 'core\ngateway\n' > "$INIT_DIR/.applied-profiles" + run bash "$CLI" test + [[ "$output" == *"profiles: core gateway"* ]] +} + +@test "profile resolution falls back to the generated header comment" { + seed_stack edge + rm -f "$INIT_DIR/.applied-profiles" + run bash "$CLI" test + [[ "$output" == *"profiles: edge"* ]] +} + +@test "profile resolution falls back to the service list" { + seed_stack core + rm -f "$INIT_DIR/.applied-profiles" + grep -v '^# Selected profiles:' "$STACK_DIR/docker-compose.yml" > "$STACK_DIR/stripped.yml" + mv "$STACK_DIR/stripped.yml" "$STACK_DIR/docker-compose.yml" + run bash "$CLI" test + [[ "$output" == *"profiles: core"* ]] +} + +@test "upgrade rewrites only the tag lines in .env" { + seed_stack core + write_manifest 9.9.9 9.1 9.2 9.3 + db_password_before="$(sed -n 's/^DEFGUARD_DB_PASSWORD=//p' "$STACK_DIR/.env")" + + run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -eq 0 ] + [[ "$output" == *"layout: current"* ]] + [ "$(sed -n 's/^DEFGUARD_CORE_TAG=//p' "$STACK_DIR/.env")" = "9.1" ] + [ "$(sed -n 's/^DEFGUARD_PROXY_TAG=//p' "$STACK_DIR/.env")" = "9.2" ] + [ "$(sed -n 's/^DEFGUARD_GATEWAY_TAG=//p' "$STACK_DIR/.env")" = "9.3" ] + [ "$(sed -n 's/^DEFGUARD_DB_PASSWORD=//p' "$STACK_DIR/.env")" = "$db_password_before" ] + [ "$(file_mode "$STACK_DIR/.env")" = "600" ] +} + +@test "upgrade regenerates the compose file for the same profile set" { + seed_stack core + write_manifest + run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -eq 0 ] + unset COMPOSE_PROFILES + [ "$(docker compose -f "$STACK_DIR/docker-compose.yml" config --services | sort | xargs)" = "core db" ] + [ ! -e "$STACK_DIR/active-profiles" ] + [ "$(cat "$INIT_DIR/.applied-profiles")" = "core" ] +} + +@test "upgrade keeps certificate bind paths relative to the live stack" { + seed_stack + write_manifest + run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -eq 0 ] + grep -Fq 'source: ./.volumes/certs/edge' "$STACK_DIR/docker-compose.yml" + grep -Fq 'source: ./.volumes/certs/gateway' "$STACK_DIR/docker-compose.yml" + ! grep -Fq '/staged-stack/.volumes/certs' "$STACK_DIR/docker-compose.yml" +} + +@test "upgrade migrates a pre-simplification full OVA" { + seed_legacy_stack full + write_manifest 3.0.0 3 3 3 + before_service="$(sha256sum "$DEFGUARD_INIT_SERVICE_FILE" | awk '{print $1}')" + + run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -eq 0 ] + upgrade_output="$output" + [[ "$upgrade_output" == *"layout: legacy -> current (migration)"* ]] + [ -f "$STACK_DIR/docker-compose.yml" ] + [ ! -e "$STACK_DIR/docker-compose.yaml" ] + [ ! -e "$STACK_DIR/docker-compose.standalone.yaml" ] + [ ! -e "$STACK_DIR/start.sh" ] + [ -f "$STACK_DIR/init/generate-env.sh" ] + [ "$(sed -n 's/^DEFGUARD_CORE_TAG=//p' "$STACK_DIR/init/.image-tags")" = "3" ] + [ "$(cat "$STACK_DIR/init/.deployment-mode")" = full ] + [ "$(sed -n 's/^DEFGUARD_ADOPT_EDGE=//p' "$STACK_DIR/.env")" = "edge:50051" ] + [ "$(sed -n 's/^DEFGUARD_ADOPT_GATEWAY=//p' "$STACK_DIR/.env")" = "host.docker.internal:50066" ] + [ "$(docker compose -f "$STACK_DIR/docker-compose.yml" config --services | sort | xargs)" = "core db edge gateway" ] + grep -Fq '/opt/stacks/defguard/init/generate-compose.sh' "$DEFGUARD_INIT_SERVICE_FILE" + backup_id="$(jq -r '.backup_id' "$OVA_HOME/state.json")" + [ -f "$OVA_HOME/backups/$backup_id/defguard-init.service" ] + + run bash "$CLI" rollback --skip-tests "$backup_id" + [ "$status" -eq 0 ] + [ -f "$STACK_DIR/docker-compose.yaml" ] + [ ! -e "$STACK_DIR/docker-compose.yml" ] + [ "$(sha256sum "$DEFGUARD_INIT_SERVICE_FILE" | awk '{print $1}')" = "$before_service" ] +} + +@test "upgrade preserves the legacy standalone mode when all profiles are selected" { + seed_legacy_stack segmented + write_manifest 3.0.0 3 3 3 + + run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -eq 0 ] + [ "$(cat "$STACK_DIR/init/.deployment-mode")" = segmented ] + deps="$(docker compose -f "$STACK_DIR/docker-compose.yml" config --format json \ + | jq -r 'if (.services.core.depends_on | type) == "array" then .services.core.depends_on[] else (.services.core.depends_on | keys[]) end' \ + | sort | xargs)" + [ "$deps" = db ] + docker compose -f "$STACK_DIR/docker-compose.yml" config --format json \ + | jq -e '.services.edge.ports[] | select(.published == "50051")' >/dev/null + [ -z "$(sed -n 's/^DEFGUARD_ADOPT_EDGE=//p' "$STACK_DIR/.env")" ] +} + +@test "failed legacy upgrade restores the old layout and startup unit" { + seed_legacy_stack segmented + write_manifest 3.0.0 3 3 3 + before_service="$(sha256sum "$DEFGUARD_INIT_SERVICE_FILE" | awk '{print $1}')" + + DOCKER_STUB_PULL_FAIL=1 run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -ne 0 ] + [ -f "$STACK_DIR/docker-compose.yaml" ] + [ -f "$STACK_DIR/docker-compose.standalone.yaml" ] + [ ! -e "$STACK_DIR/docker-compose.yml" ] + [ -f "$STACK_DIR/start.sh" ] + [ "$(sha256sum "$DEFGUARD_INIT_SERVICE_FILE" | awk '{print $1}')" = "$before_service" ] + [ "$(cat "$STACK_DIR/.volumes/db/marker")" = "seeded-db-file" ] +} + +@test "upgrade command line tags override the manifest" { + seed_stack core + write_manifest 9.9.9 9 9 9 + run bash "$CLI" upgrade --yes --skip-tests --core-tag 3.3.3 + [ "$status" -eq 0 ] + [ "$(sed -n 's/^DEFGUARD_CORE_TAG=//p' "$STACK_DIR/.env")" = "3.3.3" ] + [ "$(sed -n 's/^DEFGUARD_PROXY_TAG=//p' "$STACK_DIR/.env")" = "9" ] +} + +@test "upgrade records state.json" { + seed_stack core + write_manifest 2.1.7 2.1 2.1 2.1 + run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -eq 0 ] + [ "$(jq -r '.ova_version' "$OVA_HOME/state.json")" = "2.1.7" ] + [ "$(jq -r '.template_ref' "$OVA_HOME/state.json")" = "test-ref" ] + [ "$(jq -r '.tags.gateway' "$OVA_HOME/state.json")" = "2.1" ] + [ "$(jq -r '.backup_id' "$OVA_HOME/state.json")" != "none" ] +} + +@test "upgrade aborts without touching the stack when the fetched template is broken" { + seed_stack core + write_manifest + broken="$(mktemp -d)" + mkdir -p "$broken/ova/files" + cp "$FILES_DIR/lib.sh" "$FILES_DIR/generate-compose.sh" "$broken/ova/files/" + echo "this: is: not: a: compose: file" > "$broken/ova/files/docker-compose.template.yaml" + export DEFGUARD_OVA_SOURCE_DIR="$broken" + before="$(sha256sum "$STACK_DIR/docker-compose.yml" | awk '{print $1}')" + + run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -ne 0 ] + [[ "$output" == *"does not parse"* ]] + [ "$(sha256sum "$STACK_DIR/docker-compose.yml" | awk '{print $1}')" = "$before" ] + [ "$(ls -1 "$OVA_HOME/backups" | wc -l)" -eq 0 ] + rm -rf "$broken" +} + +@test "upgrade with an unreachable manifest changes nothing" { + seed_stack core + export DEFGUARD_OVA_MANIFEST_URL="file://$STACK_DIR/does-not-exist.json" + run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -ne 0 ] + [ "$(ls -1 "$OVA_HOME/backups" | wc -l)" -eq 0 ] +} + +@test "a failing health check exits non-zero and prints the rollback command" { + seed_stack gateway + jq -n '{ova_version: "2.1.0", template_ref: "old-ref"}' > "$OVA_HOME/state.json" + chmod 600 "$OVA_HOME/state.json" + write_manifest + DOCKER_STUB_STATE=exited run bash "$CLI" upgrade --yes + [ "$status" -ne 0 ] + [[ "$output" == *"FAIL"* ]] + [[ "$output" == *"rollback"* ]] + # Automatic recovery restores the prior state as well as the stack. + [ -f "$OVA_HOME/state.json" ] + [ "$(jq -r '.ova_version' "$OVA_HOME/state.json")" = "2.1.0" ] +} + +@test "--no-backup takes no backup" { + seed_stack core + write_manifest + run bash "$CLI" upgrade --yes --skip-tests --no-backup + [ "$status" -eq 0 ] + [ "$(ls -1 "$OVA_HOME/backups" | wc -l)" -eq 0 ] + [ "$(jq -r '.backup_id' "$OVA_HOME/state.json")" = "none" ] +} + +@test "upgrade runs a pending migration and records it in state.json" { + seed_stack core + write_manifest 5.0.0 5 5 5 '["5.0.0"]' + stage_migration 5.0.0 'echo "migrated $FROM_VERSION -> $TO_VERSION" > "$STACK_DIR/migration-ran"' + + run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -eq 0 ] + [[ "$output" == *"running migration 5.0.0"* ]] + [ -f "$STACK_DIR/migration-ran" ] + [[ "$(cat "$STACK_DIR/migration-ran")" == "migrated unknown -> 5.0.0" ]] + [ "$(jq -r '.migrations_applied | join(",")' "$OVA_HOME/state.json")" = "5.0.0" ] +} + +@test "migration changes to .env survive the staged install" { + seed_stack core + write_manifest 5.0.0 5 5 5 '["5.0.0"]' + stage_migration 5.0.0 'printf "MIGRATION_ADDED=value\\n" >> "$ENV_FILE"' + + run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -eq 0 ] + [ "$(sed -n 's/^MIGRATION_ADDED=//p' "$STACK_DIR/.env")" = "value" ] +} + +@test "migration environment changes feed regenerated Compose interpolation" { + seed_stack core + write_manifest 5.0.0 5 5 5 '["5.0.0"]' + stage_migration 5.0.0 'printf "DEFGUARD_ADOPT_EDGE=migrated-edge\\n" >> "$ENV_FILE"' + + run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -eq 0 ] + docker compose -f "$STACK_DIR/docker-compose.yml" config --format json \ + | jq -e '.services.core.environment.DEFGUARD_ADOPT_EDGE == "migrated-edge"' >/dev/null +} + +@test "upgrade does not re-run a migration already covered by the current version" { + seed_stack core + write_manifest 5.0.0 5 5 5 '["5.0.0"]' + stage_migration 5.0.0 'echo ran >> "$STACK_DIR/migration-ran"' + run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -eq 0 ] + [ "$(wc -l < "$STACK_DIR/migration-ran")" -eq 1 ] + teardown_migration_source + + write_manifest 6.0.0 6 6 6 '["5.0.0","6.0.0"]' + # Only 6.0.0.sh is staged; if 5.0.0 were (wrongly) re-fetched, the upgrade + # would fail outright since no such file exists here. + stage_migration 6.0.0 'echo ran >> "$STACK_DIR/migration-ran"' + run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -eq 0 ] + [ "$(wc -l < "$STACK_DIR/migration-ran")" -eq 2 ] + [ "$(jq -r '.migrations_applied | join(",")' "$OVA_HOME/state.json")" = "6.0.0" ] +} + +@test "a failing migration aborts the upgrade before tags or compose change" { + seed_stack core + write_manifest 5.0.0 5 5 5 '["5.0.0"]' + stage_migration 5.0.0 'exit 1' + before="$(sha256sum "$STACK_DIR/docker-compose.yml" | awk '{print $1}')" + + run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -ne 0 ] + [[ "$output" == *"migration 5.0.0 failed"* ]] + [[ "$output" == *"rollback"* ]] + [ "$(sed -n 's/^DEFGUARD_CORE_TAG=//p' "$STACK_DIR/.env")" = "2" ] # untouched, seeded before the manifest's "5" + [ "$(sha256sum "$STACK_DIR/docker-compose.yml" | awk '{print $1}')" = "$before" ] + # a backup was taken before the migration ran, so rollback is possible + [ "$(ls -1 "$OVA_HOME/backups" | wc -l)" -eq 1 ] +} + +@test "pull failure automatically restores the old stack and state" { + seed_stack core + jq -n '{ova_version: "2.1.0", template_ref: "old-ref"}' > "$OVA_HOME/state.json" + chmod 600 "$OVA_HOME/state.json" + write_manifest 3.0.0 3 3 3 + before_env="$(sha256sum "$STACK_DIR/.env" | awk '{print $1}')" + + DOCKER_STUB_PULL_FAIL=1 run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -ne 0 ] + [ "$(jq -r '.ova_version' "$OVA_HOME/state.json")" = "2.1.0" ] + [ "$(sha256sum "$STACK_DIR/.env" | awk '{print $1}')" = "$before_env" ] + [ "$(cat "$DOCKER_STUB_STATE_FILE")" = "running" ] +} + +@test "automatic rollback preserves a previously stopped stack" { + seed_stack core + printf '%s\n' exited > "$DOCKER_STUB_STATE_FILE" + jq -n '{ova_version: "2.1.0", template_ref: "old-ref"}' > "$OVA_HOME/state.json" + chmod 600 "$OVA_HOME/state.json" + write_manifest 3.0.0 3 3 3 + + DOCKER_STUB_PULL_FAIL=1 run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -ne 0 ] + [ "$(cat "$DOCKER_STUB_STATE_FILE")" = "exited" ] +} + +@test "automatic rollback reports failure when Compose down fails" { + seed_stack core + jq -n '{ova_version: "2.1.0", template_ref: "old-ref"}' > "$OVA_HOME/state.json" + chmod 600 "$OVA_HOME/state.json" + write_manifest 3.0.0 3 3 3 + + DOCKER_STUB_PULL_FAIL=1 DOCKER_STUB_DOWN_FAIL=1 run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -ne 0 ] + [[ "$output" == *"automatic rollback failed"* ]] +} + +@test "rollback reapplies the saved image digest tag" { + seed_stack core + export DOCKER_STUB_IMAGES_JSON='[{"Repository":"example/app","Tag":"2","Digest":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","ID":"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}]' + id="$(bash "$CLI" backup 2>/dev/null | tail -n1)" + + run bash "$CLI" rollback --skip-tests "$id" + [ "$status" -eq 0 ] + [[ "$(cat "$DOCKER_STUB_LOG")" == *"image tag example/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa example/app:2"* ]] +} + +@test "rollback refuses to delete volumes when stop fails" { + seed_stack core + id="$(bash "$CLI" backup 2>/dev/null | tail -n1)" + echo damaged > "$STACK_DIR/.volumes/db/marker" + + DOCKER_STUB_STOP_FAIL=1 run bash "$CLI" rollback --skip-tests "$id" + [ "$status" -ne 0 ] + [ "$(cat "$STACK_DIR/.volumes/db/marker")" = "damaged" ] +} + +@test "automatic rollback removes migration side effects outside volumes" { + seed_stack core + write_manifest 5.0.0 5 5 5 '["5.0.0"]' + stage_migration 5.0.0 'touch "$OVA_DIR/migration-side-effect"' + + DOCKER_STUB_PULL_FAIL=1 run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -ne 0 ] + [ ! -e "$OVA_HOME/migration-side-effect" ] +} + +@test "invalid manifest tags are rejected before backup" { + seed_stack core + write_manifest 3.0.0 'bad&tag' 3 3 + + run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -ne 0 ] + [[ "$output" == *"invalid Docker image tag"* ]] + [ "$(ls -1 "$OVA_HOME/backups" | wc -l)" -eq 0 ] +} + +@test "backup rejects unexpected arguments" { + seed_stack core + + run bash "$CLI" backup --label unit unexpected + [ "$status" -ne 0 ] + [[ "$output" == *"unknown option for backup"* ]] + [ "$(ls -1 "$OVA_HOME/backups" | wc -l)" -eq 0 ] +} + +@test "very large semver components do not overflow version comparison" { + seed_stack core + jq -n '{ova_version: "1.0.0", template_ref: "old-ref"}' > "$OVA_HOME/state.json" + chmod 600 "$OVA_HOME/state.json" + write_manifest 999999999999999999999999999999.0.0 3 3 3 + + run bash "$CLI" upgrade --yes --skip-tests --no-backup + [ "$status" -eq 0 ] + [ "$(jq -r '.ova_version' "$OVA_HOME/state.json")" = "999999999999999999999999999999.0.0" ] +} + +@test "malformed migration metadata is rejected" { + seed_stack core + write_manifest 3.0.0 3 3 3 '"not-an-array"' + + run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -ne 0 ] + [[ "$output" == *"invalid schema"* ]] + [ "$(ls -1 "$OVA_HOME/backups" | wc -l)" -eq 0 ] +} + +@test "downgrades are rejected before backup" { + seed_stack core + jq -n '{ova_version: "5.0.0", template_ref: "old-ref"}' > "$OVA_HOME/state.json" + chmod 600 "$OVA_HOME/state.json" + write_manifest 4.0.0 4 4 4 + + run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -ne 0 ] + [[ "$output" == *"older than installed version"* ]] + [ "$(ls -1 "$OVA_HOME/backups" | wc -l)" -eq 0 ] +} + +@test "upgrades persist the fetched init assets" { + seed_stack core + write_manifest 3.0.0 3 3 3 + expected="$(sha256sum "$FILES_DIR/generate-compose.sh" | awk '{print $1}')" + + run bash "$CLI" upgrade --yes --skip-tests + [ "$status" -eq 0 ] + [ "$(sha256sum "$INIT_DIR/generate-compose.sh" | awk '{print $1}')" = "$expected" ] +} + +@test "health checks fail when Compose service discovery fails" { + seed_stack core + echo 'not valid compose' > "$STACK_DIR/docker-compose.yml" + + run bash "$CLI" test + [ "$status" -ne 0 ] + [[ "$output" == *"could not inspect Compose services"* ]] +} + +@test "version reports installed and available versions" { + seed_stack core + write_manifest 4.5.6 4 4 4 + run bash "$CLI" version + [ "$status" -eq 0 ] + [[ "$output" == *"core tag: 2"* ]] + [[ "$output" == *"4.5.6"* ]] +} + +@test "an unknown command fails with usage" { + run bash "$CLI" definitely-not-a-command + [ "$status" -ne 0 ] + [[ "$output" == *"usage: dg-ctl"* ]] +} + +@test "commands refuse to run outside a defguard stack" { + run bash "$CLI" test + [ "$status" -ne 0 ] + [[ "$output" == *"does not look like a defguard OVA host"* ]] +} diff --git a/ova/tests/generate-compose.bats b/ova/tests/generate-compose.bats index 4a3acae..885b09e 100644 --- a/ova/tests/generate-compose.bats +++ b/ova/tests/generate-compose.bats @@ -59,6 +59,14 @@ generated_json() { [ "$(generated_json | jq '[.services[] | select(has("profiles"))] | length')" -eq 0 ] } +@test "certificate bind paths stay relative to the installed stack" { + run bash "$FILES_DIR/generate-compose.sh" + [ "$status" -eq 0 ] + grep -Fq 'source: ./.volumes/certs/edge' "$STACK_DIR/docker-compose.yml" + grep -Fq 'source: ./.volumes/certs/gateway' "$STACK_DIR/docker-compose.yml" + ! grep -Fq "$STACK_DIR/.volumes/certs" "$STACK_DIR/docker-compose.yml" +} + @test "full stack: core.depends_on includes edge and gateway" { run bash "$FILES_DIR/generate-compose.sh" [ "$status" -eq 0 ] diff --git a/ova/tests/generate-env.bats b/ova/tests/generate-env.bats index a854981..cde1ea8 100644 --- a/ova/tests/generate-env.bats +++ b/ova/tests/generate-env.bats @@ -74,6 +74,15 @@ teardown() { grep -qx 'DEFGUARD_ADOPT_GATEWAY=host.docker.internal:50066' "$STACK_DIR/.env" } +@test "segmented deployment marker suppresses all-in-one adoption defaults" { + printf 'core\nedge\ngateway\n' > "$STACK_DIR/active-profiles" + printf 'segmented\n' > "$INIT_DIR/.deployment-mode" + write_image_tags + bash "$FILES_DIR/generate-env.sh" + grep -qx 'DEFGUARD_ADOPT_EDGE=' "$STACK_DIR/.env" + grep -qx 'DEFGUARD_ADOPT_GATEWAY=' "$STACK_DIR/.env" +} + @test "active-profiles='core gateway' (no edge) -> ADOPT_EDGE/ADOPT_GATEWAY left blank" { printf 'core gateway\n' > "$STACK_DIR/active-profiles" write_image_tags diff --git a/ova/tests/helpers.bash b/ova/tests/helpers.bash index b9d4322..d223a5c 100644 --- a/ova/tests/helpers.bash +++ b/ova/tests/helpers.bash @@ -29,6 +29,167 @@ DEFGUARD_GATEWAY_TAG=${3:-test-gateway} EOF } +file_mode() { + stat -c '%a' "$1" 2>/dev/null || stat -f '%Lp' "$1" +} + +make_ova_home() { + OVA_HOME="$(mktemp -d)" + mkdir -p "$OVA_HOME/backups" + export DEFGUARD_OVA_DIR="$OVA_HOME" + export DEFGUARD_OVA_ALLOW_NONROOT=1 + # Keep tests from touching the host's systemd unit while exercising the + # legacy-layout backup/upgrade path. + export DEFGUARD_INIT_SERVICE_FILE="$OVA_HOME/defguard-init.service" +} + +teardown_ova_home() { + [ -n "${OVA_HOME:-}" ] && rm -rf "$OVA_HOME" + return 0 +} + +stub_docker() { + STUB_BIN="$(mktemp -d)" + DOCKER_REAL="$(command -v docker)" + export DOCKER_REAL + export DOCKER_STUB_LOG="$STUB_BIN/calls.log" + export DOCKER_STUB_STATE_FILE="$STUB_BIN/state" + cp "$STUB_DIR/docker-stub" "$STUB_BIN/docker" + chmod +x "$STUB_BIN/docker" + PATH="$STUB_BIN:$PATH" + export PATH +} + +teardown_stub_docker() { + [ -n "${STUB_BIN:-}" ] && rm -rf "$STUB_BIN" + return 0 +} + +seed_stack() { + if [ "$#" -gt 0 ]; then + printf '%s\n' "$@" > "$STACK_DIR/active-profiles" + fi + write_image_tags "${CORE_TAG:-2}" "${PROXY_TAG:-2}" "${GATEWAY_TAG:-2}" + bash "$FILES_DIR/generate-env.sh" >/dev/null + bash "$FILES_DIR/generate-compose.sh" >/dev/null + mkdir -p "$STACK_DIR/.volumes/db" "$STACK_DIR/.volumes/certs" + echo "seeded-db-file" > "$STACK_DIR/.volumes/db/marker" +} + +# Recreates the layout emitted before commit ff0bce1 (the Compose +# simplification): static full/standalone files, root startup scripts, and no +# init directory. The segmented fixture deliberately selects all three +# profiles because that was still a distinct standalone layout before the +# simplification. +seed_legacy_stack() { + local mode="$1" + write_env "2" "2" "2" + write_image_tags 2 2 2 + cat > "$STACK_DIR/docker-compose.yaml" <<'EOF' +services: + core: + image: ghcr.io/defguard/defguard:${DEFGUARD_CORE_TAG:?DEFGUARD_CORE_TAG is required} + env_file: .env + environment: + DEFGUARD_DB_HOST: db + DEFGUARD_DB_PORT: 5432 + DEFGUARD_ADOPT_EDGE: "edge:50051" + DEFGUARD_ADOPT_GATEWAY: "host.docker.internal:50066" + depends_on: [db, edge, gateway] + edge: + image: ghcr.io/defguard/defguard-proxy:${DEFGUARD_PROXY_TAG:?DEFGUARD_PROXY_TAG is required} + ports: ["8080:8080", "443:443", "80:80"] + gateway: + image: ghcr.io/defguard/gateway:${DEFGUARD_GATEWAY_TAG:?DEFGUARD_GATEWAY_TAG is required} + network_mode: host + dockge: + image: louislam/dockge:1 + profiles: [dockge] + db: + image: postgres:18-alpine + env_file: .env +EOF + cat > "$STACK_DIR/docker-compose.standalone.yaml" <<'EOF' +services: + core: + profiles: [core] + image: ghcr.io/defguard/defguard:${DEFGUARD_CORE_TAG:?DEFGUARD_CORE_TAG is required} + env_file: .env + depends_on: [db] + edge: + profiles: [edge] + image: ghcr.io/defguard/defguard-proxy:${DEFGUARD_PROXY_TAG:?DEFGUARD_PROXY_TAG is required} + ports: ["8080:8080", "50051:50051", "443:443", "80:80"] + gateway: + profiles: [gateway] + image: ghcr.io/defguard/gateway:${DEFGUARD_GATEWAY_TAG:?DEFGUARD_GATEWAY_TAG is required} + network_mode: host + dockge: + profiles: [dockge] + image: louislam/dockge:1 + db: + profiles: [core] + image: postgres:18-alpine + env_file: .env +EOF + cat > "$STACK_DIR/generate-env.sh" <<'EOF' +#!/bin/bash +exit 0 +EOF + cat > "$STACK_DIR/start.sh" <<'EOF' +#!/bin/bash +docker compose up -d +EOF + chmod +x "$STACK_DIR/generate-env.sh" "$STACK_DIR/start.sh" + if [ "$mode" = segmented ]; then + printf 'core\nedge\ngateway\n' > "$STACK_DIR/active-profiles" + fi + mkdir -p "$STACK_DIR/.volumes/db" + echo "seeded-db-file" > "$STACK_DIR/.volumes/db/marker" + cat > "$DEFGUARD_INIT_SERVICE_FILE" <<'EOF' +[Service] +Type=oneshot +ExecStart=/bin/bash /opt/stacks/defguard/generate-env.sh +ExecStart=/bin/bash /opt/stacks/defguard/start.sh +EOF +} + +# A file:// manifest, so upgrades can be exercised without network access. +# $5, if given, is a JSON array literal for the "migrations" field. +write_manifest() { + cat > "$STACK_DIR/manifest.json" <.sh so +# do_upgrade's fetch_ova_file picks it up, and points DEFGUARD_OVA_SOURCE_DIR +# at the copy. Caller must write_manifest with matching "migrations" first (or +# after - only DEFGUARD_OVA_SOURCE_DIR needs to be set last). +stage_migration() { + local version="$1" body="$2" + MIGRATION_SOURCE_DIR="$(mktemp -d)" + mkdir -p "$MIGRATION_SOURCE_DIR/ova/files/migrations" + cp "$FILES_DIR/docker-compose.template.yaml" "$FILES_DIR/lib.sh" "$FILES_DIR/generate-compose.sh" \ + "$MIGRATION_SOURCE_DIR/ova/files/" + printf '%s\n' "$body" > "$MIGRATION_SOURCE_DIR/ova/files/migrations/$version.sh" + export DEFGUARD_OVA_SOURCE_DIR="$MIGRATION_SOURCE_DIR" +} + +teardown_migration_source() { + [ -n "${MIGRATION_SOURCE_DIR:-}" ] && rm -rf "$MIGRATION_SOURCE_DIR" + return 0 +} + # Minimal .env so compose interpolation of the *_TAG variables succeeds. write_env() { cat > "$STACK_DIR/.env" </dev/null 2>&1; then - docker compose -f "$STACK_DIR/docker-compose.yml" down -v >/dev/null 2>&1 || true + if command -v sudo >/dev/null 2>&1; then + sudo docker compose -f "$STACK_DIR/docker-compose.yml" down -v >/dev/null 2>&1 || true + else + docker compose -f "$STACK_DIR/docker-compose.yml" down -v >/dev/null 2>&1 || true + fi + fi + if command -v sudo >/dev/null 2>&1; then + [ -z "${STACK_DIR:-}" ] || sudo rm -rf -- "$STACK_DIR" + [ -z "${OVA_HOME:-}" ] || sudo rm -rf -- "$OVA_HOME" + else + [ -z "${STACK_DIR:-}" ] || rm -rf -- "$STACK_DIR" + [ -z "${OVA_HOME:-}" ] || rm -rf -- "$OVA_HOME" fi - teardown_stack } bring_up() { @@ -52,3 +62,39 @@ wait_for_health() { bring_up wait_for_health } + +# Upgrade the core-only profile in place: cheaper than the full stack and it is +# the profile that owns the database, which is what an upgrade must not lose. +@test "dg-ctl upgrade keeps database state and passes its own health checks" { + command -v zstd >/dev/null 2>&1 || skip "zstd not installed" + command -v sudo >/dev/null 2>&1 || skip "sudo not installed" + echo "core" > "$STACK_DIR/active-profiles" + bring_up + wait_for_health + + psql() { sudo docker compose -f "$STACK_DIR/docker-compose.yml" exec -T db psql -qtAX -U defguard -d defguard "$@"; } + psql -c "CREATE TABLE ova_upgrade_probe (v text); INSERT INTO ova_upgrade_probe VALUES ('survived');" + + OVA_HOME="$(mktemp -d)" + cat > "$OVA_HOME/manifest.json" < `compose ps` reports services not running +# DOCKER_STUB_RESTARTS=2 -> `inspect` reports that many restarts +# DOCKER_STUB_PULL_FAIL=1 -> `compose pull` exits non-zero +set -uo pipefail + +: "${DOCKER_REAL:?DOCKER_REAL must point at the real docker binary}" +[ -n "${DOCKER_STUB_LOG:-}" ] && printf '%s\n' "$*" >> "$DOCKER_STUB_LOG" + +stub_state() { + if [ -n "${DOCKER_STUB_STATE:-}" ]; then + printf '%s\n' "$DOCKER_STUB_STATE" + elif [ -f "${DOCKER_STUB_STATE_FILE:-}" ]; then + cat "$DOCKER_STUB_STATE_FILE" + else + printf '%s\n' running + fi +} + +set_stub_state() { + [ -n "${DOCKER_STUB_STATE:-}" ] || printf '%s\n' "$1" >"$DOCKER_STUB_STATE_FILE" +} + +passthrough() { exec "$DOCKER_REAL" "$@"; } + +verb="" +skip_next=0 +seen_compose=0 +for a in "$@"; do + if [ "$skip_next" = 1 ]; then skip_next=0; continue; fi + case "$a" in + compose) seen_compose=1; continue ;; + -f|--project-directory|--env-file|--profile|-p) skip_next=1; continue ;; + -*) continue ;; + *) verb="$a"; break ;; + esac +done + +if [ "$seen_compose" != 1 ]; then + case "${1:-}" in + inspect) + printf '%s\n' "${DOCKER_STUB_RESTARTS:-0}" + exit 0 + ;; + image) + case "${2:-}" in + inspect|tag) exit 0 ;; + *) passthrough "$@" ;; + esac + ;; + pull) exit "${DOCKER_STUB_PULL_FAIL:-0}" ;; + *) passthrough "$@" ;; + esac +fi + +case "$verb" in + version|config) passthrough "$@" ;; + pull) exit "${DOCKER_STUB_PULL_FAIL:-0}" ;; + up|start|restart) set_stub_state running; exit 0 ;; + stop) exit "${DOCKER_STUB_STOP_FAIL:-$(set_stub_state exited; printf 0)}" ;; + down) exit "${DOCKER_STUB_DOWN_FAIL:-$(set_stub_state exited; printf 0)}" ;; + images) printf '%s\n' "${DOCKER_STUB_IMAGES_JSON:-[]}"; exit 0 ;; + port) exit 1 ;; + exec) exit "${DOCKER_STUB_EXEC_FAIL:-0}" ;; + ps) + # -q asks for container ids; --format json asks for state. + if [[ " $* " == *" -q "* ]]; then + echo "stubcontainerid" + else + printf '{"Service":"stub","State":"%s"}\n' "$(stub_state)" + fi + exit 0 + ;; + *) exit 0 ;; +esac