From 6275321bb17c9173df9bece2a07dbce69e6bc85f Mon Sep 17 00:00:00 2001 From: Marinski Date: Fri, 14 Aug 2026 00:58:25 +0300 Subject: [PATCH] feat(ops): restart VM containers that are persistently unhealthy dockurr/windows keeps the container up while the Windows guest may have crashed internally, so restart: unless-stopped never fires and every terminal API in that VM stays dead until a human acts. watchdog.sh is the host-side recovery: run from cron every few minutes, it discovers the VM containers by compose labels, and docker restart only the ones whose Docker health has stayed unhealthy for a sustained streak. Healthy or starting VMs are never touched, so running backtests are not interrupted; a cooldown prevents restart loops. This complements the in-VM MT5AutoReboot task for operators who disable it to protect long backtests. --- docs/operations.md | 45 +++++++++++++ scripts/watchdog.sh | 159 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 204 insertions(+) create mode 100755 scripts/watchdog.sh diff --git a/docs/operations.md b/docs/operations.md index 2984ee0..6ddabd0 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -10,6 +10,7 @@ The commands, ports, tunnels, locks, queues, and logs you need after the thing b - [Cloudflare Tunnel](#cloudflare-tunnel-optional) - [Project structure](#project-structure) - [Concurrency and backpressure](#concurrency-and-backpressure) +- [Auto-recovery](#auto-recovery) - [Logs](#logs) ## Make Targets @@ -222,6 +223,50 @@ Knock-on effects you'll observe: If you see persistent 503/504 from a single terminal, check `data/shared/logs/api--.log` for `mt5.* TIMEOUT` lines — that's the SDK call that wedged. +## Auto-recovery + +The Windows VM(s) run inside `dockurr/windows` containers with a Docker healthcheck +(`scripts/healthcheck.sh`) that probes every terminal port this VM owns. A crash +inside the guest — an unexpected shutdown (Event 6008), a wedged terminal, an +OOM — leaves the **container** up while the **API** is dead, so +`restart: unless-stopped` never fires and nothing recovers it on its own. + +### VM crash watchdog + +`scripts/watchdog.sh` is the host-side recovery for exactly that case. Run it +from cron or systemd every few minutes; it is idempotent and safe to overlap: + +```bash +# every 5 minutes +*/5 * * * * /home/monster/apps/mt-backtest-manager/docker/mt5-httpapi/scripts/watchdog.sh >> /var/log/mt5-httpapi-watchdog.log 2>&1 +``` + +Behavior: + +- Discovers VM containers by compose labels + (`com.docker.compose.project=mt5-httpapi` + `com.docker.compose.service`) and + verifies the image (`dockurr/windows`), so it only ever restarts the VM + containers — never nginx, wickworks, the log rotator, or other sidecars. +- Restarts a container **only** after its Docker health has stayed `unhealthy` + for `WATCHDOG_MIN_FAILING_STREAK` consecutive healthcheck failures (default + `10`, i.e. ~5 minutes at the default 30s interval). A container that is + healthy or still starting is never touched, so running backtests on a working + VM are never interrupted — the healthcheck stays green the whole time a + terminal is serving. +- Refuses to restart a container that was (re)started less than + `WATCHDOG_RESTART_COOLDOWN_SECONDS` ago (default `300`), so a VM that crashes + again immediately after recovery is not restarted into a loop. +- `WATCHDOG_DRY_RUN=1` (or `--dry-run`) prints what it would do without + touching any container — useful to sanity-check a cron line before enabling it. + +Environment overrides: `WATCHDOG_COMPOSE_PROJECT`, `WATCHDOG_IMAGE_FILTER`, +`WATCHDOG_MIN_FAILING_STREAK`, `WATCHDOG_RESTART_COOLDOWN_SECONDS`, +`WATCHDOG_DRY_RUN`. + +This complements the in-VM `MT5AutoReboot` scheduled task, which reboots on a +fixed timer and can interrupt long-running backtests; operators who disable that +task still get crash recovery from the watchdog. + ## Logs Inside the VM's shared folder (`data/shared/logs/`): diff --git a/scripts/watchdog.sh b/scripts/watchdog.sh new file mode 100755 index 0000000..eda4110 --- /dev/null +++ b/scripts/watchdog.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# +# watchdog.sh -- restart dockurr/windows VM containers whose Docker health +# has stayed RED for a sustained period. +# +# WHY THIS EXISTS +# --------------- +# dockurr/windows keeps the container "up" while the Windows guest may have +# crashed internally (e.g. Event ID 6008 "The previous system shutdown ... was +# unexpected"). Docker's healthcheck then reports `unhealthy`, but because the +# container never exits, `restart: unless-stopped` never fires and every +# terminal API inside that VM stays dead until a human restarts it. This +# script is that human, on a schedule. +# +# It deliberately restarts ONLY after the container has been continuously +# unhealthy for longer than WATCHDOG_MIN_FAILING_STREAK healthcheck failures. +# A healthy or merely-starting VM is never touched, so long-running backtests +# on a working VM are never interrupted -- the healthcheck goes green the whole +# time a terminal is serving, and only a genuinely dead VM stays red. +# +# It also refuses to restart a container in a restart loop (it just came back +# and is already unhealthy again) -- see WATCHDOG_RESTART_COOLDOWN_SECONDS. +# +# This is the host-side complement to the in-VM MT5AutoReboot scheduled task. +# That task reboots every N minutes unconditionally, which interrupts running +# backtests; operators who disable it (as the backtester does) still want the +# crash case covered. Run this on the host from cron or systemd every few +# minutes -- it is idempotent and safe to run overlapping. +# +# USAGE +# ----- +# ./scripts/watchdog.sh [--dry-run] +# +# ENV +# --- +# WATCHDOG_COMPOSE_PROJECT compose project label to filter containers +# (default: mt5-httpapi) +# WATCHDOG_IMAGE_FILTER only restart containers running this image +# (default: dockurr/windows) +# WATCHDOG_MIN_FAILING_STREAK restart only after this many consecutive +# failed healthchecks (default: 10) +# WATCHDOG_RESTART_COOLDOWN_SECONDS +# seconds a restarted container must stay healthy +# before it can be restarted again (default: 300) +# WATCHDOG_DRY_RUN "1" to only report what would happen +# +# It discovers VM containers by their compose labels +# com.docker.compose.project + com.docker.compose.service +# and verifies the image, so it only ever restarts the Windows VM containers, +# never nginx, wickworks, the log rotator, or any other sidecar. + +set -euo pipefail + +trap 'echo "[ERROR] ${BASH_SOURCE[0]}:${LINENO} - command failed (exit $?)" >&2' ERR + +PROJECT="${WATCHDOG_COMPOSE_PROJECT:-mt5-httpapi}" +IMAGE_FILTER="${WATCHDOG_IMAGE_FILTER:-dockurr/windows}" +MIN_STREAK="${WATCHDOG_MIN_FAILING_STREAK:-10}" +COOLDOWN_SECONDS="${WATCHDOG_RESTART_COOLDOWN_SECONDS:-300}" +DRY_RUN="${WATCHDOG_DRY_RUN:-0}" + +log() { + printf '[%s] [watchdog] %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" +} + +# Seconds since the container last flipped to `running`. Used to enforce a +# cooldown after a restart so a VM that crashes again immediately is not +# restarted into a loop. A start time that is missing, unparseable, or in the +# future yields "0" -> treated as "started long ago" so an unhealthy VM is not +# held back by a bad timestamp. +container_started_epoch() { + local id="$1" started epoch + started=$(docker inspect --format '{{.State.StartedAt}}' "$id" 2>/dev/null || true) + [ -z "$started" ] && { + echo "0" + return + } + epoch=$(date -u -d "$started" +%s 2>/dev/null || echo "0") + if [ "$epoch" -le 0 ] || [ "$epoch" -gt "$(date +%s)" ]; then + echo "0" + else + echo "$epoch" + fi +} + +# One pass. Returns 0 if every candidate VM container is either healthy or +# recovering (nothing needed doing), 1 if any container was restarted. +watch_once() { + local restarted=0 + local containers + + if ! command -v docker >/dev/null 2>&1; then + log "docker not found on PATH; nothing to do" + return 0 + fi + + containers=$(docker ps -q \ + --filter "label=com.docker.compose.project=${PROJECT}" \ + --filter "label=com.docker.compose.service" 2>/dev/null || true) + + if [ -z "$containers" ]; then + log "no containers for compose project '${PROJECT}'" + return 0 + fi + + local cid name image health streak started now age + for cid in $containers; do + image=$(docker inspect --format '{{.Config.Image}}' "$cid" 2>/dev/null || true) + case "$image" in + ${IMAGE_FILTER}*) ;; + *) continue ;; # not a VM container; leave sidecars alone + esac + + name=$(docker inspect --format '{{.Name}}' "$cid" 2>/dev/null | sed 's#^/##') + health=$(docker inspect --format '{{.State.Health.Status}}' "$cid" 2>/dev/null || echo 'none') + streak=$(docker inspect --format '{{.State.Health.FailingStreak}}' "$cid" 2>/dev/null || echo '0') + + if [ "$health" != "unhealthy" ]; then + continue # healthy, starting, or no healthcheck defined + fi + + if [ "$streak" -lt "$MIN_STREAK" ]; then + log "VM '${name}' unhealthy for ${streak}/${MIN_STREAK} failures - waiting" + continue + fi + + started=$(container_started_epoch "$cid") + now=$(date +%s) + age=$((now - started)) + if [ "$age" -lt "$COOLDOWN_SECONDS" ]; then + log "VM '${name}' unhealthy but restarted only ${age}s ago - skipping (cooldown ${COOLDOWN_SECONDS}s)" + continue + fi + + if [ "$DRY_RUN" = "1" ]; then + log "DRY-RUN: would restart VM '${name}' (unhealthy streak=${streak}, running ${age}s)" + continue + fi + + log "restarting VM '${name}' (unhealthy streak=${streak})" + if docker restart "$cid" >/dev/null 2>&1; then + log "VM '${name}' restart issued" + restarted=1 + else + log "WARN: 'docker restart ${name}' failed" + fi + done + + return "$restarted" +} + +main() { + if [ "${1:-}" = "--dry-run" ]; then + DRY_RUN=1 + fi + watch_once +} + +main "$@"