Add black-box container and deployment smoke tests

.github/workflows/build.yaml

@@ -88,9 +88,10 @@ jobs:
         run: make redis-integration-test
 
   deep-container-lane:
-    name: Deep container lane
+    name: Deep container and system lane
     runs-on: ubuntu-24.04
     if: github.event_name != 'pull_request'
+    timeout-minutes: 30
 
     steps:
       - name: Checkout
@@ -111,11 +112,25 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
+      - name: Set up Helm
+        uses: azure/setup-helm@v4
+
+      - name: Set up kind
+        uses: helm/kind-action@v1
+        with:
+          install_only: true
+
       - name: Install Tools and Dependencies
         run: make install_tools
 
       - name: Build Container
         run: make container
 
+      - name: Container Smoke Test
+        run: SMOKE_IMAGE=ghcr.io/darthfork/promgithub:$(awk -F= '/^VERSION=/{print $2}' version) ./test/system/container-smoke.sh
+
+      - name: Helm Deployment Smoke Test
+        run: SMOKE_IMAGE=ghcr.io/darthfork/promgithub:$(awk -F= '/^VERSION=/{print $2}' version) ./test/system/helm-smoke.sh
+
       - name: Container Security Scan
         run: make container-security

.gitignore

@@ -8,4 +8,5 @@ trivy-report.json
 secrets-report.json
 *.sarif
 *.tgz
+helm/promgithub/charts/
 values-test.yaml

.go-version

@@ -1 +1 @@
-1.25.11
+1.25.13

Makefile

@@ -1,4 +1,4 @@
-.PHONY: build container container-security cross-platform debug release test unit-test integration-test redis-integration-test test-all go-version coverage fmt lint deps security clean dev-setup
+.PHONY: build container container-security container-smoke helm-smoke system-smoke cross-platform debug release test unit-test integration-test redis-integration-test test-all go-version coverage fmt lint deps security clean dev-setup
 
 include version
 
@@ -73,6 +73,14 @@ install_tools: deps ## Install development tooling
 container: ## Build promgithub service container
 	@docker build --progress=plain --build-arg GO_VERSION=$(GO_VERSION) -t $(CONTAINER_REGISTRY):$(VERSION) .
 
+container-smoke: container ## Run black-box startup, health, metrics, and webhook checks against the container
+	@SMOKE_IMAGE=$(CONTAINER_REGISTRY):$(VERSION) ./test/system/container-smoke.sh
+
+helm-smoke: container ## Install the chart in kind and smoke test standalone and Redis-backed modes
+	@SMOKE_IMAGE=$(CONTAINER_REGISTRY):$(VERSION) ./test/system/helm-smoke.sh
+
+system-smoke: container-smoke helm-smoke ## Run all black-box container and deployment smoke tests
+
 package-helm-chart: mkdir ## Package promgithub helm chart
 	@helm package $(CHART_SOURCE) -d $(BUILDDIR)
 

docs/testing.md

@@ -8,7 +8,7 @@
 | --- | --- | --- | --- | --- |
 | Fast PR lane | `Fast PR lane` | Pull requests, pushes to `main`, weekly schedule | Static checks, security scan, build, unit tests, in-process HTTP integration tests, and coverage. This lane has no external service dependency. | `make lint`, `make security`, `make build`, `make unit-test`, `make integration-test`, `make coverage` |
 | Medium Redis lane | `Medium Redis lane` | Pull requests, pushes to `main`, weekly schedule | Redis-backed deduplication and shared run/job state behavior against a real Redis service. | `PROMGITHUB_REDIS_ADDR=127.0.0.1:6379 make redis-integration-test` |
-| Deep container lane | `Deep container lane` | Pushes to `main`, weekly schedule | Container build and container image security scan. This lane is intentionally kept out of normal pull request feedback because it is slower and Docker-dependent. | `make container`, `make container-security` |
+| Deep container and system lane | `Deep container and system lane` | Pushes to `main`, weekly schedule | Container build/security scan plus black-box container and kind/Helm deployment smoke tests. It validates startup configuration, health, metrics, signed webhook ingestion, chart readiness, and standalone and Redis-backed deployment modes. | `make system-smoke`, `make container-security` |
 
 ## Pull request expectations
 
@@ -20,7 +20,7 @@ Every pull request should keep the Fast PR lane and Medium Redis lane green. Tho
 - In-process HTTP webhook-to-metrics behavior.
 - Redis-backed delivery deduplication and run/job state behavior.
 
-The Deep container lane runs after merge to `main` and on the weekly scheduled workflow. It catches packaging and image-scan issues without making every pull request wait on Docker image work.
+The Deep container and system lane runs after merge to `main` and on the weekly scheduled workflow. It catches packaging, deployment, and image-scan issues without making every pull request wait on Docker and kind work.
 
 ## Local verification
 
@@ -34,6 +34,21 @@ make security
 
 `make test-all` already includes the unit, integration, Redis integration, coverage, security, and lint targets. Run `make lint` and `make security` separately as explicit final checks before pushing.
 
-## Future system smoke tests
+## System smoke tests
 
-Black-box container and Helm/deployment smoke tests are tracked separately in issue #56. When those tests exist, they should join the Deep container lane or a dedicated deep system lane rather than the Fast PR lane.
+Run the complete black-box suite with:
+
+```bash
+make system-smoke
+```
+
+The suite requires Docker, Helm, kind, kubectl, curl, and OpenSSL. It builds the production image and then:
+
+- starts the container from the outside and checks `/health` and `/metrics`;
+- submits a signed push webhook and waits for its exported metric;
+- confirms startup fails when `PROMGITHUB_WEBHOOK_SECRET` is missing;
+- lints and renders the Helm chart, including its bundled Redis configuration;
+- installs the chart into a disposable kind cluster and checks pod readiness and endpoints;
+- upgrades the deployment to use a real external Redis instance and repeats the webhook-to-metric check.
+
+The scripts delete their temporary container and kind cluster on exit. Override `SMOKE_IMAGE`, `SMOKE_KIND_CLUSTER`, or `SMOKE_LOCAL_PORT` when local names or ports conflict.

helm/promgithub/Chart.lock

@@ -0,0 +1,6 @@
+dependencies:
+- name: redis
+  repository: oci://registry-1.docker.io/bitnamicharts
+  version: 23.1.1
+digest: sha256:e6ffcd1baccb5e1058c8a98af891ebedfdd84733e0f684db5be12550c9f6752f
+generated: "2026-08-22T18:44:12.271939-07:00"

helm/promgithub/Chart.yaml

@@ -14,5 +14,5 @@ appVersion: "0.0.7"
 dependencies:
   - name: redis
     version: 0.0.7
-    repository: "https://charts.bitnami.com/bitnami"
+    repository: "oci://registry-1.docker.io/bitnamicharts"
     condition: redis.enabled

helm/promgithub/values.yaml

@@ -65,11 +65,11 @@ resources: {}
 # This is to setup the liveness and readiness probes
 livenessProbe:
   httpGet:
-    path: /
+    path: /health
     port: http
 readinessProbe:
   httpGet:
-    path: /
+    path: /health
     port: http
 
 # This section is for setting up autoscaling

test/system/container-smoke.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+# shellcheck source=smoke-lib.sh
+source "${SCRIPT_DIR}/smoke-lib.sh"
+
+IMAGE="${SMOKE_IMAGE:-ghcr.io/darthfork/promgithub:0.0.7}"
+CONTAINER_NAME="promgithub-system-smoke-${RANDOM}"
+FAILURE_CONTAINER_NAME="${CONTAINER_NAME}-missing-config"
+LOG_FILE="$(mktemp)"
+
+cleanup() {
+  docker rm --force "$CONTAINER_NAME" "$FAILURE_CONTAINER_NAME" >/dev/null 2>&1 || true
+  rm -f "$LOG_FILE"
+}
+trap cleanup EXIT
+
+cd "$REPO_ROOT"
+
+printf 'Starting black-box container smoke test for %s\n' "$IMAGE"
+docker run --detach --name "$CONTAINER_NAME" --publish 127.0.0.1::8080 \
+  --env "PROMGITHUB_WEBHOOK_SECRET=${SMOKE_SECRET}" \
+  "$IMAGE" >/dev/null
+
+HOST_PORT="$(docker inspect --format '{{(index (index .NetworkSettings.Ports "8080/tcp") 0).HostPort}}' "$CONTAINER_NAME")"
+assert_service_smoke "http://127.0.0.1:${HOST_PORT}" "container-smoke-${RANDOM}"
+
+printf 'Verifying startup rejects missing required configuration\n'
+if docker run --name "$FAILURE_CONTAINER_NAME" "$IMAGE" >"$LOG_FILE" 2>&1; then
+  fail 'container started without PROMGITHUB_WEBHOOK_SECRET'
+fi
+
+grep --fixed-strings --quiet 'PROMGITHUB_WEBHOOK_SECRET is not set' "$LOG_FILE" \
+  || fail 'missing-secret startup failure did not explain the invalid configuration'
+
+printf 'Container smoke test passed\n'

test/system/helm-smoke.sh

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+# shellcheck source=smoke-lib.sh
+source "${SCRIPT_DIR}/smoke-lib.sh"
+
+IMAGE="${SMOKE_IMAGE:-ghcr.io/darthfork/promgithub:0.0.7}"
+CLUSTER_NAME="${SMOKE_KIND_CLUSTER:-promgithub-smoke-${RANDOM}}"
+NAMESPACE="${SMOKE_NAMESPACE:-promgithub-smoke}"
+RELEASE_NAME="${SMOKE_RELEASE:-promgithub-smoke}"
+LOCAL_PORT="${SMOKE_LOCAL_PORT:-18080}"
+PORT_FORWARD_PID=""
+PORT_FORWARD_LOG="$(mktemp)"
+CLUSTER_CREATED=false
+
+start_port_forward() {
+  : >"$PORT_FORWARD_LOG"
+  kubectl --namespace "$NAMESPACE" port-forward \
+    "service/${RELEASE_NAME}" "${LOCAL_PORT}:8080" >"$PORT_FORWARD_LOG" 2>&1 &
+  PORT_FORWARD_PID=$!
+
+  for ((attempt = 1; attempt <= 30; attempt++)); do
+    kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1 \
+      || fail "port-forward exited before the service became reachable: $(cat "$PORT_FORWARD_LOG")"
+    if curl --silent --show-error --fail "http://127.0.0.1:${LOCAL_PORT}/health" >/dev/null 2>&1; then
+      return 0
+    fi
+    sleep 1
+  done
+
+  fail "timed out waiting for port-forwarded service health"
+}
+
+stop_port_forward() {
+  [[ -n "$PORT_FORWARD_PID" ]] || return 0
+  kill "$PORT_FORWARD_PID" >/dev/null 2>&1 || true
+  wait "$PORT_FORWARD_PID" >/dev/null 2>&1 || true
+  PORT_FORWARD_PID=""
+}
+
+cluster_diagnostics() {
+  [[ "$CLUSTER_CREATED" == true ]] || return 0
+
+  printf '\nHelm smoke failure diagnostics\n' >&2
+  kubectl --namespace "$NAMESPACE" get pods,services,endpoints,deployments >&2 || true
+  kubectl --namespace "$NAMESPACE" describe pods >&2 || true
+  kubectl --namespace "$NAMESPACE" logs \
+    --selector "app.kubernetes.io/instance=${RELEASE_NAME}" \
+    --all-containers --tail=100 >&2 || true
+  kubectl --namespace "$NAMESPACE" get events --sort-by=.lastTimestamp >&2 || true
+  if [[ -s "$PORT_FORWARD_LOG" ]]; then
+    printf '\nPort-forward output\n' >&2
+    cat "$PORT_FORWARD_LOG" >&2
+  fi
+}
+
+cleanup() {
+  local status=$?
+  trap - EXIT
+
+  if ((status != 0)); then
+    cluster_diagnostics
+  fi
+  stop_port_forward
+  if [[ "$CLUSTER_CREATED" == true ]]; then
+    kind delete cluster --name "$CLUSTER_NAME" >/dev/null 2>&1 || true
+  fi
+  rm -f "$PORT_FORWARD_LOG"
+  exit "$status"
+}
+trap cleanup EXIT
+
+cd "$REPO_ROOT"
+
+for command in curl docker helm kind kubectl openssl; do
+  command -v "$command" >/dev/null || fail "required command not found: ${command}"
+done
+
+printf 'Linting and rendering Helm chart\n'
+helm dependency build helm/promgithub
+helm lint helm/promgithub \
+  --set secrets.github_webhook_secret="$SMOKE_SECRET" \
+  --set autoscaling.enabled=false
+helm template "$RELEASE_NAME" helm/promgithub \
+  --namespace "$NAMESPACE" \
+  --set secrets.github_webhook_secret="$SMOKE_SECRET" \
+  --set autoscaling.enabled=false \
+  --set redis.enabled=true \
+  --set redis.auth.password=system-smoke-redis >/dev/null
+
+printf 'Creating kind cluster and loading %s\n' "$IMAGE"
+kind create cluster --name "$CLUSTER_NAME" --wait 120s
+CLUSTER_CREATED=true
+kind load docker-image "$IMAGE" --name "$CLUSTER_NAME"
+
+printf 'Installing standalone Helm deployment\n'
+helm install "$RELEASE_NAME" helm/promgithub \
+  --namespace "$NAMESPACE" \
+  --create-namespace \
+  --set image.repository="${IMAGE%:*}" \
+  --set image.tag="${IMAGE##*:}" \
+  --set image.pullPolicy=Never \
+  --set secrets.github_webhook_secret="$SMOKE_SECRET" \
+  --set autoscaling.enabled=false \
+  --wait \
+  --timeout 3m
+
+kubectl --namespace "$NAMESPACE" rollout status "deployment/${RELEASE_NAME}" --timeout=120s
+start_port_forward
+assert_service_smoke "http://127.0.0.1:${LOCAL_PORT}" "helm-standalone-${RANDOM}"
+
+printf 'Upgrading Helm deployment to external Redis-backed mode\n'
+kubectl --namespace "$NAMESPACE" create deployment redis --image=redis:7-alpine
+kubectl --namespace "$NAMESPACE" expose deployment redis --port=6379
+kubectl --namespace "$NAMESPACE" rollout status deployment/redis --timeout=120s
+
+helm upgrade "$RELEASE_NAME" helm/promgithub \
+  --namespace "$NAMESPACE" \
+  --reuse-values \
+  --set redisConfig.addr=redis:6379 \
+  --wait \
+  --timeout 3m
+
+kubectl --namespace "$NAMESPACE" rollout status "deployment/${RELEASE_NAME}" --timeout=120s
+stop_port_forward
+start_port_forward
+assert_service_smoke "http://127.0.0.1:${LOCAL_PORT}" "helm-redis-${RANDOM}"
+
+printf 'Helm deployment smoke test passed\n'

test/system/smoke-lib.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SMOKE_SECRET="${SMOKE_SECRET:-system-smoke-secret}"
+SMOKE_REPOSITORY="${SMOKE_REPOSITORY:-user/repo}"
+
+fail() {
+  printf 'system smoke test failed: %s\n' "$*" >&2
+  exit 1
+}
+
+wait_for_http() {
+  local url="$1"
+  local attempts="${2:-60}"
+
+  for ((attempt = 1; attempt <= attempts; attempt++)); do
+    if curl --silent --show-error --fail "$url" >/dev/null 2>&1; then
+      return 0
+    fi
+    sleep 1
+  done
+
+  fail "timed out waiting for ${url}"
+}
+
+wait_for_metric() {
+  local url="$1"
+  local metric="$2"
+  local attempts="${3:-30}"
+
+  for ((attempt = 1; attempt <= attempts; attempt++)); do
+    if curl --silent --show-error --fail "$url" | grep --fixed-strings --quiet "$metric"; then
+      return 0
+    fi
+    sleep 1
+  done
+
+  fail "metric did not appear: ${metric}"
+}
+
+signature_for() {
+  local payload="$1"
+  printf 'sha256=%s' "$(openssl dgst -sha256 -hmac "$SMOKE_SECRET" "$payload" | awk '{print $NF}')"
+}
+
+assert_service_smoke() {
+  local base_url="$1"
+  local delivery_id="$2"
+  local payload="${3:-test_data/push.json}"
+  local health
+  local signature
+
+  wait_for_http "${base_url}/health"
+
+  health="$(curl --silent --show-error --fail "${base_url}/health")"
+  grep --fixed-strings --quiet '"status":"ok"' <<<"$health" || fail "unexpected health response: ${health}"
+
+  curl --silent --show-error --fail "${base_url}/metrics" \
+    | grep --fixed-strings --quiet 'promgithub_api_calls_total' \
+    || fail "metrics endpoint did not expose service metrics"
+
+  signature="$(signature_for "$payload")"
+  curl --silent --show-error --fail \
+    --request POST \
+    --header 'Content-Type: application/json' \
+    --header 'X-GitHub-Event: push' \
+    --header "X-GitHub-Delivery: ${delivery_id}" \
+    --header "X-Hub-Signature-256: ${signature}" \
+    --data-binary "@${payload}" \
+    "${base_url}/webhook" >/dev/null
+
+  wait_for_metric \
+    "${base_url}/metrics" \
+    "promgithub_commit_pushed{repository=\"${SMOKE_REPOSITORY}\"} 1"
+}