From e24e7a5683ebb8ee2845f0f4c35bab83683dabdf Mon Sep 17 00:00:00 2001 From: Abhishek Rai Date: Sat, 22 Aug 2026 18:58:14 -0700 Subject: [PATCH 1/2] Add black-box system smoke tests --- .github/workflows/build.yaml | 17 ++++- .gitignore | 1 + Makefile | 10 ++- docs/testing.md | 23 +++++- helm/promgithub/Chart.lock | 6 ++ helm/promgithub/Chart.yaml | 2 +- helm/promgithub/values.yaml | 4 +- test/system/container-smoke.sh | 39 ++++++++++ test/system/helm-smoke.sh | 132 +++++++++++++++++++++++++++++++++ test/system/smoke-lib.sh | 76 +++++++++++++++++++ 10 files changed, 301 insertions(+), 9 deletions(-) create mode 100644 helm/promgithub/Chart.lock create mode 100755 test/system/container-smoke.sh create mode 100755 test/system/helm-smoke.sh create mode 100755 test/system/smoke-lib.sh diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 0b739a8..e94748e 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -88,9 +88,10 @@ jobs: run: make redis-integration-test deep-container-lane: - name: Deep container lane + name: Deep container and system lane runs-on: ubuntu-24.04 if: github.event_name != 'pull_request' + timeout-minutes: 30 steps: - name: Checkout @@ -111,11 +112,25 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Set up Helm + uses: azure/setup-helm@v4 + + - name: Set up kind + uses: helm/kind-action@v1 + with: + install_only: true + - name: Install Tools and Dependencies run: make install_tools - name: Build Container run: make container + - name: Container Smoke Test + run: SMOKE_IMAGE=ghcr.io/darthfork/promgithub:$(awk -F= '/^VERSION=/{print $2}' version) ./test/system/container-smoke.sh + + - name: Helm Deployment Smoke Test + run: SMOKE_IMAGE=ghcr.io/darthfork/promgithub:$(awk -F= '/^VERSION=/{print $2}' version) ./test/system/helm-smoke.sh + - name: Container Security Scan run: make container-security diff --git a/.gitignore b/.gitignore index 47f0da7..d60c5f6 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ trivy-report.json secrets-report.json *.sarif *.tgz +helm/promgithub/charts/ values-test.yaml diff --git a/Makefile b/Makefile index fa3648e..95c6194 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build container container-security cross-platform debug release test unit-test integration-test redis-integration-test test-all go-version coverage fmt lint deps security clean dev-setup +.PHONY: build container container-security container-smoke helm-smoke system-smoke cross-platform debug release test unit-test integration-test redis-integration-test test-all go-version coverage fmt lint deps security clean dev-setup include version @@ -73,6 +73,14 @@ install_tools: deps ## Install development tooling container: ## Build promgithub service container @docker build --progress=plain --build-arg GO_VERSION=$(GO_VERSION) -t $(CONTAINER_REGISTRY):$(VERSION) . +container-smoke: container ## Run black-box startup, health, metrics, and webhook checks against the container + @SMOKE_IMAGE=$(CONTAINER_REGISTRY):$(VERSION) ./test/system/container-smoke.sh + +helm-smoke: container ## Install the chart in kind and smoke test standalone and Redis-backed modes + @SMOKE_IMAGE=$(CONTAINER_REGISTRY):$(VERSION) ./test/system/helm-smoke.sh + +system-smoke: container-smoke helm-smoke ## Run all black-box container and deployment smoke tests + package-helm-chart: mkdir ## Package promgithub helm chart @helm package $(CHART_SOURCE) -d $(BUILDDIR) diff --git a/docs/testing.md b/docs/testing.md index d12ff3f..61ebefe 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -8,7 +8,7 @@ | --- | --- | --- | --- | --- | | Fast PR lane | `Fast PR lane` | Pull requests, pushes to `main`, weekly schedule | Static checks, security scan, build, unit tests, in-process HTTP integration tests, and coverage. This lane has no external service dependency. | `make lint`, `make security`, `make build`, `make unit-test`, `make integration-test`, `make coverage` | | Medium Redis lane | `Medium Redis lane` | Pull requests, pushes to `main`, weekly schedule | Redis-backed deduplication and shared run/job state behavior against a real Redis service. | `PROMGITHUB_REDIS_ADDR=127.0.0.1:6379 make redis-integration-test` | -| Deep container lane | `Deep container lane` | Pushes to `main`, weekly schedule | Container build and container image security scan. This lane is intentionally kept out of normal pull request feedback because it is slower and Docker-dependent. | `make container`, `make container-security` | +| Deep container and system lane | `Deep container and system lane` | Pushes to `main`, weekly schedule | Container build/security scan plus black-box container and kind/Helm deployment smoke tests. It validates startup configuration, health, metrics, signed webhook ingestion, chart readiness, and standalone and Redis-backed deployment modes. | `make system-smoke`, `make container-security` | ## Pull request expectations @@ -20,7 +20,7 @@ Every pull request should keep the Fast PR lane and Medium Redis lane green. Tho - In-process HTTP webhook-to-metrics behavior. - Redis-backed delivery deduplication and run/job state behavior. -The Deep container lane runs after merge to `main` and on the weekly scheduled workflow. It catches packaging and image-scan issues without making every pull request wait on Docker image work. +The Deep container and system lane runs after merge to `main` and on the weekly scheduled workflow. It catches packaging, deployment, and image-scan issues without making every pull request wait on Docker and kind work. ## Local verification @@ -34,6 +34,21 @@ make security `make test-all` already includes the unit, integration, Redis integration, coverage, security, and lint targets. Run `make lint` and `make security` separately as explicit final checks before pushing. -## Future system smoke tests +## System smoke tests -Black-box container and Helm/deployment smoke tests are tracked separately in issue #56. When those tests exist, they should join the Deep container lane or a dedicated deep system lane rather than the Fast PR lane. +Run the complete black-box suite with: + +```bash +make system-smoke +``` + +The suite requires Docker, Helm, kind, kubectl, curl, and OpenSSL. It builds the production image and then: + +- starts the container from the outside and checks `/health` and `/metrics`; +- submits a signed push webhook and waits for its exported metric; +- confirms startup fails when `PROMGITHUB_WEBHOOK_SECRET` is missing; +- lints and renders the Helm chart, including its bundled Redis configuration; +- installs the chart into a disposable kind cluster and checks pod readiness and endpoints; +- upgrades the deployment to use a real external Redis instance and repeats the webhook-to-metric check. + +The scripts delete their temporary container and kind cluster on exit. Override `SMOKE_IMAGE`, `SMOKE_KIND_CLUSTER`, or `SMOKE_LOCAL_PORT` when local names or ports conflict. diff --git a/helm/promgithub/Chart.lock b/helm/promgithub/Chart.lock new file mode 100644 index 0000000..08114c3 --- /dev/null +++ b/helm/promgithub/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: redis + repository: oci://registry-1.docker.io/bitnamicharts + version: 23.1.1 +digest: sha256:e6ffcd1baccb5e1058c8a98af891ebedfdd84733e0f684db5be12550c9f6752f +generated: "2026-08-22T18:44:12.271939-07:00" diff --git a/helm/promgithub/Chart.yaml b/helm/promgithub/Chart.yaml index ec058af..e284307 100644 --- a/helm/promgithub/Chart.yaml +++ b/helm/promgithub/Chart.yaml @@ -14,5 +14,5 @@ appVersion: "0.0.7" dependencies: - name: redis version: 0.0.7 - repository: "https://charts.bitnami.com/bitnami" + repository: "oci://registry-1.docker.io/bitnamicharts" condition: redis.enabled diff --git a/helm/promgithub/values.yaml b/helm/promgithub/values.yaml index f69931a..fd9ec34 100644 --- a/helm/promgithub/values.yaml +++ b/helm/promgithub/values.yaml @@ -65,11 +65,11 @@ resources: {} # This is to setup the liveness and readiness probes livenessProbe: httpGet: - path: / + path: /health port: http readinessProbe: httpGet: - path: / + path: /health port: http # This section is for setting up autoscaling diff --git a/test/system/container-smoke.sh b/test/system/container-smoke.sh new file mode 100755 index 0000000..3edb272 --- /dev/null +++ b/test/system/container-smoke.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +# shellcheck source=smoke-lib.sh +source "${SCRIPT_DIR}/smoke-lib.sh" + +IMAGE="${SMOKE_IMAGE:-ghcr.io/darthfork/promgithub:0.0.7}" +CONTAINER_NAME="promgithub-system-smoke-${RANDOM}" +FAILURE_CONTAINER_NAME="${CONTAINER_NAME}-missing-config" +LOG_FILE="$(mktemp)" + +cleanup() { + docker rm --force "$CONTAINER_NAME" "$FAILURE_CONTAINER_NAME" >/dev/null 2>&1 || true + rm -f "$LOG_FILE" +} +trap cleanup EXIT + +cd "$REPO_ROOT" + +printf 'Starting black-box container smoke test for %s\n' "$IMAGE" +docker run --detach --name "$CONTAINER_NAME" --publish 127.0.0.1::8080 \ + --env "PROMGITHUB_WEBHOOK_SECRET=${SMOKE_SECRET}" \ + "$IMAGE" >/dev/null + +HOST_PORT="$(docker inspect --format '{{(index (index .NetworkSettings.Ports "8080/tcp") 0).HostPort}}' "$CONTAINER_NAME")" +assert_service_smoke "http://127.0.0.1:${HOST_PORT}" "container-smoke-${RANDOM}" + +printf 'Verifying startup rejects missing required configuration\n' +if docker run --name "$FAILURE_CONTAINER_NAME" "$IMAGE" >"$LOG_FILE" 2>&1; then + fail 'container started without PROMGITHUB_WEBHOOK_SECRET' +fi + +grep --fixed-strings --quiet 'PROMGITHUB_WEBHOOK_SECRET is not set' "$LOG_FILE" \ + || fail 'missing-secret startup failure did not explain the invalid configuration' + +printf 'Container smoke test passed\n' diff --git a/test/system/helm-smoke.sh b/test/system/helm-smoke.sh new file mode 100755 index 0000000..fcdc9c5 --- /dev/null +++ b/test/system/helm-smoke.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +# shellcheck source=smoke-lib.sh +source "${SCRIPT_DIR}/smoke-lib.sh" + +IMAGE="${SMOKE_IMAGE:-ghcr.io/darthfork/promgithub:0.0.7}" +CLUSTER_NAME="${SMOKE_KIND_CLUSTER:-promgithub-smoke-${RANDOM}}" +NAMESPACE="${SMOKE_NAMESPACE:-promgithub-smoke}" +RELEASE_NAME="${SMOKE_RELEASE:-promgithub-smoke}" +LOCAL_PORT="${SMOKE_LOCAL_PORT:-18080}" +PORT_FORWARD_PID="" +PORT_FORWARD_LOG="$(mktemp)" +CLUSTER_CREATED=false + +start_port_forward() { + : >"$PORT_FORWARD_LOG" + kubectl --namespace "$NAMESPACE" port-forward \ + "service/${RELEASE_NAME}" "${LOCAL_PORT}:8080" >"$PORT_FORWARD_LOG" 2>&1 & + PORT_FORWARD_PID=$! + + for ((attempt = 1; attempt <= 30; attempt++)); do + kill -0 "$PORT_FORWARD_PID" >/dev/null 2>&1 \ + || fail "port-forward exited before the service became reachable: $(cat "$PORT_FORWARD_LOG")" + if curl --silent --show-error --fail "http://127.0.0.1:${LOCAL_PORT}/health" >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + + fail "timed out waiting for port-forwarded service health" +} + +stop_port_forward() { + [[ -n "$PORT_FORWARD_PID" ]] || return 0 + kill "$PORT_FORWARD_PID" >/dev/null 2>&1 || true + wait "$PORT_FORWARD_PID" >/dev/null 2>&1 || true + PORT_FORWARD_PID="" +} + +cluster_diagnostics() { + [[ "$CLUSTER_CREATED" == true ]] || return 0 + + printf '\nHelm smoke failure diagnostics\n' >&2 + kubectl --namespace "$NAMESPACE" get pods,services,endpoints,deployments >&2 || true + kubectl --namespace "$NAMESPACE" describe pods >&2 || true + kubectl --namespace "$NAMESPACE" logs \ + --selector "app.kubernetes.io/instance=${RELEASE_NAME}" \ + --all-containers --tail=100 >&2 || true + kubectl --namespace "$NAMESPACE" get events --sort-by=.lastTimestamp >&2 || true + if [[ -s "$PORT_FORWARD_LOG" ]]; then + printf '\nPort-forward output\n' >&2 + cat "$PORT_FORWARD_LOG" >&2 + fi +} + +cleanup() { + local status=$? + trap - EXIT + + if ((status != 0)); then + cluster_diagnostics + fi + stop_port_forward + if [[ "$CLUSTER_CREATED" == true ]]; then + kind delete cluster --name "$CLUSTER_NAME" >/dev/null 2>&1 || true + fi + rm -f "$PORT_FORWARD_LOG" + exit "$status" +} +trap cleanup EXIT + +cd "$REPO_ROOT" + +for command in curl docker helm kind kubectl openssl; do + command -v "$command" >/dev/null || fail "required command not found: ${command}" +done + +printf 'Linting and rendering Helm chart\n' +helm dependency build helm/promgithub +helm lint helm/promgithub \ + --set secrets.github_webhook_secret="$SMOKE_SECRET" \ + --set autoscaling.enabled=false +helm template "$RELEASE_NAME" helm/promgithub \ + --namespace "$NAMESPACE" \ + --set secrets.github_webhook_secret="$SMOKE_SECRET" \ + --set autoscaling.enabled=false \ + --set redis.enabled=true \ + --set redis.auth.password=system-smoke-redis >/dev/null + +printf 'Creating kind cluster and loading %s\n' "$IMAGE" +kind create cluster --name "$CLUSTER_NAME" --wait 120s +CLUSTER_CREATED=true +kind load docker-image "$IMAGE" --name "$CLUSTER_NAME" + +printf 'Installing standalone Helm deployment\n' +helm install "$RELEASE_NAME" helm/promgithub \ + --namespace "$NAMESPACE" \ + --create-namespace \ + --set image.repository="${IMAGE%:*}" \ + --set image.tag="${IMAGE##*:}" \ + --set image.pullPolicy=Never \ + --set secrets.github_webhook_secret="$SMOKE_SECRET" \ + --set autoscaling.enabled=false \ + --wait \ + --timeout 3m + +kubectl --namespace "$NAMESPACE" rollout status "deployment/${RELEASE_NAME}" --timeout=120s +start_port_forward +assert_service_smoke "http://127.0.0.1:${LOCAL_PORT}" "helm-standalone-${RANDOM}" + +printf 'Upgrading Helm deployment to external Redis-backed mode\n' +kubectl --namespace "$NAMESPACE" create deployment redis --image=redis:7-alpine +kubectl --namespace "$NAMESPACE" expose deployment redis --port=6379 +kubectl --namespace "$NAMESPACE" rollout status deployment/redis --timeout=120s + +helm upgrade "$RELEASE_NAME" helm/promgithub \ + --namespace "$NAMESPACE" \ + --reuse-values \ + --set redisConfig.addr=redis:6379 \ + --wait \ + --timeout 3m + +kubectl --namespace "$NAMESPACE" rollout status "deployment/${RELEASE_NAME}" --timeout=120s +stop_port_forward +start_port_forward +assert_service_smoke "http://127.0.0.1:${LOCAL_PORT}" "helm-redis-${RANDOM}" + +printf 'Helm deployment smoke test passed\n' diff --git a/test/system/smoke-lib.sh b/test/system/smoke-lib.sh new file mode 100755 index 0000000..484bde2 --- /dev/null +++ b/test/system/smoke-lib.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SMOKE_SECRET="${SMOKE_SECRET:-system-smoke-secret}" +SMOKE_REPOSITORY="${SMOKE_REPOSITORY:-user/repo}" + +fail() { + printf 'system smoke test failed: %s\n' "$*" >&2 + exit 1 +} + +wait_for_http() { + local url="$1" + local attempts="${2:-60}" + + for ((attempt = 1; attempt <= attempts; attempt++)); do + if curl --silent --show-error --fail "$url" >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + + fail "timed out waiting for ${url}" +} + +wait_for_metric() { + local url="$1" + local metric="$2" + local attempts="${3:-30}" + + for ((attempt = 1; attempt <= attempts; attempt++)); do + if curl --silent --show-error --fail "$url" | grep --fixed-strings --quiet "$metric"; then + return 0 + fi + sleep 1 + done + + fail "metric did not appear: ${metric}" +} + +signature_for() { + local payload="$1" + printf 'sha256=%s' "$(openssl dgst -sha256 -hmac "$SMOKE_SECRET" "$payload" | awk '{print $NF}')" +} + +assert_service_smoke() { + local base_url="$1" + local delivery_id="$2" + local payload="${3:-test_data/push.json}" + local health + local signature + + wait_for_http "${base_url}/health" + + health="$(curl --silent --show-error --fail "${base_url}/health")" + grep --fixed-strings --quiet '"status":"ok"' <<<"$health" || fail "unexpected health response: ${health}" + + curl --silent --show-error --fail "${base_url}/metrics" \ + | grep --fixed-strings --quiet 'promgithub_api_calls_total' \ + || fail "metrics endpoint did not expose service metrics" + + signature="$(signature_for "$payload")" + curl --silent --show-error --fail \ + --request POST \ + --header 'Content-Type: application/json' \ + --header 'X-GitHub-Event: push' \ + --header "X-GitHub-Delivery: ${delivery_id}" \ + --header "X-Hub-Signature-256: ${signature}" \ + --data-binary "@${payload}" \ + "${base_url}/webhook" >/dev/null + + wait_for_metric \ + "${base_url}/metrics" \ + "promgithub_commit_pushed{repository=\"${SMOKE_REPOSITORY}\"} 1" +} From 8e24d0bbf27f5f5568784073932c3bafd946a3e3 Mon Sep 17 00:00:00 2001 From: Abhishek Rai Date: Sat, 22 Aug 2026 19:06:24 -0700 Subject: [PATCH 2/2] Bump Go toolchain to 1.25.13 for stdlib security fixes. govulncheck fails the Fast PR lane on four reachable standard-library CVEs in 1.25.11; 1.25.13 is the first patch that remediates all of them. Co-authored-by: Cursor --- .go-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.go-version b/.go-version index 4fd1625..d7fe375 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.25.11 +1.25.13