From 86b5a03557dea8fb0f0a07f0dffac31ff4ba825e Mon Sep 17 00:00:00 2001 From: Rayhan Hossain Date: Tue, 7 Jul 2026 13:34:39 -0700 Subject: [PATCH] feat: add secret and exposure scanning for PRs Signed-off-by: Rayhan Hossain --- .github/copilot-review-instructions.md | 15 +++ .github/workflows/secret-scan.yml | 135 +++++++++++++++++++++++++ .gitleaks.toml | 78 ++++++++++++++ .pre-commit-config.yaml | 24 +++++ scripts/security/test-secret-scan.sh | 112 ++++++++++++++++++++ 5 files changed, 364 insertions(+) create mode 100644 .github/workflows/secret-scan.yml create mode 100644 .gitleaks.toml create mode 100644 .pre-commit-config.yaml create mode 100755 scripts/security/test-secret-scan.sh diff --git a/.github/copilot-review-instructions.md b/.github/copilot-review-instructions.md index 64eee5845..b726a2848 100644 --- a/.github/copilot-review-instructions.md +++ b/.github/copilot-review-instructions.md @@ -7,6 +7,21 @@ These instructions guide GitHub Copilot's automated pull request reviews for the - Use the severity levels defined in `.github/copilot-instructions.md`: 🔴 Critical, 🟠 Major, 🟡 Minor, 🟢 Nitpick. - Focus on correctness, security, and maintainability. Don't flag purely stylistic preferences. +## Security & secret-leak review + +Treat any of the following as 🔴 **Critical** and request changes so the PR is not merged until resolved. This AI review is a broad, contextual second layer; the deterministic `secret-scan` workflow (gitleaks) is the enforced gate. Flag anything the scanner might miss, and confirm findings the scanner reports. + +- **Hardcoded credentials:** passwords, API keys, tokens, or secrets assigned to a variable, env var, YAML value, or CLI flag. This includes base64-encoded secret values committed in `Secret` manifests (real values must come from a secret store, not the repo). +- **Private keys & certificates:** any `-----BEGIN ... PRIVATE KEY-----` block, `.pem`/`.key`/`.pfx`/`.p12` files, or embedded TLS private material. Public certs/CA bundles are fine; private keys are not. +- **Credentialed connection strings:** URIs embedding a username and password, e.g. `mongodb://user:pass@host`, `postgres://…`, or `redis://…`. +- **DocumentDB connection strings:** a full DocumentDB/Cosmos Mongo connection string with real credentials — e.g. `mongodb://:@:10260/…` (gateway port), `mongodb+srv://:@.mongocluster.cosmos.azure.com/…`, or a hardcoded value for the CRD `status.connectionString`. These must be read from a Secret at runtime, not committed. Ignore obvious placeholders (`username:password`, `user:pass`, `<...>`). +- **Azure Application Insights keys:** a bare `InstrumentationKey` GUID, a full App Insights connection string (`InstrumentationKey=;IngestionEndpoint=https://…`), or a value assigned to `APPLICATIONINSIGHTS_CONNECTION_STRING` / `APPINSIGHTS_INSTRUMENTATIONKEY`. These must come from a Secret or config store, never be committed inline. +- **Azure subscription IDs:** a subscription GUID assigned to a `subscriptionId` / `AZURE_SUBSCRIPTION_ID` / `ARM_SUBSCRIPTION_ID` field or embedded in an ARM resource ID (`/subscriptions//…`). Flag hardcoded values; they should be parameterized via env vars, variables, or secrets rather than committed. +- **Accidental public exposure:** Kubernetes `Service` with `type: LoadBalancer` (or a public IP / external DNS annotation) that lacks an internal-LB annotation (`azure-load-balancer-internal: "true"`, `aws-load-balancer-internal`, `networking.gke.io/load-balancer-type: "Internal"`). Confirm whether public exposure is intentional and called out in the PR description; if not, request changes. +- **Overly permissive settings:** `0.0.0.0/0` ingress/allowlists, disabled TLS verification, wildcard RBAC (`verbs: ["*"]` on `resources: ["*"]`), or debug/insecure flags left enabled. + +When a match appears in `test/`, `e2e/`, `examples/`, `documentdb-playground/`, or docs and is clearly a placeholder (`changeme`, `example`, `<...>`, `${VAR}`), do not flag it. When in doubt, comment asking the author to confirm the value is not a real secret rather than staying silent. + ## Code reviews For the full code review checklist — including Kubernetes operator patterns, security, performance, and testing standards — see [`.github/agents/code-review-agent.md`](agents/code-review-agent.md). diff --git a/.github/workflows/secret-scan.yml b/.github/workflows/secret-scan.yml new file mode 100644 index 000000000..ac4287801 --- /dev/null +++ b/.github/workflows/secret-scan.yml @@ -0,0 +1,135 @@ +name: secret-scan + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + config-selftest: + name: gitleaks config self-test + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Install gitleaks + run: | + set -euo pipefail + GL_VER=8.21.2 + curl -sSL "https://github.com/gitleaks/gitleaks/releases/download/v${GL_VER}/gitleaks_${GL_VER}_linux_x64.tar.gz" \ + | tar xz gitleaks + sudo mv gitleaks /usr/local/bin/gitleaks + gitleaks version + + - name: Verify config detects known leak patterns + run: scripts/security/test-secret-scan.sh + + gitleaks: + name: gitleaks (secrets, keys, certs) + runs-on: ubuntu-22.04 + permissions: + contents: read + # Allows uploading findings to the GitHub Security > Code scanning tab. + security-events: write + steps: + - uses: actions/checkout@v4 + with: + # Full history so PR commits can be diffed for leaks that were + # added and later "removed" within the same branch. + fetch-depth: 0 + persist-credentials: false + + - name: Install gitleaks + run: | + set -euo pipefail + GL_VER=8.21.2 + curl -sSL "https://github.com/gitleaks/gitleaks/releases/download/v${GL_VER}/gitleaks_${GL_VER}_linux_x64.tar.gz" \ + | tar xz gitleaks + sudo mv gitleaks /usr/local/bin/gitleaks + gitleaks version + + # Uses the MIT-licensed gitleaks CLI directly (NOT gitleaks-action, which + # requires a paid GITLEAKS_LICENSE secret for GitHub organizations). + # Scans only the commits introduced by this PR/push so that pre-existing + # findings in git history don't block unrelated changes. + - name: Run gitleaks + id: gitleaks + run: | + set +e + LOGOPTS="" + if [ "${{ github.event_name }}" = "pull_request" ]; then + git fetch --no-tags --depth=1 origin "${{ github.base_ref }}" || git fetch --no-tags origin "${{ github.base_ref }}" + LOGOPTS="--log-opts=origin/${{ github.base_ref }}..HEAD" + elif [ -n "${{ github.event.before }}" ] && [ "${{ github.event.before }}" != "0000000000000000000000000000000000000000" ]; then + LOGOPTS="--log-opts=${{ github.event.before }}..${{ github.sha }}" + fi + echo "Scanning range: ${LOGOPTS:-}" + gitleaks detect \ + --source . \ + --config .gitleaks.toml \ + --redact \ + --report-format sarif \ + --report-path gitleaks.sarif \ + --exit-code 1 \ + --verbose \ + $LOGOPTS + echo "exit_code=$?" >> "$GITHUB_OUTPUT" + + - name: Upload SARIF to code scanning + if: always() && hashFiles('gitleaks.sarif') != '' + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: gitleaks.sarif + category: gitleaks + + - name: Fail if secrets were found + if: steps.gitleaks.outputs.exit_code != '0' + run: | + echo "::error::gitleaks detected potential secrets. See the job log and the Security > Code scanning tab." + exit 1 + + loadbalancer-exposure: + name: public LoadBalancer / external exposure check + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Scan for accidental public LoadBalancer exposure + shell: bash + run: | + set -euo pipefail + echo "Scanning YAML/manifests for public Service type: LoadBalancer..." + + # Find manifests that expose a public LoadBalancer WITHOUT an + # internal-LB annotation. Adjust the allowlist paths as needed. + mapfile -t files < <(git ls-files '*.yaml' '*.yml' \ + | grep -v -E '(^|/)(test|tests|e2e|examples?|documentdb-playground)/' || true) + + violations=0 + for f in "${files[@]}"; do + [ -f "$f" ] || continue + if grep -Eq '^\s*type:\s*LoadBalancer\s*$' "$f"; then + # Allow if it is explicitly marked internal (Azure/AWS/GCP annotations). + if grep -Eq 'azure-load-balancer-internal:\s*"?true"?|aws-load-balancer-internal|networking.gke.io/load-balancer-type:\s*"?Internal"?' "$f"; then + echo "OK (internal): $f" + else + echo "::error file=$f::Service type LoadBalancer without an internal-LB annotation (potential public exposure)" + violations=$((violations+1)) + fi + fi + done + + if [ "$violations" -gt 0 ]; then + echo "Found $violations potential public LoadBalancer exposure(s)." + echo "If intentional, add the file/path to the allowlist in this workflow." + exit 1 + fi + echo "No unexpected public LoadBalancer exposure found." diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 000000000..881b46dd9 --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,78 @@ +# Gitleaks configuration for the DocumentDB Kubernetes Operator. +# Extends the built-in ruleset (API keys, cloud creds, tokens, PEM keys, etc.) +# and adds project-specific rules for passwords and certificates. +# +# Run locally: gitleaks detect --config .gitleaks.toml --redact -v +# Scan staged: gitleaks protect --staged --config .gitleaks.toml --redact -v + +[extend] +useDefault = true + +[[rules]] +id = "generic-password-assignment" +description = "Hardcoded password / secret assignment" +regex = '''(?i)(password|passwd|pwd|secret|token|apikey|api_key)\s*[:=]\s*['"][^'"\s]{6,}['"]''' +keywords = ["password", "passwd", "pwd", "secret", "token", "apikey", "api_key"] + +[[rules]] +id = "private-key-block" +description = "PEM private key / certificate private material" +regex = '''-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY-----''' +keywords = ["PRIVATE KEY"] + +[[rules]] +id = "connection-string-with-credentials" +description = "Connection string embedding credentials (e.g. mongodb://user:pass@host)" +regex = '''(?i)(mongodb|postgres|postgresql|amqp|redis|https?)://[^:\s]+:[^@\s]+@''' +keywords = ["mongodb://", "postgres://", "postgresql://", "redis://"] + +[[rules]] +id = "documentdb-connection-string" +description = "DocumentDB / Cosmos Mongo connection string with embedded credentials (gateway port 10260, mongodb+srv, or *.mongocluster.cosmos.azure.com)" +regex = '''(?i)mongodb(\+srv)?://[^:\s/'"]+:[^@\s/'"]+@[^\s/'"]*(:10260|\.mongocluster\.cosmos\.azure\.com)''' +keywords = ["mongodb://", "mongodb+srv://", "10260", "mongocluster.cosmos.azure.com"] + +[[rules]] +id = "azure-appinsights-instrumentation-key" +description = "Azure Application Insights instrumentation key (GUID)" +regex = '''(?i)instrumentationkey\s*[:=]\s*['"]?[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}['"]?''' +keywords = ["instrumentationkey"] + +[[rules]] +id = "azure-appinsights-connection-string" +description = "Azure Application Insights connection string (InstrumentationKey + IngestionEndpoint)" +regex = '''(?i)InstrumentationKey=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12};.*IngestionEndpoint=https?://[^\s'";]+''' +keywords = ["instrumentationkey", "ingestionendpoint"] + +[[rules]] +id = "azure-appinsights-connstring-env" +description = "Application Insights connection string in APPLICATIONINSIGHTS_CONNECTION_STRING" +regex = '''(?i)APPLICATIONINSIGHTS_CONNECTION_STRING\s*[:=]\s*['"]?InstrumentationKey=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}''' +keywords = ["applicationinsights_connection_string"] + +[[rules]] +id = "azure-subscription-id" +description = "Azure subscription ID (GUID assigned to a subscription field or in a resource ID)" +regex = '''(?i)(subscription[_-]?id|subscriptionid|azure_subscription_id|arm_subscription_id|/subscriptions/)\s*['":=/]{0,3}\s*['"]?[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}['"]?''' +keywords = ["subscription_id", "subscriptionid", "arm_subscription_id", "azure_subscription_id", "/subscriptions/"] + +[allowlist] +description = "Paths and patterns that are safe to ignore" +# Match the regexes below against the full line, not just the extracted secret, +# so context-prefixed placeholders (e.g. password = "changeme") are recognized. +regexTarget = "line" +paths = [ + '''(^|/)test(s)?/''', + '''(^|/)e2e/''', + '''(^|/)examples?/''', + '''(^|/)documentdb-playground/''', + '''\.md$''', +] +# Common non-secret placeholders used in docs/tests. +regexes = [ + '''(?i)(password|secret|token)\s*[:=]\s*['"]?(changeme|example|placeholder|redacted|xxx+|<[^>]+>|\$\{[^}]+\})['"]?''', + # Placeholder / empty instrumentation-key GUIDs used in docs and samples. + '''(?i)instrumentationkey\s*[:=]\s*['"]?0{8}-0{4}-0{4}-0{4}-0{12}['"]?''', + # Placeholder credentials in example connection strings (username:password, user:pass, <...>). + '''(?i)mongodb(\+srv)?://(<[^>]+>|username|user):(<[^>]+>|password|pass)@''', +] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..4579a8906 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,24 @@ +# Local pre-commit hooks. Install once per clone: +# +# pip install pre-commit # or: brew install pre-commit +# pre-commit install # enables the git commit hook +# +# Run against everything on demand: +# +# pre-commit run --all-files +# +# This blocks a commit locally before secrets ever reach a PR. +repos: + - repo: https://github.com/gitleaks/gitleaks + rev: v8.21.2 + hooks: + - id: gitleaks + name: Detect hardcoded secrets, keys and certs + args: ["protect", "--staged", "--redact", "--config", ".gitleaks.toml"] + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: detect-private-key + - id: detect-aws-credentials + args: ["--allow-missing-credentials"] diff --git a/scripts/security/test-secret-scan.sh b/scripts/security/test-secret-scan.sh new file mode 100755 index 000000000..d52a35ad3 --- /dev/null +++ b/scripts/security/test-secret-scan.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# +# Verifies that the gitleaks configuration in .gitleaks.toml actually detects +# the secret types we care about (and ignores known placeholders). +# +# It works by generating throwaway fixture files containing FAKE secrets in a +# temp directory, running gitleaks against them, and asserting the expected +# rules fire. No real or fixture secrets are ever committed to the repo. +# +# Usage: +# scripts/security/test-secret-scan.sh # uses gitleaks on PATH +# GITLEAKS_BIN=/tmp/gitleaks scripts/security/test-secret-scan.sh +# +# Exit code 0 = all assertions passed. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +CONFIG="${REPO_ROOT}/.gitleaks.toml" +GITLEAKS_BIN="${GITLEAKS_BIN:-gitleaks}" + +if ! command -v "${GITLEAKS_BIN}" >/dev/null 2>&1; then + echo "ERROR: gitleaks not found (set GITLEAKS_BIN or install gitleaks)." >&2 + exit 2 +fi +if [ ! -f "${CONFIG}" ]; then + echo "ERROR: config not found at ${CONFIG}" >&2 + exit 2 +fi + +WORK="$(mktemp -d)" +trap 'rm -rf "${WORK}"' EXIT + +# ---- Fixtures that MUST be flagged (fake but pattern-valid) ----------------- +cat > "${WORK}/leaks.txt" <<'EOF' +password = "sup3rSecretValue" +mongodb://admin:hunter2@db.internal:27017/app +-----BEGIN RSA PRIVATE KEY----- +MIIEvfakekeymaterialdoesnotmatterforregexmatching +-----END RSA PRIVATE KEY----- +InstrumentationKey=11111111-2222-3333-4444-555555555555 +APPLICATIONINSIGHTS_CONNECTION_STRING=InstrumentationKey=11111111-2222-3333-4444-555555555555;IngestionEndpoint=https://eastus.in.applicationinsights.azure.com/ +subscriptionId: 99999999-8888-7777-6666-555555555555 +/subscriptions/12345678-90ab-cdef-1234-567890abcdef/resourceGroups/rg +mongodb://docdbadmin:R3alSecret!@10.0.0.5:10260/?tls=true +mongodb+srv://docdbadmin:R3alSecret!@my-cluster.mongocluster.cosmos.azure.com/ +EOF + +# ---- Fixtures that must NOT be flagged (placeholders) ----------------------- +cat > "${WORK}/placeholders.txt" <<'EOF' +password = "changeme" +password: ${DB_PASSWORD} +InstrumentationKey=00000000-0000-0000-0000-000000000000 +mongodb://username:password@:10260/ +mongodb+srv://user:pass@my-cluster.mongocluster.cosmos.azure.com/ +EOF + +REPORT="${WORK}/report.json" +set +e +"${GITLEAKS_BIN}" detect \ + --source "${WORK}" \ + --config "${CONFIG}" \ + --no-git \ + --report-format json \ + --report-path "${REPORT}" \ + --redact >/dev/null 2>&1 +set -e + +if [ ! -s "${REPORT}" ]; then + echo "FAIL: gitleaks produced no findings; expected several." >&2 + exit 1 +fi + +fail=0 + +assert_rule_present() { + local rule="$1" + if grep -q "\"RuleID\": *\"${rule}\"" "${REPORT}"; then + echo "PASS detected: ${rule}" + else + echo "FAIL missing: ${rule}" >&2 + fail=1 + fi +} + +assert_no_finding_in() { + # Assert no finding references the placeholder file. + local file="$1" + if grep -q "placeholders.txt" "${REPORT}"; then + echo "FAIL placeholder in ${file} was flagged (false positive)" >&2 + fail=1 + else + echo "PASS placeholders ignored" + fi +} + +assert_rule_present "generic-password-assignment" +assert_rule_present "private-key-block" +assert_rule_present "connection-string-with-credentials" +assert_rule_present "azure-appinsights-instrumentation-key" +assert_rule_present "azure-appinsights-connection-string" +assert_rule_present "azure-subscription-id" +assert_rule_present "documentdb-connection-string" +assert_no_finding_in "placeholders.txt" + +if [ "${fail}" -ne 0 ]; then + echo "" >&2 + echo "Secret-scan verification FAILED. Review .gitleaks.toml rules." >&2 + exit 1 +fi + +echo "" +echo "All secret-scan assertions passed."