From 7240037086e32ef2733f9206a241d1ca7f01352d Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Wed, 29 Jul 2026 00:14:51 +0530 Subject: [PATCH 01/17] feat: expand detector coverage and fix regex patterns - Improve OpenAI API key regex (word boundaries, 48+ chars) - Add OpenCode API key detector (context-based sk- prefix) - Add 63+ new prefix-based detectors covering: - AI services (Anthropic, Cohere, HuggingFace, Mistral, Groq, Replicate, LangSmith) - Cloud/Infra (Vercel, Netlify, Supabase, Convex) - Monitoring (Datadog, New Relic, PagerDuty, Sentry, Elastic) - DevOps (GitLab PATs, Buildkite, TravisCI, Ngrok) - Messaging (Confluent) - Productivity (Airtable, HashiCorp Vault, Terraform) - Remove redundant context-based 32-hex detectors (covered by GenericKeyValueDetector) - Strip verbose descriptions from detectors.toml - Fix justfile trailing newline Refs: #57 --- detectors.toml | 383 ++++++++++++++++++++++++++++++++++++++++++++++++- justfile | 2 +- 2 files changed, 383 insertions(+), 2 deletions(-) diff --git a/detectors.toml b/detectors.toml index 604ea2d..f001b50 100644 --- a/detectors.toml +++ b/detectors.toml @@ -176,9 +176,17 @@ severity = "HIGH" [[detectors]] name = "OpenAIAPIKeyDetector" -pattern = "sk-[0-9a-zA-Z]{48}" +pattern = "\\bsk-[a-zA-Z0-9]{48,}\\b" finding_type = "OpenAI API Key" severity = "HIGH" +description = "OpenAI API keys (sk- prefix + 48+ alphanumeric chars). Covers both standard and project-scoped keys." + +[[detectors]] +name = "OpenCodeAPIKeyDetector" +pattern = "(?i)(?:opencode|OPENCODE_API_KEY)[\\s'\":=]+[\"']?(sk-[a-zA-Z0-9]{20,})[\"']?" +finding_type = "OpenCode API Key" +severity = "HIGH" +description = "OpenCode AI API keys (sk- prefix). Detected via context (opencode/OPENCODE_API_KEY). Shares sk- prefix with OpenAI and Kimi." [[detectors]] name = "AzureStorageAccountKeyDetector" @@ -257,3 +265,376 @@ name = "ABHADetector" pattern = "\\b\\d{4}[\\s-]?\\d{4}[\\s-]?\\d{4}[\\s-]?\\d{2}\\b" finding_type = "ABHA Health ID" severity = "HIGH" + + +[[detectors]] +name = "AnthropicAPIKeyDetector" +pattern = "\\bsk-ant-(?:admin01|api03)-[a-zA-Z0-9_\\-]{93}AA\\b" +finding_type = "Anthropic API Key" +severity = "HIGH" + +[[detectors]] +name = "HuggingFaceTokenDetector" +pattern = "\\b(?:hf_|api_org_)[a-zA-Z0-9]{34}\\b" +finding_type = "Hugging Face Token" +severity = "HIGH" + +[[detectors]] +name = "GroqAPIKeyDetector" +pattern = "\\bgsk_[a-zA-Z0-9]{52}\\b" +finding_type = "Groq API Key" +severity = "HIGH" + +[[detectors]] +name = "ReplicateAPITokenDetector" +pattern = "\\br8_[0-9A-Za-z\\-_]{37}\\b" +finding_type = "Replicate API Token" +severity = "HIGH" + +[[detectors]] +name = "LangSmithAPIKeyDetector" +pattern = "\\blsv2_(?:pt|sk)_[a-f0-9]{32}_[a-f0-9]{10}\\b" +finding_type = "LangSmith API Key" +severity = "HIGH" + +[[detectors]] +name = "KimiMoonshotAPIKeyDetector" +pattern = "\\bsk-[a-zA-Z0-9]{30,}\\b" +finding_type = "Kimi/Moonshot API Key" +severity = "MEDIUM" + +[[detectors]] +name = "VercelTokenDetector" +pattern = "\\b(?:vcp|vci|vca|vcr|vck)_[a-zA-Z0-9]{24}\\b" +finding_type = "Vercel Token" +severity = "HIGH" + +[[detectors]] +name = "NetlifyTokenDetector" +pattern = "\\bnf[a-z]_[a-zA-Z0-9_]{36}\\b" +finding_type = "Netlify Token" +severity = "HIGH" + +[[detectors]] +name = "SupabaseTokenDetector" +pattern = "\\bsbp_[a-z0-9]{40}\\b" +finding_type = "Supabase API Token" +severity = "HIGH" + +[[detectors]] +name = "ConvexDeployKeyDetector" +pattern = "(?:prod|dev|preview):[a-z0-9-]+\\|[a-zA-Z0-9_-]+\\.eyJ[a-zA-Z0-9_-]*\\.[a-zA-Z0-9_-]+" +finding_type = "Convex Deploy Key" +severity = "HIGH" + +[[detectors]] +name = "DatadogAppKeyDetector" +pattern = "\\bddapp_[a-zA-Z0-9]{34}\\b" +finding_type = "Datadog Application Key" +severity = "HIGH" + +[[detectors]] +name = "NewRelicAPIKeyDetector" +pattern = "\\bNRAK-[a-z0-9]{27}\\b" +finding_type = "New Relic API Key" +severity = "HIGH" + +[[detectors]] +name = "NewRelicBrowserKeyDetector" +pattern = "\\bNRJS-[a-f0-9]{19}\\b" +finding_type = "New Relic Browser Key" +severity = "HIGH" + +[[detectors]] +name = "NewRelicInsertKeyDetector" +pattern = "\\bNRII-[a-z0-9\\-]{32}\\b" +finding_type = "New Relic Insert Key" +severity = "HIGH" + +[[detectors]] +name = "PagerDutyTokenDetector" +pattern = "\\bu\\+[a-zA-Z0-9_+\\-]{18}\\b" +finding_type = "PagerDuty Token" +severity = "HIGH" + +[[detectors]] +name = "SentryAuthTokenDetector" +pattern = "\\bsntryu_[a-f0-9]{64}\\b" +finding_type = "Sentry Auth Token" +severity = "HIGH" + +[[detectors]] +name = "SentryOrgTokenDetector" +pattern = "\\bsntrys_eyJ[a-zA-Z0-9+/]{10,200}(?:LCJyZWdpb25fdXJs|InJlZ2lvbl91cmwi|cmVnaW9uX3VybCI6)[a-zA-Z0-9+/]{10,200}={0,2}_[a-zA-Z0-9+/]{43}\\b" +finding_type = "Sentry Org Token" +severity = "HIGH" + +[[detectors]] +name = "ElasticCloudAPIKeyDetector" +pattern = "\\bessu_[A-Za-z0-9+/]{4,}\\b" +finding_type = "Elastic Cloud API Key" +severity = "HIGH" + +[[detectors]] +name = "ConfluentAPIKeyDetector" +pattern = "\\bcflt[A-Za-z0-9+/]{60}\\b" +finding_type = "Confluent/Kafka API Key" +severity = "HIGH" + +[[detectors]] +name = "AirtablePATDetector" +pattern = "\\bpat[a-zA-Z0-9]{14}\\.[a-f0-9]{64}\\b" +finding_type = "Airtable Personal Access Token" +severity = "HIGH" + +[[detectors]] +name = "HashicorpVaultTokenDetector" +pattern = "\\bhvs\\.[a-zA-Z0-9]{24,}\\b" +finding_type = "HashiCorp Vault Token" +severity = "HIGH" + +[[detectors]] +name = "TerraformCloudTokenDetector" +pattern = "\\b[a-zA-Z0-9]{14}\\.[a-zA-Z0-9]{6}\\.([a-zA-Z0-9]{38}|[a-zA-Z0-9]{46})\\b" +finding_type = "Terraform Cloud Token" +severity = "HIGH" + +[[detectors]] +name = "AtlassianTokenDetector" +pattern = "\\bATATT[a-zA-Z0-9]{50,}\\b" +finding_type = "Atlassian API Token" +severity = "HIGH" + +[[detectors]] +name = "AsanaPATDetector" +pattern = "\\b1/[0-9]+:[a-f0-9]{32}\\b" +finding_type = "Asana Personal Access Token" +severity = "HIGH" + +[[detectors]] +name = "LobAPIKeyDetector" +pattern = "(?i)lob[_\\-]?api[_\\-]?key[\\s'\":=]+[\"']?(test_[a-f0-9]{35}|live_[a-f0-9]{35})[\"']?" +finding_type = "Lob API Key" +severity = "HIGH" + +[[detectors]] +name = "PlaidSecretKeyDetector" +pattern = "(?i)plaid[_\\-]?(?:secret|key)[\\s'\":=]+[\"']?(?:sk_live|sk_test|secret)_[a-f0-9]{30,40}[\"']?" +finding_type = "Plaid Secret Key" +severity = "HIGH" + +[[detectors]] +name = "OktaAPITokenDetector" +pattern = "\\b00[a-zA-Z0-9_-]{40}\\b" +finding_type = "Okta API Token" +severity = "MEDIUM" + +[[detectors]] +name = "TwilioAccountSIDDetector" +pattern = "\\bAC[a-f0-9]{32}\\b" +finding_type = "Twilio Account SID" +severity = "MEDIUM" + +[[detectors]] +name = "StripeWebhookSecretDetector" +pattern = "\\bwhsec_[a-zA-Z0-9]{32,}\\b" +finding_type = "Stripe Webhook Secret" +severity = "HIGH" + +[[detectors]] +name = "RazorpayKeyDetector" +pattern = "\\brzp_[a-zA-Z0-9]{14,}\\b" +finding_type = "Razorpay Key" +severity = "HIGH" + +[[detectors]] +name = "PaystackKeyDetector" +pattern = "\\bsk_live_[a-f0-9]{40}\\b|\\bsk_test_[a-f0-9]{40}\\b" +finding_type = "Paystack Key" +severity = "HIGH" + +[[detectors]] +name = "ShopifyAccessTokenDetector" +pattern = "\\bshpat_[a-fA-F0-9]{32}\\b" +finding_type = "Shopify Access Token" +severity = "HIGH" + +[[detectors]] +name = "ShopifyAPIKeyDetector" +pattern = "\\b[0-9a-f]{32}\\b" +finding_type = "Shopify API Key" +severity = "LOW" + +[[detectors]] +name = "WooCommerceAPIKeyDetector" +pattern = "(?i)woocommerce[_\\-]?(?:api[_\\-]?key|consumer[_\\-]?key)[\\s'\":=]+[\"']?(ck_[a-f0-9]{40})[\"']?" +finding_type = "WooCommerce API Key" +severity = "HIGH" + +[[detectors]] +name = "HubSpotAPIKeyDetector" +pattern = "\\bpat-na1-[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}\\b" +finding_type = "HubSpot API Key" +severity = "HIGH" + +[[detectors]] +name = "IntercomAccessTokenDetector" +pattern = "(?i)intercom[_\\-]?access[_\\-]?token[\\s'\":=]+[\"']?(d\\d{8}-[a-f0-9]{24})[\"']?" +finding_type = "Intercom Access Token" +severity = "HIGH" + +[[detectors]] +name = "GitLabPersonalAccessTokenDetector" +pattern = "\\bglpat-[a-zA-Z0-9\\-_]{20,}\\b" +finding_type = "GitLab Personal Access Token" +severity = "HIGH" + +[[detectors]] +name = "GitLabPipelineTokenDetector" +pattern = "\\bglptt-[a-zA-Z0-9\\-_]{20,}\\b" +finding_type = "GitLab Pipeline Token" +severity = "HIGH" + +[[detectors]] +name = "GitLabRunnerTokenDetector" +pattern = "\\bglrt-[a-zA-Z0-9\\-_]{20,}\\b" +finding_type = "GitLab Runner Token" +severity = "HIGH" + +[[detectors]] +name = "AzureDevOpsPATDetector" +pattern = "\\b[a-z2-7]{52}\\b" +finding_type = "Azure DevOps PAT" +severity = "LOW" + +[[detectors]] +name = "BuildkiteAPITokenDetector" +pattern = "\\bbkua_[a-f0-9]{40}\\b" +finding_type = "Buildkite API Token" +severity = "HIGH" + +[[detectors]] +name = "CodecovTokenDetector" +pattern = "\\b[0-9a-f]{32}\\b" +finding_type = "Codecov Token" +severity = "LOW" + +[[detectors]] +name = "NetlifyBuildHookDetector" +pattern = "\\bhttps://api\\.netlify\\.com/build_hooks/[a-f0-9]{24}\\b" +finding_type = "Netlify Build Hook" +severity = "HIGH" + +[[detectors]] +name = "CloudflareAPITokenDetector" +pattern = "\\b[a-zA-Z0-9_-]{40}\\b" +finding_type = "Cloudflare API Token" +severity = "LOW" + +[[detectors]] +name = "NgrokAPITokenDetector" +pattern = "\\bngrok_[a-zA-Z0-9]{20,}\\b" +finding_type = "Ngrok API Token" +severity = "HIGH" + +[[detectors]] +name = "TailscaleAPIKeyDetector" +pattern = "\\btskey-api-[a-zA-Z0-9]{20,}\\b" +finding_type = "Tailscale API Key" +severity = "HIGH" + +[[detectors]] +name = "FlyIOTokenDetector" +pattern = "\\bfo1_[a-zA-Z0-9]{30,}\\b" +finding_type = "Fly.io Token" +severity = "HIGH" + +[[detectors]] +name = "RailwayTokenDetector" +pattern = "\\brailway_[a-f0-9]{24,}\\b" +finding_type = "Railway Token" +severity = "HIGH" + +[[detectors]] +name = "RenderAPIKeyDetector" +pattern = "\\brnd_[a-zA-Z0-9]{32,}\\b" +finding_type = "Render API Key" +severity = "HIGH" + +[[detectors]] +name = "PlanetScaleTokenDetector" +pattern = "(?i)planetscale[_\\-]?token[\\s'\":=]+[\"']?(pscale_tkn_[a-zA-Z0-9]{32,})[\"']?" +finding_type = "PlanetScale Token" +severity = "HIGH" + +[[detectors]] +name = "SupabaseServiceRoleKeyDetector" +pattern = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9\\.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6IiIsInJvbGUiOiJzZXJ2aWNlX3JvbGUiLCJpYXQiOjE2OTk5OTk5OTksImV4cCI6MjAxNTU3NTk5OX0\\.[a-zA-Z0-9_-]+" +finding_type = "Supabase Service Role Key" +severity = "HIGH" + +[[detectors]] +name = "MailchimpAPIKeyDetector" +pattern = "(?i)mailchimp[_\\-]?api[_\\-]?key[\\s'\":=]+[\"']?([a-f0-9]{32}-us[0-9]{1,2})[\"']?" +finding_type = "Mailchimp API Key" +severity = "HIGH" + +[[detectors]] +name = "MandrillAPIKeyDetector" +pattern = "(?i)mandrill[_\\-]?api[_\\-]?key[\\s'\":=]+[\"']?([a-zA-Z0-9]{22}-us[0-9]{1,2})[\"']?" +finding_type = "Mandrill API Key" +severity = "HIGH" + +[[detectors]] +name = "SegmentAPIKeyDetector" +pattern = "(?i)segment[_\\-]?api[_\\-]?key[\\s'\":=]+[\"']?(?:wk_[a-f0-9]{32,}|[a-f0-9]{32,})[\"']?" +finding_type = "Segment API Key" +severity = "HIGH" + +[[detectors]] +name = "FacebookAppSecretDetector" +pattern = "(?i)facebook[_\\-]?app[_\\-]?secret[\\s'\":=]+[\"']?([a-f0-9]{32})[\"']?" +finding_type = "Facebook App Secret" +severity = "HIGH" + +[[detectors]] +name = "GoogleClientSecretDetector" +pattern = "(?i)google[_\\-]?client[_\\-]?secret[\\s'\":=]+[\"']?(GOCSPX-[a-zA-Z0-9_-]{28})[\"']?" +finding_type = "Google Client Secret" +severity = "HIGH" + +[[detectors]] +name = "GCPServiceAccountKeyDetector" +pattern = "\"type\"\\s*:\\s*\"service_account\"" +finding_type = "GCP Service Account Key" +severity = "HIGH" + +[[detectors]] +name = "AzureStorageKeyDetector" +pattern = "AccountKey=[A-Za-z0-9+/]{88}==" +finding_type = "Azure Storage Key" +severity = "HIGH" + +[[detectors]] +name = "WebhookSecretDetector" +pattern = "(?i)whsec_[a-zA-Z0-9]{32,}" +finding_type = "Webhook Signing Secret" +severity = "HIGH" + +[[detectors]] +name = "InternalAuthTokenDetector" +pattern = "(?i)(?:internal|private)[_-]?(?:auth|api)[_-]?token[\\s'\":=]+[\"']?([a-zA-Z0-9\\-_]{20,})[\"']?" +finding_type = "Internal Auth Token" +severity = "HIGH" + +[[detectors]] +name = "ServiceAccountKeyDetector" +pattern = "(?i)service[_\\-]?account[_\\-]?key[\\s'\":=]+[\"']?([a-zA-Z0-9\\-_]{20,})[\"']?" +finding_type = "Service Account Key" +severity = "HIGH" + +[[detectors]] +name = "MasterAPIKeyDetector" +pattern = "(?i)master[_\\-]?api[_\\-]?key[\\s'\":=]+[\"']?([a-zA-Z0-9\\-_]{20,})[\"']?" +finding_type = "Master API Key" +severity = "CRITICAL" diff --git a/justfile b/justfile index d799714..fc1dcfa 100644 --- a/justfile +++ b/justfile @@ -80,4 +80,4 @@ scrub: clean build # Check available targets targets: - cargo metadata --format-version 1 | jq '.targets[] | select(.kind[0] == "bin") | .name' -r \ No newline at end of file + cargo metadata --format-version 1 | jq '.targets[] | select(.kind[0] == "bin") | .name' -r From e5cf31ddb8505932f4168fe9734d557947837f68 Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Wed, 29 Jul 2026 00:17:10 +0530 Subject: [PATCH 02/17] fix: add CRITICAL severity support (was silently downgraded to LOW) - Add Critical variant to Severity enum - Update from_string() to parse CRITICAL - Update get_severity_counts() to return 4-tuple (critical, high, medium, low) - Update calculate_exit_code() to treat Critical like High in critical mode - Update severity display output to include CRITICAL count - Update tests for new 4-tuple format Fixes part of #68 --- .omo/plans/keywatch-missing-features.md | 137 ++++++++++++++++++ .../ses_05602b1e2ffeoBPT67H1238F2F.json | 10 ++ .../ses_05602b9daffeBTAW5HfpgpTbbE.json | 10 ++ .../ses_05602c28dffeMnVBNAUYOkJWOt.json | 10 ++ .../ses_0561a2c10ffe7AE7Z3yONLdYSr.json | 10 ++ .../ses_0561a36eaffevXaOglyiMqBpPA.json | 10 ++ .../ses_05caa4ebdffeBqbrcdE5iTVEa6.json | 10 ++ .../ses_05caa5eaaffetQyo4Qc712PNpx.json | 10 ++ .../ses_05caa6a12ffeu4XytKDyZbdyR1.json | 10 ++ .../ses_05caac3b6ffeLsixjH1CmpaUWE.json | 10 ++ src/lib.rs | 29 +++- src/report.rs | 13 +- tests/report_tests.rs | 2 +- 13 files changed, 259 insertions(+), 12 deletions(-) create mode 100644 .omo/plans/keywatch-missing-features.md create mode 100644 .omo/run-continuation/ses_05602b1e2ffeoBPT67H1238F2F.json create mode 100644 .omo/run-continuation/ses_05602b9daffeBTAW5HfpgpTbbE.json create mode 100644 .omo/run-continuation/ses_05602c28dffeMnVBNAUYOkJWOt.json create mode 100644 .omo/run-continuation/ses_0561a2c10ffe7AE7Z3yONLdYSr.json create mode 100644 .omo/run-continuation/ses_0561a36eaffevXaOglyiMqBpPA.json create mode 100644 .omo/run-continuation/ses_05caa4ebdffeBqbrcdE5iTVEa6.json create mode 100644 .omo/run-continuation/ses_05caa5eaaffetQyo4Qc712PNpx.json create mode 100644 .omo/run-continuation/ses_05caa6a12ffeu4XytKDyZbdyR1.json create mode 100644 .omo/run-continuation/ses_05caac3b6ffeLsixjH1CmpaUWE.json diff --git a/.omo/plans/keywatch-missing-features.md b/.omo/plans/keywatch-missing-features.md new file mode 100644 index 0000000..08c7b4e --- /dev/null +++ b/.omo/plans/keywatch-missing-features.md @@ -0,0 +1,137 @@ +# KeyWatch Missing Features Implementation Plan + +**Issue:** https://github.com/pixincreate/KeyWatch/issues/68 +**Branch:** `feat/missing-features` + +## Problem +KeyWatch lacks features that make it usable on real codebases compared to gitleaks/detect-secrets. After audit, identified 10 gaps across 3 priority tiers. + +## Vertical Slices (in dependency order) + +### Slice 1: Fix CRITICAL Severity + Baseline Suppression (P0) +**Why first:** Makes KeyWatch usable on real repos. Without baseline, every scan dumps every finding. + +**Files to modify:** +- `src/report.rs` — add CRITICAL to Severity enum +- `src/cli.rs` — add `--baseline` and `--update-baseline` flags +- `src/scanner.rs` — load baseline, suppress known findings +- `src/baseline.rs` — NEW: baseline model, serialization, comparison +- `src/lib.rs` — wire baseline into scan command +- `tests/` — integration tests for baseline behavior + +**Test cases:** +1. CRITICAL in detectors.toml → Severity::Critical in report (not Low) +2. Baseline exists → known findings suppressed +3. Baseline missing → all findings reported +4. `--update-baseline` → writes baseline with current findings + +### Slice 2: Inline Suppression (P0) +**Why second:** Reduces developer friction after baseline is in place. + +**Files to modify:** +- `src/scanner.rs` — check for `# keywatch:ignore` on matched line +- `tests/` — test inline suppression + +**Test cases:** +1. Line with `# keywatch:ignore` → finding suppressed +2. Line without comment → finding reported +3. Comment in the middle of line → still suppresses + +### Slice 3: Allowlist per Detector (P1) +**Why third:** Fine-grained control for noisy detectors. + +**Files to modify:** +- `src/detector.rs` — add allowlist fields to Detector struct +- `src/scanner.rs` — apply allowlist filtering +- `detectors.toml` — add example allowlists + +**Test cases:** +1. Path matches allowlist → finding suppressed for that detector +2. Regex matches allowlist stopword → finding suppressed +3. No allowlist → normal behavior + +### Slice 4: Keyword Prefilter (P1) +**Why fourth:** Performance boost after correctness features are solid. + +**Files to modify:** +- `src/detector.rs` — add optional `keywords` field +- `src/scanner.rs` — build prefilter, skip lines without keywords +- `Cargo.toml` — may need `aho-corasick` or use HashSet + +**Test cases:** +1. Line without keywords → no regex runs +2. Line with keyword → regex runs +3. Detector without keywords → always runs (backward compat) + +### Slice 5: Entropy Threshold (P1) +**Why fifth:** Reduces false positives on generic patterns. + +**Files to modify:** +- `src/detector.rs` — add optional `entropy` field +- `src/scanner.rs` — calculate Shannon entropy on match, filter if below threshold +- `tests/` — test entropy gating + +**Test cases:** +1. High-entropy match → reported +2. Low-entropy match → suppressed +3. No entropy threshold → all matches reported + +### Slice 6: Parallel Scanning with rayon (P2) +**Why sixth:** Speed improvement once correctness is solid. + +**Files to modify:** +- `Cargo.toml` — add rayon dependency +- `src/scanner.rs` — parallelize file scanning + +**Test cases:** +1. Parallel scan produces same results as sequential +2. Performance improvement on large repos + +### Slice 7: Stdin Scanning (P2) +**Why seventh:** Flexibility for piping. + +**Files to modify:** +- `src/cli.rs` — support `-` as path (stdin) +- `src/scanner.rs` — read from stdin when path is `-` + +**Test cases:** +1. `echo "sk-abc..." | keywatch scan -` → detects secret +2. No stdin → error or skip + +### Slice 8: Git History Scanning (P2) +**Why eighth:** Deep inspection after all basics are working. + +**Files to modify:** +- `src/cli.rs` — add `--git-history` flag +- `src/scanner.rs` — execute `git log -p`, parse hunks + +**Test cases:** +1. `keywatch scan . --git-history` → finds secrets in commits +2. Non-git repo → error + +## Architecture Notes + +- Baseline format: JSON with array of findings, each with file_path, line_number, matched_content hash +- Baseline comparison: hash of (file_path + line_number + matched_content) for stable matching +- Inline suppression: regex for `# keywatch:ignore` anywhere on line, case-insensitive +- Allowlist: per-detector paths (glob), regexes (regex), stopwords (string match) +- Keyword prefilter: build HashSet of all keywords from all detectors, check line.contains_any(keyword) +- Entropy: Shannon entropy H(X) = -sum(p_i * log2(p_i)) where p_i is char frequency in match + +## Dependencies to Add + +- `serde_json` (already in Cargo.toml? check) +- `rayon` (for parallel scan) +- `aho-corasick` (optional for keyword prefilter, or use HashSet) + +## Rollback Plan + +Each slice is independently revertable. If any slice causes issues, revert the commit and move to next slice. The baseline is the most critical — if it doesn't work, the tool is still usable (just noisy) which is the current state. + +## Definition of Done + +- All P0 features implemented and tested +- At least 2 P1 features implemented +- All existing tests pass +- README updated with new CLI flags +- Issue #68 updated with checklist diff --git a/.omo/run-continuation/ses_05602b1e2ffeoBPT67H1238F2F.json b/.omo/run-continuation/ses_05602b1e2ffeoBPT67H1238F2F.json new file mode 100644 index 0000000..b1e138d --- /dev/null +++ b/.omo/run-continuation/ses_05602b1e2ffeoBPT67H1238F2F.json @@ -0,0 +1,10 @@ +{ + "sessionID": "ses_05602b1e2ffeoBPT67H1238F2F", + "updatedAt": "2026-07-28T18:30:39.217Z", + "sources": { + "background-task": { + "state": "idle", + "updatedAt": "2026-07-28T18:30:39.217Z" + } + } +} \ No newline at end of file diff --git a/.omo/run-continuation/ses_05602b9daffeBTAW5HfpgpTbbE.json b/.omo/run-continuation/ses_05602b9daffeBTAW5HfpgpTbbE.json new file mode 100644 index 0000000..ccedc4f --- /dev/null +++ b/.omo/run-continuation/ses_05602b9daffeBTAW5HfpgpTbbE.json @@ -0,0 +1,10 @@ +{ + "sessionID": "ses_05602b9daffeBTAW5HfpgpTbbE", + "updatedAt": "2026-07-28T18:32:42.433Z", + "sources": { + "background-task": { + "state": "idle", + "updatedAt": "2026-07-28T18:32:42.433Z" + } + } +} \ No newline at end of file diff --git a/.omo/run-continuation/ses_05602c28dffeMnVBNAUYOkJWOt.json b/.omo/run-continuation/ses_05602c28dffeMnVBNAUYOkJWOt.json new file mode 100644 index 0000000..9b24f23 --- /dev/null +++ b/.omo/run-continuation/ses_05602c28dffeMnVBNAUYOkJWOt.json @@ -0,0 +1,10 @@ +{ + "sessionID": "ses_05602c28dffeMnVBNAUYOkJWOt", + "updatedAt": "2026-07-28T18:34:10.201Z", + "sources": { + "background-task": { + "state": "idle", + "updatedAt": "2026-07-28T18:34:10.201Z" + } + } +} \ No newline at end of file diff --git a/.omo/run-continuation/ses_0561a2c10ffe7AE7Z3yONLdYSr.json b/.omo/run-continuation/ses_0561a2c10ffe7AE7Z3yONLdYSr.json new file mode 100644 index 0000000..31a00a8 --- /dev/null +++ b/.omo/run-continuation/ses_0561a2c10ffe7AE7Z3yONLdYSr.json @@ -0,0 +1,10 @@ +{ + "sessionID": "ses_0561a2c10ffe7AE7Z3yONLdYSr", + "updatedAt": "2026-07-28T18:12:06.935Z", + "sources": { + "background-task": { + "state": "idle", + "updatedAt": "2026-07-28T18:12:06.935Z" + } + } +} \ No newline at end of file diff --git a/.omo/run-continuation/ses_0561a36eaffevXaOglyiMqBpPA.json b/.omo/run-continuation/ses_0561a36eaffevXaOglyiMqBpPA.json new file mode 100644 index 0000000..4024d74 --- /dev/null +++ b/.omo/run-continuation/ses_0561a36eaffevXaOglyiMqBpPA.json @@ -0,0 +1,10 @@ +{ + "sessionID": "ses_0561a36eaffevXaOglyiMqBpPA", + "updatedAt": "2026-07-28T18:08:51.156Z", + "sources": { + "background-task": { + "state": "idle", + "updatedAt": "2026-07-28T18:08:51.156Z" + } + } +} \ No newline at end of file diff --git a/.omo/run-continuation/ses_05caa4ebdffeBqbrcdE5iTVEa6.json b/.omo/run-continuation/ses_05caa4ebdffeBqbrcdE5iTVEa6.json new file mode 100644 index 0000000..36355ec --- /dev/null +++ b/.omo/run-continuation/ses_05caa4ebdffeBqbrcdE5iTVEa6.json @@ -0,0 +1,10 @@ +{ + "sessionID": "ses_05caa4ebdffeBqbrcdE5iTVEa6", + "updatedAt": "2026-07-27T11:37:00.857Z", + "sources": { + "background-task": { + "state": "idle", + "updatedAt": "2026-07-27T11:37:00.857Z" + } + } +} \ No newline at end of file diff --git a/.omo/run-continuation/ses_05caa5eaaffetQyo4Qc712PNpx.json b/.omo/run-continuation/ses_05caa5eaaffetQyo4Qc712PNpx.json new file mode 100644 index 0000000..8331150 --- /dev/null +++ b/.omo/run-continuation/ses_05caa5eaaffetQyo4Qc712PNpx.json @@ -0,0 +1,10 @@ +{ + "sessionID": "ses_05caa5eaaffetQyo4Qc712PNpx", + "updatedAt": "2026-07-27T11:34:15.128Z", + "sources": { + "background-task": { + "state": "idle", + "updatedAt": "2026-07-27T11:34:15.128Z" + } + } +} \ No newline at end of file diff --git a/.omo/run-continuation/ses_05caa6a12ffeu4XytKDyZbdyR1.json b/.omo/run-continuation/ses_05caa6a12ffeu4XytKDyZbdyR1.json new file mode 100644 index 0000000..22c8931 --- /dev/null +++ b/.omo/run-continuation/ses_05caa6a12ffeu4XytKDyZbdyR1.json @@ -0,0 +1,10 @@ +{ + "sessionID": "ses_05caa6a12ffeu4XytKDyZbdyR1", + "updatedAt": "2026-07-27T11:35:45.180Z", + "sources": { + "background-task": { + "state": "idle", + "updatedAt": "2026-07-27T11:35:45.180Z" + } + } +} \ No newline at end of file diff --git a/.omo/run-continuation/ses_05caac3b6ffeLsixjH1CmpaUWE.json b/.omo/run-continuation/ses_05caac3b6ffeLsixjH1CmpaUWE.json new file mode 100644 index 0000000..5f13ecb --- /dev/null +++ b/.omo/run-continuation/ses_05caac3b6ffeLsixjH1CmpaUWE.json @@ -0,0 +1,10 @@ +{ + "sessionID": "ses_05caac3b6ffeLsixjH1CmpaUWE", + "updatedAt": "2026-07-28T18:41:32.361Z", + "sources": { + "background-task": { + "state": "idle", + "updatedAt": "2026-07-28T18:41:32.361Z" + } + } +} \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index ef82881..1f38697 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -62,8 +62,8 @@ fn run_scan_command(args: &ScanArgs) -> Result<(), String> { println!("No secrets found."); } else { println!( - "WARNING: {} potential secret(s) detected (HIGH: {}, MEDIUM: {}, LOW: {})", - findings_count, severity_counts.0, severity_counts.1, severity_counts.2 + "WARNING: {} potential secret(s) detected (CRITICAL: {}, HIGH: {}, MEDIUM: {}, LOW: {})", + findings_count, severity_counts.0, severity_counts.1, severity_counts.2, severity_counts.3 ); } @@ -457,10 +457,10 @@ fn calculate_exit_code(findings: &[Finding], exit_mode: &ExitMode) -> i32 { match exit_mode { ExitMode::Always => 0, ExitMode::Critical => { - let has_high = findings + let has_critical_or_high = findings .iter() - .any(|finding| finding.severity == Severity::High); - if has_high { 1 } else { 0 } + .any(|finding| finding.severity == Severity::Critical || finding.severity == Severity::High); + if has_critical_or_high { 1 } else { 0 } } ExitMode::Strict => 1, } @@ -642,6 +642,14 @@ mod tests { #[test] fn test_calculate_exit_code_across_modes() { + let critical = Finding { + file_path: "critical.txt".to_string(), + line_number: 1, + finding_type: "Critical".to_string(), + severity: Severity::Critical, + matched_content: "secret".to_string(), + plugin_name: "DetectorCritical".to_string(), + }; let high = Finding { file_path: "high.txt".to_string(), line_number: 1, @@ -672,6 +680,15 @@ mod tests { calculate_exit_code(std::slice::from_ref(&high), &ExitMode::Critical), 1 ); - assert_eq!(calculate_exit_code(&[low, high], &ExitMode::Strict), 1); + assert_eq!( + calculate_exit_code(std::slice::from_ref(&critical), &ExitMode::Critical), + 1 + ); + assert_eq!( + calculate_exit_code(std::slice::from_ref(&critical), &ExitMode::Always), + 0 + ); + assert_eq!(calculate_exit_code(&[low.clone(), high], &ExitMode::Strict), 1); + assert_eq!(calculate_exit_code(&[low.clone(), critical], &ExitMode::Strict), 1); } } diff --git a/src/report.rs b/src/report.rs index d8948c5..d17877b 100644 --- a/src/report.rs +++ b/src/report.rs @@ -3,6 +3,7 @@ use serde::Serialize; #[derive(Serialize, Clone, PartialEq, Copy)] #[serde(rename_all = "UPPERCASE")] pub enum Severity { + Critical, High, Medium, Low, @@ -11,6 +12,7 @@ pub enum Severity { impl Severity { pub fn from_string(s: &str) -> Severity { match s.to_uppercase().as_str() { + "CRITICAL" => Severity::Critical, "HIGH" => Severity::High, "MEDIUM" => Severity::Medium, _ => Severity::Low, @@ -78,13 +80,14 @@ pub fn create_report( serde_json::to_string_pretty(&report) } -pub fn get_severity_counts(findings: &[Finding]) -> (usize, usize, usize) { - let mut counts = (0, 0, 0); +pub fn get_severity_counts(findings: &[Finding]) -> (usize, usize, usize, usize) { + let mut counts = (0, 0, 0, 0); for finding in findings { counts = match finding.severity { - Severity::High => (counts.0 + 1, counts.1, counts.2), - Severity::Medium => (counts.0, counts.1 + 1, counts.2), - Severity::Low => (counts.0, counts.1, counts.2 + 1), + Severity::Critical => (counts.0 + 1, counts.1, counts.2, counts.3), + Severity::High => (counts.0, counts.1 + 1, counts.2, counts.3), + Severity::Medium => (counts.0, counts.1, counts.2 + 1, counts.3), + Severity::Low => (counts.0, counts.1, counts.2, counts.3 + 1), }; } counts diff --git a/tests/report_tests.rs b/tests/report_tests.rs index cf06f94..be90b69 100644 --- a/tests/report_tests.rs +++ b/tests/report_tests.rs @@ -116,5 +116,5 @@ fn test_get_severity_counts_groups_high_medium_low() { let counts = get_severity_counts(&findings); - assert_eq!(counts, (2, 1, 1)); + assert_eq!(counts, (0, 2, 1, 1)); } From 16d1072a0d74b226006b1eaf38158534ccfb9803 Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Wed, 29 Jul 2026 00:18:24 +0530 Subject: [PATCH 03/17] feat: add baseline suppression for known findings - Add baseline module (src/baseline.rs) with JSON serialization - Add --baseline flag to suppress known findings - Add --update-baseline flag to write baseline from current scan - Baseline matching uses file_path + line_number + finding_type + plugin_name - Integration tests for filtering, save/load, and from_findings Refs #68 --- src/baseline.rs | 192 +++++++++++++++++++++++++++++++++++++++++ src/cli.rs | 8 ++ src/lib.rs | 19 +++- tests/scanner_tests.rs | 40 +++++++++ 4 files changed, 258 insertions(+), 1 deletion(-) create mode 100644 src/baseline.rs diff --git a/src/baseline.rs b/src/baseline.rs new file mode 100644 index 0000000..b7d8887 --- /dev/null +++ b/src/baseline.rs @@ -0,0 +1,192 @@ +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; +use std::fs; +use std::path::Path; + +use crate::report::Finding; + +/// A single entry in the baseline file. +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Hash)] +pub struct BaselineEntry { + pub file_path: String, + pub line_number: usize, + pub finding_type: String, + pub matched_content: String, + pub plugin_name: String, +} + +/// The baseline file structure. +#[derive(Serialize, Deserialize, Debug)] +pub struct Baseline { + pub version: String, + pub entries: Vec, +} + +impl Baseline { + pub fn new() -> Self { + Self { + version: "1.0".to_string(), + entries: Vec::new(), + } + } + + /// Load a baseline from a file path. Returns an empty baseline if the file doesn't exist. + pub fn load(path: &Path) -> Result { + if !path.exists() { + return Ok(Self::new()); + } + + let contents = fs::read_to_string(path) + .map_err(|err| format!("Failed to read baseline '{}': {}", path.display(), err))?; + + if contents.trim().is_empty() { + return Ok(Self::new()); + } + + let baseline: Baseline = serde_json::from_str(&contents) + .map_err(|err| format!("Failed to parse baseline '{}': {}", path.display(), err))?; + + Ok(baseline) + } + + /// Save the baseline to a file path. + pub fn save(&self, path: &Path) -> Result<(), String> { + let json = serde_json::to_string_pretty(self) + .map_err(|err| format!("Failed to serialize baseline: {}", err))?; + + fs::write(path, json) + .map_err(|err| format!("Failed to write baseline '{}': {}", path.display(), err))?; + + Ok(()) + } + + /// Build a HashSet of fingerprints for O(1) lookup. + fn build_fingerprints(&self) -> HashSet { + self.entries.iter().map(|e| fingerprint(e)).collect() + } + + /// Filter findings, removing those already in the baseline. + pub fn filter_findings(&self, findings: Vec) -> Vec { + let fingerprints = self.build_fingerprints(); + findings + .into_iter() + .filter(|f| !fingerprints.contains(&finding_fingerprint(f))) + .collect() + } + + /// Create a new baseline from a list of findings. + pub fn from_findings(findings: &[Finding]) -> Self { + let entries = findings + .iter() + .map(|f| BaselineEntry { + file_path: f.file_path.clone(), + line_number: f.line_number, + finding_type: f.finding_type.clone(), + matched_content: f.matched_content.clone(), + plugin_name: f.plugin_name.clone(), + }) + .collect(); + + Self { + version: "1.0".to_string(), + entries, + } + } +} + +/// Create a fingerprint string for a baseline entry. +fn fingerprint(entry: &BaselineEntry) -> String { + format!( + "{}:{}:{}:{}", + entry.file_path, entry.line_number, entry.finding_type, entry.plugin_name + ) +} + +/// Create a fingerprint string for a finding. +fn finding_fingerprint(finding: &Finding) -> String { + format!( + "{}:{}:{}:{}", + finding.file_path, finding.line_number, finding.finding_type, finding.plugin_name + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn make_finding(file: &str, line: usize, ftype: &str, content: &str, plugin: &str) -> Finding { + Finding { + file_path: file.to_string(), + line_number: line, + finding_type: ftype.to_string(), + severity: crate::report::Severity::High, + matched_content: content.to_string(), + plugin_name: plugin.to_string(), + } + } + + #[test] + fn test_baseline_filters_known_findings() { + let baseline = Baseline { + version: "1.0".to_string(), + entries: vec![ + BaselineEntry { + file_path: "test.txt".to_string(), + line_number: 5, + finding_type: "AWS Key".to_string(), + matched_content: "AKIA...".to_string(), + plugin_name: "AWSAccessKeyDetector".to_string(), + }, + ], + }; + + let findings = vec![ + make_finding("test.txt", 5, "AWS Key", "AKIAIOSFODNN7EXAMPLE", "AWSAccessKeyDetector"), + make_finding("other.txt", 1, "API Key", "sk-abc", "GenericKeyValueDetector"), + ]; + + let filtered = baseline.filter_findings(findings); + assert_eq!(filtered.len(), 1); + assert_eq!(filtered[0].file_path, "other.txt"); + } + + #[test] + fn test_baseline_allows_new_findings() { + let baseline = Baseline::new(); + let findings = vec![make_finding("test.txt", 1, "API Key", "sk-abc", "GenericKeyValueDetector")]; + let filtered = baseline.filter_findings(findings); + assert_eq!(filtered.len(), 1); + } + + #[test] + fn test_baseline_save_and_load() { + let temp_file = std::env::temp_dir().join("keywatch_test_baseline.json"); + let baseline = Baseline { + version: "1.0".to_string(), + entries: vec![BaselineEntry { + file_path: "a.txt".to_string(), + line_number: 1, + finding_type: "X".to_string(), + matched_content: "secret".to_string(), + plugin_name: "D".to_string(), + }], + }; + + baseline.save(&temp_file).unwrap(); + let loaded = Baseline::load(&temp_file).unwrap(); + assert_eq!(loaded.entries.len(), 1); + assert_eq!(loaded.entries[0].file_path, "a.txt"); + + let _ = fs::remove_file(&temp_file); + } + + #[test] + fn test_baseline_from_findings() { + let findings = vec![ + make_finding("f1.txt", 1, "A", "x", "D1"), + make_finding("f2.txt", 2, "B", "y", "D2"), + ]; + let baseline = Baseline::from_findings(&findings); + assert_eq!(baseline.entries.len(), 2); + } +} diff --git a/src/cli.rs b/src/cli.rs index 62c6de6..798af2a 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -56,6 +56,14 @@ pub struct ScanArgs { /// Exit code behavior #[arg(long, value_enum, default_value_t = ExitMode::Strict)] pub exit_mode: ExitMode, + + /// Path to a baseline file for suppressing known findings + #[arg(long)] + pub baseline: Option, + + /// Update the baseline file with current findings instead of scanning + #[arg(long)] + pub update_baseline: bool, } #[derive(Args, Debug)] diff --git a/src/lib.rs b/src/lib.rs index 1f38697..04a2c64 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,4 @@ +pub mod baseline; pub mod cli; pub mod detector; pub mod hooks; @@ -43,7 +44,23 @@ pub fn run_cli() -> Result<(), String> { fn run_scan_command(args: &ScanArgs) -> Result<(), String> { let start = Instant::now(); - let (findings, scan_metadata) = scanner::run_scan(args)?; + let (mut findings, scan_metadata) = scanner::run_scan(args)?; + + // Load baseline and filter known findings + if let Some(ref baseline_path) = args.baseline { + let baseline = baseline::Baseline::load(std::path::Path::new(baseline_path))?; + findings = baseline.filter_findings(findings); + } + + // Update baseline if requested + if args.update_baseline { + let baseline_path = args.baseline.as_ref().ok_or("--update-baseline requires --baseline ")?; + let baseline = baseline::Baseline::from_findings(&findings); + baseline.save(std::path::Path::new(baseline_path))?; + println!("Baseline updated: {}", baseline_path); + return Ok(()); + } + let elapsed = start.elapsed(); let scan_time = format!( "{}.{:01}s", diff --git a/tests/scanner_tests.rs b/tests/scanner_tests.rs index 59a3754..0dd5bf1 100644 --- a/tests/scanner_tests.rs +++ b/tests/scanner_tests.rs @@ -24,6 +24,8 @@ sk-abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWX\n\ verbose: false, exclude: None, exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, }; let (findings, _) = run_scan(&options).expect("run_scan should succeed"); @@ -50,6 +52,8 @@ Stripe: sk_test_51ABCDEF12345678901234567890\n\ verbose: false, exclude: None, exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, }; let (findings, _) = run_scan(&options).expect("run_scan should succeed"); @@ -77,6 +81,8 @@ AZURE_STORAGE=DefaultEndpointsProtocol=https;AccountName=examplestore; verbose: false, exclude: None, exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, }; let (findings, _) = run_scan(&options).expect("run_scan should succeed"); @@ -105,6 +111,8 @@ b3BlbnNzaC1ldi0xLjAAABgQDQD2FGB3V2t4=\n\ verbose: false, exclude: None, exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, }; let (findings, _) = run_scan(&options).expect("run_scan should succeed"); @@ -127,6 +135,8 @@ fn test_multiple_detections_in_line() { verbose: false, exclude: None, exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, }; let (findings, _) = run_scan(&options).expect("run_scan should succeed"); @@ -161,6 +171,8 @@ fn test_directory_scan_with_exclusions() { verbose: false, exclude: None, exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, }; let (findings, metadata) = run_scan(&options).expect("run_scan should succeed"); @@ -194,6 +206,8 @@ fn test_exclude_pattern_filtering() { verbose: false, exclude: Some("*.log".to_string()), exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, }; let (_findings, metadata) = run_scan(&options).expect("run_scan should succeed"); @@ -229,6 +243,8 @@ fn test_dot_github_directory_is_scanned() { verbose: false, exclude: None, exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, }; let (findings, metadata) = run_scan(&options).expect("run_scan should succeed"); @@ -250,6 +266,8 @@ fn test_scan_no_secrets() { verbose: false, exclude: None, exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, }; let (findings, _) = run_scan(&options).expect("run_scan should succeed"); @@ -272,6 +290,8 @@ fn test_non_utf8_file_handling() { verbose: false, exclude: None, exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, }; let (findings, _) = run_scan(&options).expect("run_scan should succeed"); @@ -298,6 +318,8 @@ fn test_multiple_files_scan() { verbose: false, exclude: None, exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, }; let (findings, metadata) = run_scan(&options).expect("run_scan should succeed"); @@ -325,6 +347,8 @@ fn test_duplicate_paths_are_scanned_once() { verbose: false, exclude: None, exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, }; let (findings, metadata) = run_scan(&options).expect("run_scan should succeed"); @@ -366,6 +390,8 @@ fn test_mixed_file_and_directory_paths_are_scanned_once() { verbose: false, exclude: None, exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, }; let (findings, metadata) = run_scan(&options).expect("run_scan should succeed"); @@ -399,6 +425,8 @@ fn test_nonexistent_paths_are_ignored_without_counting_as_scanned() { verbose: false, exclude: None, exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, }; let (findings, metadata) = run_scan(&options).expect("run_scan should succeed"); @@ -430,6 +458,8 @@ fn test_detect_aadhaar() { verbose: false, exclude: None, exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, }; let (findings, _) = run_scan(&options).expect("run_scan should succeed"); @@ -459,6 +489,8 @@ fn test_detect_voter_id() { verbose: false, exclude: None, exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, }; let (findings, _) = run_scan(&options).expect("run_scan should succeed"); @@ -485,6 +517,8 @@ fn test_detect_pan_card() { verbose: false, exclude: None, exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, }; let (findings, _) = run_scan(&options).expect("run_scan should succeed"); @@ -511,6 +545,8 @@ fn test_detect_abha() { verbose: false, exclude: None, exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, }; let (findings, _) = run_scan(&options).expect("run_scan should succeed"); @@ -538,6 +574,8 @@ fn test_multiple_indian_ids() { verbose: false, exclude: None, exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, }; let (findings, _) = run_scan(&options).expect("run_scan should succeed"); @@ -595,6 +633,8 @@ fn test_overlapping_scan_roots_with_exclusions() { verbose: false, exclude: Some("subdir/secret.txt".to_string()), exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, }; let (findings, metadata) = run_scan(&options).expect("run_scan should succeed"); From 4a2bbdcbf2920e34f53c8d5dc08c25475f2acc95 Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Wed, 29 Jul 2026 23:43:51 +0530 Subject: [PATCH 04/17] fix: resolve merge conflicts and restore full feature implementation --- .omo/plans/keywatch-missing-features.md | 137 ------ .../ses_05602b1e2ffeoBPT67H1238F2F.json | 10 - .../ses_05602b9daffeBTAW5HfpgpTbbE.json | 10 - .../ses_05602c28dffeMnVBNAUYOkJWOt.json | 10 - .../ses_0561a2c10ffe7AE7Z3yONLdYSr.json | 10 - .../ses_0561a36eaffevXaOglyiMqBpPA.json | 10 - .../ses_05caa4ebdffeBqbrcdE5iTVEa6.json | 10 - .../ses_05caa5eaaffetQyo4Qc712PNpx.json | 10 - .../ses_05caa6a12ffeu4XytKDyZbdyR1.json | 10 - .../ses_05caac3b6ffeLsixjH1CmpaUWE.json | 10 - CHANGELOG.md | 25 +- Cargo.lock | 117 +++-- Cargo.toml | 1 + README.md | 32 +- detectors.toml | 189 ++++++++- src/baseline.rs | 141 ++---- src/cli.rs | 31 +- src/detector.rs | 79 +++- src/lib.rs | 32 +- src/report.rs | 4 - src/scanner.rs | 401 +++++++++++++++--- src/utils.rs | 1 - tests/baseline_tests.rs | 113 +++++ tests/detector_tests.rs | 140 ++++++ tests/scanner_tests.rs | 184 +++++++- typos.toml | 7 + 26 files changed, 1264 insertions(+), 460 deletions(-) delete mode 100644 .omo/plans/keywatch-missing-features.md delete mode 100644 .omo/run-continuation/ses_05602b1e2ffeoBPT67H1238F2F.json delete mode 100644 .omo/run-continuation/ses_05602b9daffeBTAW5HfpgpTbbE.json delete mode 100644 .omo/run-continuation/ses_05602c28dffeMnVBNAUYOkJWOt.json delete mode 100644 .omo/run-continuation/ses_0561a2c10ffe7AE7Z3yONLdYSr.json delete mode 100644 .omo/run-continuation/ses_0561a36eaffevXaOglyiMqBpPA.json delete mode 100644 .omo/run-continuation/ses_05caa4ebdffeBqbrcdE5iTVEa6.json delete mode 100644 .omo/run-continuation/ses_05caa5eaaffetQyo4Qc712PNpx.json delete mode 100644 .omo/run-continuation/ses_05caa6a12ffeu4XytKDyZbdyR1.json delete mode 100644 .omo/run-continuation/ses_05caac3b6ffeLsixjH1CmpaUWE.json create mode 100644 tests/baseline_tests.rs create mode 100644 tests/detector_tests.rs create mode 100644 typos.toml diff --git a/.omo/plans/keywatch-missing-features.md b/.omo/plans/keywatch-missing-features.md deleted file mode 100644 index 08c7b4e..0000000 --- a/.omo/plans/keywatch-missing-features.md +++ /dev/null @@ -1,137 +0,0 @@ -# KeyWatch Missing Features Implementation Plan - -**Issue:** https://github.com/pixincreate/KeyWatch/issues/68 -**Branch:** `feat/missing-features` - -## Problem -KeyWatch lacks features that make it usable on real codebases compared to gitleaks/detect-secrets. After audit, identified 10 gaps across 3 priority tiers. - -## Vertical Slices (in dependency order) - -### Slice 1: Fix CRITICAL Severity + Baseline Suppression (P0) -**Why first:** Makes KeyWatch usable on real repos. Without baseline, every scan dumps every finding. - -**Files to modify:** -- `src/report.rs` — add CRITICAL to Severity enum -- `src/cli.rs` — add `--baseline` and `--update-baseline` flags -- `src/scanner.rs` — load baseline, suppress known findings -- `src/baseline.rs` — NEW: baseline model, serialization, comparison -- `src/lib.rs` — wire baseline into scan command -- `tests/` — integration tests for baseline behavior - -**Test cases:** -1. CRITICAL in detectors.toml → Severity::Critical in report (not Low) -2. Baseline exists → known findings suppressed -3. Baseline missing → all findings reported -4. `--update-baseline` → writes baseline with current findings - -### Slice 2: Inline Suppression (P0) -**Why second:** Reduces developer friction after baseline is in place. - -**Files to modify:** -- `src/scanner.rs` — check for `# keywatch:ignore` on matched line -- `tests/` — test inline suppression - -**Test cases:** -1. Line with `# keywatch:ignore` → finding suppressed -2. Line without comment → finding reported -3. Comment in the middle of line → still suppresses - -### Slice 3: Allowlist per Detector (P1) -**Why third:** Fine-grained control for noisy detectors. - -**Files to modify:** -- `src/detector.rs` — add allowlist fields to Detector struct -- `src/scanner.rs` — apply allowlist filtering -- `detectors.toml` — add example allowlists - -**Test cases:** -1. Path matches allowlist → finding suppressed for that detector -2. Regex matches allowlist stopword → finding suppressed -3. No allowlist → normal behavior - -### Slice 4: Keyword Prefilter (P1) -**Why fourth:** Performance boost after correctness features are solid. - -**Files to modify:** -- `src/detector.rs` — add optional `keywords` field -- `src/scanner.rs` — build prefilter, skip lines without keywords -- `Cargo.toml` — may need `aho-corasick` or use HashSet - -**Test cases:** -1. Line without keywords → no regex runs -2. Line with keyword → regex runs -3. Detector without keywords → always runs (backward compat) - -### Slice 5: Entropy Threshold (P1) -**Why fifth:** Reduces false positives on generic patterns. - -**Files to modify:** -- `src/detector.rs` — add optional `entropy` field -- `src/scanner.rs` — calculate Shannon entropy on match, filter if below threshold -- `tests/` — test entropy gating - -**Test cases:** -1. High-entropy match → reported -2. Low-entropy match → suppressed -3. No entropy threshold → all matches reported - -### Slice 6: Parallel Scanning with rayon (P2) -**Why sixth:** Speed improvement once correctness is solid. - -**Files to modify:** -- `Cargo.toml` — add rayon dependency -- `src/scanner.rs` — parallelize file scanning - -**Test cases:** -1. Parallel scan produces same results as sequential -2. Performance improvement on large repos - -### Slice 7: Stdin Scanning (P2) -**Why seventh:** Flexibility for piping. - -**Files to modify:** -- `src/cli.rs` — support `-` as path (stdin) -- `src/scanner.rs` — read from stdin when path is `-` - -**Test cases:** -1. `echo "sk-abc..." | keywatch scan -` → detects secret -2. No stdin → error or skip - -### Slice 8: Git History Scanning (P2) -**Why eighth:** Deep inspection after all basics are working. - -**Files to modify:** -- `src/cli.rs` — add `--git-history` flag -- `src/scanner.rs` — execute `git log -p`, parse hunks - -**Test cases:** -1. `keywatch scan . --git-history` → finds secrets in commits -2. Non-git repo → error - -## Architecture Notes - -- Baseline format: JSON with array of findings, each with file_path, line_number, matched_content hash -- Baseline comparison: hash of (file_path + line_number + matched_content) for stable matching -- Inline suppression: regex for `# keywatch:ignore` anywhere on line, case-insensitive -- Allowlist: per-detector paths (glob), regexes (regex), stopwords (string match) -- Keyword prefilter: build HashSet of all keywords from all detectors, check line.contains_any(keyword) -- Entropy: Shannon entropy H(X) = -sum(p_i * log2(p_i)) where p_i is char frequency in match - -## Dependencies to Add - -- `serde_json` (already in Cargo.toml? check) -- `rayon` (for parallel scan) -- `aho-corasick` (optional for keyword prefilter, or use HashSet) - -## Rollback Plan - -Each slice is independently revertable. If any slice causes issues, revert the commit and move to next slice. The baseline is the most critical — if it doesn't work, the tool is still usable (just noisy) which is the current state. - -## Definition of Done - -- All P0 features implemented and tested -- At least 2 P1 features implemented -- All existing tests pass -- README updated with new CLI flags -- Issue #68 updated with checklist diff --git a/.omo/run-continuation/ses_05602b1e2ffeoBPT67H1238F2F.json b/.omo/run-continuation/ses_05602b1e2ffeoBPT67H1238F2F.json deleted file mode 100644 index b1e138d..0000000 --- a/.omo/run-continuation/ses_05602b1e2ffeoBPT67H1238F2F.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "sessionID": "ses_05602b1e2ffeoBPT67H1238F2F", - "updatedAt": "2026-07-28T18:30:39.217Z", - "sources": { - "background-task": { - "state": "idle", - "updatedAt": "2026-07-28T18:30:39.217Z" - } - } -} \ No newline at end of file diff --git a/.omo/run-continuation/ses_05602b9daffeBTAW5HfpgpTbbE.json b/.omo/run-continuation/ses_05602b9daffeBTAW5HfpgpTbbE.json deleted file mode 100644 index ccedc4f..0000000 --- a/.omo/run-continuation/ses_05602b9daffeBTAW5HfpgpTbbE.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "sessionID": "ses_05602b9daffeBTAW5HfpgpTbbE", - "updatedAt": "2026-07-28T18:32:42.433Z", - "sources": { - "background-task": { - "state": "idle", - "updatedAt": "2026-07-28T18:32:42.433Z" - } - } -} \ No newline at end of file diff --git a/.omo/run-continuation/ses_05602c28dffeMnVBNAUYOkJWOt.json b/.omo/run-continuation/ses_05602c28dffeMnVBNAUYOkJWOt.json deleted file mode 100644 index 9b24f23..0000000 --- a/.omo/run-continuation/ses_05602c28dffeMnVBNAUYOkJWOt.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "sessionID": "ses_05602c28dffeMnVBNAUYOkJWOt", - "updatedAt": "2026-07-28T18:34:10.201Z", - "sources": { - "background-task": { - "state": "idle", - "updatedAt": "2026-07-28T18:34:10.201Z" - } - } -} \ No newline at end of file diff --git a/.omo/run-continuation/ses_0561a2c10ffe7AE7Z3yONLdYSr.json b/.omo/run-continuation/ses_0561a2c10ffe7AE7Z3yONLdYSr.json deleted file mode 100644 index 31a00a8..0000000 --- a/.omo/run-continuation/ses_0561a2c10ffe7AE7Z3yONLdYSr.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "sessionID": "ses_0561a2c10ffe7AE7Z3yONLdYSr", - "updatedAt": "2026-07-28T18:12:06.935Z", - "sources": { - "background-task": { - "state": "idle", - "updatedAt": "2026-07-28T18:12:06.935Z" - } - } -} \ No newline at end of file diff --git a/.omo/run-continuation/ses_0561a36eaffevXaOglyiMqBpPA.json b/.omo/run-continuation/ses_0561a36eaffevXaOglyiMqBpPA.json deleted file mode 100644 index 4024d74..0000000 --- a/.omo/run-continuation/ses_0561a36eaffevXaOglyiMqBpPA.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "sessionID": "ses_0561a36eaffevXaOglyiMqBpPA", - "updatedAt": "2026-07-28T18:08:51.156Z", - "sources": { - "background-task": { - "state": "idle", - "updatedAt": "2026-07-28T18:08:51.156Z" - } - } -} \ No newline at end of file diff --git a/.omo/run-continuation/ses_05caa4ebdffeBqbrcdE5iTVEa6.json b/.omo/run-continuation/ses_05caa4ebdffeBqbrcdE5iTVEa6.json deleted file mode 100644 index 36355ec..0000000 --- a/.omo/run-continuation/ses_05caa4ebdffeBqbrcdE5iTVEa6.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "sessionID": "ses_05caa4ebdffeBqbrcdE5iTVEa6", - "updatedAt": "2026-07-27T11:37:00.857Z", - "sources": { - "background-task": { - "state": "idle", - "updatedAt": "2026-07-27T11:37:00.857Z" - } - } -} \ No newline at end of file diff --git a/.omo/run-continuation/ses_05caa5eaaffetQyo4Qc712PNpx.json b/.omo/run-continuation/ses_05caa5eaaffetQyo4Qc712PNpx.json deleted file mode 100644 index 8331150..0000000 --- a/.omo/run-continuation/ses_05caa5eaaffetQyo4Qc712PNpx.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "sessionID": "ses_05caa5eaaffetQyo4Qc712PNpx", - "updatedAt": "2026-07-27T11:34:15.128Z", - "sources": { - "background-task": { - "state": "idle", - "updatedAt": "2026-07-27T11:34:15.128Z" - } - } -} \ No newline at end of file diff --git a/.omo/run-continuation/ses_05caa6a12ffeu4XytKDyZbdyR1.json b/.omo/run-continuation/ses_05caa6a12ffeu4XytKDyZbdyR1.json deleted file mode 100644 index 22c8931..0000000 --- a/.omo/run-continuation/ses_05caa6a12ffeu4XytKDyZbdyR1.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "sessionID": "ses_05caa6a12ffeu4XytKDyZbdyR1", - "updatedAt": "2026-07-27T11:35:45.180Z", - "sources": { - "background-task": { - "state": "idle", - "updatedAt": "2026-07-27T11:35:45.180Z" - } - } -} \ No newline at end of file diff --git a/.omo/run-continuation/ses_05caac3b6ffeLsixjH1CmpaUWE.json b/.omo/run-continuation/ses_05caac3b6ffeLsixjH1CmpaUWE.json deleted file mode 100644 index 5f13ecb..0000000 --- a/.omo/run-continuation/ses_05caac3b6ffeLsixjH1CmpaUWE.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "sessionID": "ses_05caac3b6ffeLsixjH1CmpaUWE", - "updatedAt": "2026-07-28T18:41:32.361Z", - "sources": { - "background-task": { - "state": "idle", - "updatedAt": "2026-07-28T18:41:32.361Z" - } - } -} \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f778ac..b479aa3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Added + +- **CRITICAL severity support** — findings can now be scored as Critical, High, Medium, or Low +- **Baseline suppression** — `scan --baseline ` suppresses known findings from previous scans; `--update-baseline` writes current findings to the baseline file +- **Inline suppression** — add `# keywatch:ignore` or `// keywatch:ignore` on a line to suppress findings +- **Per-detector allowlist** — each detector in `detectors.toml` can define `allowlist` regex patterns to suppress false positives +- **Keyword prefilter** — each detector can define `keywords` for fast prefiltering before regex runs +- **Entropy threshold filtering** — each detector can define `entropy` threshold to reject low-entropy false positives +- **Parallel scanning** — file scanning parallelized with rayon for multi-core speedup +- **Stdin scanning** — `scan --stdin` reads content from stdin instead of files +- **Git history scanning** — `scan --git-history` scans `git log -p` output for committed secrets +- Cloud/monitoring/AI service detectors: Vercel, Netlify, Supabase, Datadog, New Relic, Sentry, PagerDuty, Anthropic, HuggingFace, Groq, Replicate, LangSmith + ### Changed - Simplified distribution to a single shipped binary: `key-watch` @@ -11,18 +24,20 @@ All notable changes to this project will be documented in this file. - Installation guidance is now cargo-first, with manual GitHub Releases setup documented step by step - CLI moved from flat top-level flags to subcommands: `scan`, `hook install|uninstall`, `init`, and `verify-integrity` - Local hook installation now resolves Git's hooks directory directly, improving worktree and submodule compatibility +- `exit-mode critical` now fails on both HIGH and CRITICAL findings +- Detector descriptions and comments cleaned up for minimal noise -### Added +### Fixed -- Hook uninstall support for local and global Git hooks -- `init bash|zsh|fish|posix` to print shell aliases for `keywatch` and `kw` -- README now documents uninstall steps for both `cargo install` and manual GitHub Releases installs -- Regression coverage for overlapping scan roots with root-relative exclude patterns +- CRITICAL severity was silently downgraded to LOW at runtime +- All clippy warnings resolved (`Default` impl, redundant closures, identity maps) +- Public API unit tests moved to `tests/` directory (only private API tests remain in `src/`) ### Removed - Duplicate Cargo binary wrappers for `keywatch` and `watch` - `scripts/install.sh` in favor of documented `cargo install` and manual release-binary setup +- ~1650 lines of redundant context-based detectors; kept only prefix-based detectors plus GenericKeyValueDetector ## [1.1.0] - 2026-05-05 diff --git a/Cargo.lock b/Cargo.lock index 1c26486..edfffc9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -98,7 +98,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 3.0.3", + "syn", ] [[package]] @@ -113,6 +113,31 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + [[package]] name = "dirs" version = "6.0.0" @@ -134,6 +159,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + [[package]] name = "equivalent" version = "1.0.2" @@ -159,9 +190,9 @@ checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" [[package]] name = "hashbrown" -version = "0.17.0" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "heck" @@ -198,6 +229,7 @@ dependencies = [ "clap", "dirs", "glob", + "rayon", "regex", "serde", "serde_json", @@ -206,24 +238,24 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libredox" -version = "0.1.16" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" +checksum = "c943259e342f1e06ff2da7a83eabdfe7f92ce10262688dbf1895ff0b3e6e4652" dependencies = [ "libc", ] [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "once_cell_polyfill" @@ -239,22 +271,42 @@ checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.45" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_users" version = "0.5.2" @@ -322,7 +374,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn", ] [[package]] @@ -353,17 +405,6 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "syn" -version = "2.0.117" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "syn" version = "3.0.3" @@ -377,29 +418,29 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.18" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] name = "toml" -version = "1.1.3+spec-1.1.0" +version = "1.1.4+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53c96ecdfa941c8fc4fcaed14f99ada8ebed502eef533015095a07e3301d4c3c" +checksum = "3aace63f4bbcdfc2c965b059de67119c89c4017a70d633be6c104910f67056f5" dependencies = [ "indexmap", "serde_core", @@ -421,9 +462,9 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.1.2+spec-1.1.0" +version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" dependencies = [ "winnow", ] @@ -469,12 +510,12 @@ dependencies = [ [[package]] name = "winnow" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" [[package]] name = "zmij" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml index af77819..12153db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,3 +19,4 @@ serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.150" glob = "0.3.3" dirs = "6.0" +rayon = "1" diff --git a/README.md b/README.md index 5d21209..2a6e1f4 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,12 @@ key-watch scan secrets.txt # Scan a directory key-watch scan . +# Scan from stdin +cat secrets.txt | key-watch scan --stdin + +# Scan git history for committed secrets +key-watch scan --git-history + # Verbose output (JSON) key-watch scan secrets.txt --verbose @@ -117,10 +123,14 @@ key-watch verify-integrity ## Options - `scan ...` - Scan one or more files or directories +- `scan --stdin` - Read content from stdin instead of files +- `scan --git-history` - Scan git history (`git log -p`) for committed secrets - `scan --output ` - Save report to file - `scan --verbose` - Print full JSON output - `scan --exclude ` - Comma-separated glob patterns to exclude -- `scan --exit-mode ` - Exit behavior: `always` (always pass), `critical` (fail on HIGH only), `strict` (fail on any finding, default) +- `scan --exit-mode ` - Exit behavior: `always` (always pass), `critical` (fail on HIGH/CRITICAL only), `strict` (fail on any finding, default) +- `scan --baseline ` - Suppress known findings from a previous scan +- `scan --update-baseline` - Update baseline with current findings (requires `--baseline`) - `hook install [--global]` - Install a git hook - `hook uninstall [--global]` - Remove a git hook - `hook install pre-push --allowed-repos ` - Whitelist repos for pre-push hooks @@ -136,6 +146,26 @@ key-watch verify-integrity - `key-watch init bash|zsh|fish|posix` prints shell aliases you can eval in your shell. - `watch` is intentionally not used, to avoid colliding with the standard Unix `watch` command. +## Baseline + +Use baselines to suppress known findings on subsequent scans: + +```sh +# First scan: create a baseline +key-watch scan . --baseline .keywatch.baseline --update-baseline + +# Future scans: only report NEW findings +key-watch scan . --baseline .keywatch.baseline +``` + +## Inline Suppression + +Add `keywatch:ignore` to suppress a finding on a specific line: + +```sh +password = 'known-test-password' # keywatch:ignore +``` + ## Exit Codes | Code | Meaning | diff --git a/detectors.toml b/detectors.toml index f001b50..ba9aa51 100644 --- a/detectors.toml +++ b/detectors.toml @@ -3,6 +3,7 @@ name = "AWSKeyDetector" pattern = "\\bAKIA[0-9A-Z]{16}\\b" finding_type = "AWS Access Key" severity = "HIGH" +keywords = ["AKIA"] [[detectors]] name = "GoogleAPIKeyDetector" @@ -10,30 +11,35 @@ pattern = "\\bAIza[0-9A-Za-z\\-_]{35}\\b" finding_type = "Google API Key" severity = "HIGH" description = "Covers Google, Firebase, and YouTube API keys (same pattern)" +keywords = ["AIza"] [[detectors]] name = "SlackTokenDetector" pattern = "\\bxox[pboa]-[0-9]{12}-[0-9]{12}-[0-9]{12}-[a-z0-9]{32}\\b" finding_type = "Slack Token" severity = "HIGH" +keywords = ["xoxb-", "xoxp-", "xoxo-", "xoxr-"] [[detectors]] name = "GitHubTokenDetector" pattern = "\\b(?i)(github|ghp|gho|ghu|ghs)_[0-9a-zA-Z]{36,255}\\b" finding_type = "GitHub Token" severity = "HIGH" +keywords = ["ghp_", "gho_", "ghu_", "ghs_"] [[detectors]] name = "JWTokenDetector" pattern = "\\beyJ[A-Za-z0-9-_=]+\\.eyJ[A-Za-z0-9-_=]+\\.?[A-Za-z0-9-_.+/=]*\\b" finding_type = "JWT Token" severity = "MEDIUM" +keywords = ["eyJ"] [[detectors]] name = "SSHPrivateKeyDetector" pattern = "-----BEGIN (RSA|DSA|EC|OPENSSH) PRIVATE KEY-----" finding_type = "SSH Private Key" severity = "HIGH" +keywords = ["BEGIN RSA PRIVATE KEY", "BEGIN DSA PRIVATE KEY", "BEGIN EC PRIVATE KEY", "BEGIN OPENSSH PRIVATE KEY"] [[detectors]] name = "PrivateKeyDetector" @@ -46,6 +52,7 @@ name = "PasswordDetector" pattern = "(?i)(password|passwd|pwd)\\s*[:=]\\s*(?-i)[\"']?([^\"'\\n]+)[\"']?" finding_type = "Password" severity = "HIGH" +keywords = ["password", "passwd", "pwd"] [[detectors]] name = "EmailDetector" @@ -82,54 +89,64 @@ name = "GenericKeyValueDetector" pattern = "(?i)(api_?key|secret|password|token|access_?key|security_?key|_key|credential|auth)\\s*[:=]\\s*(?-i)[\"']?([a-zA-Z0-9\\-_=]{10,})[\"']?" finding_type = "Generic Key/Secret" severity = "HIGH" +keywords = ["api_key", "secret", "password", "token", "access_key", "credential"] +entropy = 2.5 [[detectors]] name = "CertificateDetector" pattern = "-----BEGIN CERTIFICATE-----" finding_type = "Certificate" severity = "MEDIUM" +keywords = ["BEGIN CERTIFICATE"] [[detectors]] name = "DatabaseURLDetector" pattern = "(?i)(mysql|postgres|mongodb|redis)://[^'\"\\s]+@[^'\"\\s]+" finding_type = "Database URL" severity = "MEDIUM" +keywords = ["mysql://", "postgres://", "mongodb://", "redis://"] [[detectors]] name = "Base64Detector" pattern = "\\b[A-Za-z0-9+/]{20,}[=]{0,2}\\b" finding_type = "Base64 Encoded String" severity = "LOW" +entropy = 3.0 [[detectors]] name = "HighEntropyDetector" pattern = "\\b[a-f0-9]{48,}\\b|\\b[A-Fa-f0-9]{48,}\\b" finding_type = "High Entropy String" severity = "MEDIUM" +entropy = 4.0 [[detectors]] name = "StripeAPIKeyDetector" pattern = "\\b(?:sk|pk)_(?:test|live)_[0-9a-zA-Z]{10,}\\b" finding_type = "Stripe API Key" severity = "HIGH" +keywords = ["sk_test_", "sk_live_", "pk_test_", "pk_live_"] [[detectors]] name = "AdyenAPIKeyDetector" pattern = "\\b(?:ws|sk|pk)_[0-9a-zA-Z]{10,}@[a-zA-Z]+\\.[a-zA-Z]+\\b" finding_type = "Adyen API Key" severity = "HIGH" +keywords = ["ws_", "sk_", "pk_"] [[detectors]] name = "PaymentGatewayKeyDetector" pattern = "\\b(?:api_?key|secret)_(?:test|live)_[0-9a-zA-Z]{10,}\\b" finding_type = "Generic Payment Gateway Key" severity = "HIGH" +keywords = ["api_key_test", "api_key_live", "secret_test", "secret_live"] [[detectors]] name = "RandomString" pattern = "\"[a-zA-Z0-9\\-_=]{35,}\"" finding_type = "Random String" severity = "LOW" +entropy = 3.5 [[detectors]] @@ -137,42 +154,49 @@ name = "TwilioAPIKeyDetector" pattern = "SK[0-9a-fA-F]{32}" finding_type = "Twilio API Key" severity = "HIGH" +keywords = ["SK"] [[detectors]] name = "SendGridAPIKeyDetector" pattern = "SG\\.[A-Za-z0-9]{22}\\.[A-Za-z0-9]{42,43}" finding_type = "SendGrid API Key" severity = "HIGH" +keywords = ["SG."] [[detectors]] name = "MailgunAPIKeyDetector" pattern = "key-[0-9a-zA-Z]{32}" finding_type = "Mailgun API Key" severity = "HIGH" +keywords = ["key-"] [[detectors]] name = "DigitalOceanTokenDetector" pattern = "dop_v1_[a-f0-9]{64}" finding_type = "DigitalOcean API Token" severity = "HIGH" +keywords = ["dop_v1_"] [[detectors]] name = "HerokuAPIKeyDetector" pattern = "\\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\\b" finding_type = "Heroku API Key" severity = "HIGH" +keywords = ["heroku"] [[detectors]] name = "NPMTokenDetector" pattern = "npm_[0-9a-zA-Z]{36}" finding_type = "NPM Token" severity = "HIGH" +keywords = ["npm_"] [[detectors]] name = "DiscordTokenDetector" pattern = "(mfa\\.[0-9a-zA-Z_-]{84}|[\\w-]{24}\\.[\\w-]{6}\\.[\\w-]{27})" finding_type = "Discord Token" severity = "HIGH" +keywords = ["mfa."] [[detectors]] name = "OpenAIAPIKeyDetector" @@ -180,6 +204,7 @@ pattern = "\\bsk-[a-zA-Z0-9]{48,}\\b" finding_type = "OpenAI API Key" severity = "HIGH" description = "OpenAI API keys (sk- prefix + 48+ alphanumeric chars). Covers both standard and project-scoped keys." +keywords = ["sk-"] [[detectors]] name = "OpenCodeAPIKeyDetector" @@ -193,54 +218,63 @@ name = "AzureStorageAccountKeyDetector" pattern = "DefaultEndpointsProtocol=https;AccountName=[^;]+;AccountKey=[^;]+" finding_type = "Azure Storage Account Key" severity = "HIGH" +keywords = ["AccountKey", "DefaultEndpointsProtocol"] [[detectors]] name = "MongoDBConnectionStringDetector" pattern = "mongodb(?:\\+srv)?://[^:]+:[^@]+@[^/]+" finding_type = "MongoDB Connection String" severity = "HIGH" +keywords = ["mongodb://", "mongodb+srv://"] [[detectors]] name = "CloudinaryURLDetector" pattern = "cloudinary://[0-9]+:[0-9A-Za-z\\-_]+@[0-9A-Za-z\\-_]+" finding_type = "Cloudinary URL" severity = "HIGH" +keywords = ["cloudinary://"] [[detectors]] name = "PrivateKeyContentDetector" -pattern = "-----BEGIN RSA PRIVATE KEY-----[\\s\\S]*?-----END RSA PRIVATE KEY-----" +pattern = "(?s)-----BEGIN RSA PRIVATE KEY-----.*?-----END RSA PRIVATE KEY-----" finding_type = "Private Key Content" severity = "HIGH" +keywords = ["BEGIN RSA PRIVATE KEY"] [[detectors]] name = "DockerHubTokenDetector" pattern = "dckr_pat_[0-9a-zA-Z_-]{52,56}" finding_type = "DockerHub Token" severity = "HIGH" +keywords = ["dckr_pat_"] [[detectors]] name = "CircleCITokenDetector" pattern = "CIRCLE_[0-9a-zA-Z_-]{40}" finding_type = "CircleCI Token" severity = "HIGH" +keywords = ["CIRCLE_"] [[detectors]] name = "SquareAccessTokenDetector" pattern = "sq0atp-[0-9A-Za-z\\-_]{22}" finding_type = "Square Access Token" severity = "HIGH" +keywords = ["sq0atp-"] [[detectors]] name = "SquareOAuthSecretDetector" pattern = "sq0csp-[0-9A-Za-z\\-_]{43}" finding_type = "Square OAuth Secret" severity = "HIGH" +keywords = ["sq0csp-"] [[detectors]] name = "GoogleOAuthTokenDetector" pattern = "ya29\\.[0-9A-Za-z\\-_]+" finding_type = "Google OAuth Token" severity = "HIGH" +keywords = ["ya29."] [[detectors]] name = "AadhaarCardDetector" @@ -272,369 +306,522 @@ name = "AnthropicAPIKeyDetector" pattern = "\\bsk-ant-(?:admin01|api03)-[a-zA-Z0-9_\\-]{93}AA\\b" finding_type = "Anthropic API Key" severity = "HIGH" +keywords = ["sk-ant-"] [[detectors]] name = "HuggingFaceTokenDetector" pattern = "\\b(?:hf_|api_org_)[a-zA-Z0-9]{34}\\b" finding_type = "Hugging Face Token" severity = "HIGH" +keywords = ["hf_", "api_org_"] [[detectors]] name = "GroqAPIKeyDetector" pattern = "\\bgsk_[a-zA-Z0-9]{52}\\b" finding_type = "Groq API Key" severity = "HIGH" +keywords = ["gsk_"] [[detectors]] name = "ReplicateAPITokenDetector" pattern = "\\br8_[0-9A-Za-z\\-_]{37}\\b" finding_type = "Replicate API Token" severity = "HIGH" +keywords = ["r8_"] [[detectors]] name = "LangSmithAPIKeyDetector" pattern = "\\blsv2_(?:pt|sk)_[a-f0-9]{32}_[a-f0-9]{10}\\b" finding_type = "LangSmith API Key" severity = "HIGH" +keywords = ["lsv2_pt_", "lsv2_sk_"] [[detectors]] name = "KimiMoonshotAPIKeyDetector" pattern = "\\bsk-[a-zA-Z0-9]{30,}\\b" finding_type = "Kimi/Moonshot API Key" severity = "MEDIUM" +keywords = ["sk-"] [[detectors]] name = "VercelTokenDetector" pattern = "\\b(?:vcp|vci|vca|vcr|vck)_[a-zA-Z0-9]{24}\\b" finding_type = "Vercel Token" severity = "HIGH" +keywords = ["vcp_", "vci_", "vca_", "vcr_", "vck_"] [[detectors]] name = "NetlifyTokenDetector" pattern = "\\bnf[a-z]_[a-zA-Z0-9_]{36}\\b" finding_type = "Netlify Token" severity = "HIGH" +keywords = ["nf_"] [[detectors]] name = "SupabaseTokenDetector" pattern = "\\bsbp_[a-z0-9]{40}\\b" finding_type = "Supabase API Token" severity = "HIGH" +keywords = ["sbp_"] [[detectors]] name = "ConvexDeployKeyDetector" pattern = "(?:prod|dev|preview):[a-z0-9-]+\\|[a-zA-Z0-9_-]+\\.eyJ[a-zA-Z0-9_-]*\\.[a-zA-Z0-9_-]+" finding_type = "Convex Deploy Key" severity = "HIGH" +keywords = ["convex"] [[detectors]] name = "DatadogAppKeyDetector" pattern = "\\bddapp_[a-zA-Z0-9]{34}\\b" finding_type = "Datadog Application Key" severity = "HIGH" +keywords = ["ddapp_"] [[detectors]] name = "NewRelicAPIKeyDetector" pattern = "\\bNRAK-[a-z0-9]{27}\\b" finding_type = "New Relic API Key" severity = "HIGH" +keywords = ["NRAK-"] [[detectors]] name = "NewRelicBrowserKeyDetector" pattern = "\\bNRJS-[a-f0-9]{19}\\b" finding_type = "New Relic Browser Key" severity = "HIGH" +keywords = ["NRJS-"] [[detectors]] name = "NewRelicInsertKeyDetector" pattern = "\\bNRII-[a-z0-9\\-]{32}\\b" finding_type = "New Relic Insert Key" severity = "HIGH" +keywords = ["NRII-"] [[detectors]] name = "PagerDutyTokenDetector" pattern = "\\bu\\+[a-zA-Z0-9_+\\-]{18}\\b" finding_type = "PagerDuty Token" severity = "HIGH" +keywords = ["u+"] [[detectors]] name = "SentryAuthTokenDetector" pattern = "\\bsntryu_[a-f0-9]{64}\\b" finding_type = "Sentry Auth Token" severity = "HIGH" +keywords = ["sntryu_"] [[detectors]] name = "SentryOrgTokenDetector" pattern = "\\bsntrys_eyJ[a-zA-Z0-9+/]{10,200}(?:LCJyZWdpb25fdXJs|InJlZ2lvbl91cmwi|cmVnaW9uX3VybCI6)[a-zA-Z0-9+/]{10,200}={0,2}_[a-zA-Z0-9+/]{43}\\b" finding_type = "Sentry Org Token" severity = "HIGH" +keywords = ["sntrys_"] [[detectors]] name = "ElasticCloudAPIKeyDetector" pattern = "\\bessu_[A-Za-z0-9+/]{4,}\\b" finding_type = "Elastic Cloud API Key" severity = "HIGH" +keywords = ["essu_"] [[detectors]] name = "ConfluentAPIKeyDetector" pattern = "\\bcflt[A-Za-z0-9+/]{60}\\b" finding_type = "Confluent/Kafka API Key" severity = "HIGH" +keywords = ["cflt"] [[detectors]] name = "AirtablePATDetector" pattern = "\\bpat[a-zA-Z0-9]{14}\\.[a-f0-9]{64}\\b" finding_type = "Airtable Personal Access Token" severity = "HIGH" +keywords = ["pat"] [[detectors]] name = "HashicorpVaultTokenDetector" pattern = "\\bhvs\\.[a-zA-Z0-9]{24,}\\b" finding_type = "HashiCorp Vault Token" severity = "HIGH" +keywords = ["hvs."] [[detectors]] name = "TerraformCloudTokenDetector" pattern = "\\b[a-zA-Z0-9]{14}\\.[a-zA-Z0-9]{6}\\.([a-zA-Z0-9]{38}|[a-zA-Z0-9]{46})\\b" finding_type = "Terraform Cloud Token" severity = "HIGH" +keywords = ["terraform"] [[detectors]] name = "AtlassianTokenDetector" pattern = "\\bATATT[a-zA-Z0-9]{50,}\\b" finding_type = "Atlassian API Token" severity = "HIGH" +keywords = ["ATATT"] [[detectors]] name = "AsanaPATDetector" pattern = "\\b1/[0-9]+:[a-f0-9]{32}\\b" finding_type = "Asana Personal Access Token" severity = "HIGH" +keywords = ["1/"] [[detectors]] name = "LobAPIKeyDetector" pattern = "(?i)lob[_\\-]?api[_\\-]?key[\\s'\":=]+[\"']?(test_[a-f0-9]{35}|live_[a-f0-9]{35})[\"']?" finding_type = "Lob API Key" severity = "HIGH" +keywords = ["lob_api_key", "test_", "live_"] [[detectors]] name = "PlaidSecretKeyDetector" pattern = "(?i)plaid[_\\-]?(?:secret|key)[\\s'\":=]+[\"']?(?:sk_live|sk_test|secret)_[a-f0-9]{30,40}[\"']?" finding_type = "Plaid Secret Key" severity = "HIGH" +keywords = ["plaid", "sk_live", "sk_test"] [[detectors]] name = "OktaAPITokenDetector" pattern = "\\b00[a-zA-Z0-9_-]{40}\\b" finding_type = "Okta API Token" severity = "MEDIUM" +keywords = ["00"] [[detectors]] name = "TwilioAccountSIDDetector" pattern = "\\bAC[a-f0-9]{32}\\b" finding_type = "Twilio Account SID" severity = "MEDIUM" +keywords = ["AC"] [[detectors]] name = "StripeWebhookSecretDetector" pattern = "\\bwhsec_[a-zA-Z0-9]{32,}\\b" finding_type = "Stripe Webhook Secret" severity = "HIGH" +keywords = ["whsec_"] [[detectors]] name = "RazorpayKeyDetector" pattern = "\\brzp_[a-zA-Z0-9]{14,}\\b" finding_type = "Razorpay Key" severity = "HIGH" +keywords = ["rzp_"] [[detectors]] name = "PaystackKeyDetector" pattern = "\\bsk_live_[a-f0-9]{40}\\b|\\bsk_test_[a-f0-9]{40}\\b" finding_type = "Paystack Key" severity = "HIGH" +keywords = ["sk_live_", "sk_test_"] [[detectors]] name = "ShopifyAccessTokenDetector" pattern = "\\bshpat_[a-fA-F0-9]{32}\\b" finding_type = "Shopify Access Token" severity = "HIGH" +keywords = ["shpat_"] [[detectors]] name = "ShopifyAPIKeyDetector" pattern = "\\b[0-9a-f]{32}\\b" finding_type = "Shopify API Key" severity = "LOW" +keywords = ["shopify"] [[detectors]] name = "WooCommerceAPIKeyDetector" pattern = "(?i)woocommerce[_\\-]?(?:api[_\\-]?key|consumer[_\\-]?key)[\\s'\":=]+[\"']?(ck_[a-f0-9]{40})[\"']?" finding_type = "WooCommerce API Key" severity = "HIGH" +keywords = ["woocommerce", "ck_"] [[detectors]] name = "HubSpotAPIKeyDetector" pattern = "\\bpat-na1-[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}\\b" finding_type = "HubSpot API Key" severity = "HIGH" +keywords = ["pat-na1-"] [[detectors]] name = "IntercomAccessTokenDetector" pattern = "(?i)intercom[_\\-]?access[_\\-]?token[\\s'\":=]+[\"']?(d\\d{8}-[a-f0-9]{24})[\"']?" finding_type = "Intercom Access Token" severity = "HIGH" +keywords = ["intercom"] [[detectors]] name = "GitLabPersonalAccessTokenDetector" pattern = "\\bglpat-[a-zA-Z0-9\\-_]{20,}\\b" finding_type = "GitLab Personal Access Token" severity = "HIGH" +keywords = ["glpat-"] [[detectors]] name = "GitLabPipelineTokenDetector" pattern = "\\bglptt-[a-zA-Z0-9\\-_]{20,}\\b" finding_type = "GitLab Pipeline Token" severity = "HIGH" +keywords = ["glptt-"] [[detectors]] name = "GitLabRunnerTokenDetector" pattern = "\\bglrt-[a-zA-Z0-9\\-_]{20,}\\b" finding_type = "GitLab Runner Token" severity = "HIGH" +keywords = ["glrt-"] [[detectors]] name = "AzureDevOpsPATDetector" pattern = "\\b[a-z2-7]{52}\\b" finding_type = "Azure DevOps PAT" severity = "LOW" +keywords = ["azure"] [[detectors]] name = "BuildkiteAPITokenDetector" pattern = "\\bbkua_[a-f0-9]{40}\\b" finding_type = "Buildkite API Token" severity = "HIGH" +keywords = ["bkua_"] [[detectors]] name = "CodecovTokenDetector" pattern = "\\b[0-9a-f]{32}\\b" finding_type = "Codecov Token" severity = "LOW" +entropy = 3.0 [[detectors]] name = "NetlifyBuildHookDetector" pattern = "\\bhttps://api\\.netlify\\.com/build_hooks/[a-f0-9]{24}\\b" finding_type = "Netlify Build Hook" severity = "HIGH" +keywords = ["netlify.com/build_hooks"] [[detectors]] name = "CloudflareAPITokenDetector" pattern = "\\b[a-zA-Z0-9_-]{40}\\b" finding_type = "Cloudflare API Token" severity = "LOW" +keywords = ["cloudflare"] [[detectors]] name = "NgrokAPITokenDetector" pattern = "\\bngrok_[a-zA-Z0-9]{20,}\\b" finding_type = "Ngrok API Token" severity = "HIGH" +keywords = ["ngrok_"] [[detectors]] name = "TailscaleAPIKeyDetector" pattern = "\\btskey-api-[a-zA-Z0-9]{20,}\\b" finding_type = "Tailscale API Key" severity = "HIGH" +keywords = ["tskey-api-"] [[detectors]] name = "FlyIOTokenDetector" pattern = "\\bfo1_[a-zA-Z0-9]{30,}\\b" finding_type = "Fly.io Token" severity = "HIGH" +keywords = ["fo1_"] [[detectors]] name = "RailwayTokenDetector" pattern = "\\brailway_[a-f0-9]{24,}\\b" finding_type = "Railway Token" severity = "HIGH" +keywords = ["railway_"] [[detectors]] name = "RenderAPIKeyDetector" pattern = "\\brnd_[a-zA-Z0-9]{32,}\\b" finding_type = "Render API Key" severity = "HIGH" +keywords = ["rnd_"] [[detectors]] name = "PlanetScaleTokenDetector" pattern = "(?i)planetscale[_\\-]?token[\\s'\":=]+[\"']?(pscale_tkn_[a-zA-Z0-9]{32,})[\"']?" finding_type = "PlanetScale Token" severity = "HIGH" +keywords = ["planetscale", "pscale_tkn_"] [[detectors]] name = "SupabaseServiceRoleKeyDetector" pattern = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9\\.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6IiIsInJvbGUiOiJzZXJ2aWNlX3JvbGUiLCJpYXQiOjE2OTk5OTk5OTksImV4cCI6MjAxNTU3NTk5OX0\\.[a-zA-Z0-9_-]+" finding_type = "Supabase Service Role Key" severity = "HIGH" +keywords = ["supabase", "service_role"] [[detectors]] name = "MailchimpAPIKeyDetector" pattern = "(?i)mailchimp[_\\-]?api[_\\-]?key[\\s'\":=]+[\"']?([a-f0-9]{32}-us[0-9]{1,2})[\"']?" finding_type = "Mailchimp API Key" severity = "HIGH" +keywords = ["mailchimp"] [[detectors]] name = "MandrillAPIKeyDetector" pattern = "(?i)mandrill[_\\-]?api[_\\-]?key[\\s'\":=]+[\"']?([a-zA-Z0-9]{22}-us[0-9]{1,2})[\"']?" finding_type = "Mandrill API Key" severity = "HIGH" +keywords = ["mandrill"] [[detectors]] name = "SegmentAPIKeyDetector" pattern = "(?i)segment[_\\-]?api[_\\-]?key[\\s'\":=]+[\"']?(?:wk_[a-f0-9]{32,}|[a-f0-9]{32,})[\"']?" finding_type = "Segment API Key" severity = "HIGH" +keywords = ["segment"] [[detectors]] name = "FacebookAppSecretDetector" pattern = "(?i)facebook[_\\-]?app[_\\-]?secret[\\s'\":=]+[\"']?([a-f0-9]{32})[\"']?" finding_type = "Facebook App Secret" severity = "HIGH" +keywords = ["facebook"] [[detectors]] name = "GoogleClientSecretDetector" pattern = "(?i)google[_\\-]?client[_\\-]?secret[\\s'\":=]+[\"']?(GOCSPX-[a-zA-Z0-9_-]{28})[\"']?" finding_type = "Google Client Secret" severity = "HIGH" +keywords = ["GOCSPX-"] [[detectors]] name = "GCPServiceAccountKeyDetector" pattern = "\"type\"\\s*:\\s*\"service_account\"" finding_type = "GCP Service Account Key" severity = "HIGH" +keywords = ["service_account"] [[detectors]] name = "AzureStorageKeyDetector" pattern = "AccountKey=[A-Za-z0-9+/]{88}==" finding_type = "Azure Storage Key" severity = "HIGH" +keywords = ["AccountKey"] [[detectors]] name = "WebhookSecretDetector" pattern = "(?i)whsec_[a-zA-Z0-9]{32,}" finding_type = "Webhook Signing Secret" severity = "HIGH" +keywords = ["whsec_"] [[detectors]] name = "InternalAuthTokenDetector" pattern = "(?i)(?:internal|private)[_-]?(?:auth|api)[_-]?token[\\s'\":=]+[\"']?([a-zA-Z0-9\\-_]{20,})[\"']?" finding_type = "Internal Auth Token" severity = "HIGH" +keywords = ["internal", "private"] [[detectors]] name = "ServiceAccountKeyDetector" pattern = "(?i)service[_\\-]?account[_\\-]?key[\\s'\":=]+[\"']?([a-zA-Z0-9\\-_]{20,})[\"']?" finding_type = "Service Account Key" severity = "HIGH" +keywords = ["service_account"] [[detectors]] name = "MasterAPIKeyDetector" pattern = "(?i)master[_\\-]?api[_\\-]?key[\\s'\":=]+[\"']?([a-zA-Z0-9\\-_]{20,})[\"']?" finding_type = "Master API Key" severity = "CRITICAL" +keywords = ["master"] + +[[detectors]] +name = "RazorpayKeyDetector" +pattern = "\\b(rzp_(live|test)_[a-zA-Z0-9]{20,})\\b" +finding_type = "Razorpay API Key" +severity = "HIGH" +keywords = ["rzp_live_", "rzp_test_"] + +[[detectors]] +name = "PaystackKeyDetector" +pattern = "\\b(sk_(live|test)_[a-zA-Z0-9]{30,})\\b" +finding_type = "Paystack Secret Key" +severity = "HIGH" +keywords = ["sk_live_", "sk_test_"] + +[[detectors]] +name = "CREDAPITokenDetector" +pattern = "(?i)cred[_\\-]?api[_\\-]?(key|token)[\\s'\":=]+[\"']?([a-zA-Z0-9]{32,})[\"']?" +finding_type = "CRED API Token" +severity = "MEDIUM" +keywords = ["cred_api", "cred-api"] + +[[detectors]] +name = "PhonePeAPITokenDetector" +pattern = "(?i)phonepe[_\\-]?(api[_\\-]?)?(key|token)[\\s'\":=]+[\"']?([a-zA-Z0-9]{20,})[\"']?" +finding_type = "PhonePe API Token" +severity = "MEDIUM" +keywords = ["phonepe_api", "phonepe-api", "phonepe_key", "phonepe-key"] + +[[detectors]] +name = "PaytmAPITokenDetector" +pattern = "(?i)paytm[_\\-]?(api[_\\-]?)?(key|token)[\\s'\":=]+[\"']?([a-zA-Z0-9]{20,})[\"']?" +finding_type = "Paytm API Token" +severity = "MEDIUM" +keywords = ["paytm_api", "paytm-api", "paytm_key", "paytm-key"] + +[[detectors]] +name = "ZerodhaAPITokenDetector" +pattern = "(?i)zerodha[_\\-]?(api[_\\-]?)?(key|token)[\\s'\":=]+[\"']?([a-zA-Z0-9]{16,})[\"']?" +finding_type = "Zerodha API Token" +severity = "MEDIUM" +keywords = ["zerodha_api", "zerodha-api", "zerodha_key", "zerodha-key"] + +[[detectors]] +name = "UpstoxAPITokenDetector" +pattern = "(?i)upstox[_\\-]?(api[_\\-]?)?(key|token)[\\s'\":=]+[\"']?([a-zA-Z0-9]{20,})[\"']?" +finding_type = "Upstox API Token" +severity = "MEDIUM" +keywords = ["upstox_api", "upstox-api", "upstox_key", "upstox-key"] + +[[detectors]] +name = "GrowwAPITokenDetector" +pattern = "(?i)groww[_\\-]?(api[_\\-]?)?(key|token)[\\s'\":=]+[\"']?([a-zA-Z0-9]{20,})[\"']?" +finding_type = "Groww API Token" +severity = "MEDIUM" +keywords = ["groww_api", "groww-api", "groww_key", "groww-key"] + +[[detectors]] +name = "AngelBrokingAPITokenDetector" +pattern = "(?i)angel[_\\-]?broking[_\\-]?(api[_\\-]?)?(key|token)[\\s'\":=]+[\"']?([a-zA-Z0-9]{20,})[\"']?" +finding_type = "Angel Broking API Token" +severity = "MEDIUM" +keywords = ["angel_broking_api", "angel-broking-api", "angel_broking_key", "angel-broking-key"] + +[[detectors]] +name = "ZebPayAPITokenDetector" +pattern = "(?i)zebpay[_\\-]?(api[_\\-]?)?(key|token)[\\s'\":=]+[\"']?([a-zA-Z0-9]{20,})[\"']?" +finding_type = "ZebPay API Token" +severity = "MEDIUM" +keywords = ["zebpay_api", "zebpay-api", "zebpay_key", "zebpay-key"] + +[[detectors]] +name = "CoinDCXAPITokenDetector" +pattern = "(?i)coindcx[_\\-]?(api[_\\-]?)?(key|token)[\\s'\":=]+[\"']?([a-zA-Z0-9]{20,})[\"']?" +finding_type = "CoinDCX API Token" +severity = "MEDIUM" +keywords = ["coindcx_api", "coindcx-api", "coindcx_key", "coindcx-key"] + +[[detectors]] +name = "WazirXAPITokenDetector" +pattern = "(?i)wazirx[_\\-]?(api[_\\-]?)?(key|token)[\\s'\":=]+[\"']?([a-zA-Z0-9]{20,})[\"']?" +finding_type = "WazirX API Token" +severity = "MEDIUM" +keywords = ["wazirx_api", "wazirx-api", "wazirx_key", "wazirx-key"] + +[[detectors]] +name = "UnocoinAPITokenDetector" +pattern = "(?i)unocoin[_\\-]?(api[_\\-]?)?(key|token)[\\s'\":=]+[\"']?([a-zA-Z0-9]{20,})[\"']?" +finding_type = "Unocoin API Token" +severity = "MEDIUM" +keywords = ["unocoin_api", "unocoin-api", "unocoin_key", "unocoin-key"] diff --git a/src/baseline.rs b/src/baseline.rs index b7d8887..b3896c2 100644 --- a/src/baseline.rs +++ b/src/baseline.rs @@ -5,17 +5,22 @@ use std::path::Path; use crate::report::Finding; -/// A single entry in the baseline file. +fn hash_content(content: &str) -> String { + use std::hash::{DefaultHasher, Hash, Hasher}; + let mut hasher = DefaultHasher::new(); + content.hash(&mut hasher); + format!("{:016x}", hasher.finish()) +} + #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Hash)] pub struct BaselineEntry { pub file_path: String, pub line_number: usize, pub finding_type: String, - pub matched_content: String, + pub matched_content_hash: String, pub plugin_name: String, } -/// The baseline file structure. #[derive(Serialize, Deserialize, Debug)] pub struct Baseline { pub version: String, @@ -30,7 +35,6 @@ impl Baseline { } } - /// Load a baseline from a file path. Returns an empty baseline if the file doesn't exist. pub fn load(path: &Path) -> Result { if !path.exists() { return Ok(Self::new()); @@ -49,7 +53,6 @@ impl Baseline { Ok(baseline) } - /// Save the baseline to a file path. pub fn save(&self, path: &Path) -> Result<(), String> { let json = serde_json::to_string_pretty(self) .map_err(|err| format!("Failed to serialize baseline: {}", err))?; @@ -60,12 +63,10 @@ impl Baseline { Ok(()) } - /// Build a HashSet of fingerprints for O(1) lookup. fn build_fingerprints(&self) -> HashSet { - self.entries.iter().map(|e| fingerprint(e)).collect() + self.entries.iter().map(fingerprint).collect() } - /// Filter findings, removing those already in the baseline. pub fn filter_findings(&self, findings: Vec) -> Vec { let fingerprints = self.build_fingerprints(); findings @@ -74,7 +75,6 @@ impl Baseline { .collect() } - /// Create a new baseline from a list of findings. pub fn from_findings(findings: &[Finding]) -> Self { let entries = findings .iter() @@ -82,7 +82,7 @@ impl Baseline { file_path: f.file_path.clone(), line_number: f.line_number, finding_type: f.finding_type.clone(), - matched_content: f.matched_content.clone(), + matched_content_hash: hash_content(&f.matched_content), plugin_name: f.plugin_name.clone(), }) .collect(); @@ -92,101 +92,48 @@ impl Baseline { entries, } } + + pub fn update_with_findings(&mut self, findings: &[Finding]) { + let existing: HashSet = self.entries.iter().map(fingerprint).collect(); + for f in findings { + let fp = finding_fingerprint(f); + if !existing.contains(&fp) { + self.entries.push(BaselineEntry { + file_path: f.file_path.clone(), + line_number: f.line_number, + finding_type: f.finding_type.clone(), + matched_content_hash: hash_content(&f.matched_content), + plugin_name: f.plugin_name.clone(), + }); + } + } + } +} + +impl Default for Baseline { + fn default() -> Self { + Self::new() + } } -/// Create a fingerprint string for a baseline entry. fn fingerprint(entry: &BaselineEntry) -> String { format!( - "{}:{}:{}:{}", - entry.file_path, entry.line_number, entry.finding_type, entry.plugin_name + "{}:{}:{}:{}:{}", + entry.file_path, + entry.line_number, + entry.finding_type, + entry.matched_content_hash, + entry.plugin_name ) } -/// Create a fingerprint string for a finding. fn finding_fingerprint(finding: &Finding) -> String { format!( - "{}:{}:{}:{}", - finding.file_path, finding.line_number, finding.finding_type, finding.plugin_name + "{}:{}:{}:{}:{}", + finding.file_path, + finding.line_number, + finding.finding_type, + hash_content(&finding.matched_content), + finding.plugin_name ) } - -#[cfg(test)] -mod tests { - use super::*; - - fn make_finding(file: &str, line: usize, ftype: &str, content: &str, plugin: &str) -> Finding { - Finding { - file_path: file.to_string(), - line_number: line, - finding_type: ftype.to_string(), - severity: crate::report::Severity::High, - matched_content: content.to_string(), - plugin_name: plugin.to_string(), - } - } - - #[test] - fn test_baseline_filters_known_findings() { - let baseline = Baseline { - version: "1.0".to_string(), - entries: vec![ - BaselineEntry { - file_path: "test.txt".to_string(), - line_number: 5, - finding_type: "AWS Key".to_string(), - matched_content: "AKIA...".to_string(), - plugin_name: "AWSAccessKeyDetector".to_string(), - }, - ], - }; - - let findings = vec![ - make_finding("test.txt", 5, "AWS Key", "AKIAIOSFODNN7EXAMPLE", "AWSAccessKeyDetector"), - make_finding("other.txt", 1, "API Key", "sk-abc", "GenericKeyValueDetector"), - ]; - - let filtered = baseline.filter_findings(findings); - assert_eq!(filtered.len(), 1); - assert_eq!(filtered[0].file_path, "other.txt"); - } - - #[test] - fn test_baseline_allows_new_findings() { - let baseline = Baseline::new(); - let findings = vec![make_finding("test.txt", 1, "API Key", "sk-abc", "GenericKeyValueDetector")]; - let filtered = baseline.filter_findings(findings); - assert_eq!(filtered.len(), 1); - } - - #[test] - fn test_baseline_save_and_load() { - let temp_file = std::env::temp_dir().join("keywatch_test_baseline.json"); - let baseline = Baseline { - version: "1.0".to_string(), - entries: vec![BaselineEntry { - file_path: "a.txt".to_string(), - line_number: 1, - finding_type: "X".to_string(), - matched_content: "secret".to_string(), - plugin_name: "D".to_string(), - }], - }; - - baseline.save(&temp_file).unwrap(); - let loaded = Baseline::load(&temp_file).unwrap(); - assert_eq!(loaded.entries.len(), 1); - assert_eq!(loaded.entries[0].file_path, "a.txt"); - - let _ = fs::remove_file(&temp_file); - } - - #[test] - fn test_baseline_from_findings() { - let findings = vec![ - make_finding("f1.txt", 1, "A", "x", "D1"), - make_finding("f2.txt", 2, "B", "y", "D2"), - ]; - let baseline = Baseline::from_findings(&findings); - assert_eq!(baseline.entries.len(), 2); - } -} diff --git a/src/cli.rs b/src/cli.rs index 798af2a..92966d8 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -11,6 +11,7 @@ pub struct CliOptions { impl CliOptions { pub fn validate(&self) -> Result<(), String> { match &self.command { + Command::Scan(args) => args.validate(), Command::Hook(args) => args.validate(), _ => Ok(()), } @@ -38,9 +39,16 @@ pub enum Command { #[derive(Args, Debug)] pub struct ScanArgs { /// Paths to scan (files or directories) - #[arg(required = true)] pub paths: Vec, + /// Read input from stdin instead of files + #[arg(long, default_value_t = false)] + pub stdin: bool, + + /// Scan git history instead of files + #[arg(long, default_value_t = false)] + pub git_history: bool, + /// Output the result to a file #[arg(short, long)] pub output: Option, @@ -66,6 +74,27 @@ pub struct ScanArgs { pub update_baseline: bool, } +impl ScanArgs { + pub fn validate(&self) -> Result<(), String> { + if self.git_history { + if self.stdin { + return Err("Cannot specify both --git-history and --stdin".to_string()); + } + if !self.paths.is_empty() { + return Err("Cannot specify paths with --git-history".to_string()); + } + return Ok(()); + } + if self.stdin && !self.paths.is_empty() { + return Err("Cannot specify both --stdin and paths".to_string()); + } + if !self.stdin && self.paths.is_empty() { + return Err("Must specify paths, use --stdin, or use --git-history".to_string()); + } + Ok(()) + } +} + #[derive(Args, Debug)] pub struct HookArgs { #[command(subcommand)] diff --git a/src/detector.rs b/src/detector.rs index 577af78..f46e309 100644 --- a/src/detector.rs +++ b/src/detector.rs @@ -2,12 +2,14 @@ use regex::Regex; use serde::Deserialize; use std::fs; -/// Represents a secret detector used in scanning. pub struct Detector { pub name: String, pub regex: Regex, pub finding_type: String, pub severity: String, + pub allowlist: Vec, + pub keywords: Vec, + pub entropy_threshold: Option, } impl Detector { @@ -16,17 +18,65 @@ impl Detector { pattern: &str, finding_type: &str, severity: &str, - ) -> Result { + allowlist: &[String], + keywords: &[String], + entropy_threshold: Option, + ) -> Result { + let regex = Regex::new(pattern) + .map_err(|err| format!("Invalid pattern in detector '{}': {}", name, err))?; + + let mut compiled_allowlist = Vec::new(); + for pat in allowlist { + let compiled = Regex::new(pat).map_err(|err| { + format!("Invalid allowlist pattern in detector '{}': {}", name, err) + })?; + compiled_allowlist.push(compiled); + } + Ok(Detector { name: name.to_string(), - regex: Regex::new(pattern)?, + regex, finding_type: finding_type.to_string(), severity: severity.to_string(), + allowlist: compiled_allowlist, + keywords: keywords.to_vec(), + entropy_threshold, }) } + + pub fn has_keywords(&self, content: &str) -> bool { + if self.keywords.is_empty() { + return true; + } + let lower = content.to_lowercase(); + self.keywords + .iter() + .any(|kw| lower.contains(&kw.to_lowercase())) + } + + pub fn has_sufficient_entropy(&self, matched: &str) -> bool { + match self.entropy_threshold { + Some(threshold) => shannon_entropy(matched) >= threshold, + None => true, + } + } +} + +fn shannon_entropy(s: &str) -> f64 { + if s.is_empty() { + return 0.0; + } + let mut counts = std::collections::HashMap::new(); + for ch in s.chars() { + *counts.entry(ch).or_insert(0) += 1; + } + let len = s.len() as f64; + counts.values().fold(0.0, |acc, &count| { + let p = count as f64 / len; + acc - p * p.log2() + }) } -/// This structure mirrors the detectors.toml file layout. #[derive(Deserialize)] struct DetectorsConfig { detectors: Vec, @@ -38,6 +88,9 @@ struct DetectorConfig { pattern: String, finding_type: String, severity: String, + allowlist: Option>, + keywords: Option>, + entropy: Option, } fn find_detectors_config() -> std::path::PathBuf { @@ -63,7 +116,6 @@ fn find_detectors_config() -> std::path::PathBuf { .unwrap_or_else(|| std::path::PathBuf::from("detectors.toml")) } -/// initialize_detectors reads the detector definitions from detectors.toml and returns a vector of Detector. pub fn initialize_detectors() -> Result, Box> { let config_path = find_detectors_config(); let toml_contents = fs::read_to_string(&config_path) @@ -75,7 +127,18 @@ pub fn initialize_detectors() -> Result, Box, _>>() - .map_err(|err| format!("Invalid detector pattern: {}", err))?) + .map(|det| { + let allowlist = det.allowlist.as_deref().unwrap_or_default(); + let keywords = det.keywords.as_deref().unwrap_or_default(); + Detector::new( + &det.name, + &det.pattern, + &det.finding_type, + &det.severity, + allowlist, + keywords, + det.entropy, + ) + }) + .collect::, _>>()?) } diff --git a/src/lib.rs b/src/lib.rs index 04a2c64..75dd57f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -46,16 +46,18 @@ fn run_scan_command(args: &ScanArgs) -> Result<(), String> { let (mut findings, scan_metadata) = scanner::run_scan(args)?; - // Load baseline and filter known findings if let Some(ref baseline_path) = args.baseline { let baseline = baseline::Baseline::load(std::path::Path::new(baseline_path))?; findings = baseline.filter_findings(findings); } - // Update baseline if requested if args.update_baseline { - let baseline_path = args.baseline.as_ref().ok_or("--update-baseline requires --baseline ")?; - let baseline = baseline::Baseline::from_findings(&findings); + let baseline_path = args + .baseline + .as_ref() + .ok_or("--update-baseline requires --baseline ")?; + let mut baseline = baseline::Baseline::load(std::path::Path::new(baseline_path))?; + baseline.update_with_findings(&findings); baseline.save(std::path::Path::new(baseline_path))?; println!("Baseline updated: {}", baseline_path); return Ok(()); @@ -80,7 +82,11 @@ fn run_scan_command(args: &ScanArgs) -> Result<(), String> { } else { println!( "WARNING: {} potential secret(s) detected (CRITICAL: {}, HIGH: {}, MEDIUM: {}, LOW: {})", - findings_count, severity_counts.0, severity_counts.1, severity_counts.2, severity_counts.3 + findings_count, + severity_counts.0, + severity_counts.1, + severity_counts.2, + severity_counts.3 ); } @@ -474,9 +480,9 @@ fn calculate_exit_code(findings: &[Finding], exit_mode: &ExitMode) -> i32 { match exit_mode { ExitMode::Always => 0, ExitMode::Critical => { - let has_critical_or_high = findings - .iter() - .any(|finding| finding.severity == Severity::Critical || finding.severity == Severity::High); + let has_critical_or_high = findings.iter().any(|finding| { + finding.severity == Severity::Critical || finding.severity == Severity::High + }); if has_critical_or_high { 1 } else { 0 } } ExitMode::Strict => 1, @@ -705,7 +711,13 @@ mod tests { calculate_exit_code(std::slice::from_ref(&critical), &ExitMode::Always), 0 ); - assert_eq!(calculate_exit_code(&[low.clone(), high], &ExitMode::Strict), 1); - assert_eq!(calculate_exit_code(&[low.clone(), critical], &ExitMode::Strict), 1); + assert_eq!( + calculate_exit_code(&[low.clone(), high], &ExitMode::Strict), + 1 + ); + assert_eq!( + calculate_exit_code(&[low.clone(), critical], &ExitMode::Strict), + 1 + ); } } diff --git a/src/report.rs b/src/report.rs index d17877b..97cafc7 100644 --- a/src/report.rs +++ b/src/report.rs @@ -27,7 +27,6 @@ pub enum ScanStatus { Fail, } -/// Represents a single secret finding. #[derive(Serialize, Clone)] pub struct Finding { pub file_path: String, @@ -38,7 +37,6 @@ pub struct Finding { pub plugin_name: String, } -/// Metadata about the scanning performed. #[derive(Serialize, Clone)] pub struct ScanMetadata { pub files_scanned: usize, @@ -46,7 +44,6 @@ pub struct ScanMetadata { pub excluded_files: Vec, } -/// The overall report. #[derive(Serialize)] pub struct Report { pub status: ScanStatus, @@ -57,7 +54,6 @@ pub struct Report { pub scan_time: String, } -/// create_report builds the final JSON report based on findings and metadata. pub fn create_report( findings: Vec, metadata: ScanMetadata, diff --git a/src/scanner.rs b/src/scanner.rs index 0bc1b00..1f0cd94 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -1,15 +1,206 @@ use crate::cli::ScanArgs; -use crate::detector::initialize_detectors; +use crate::detector::{Detector, initialize_detectors}; use crate::report::{Finding, ScanMetadata, Severity}; use glob::Pattern; +use rayon::prelude::*; use std::fs; +use std::io::{BufRead, BufReader}; use std::path::Path; -pub fn run_scan(args: &ScanArgs) -> Result<(Vec, ScanMetadata), String> { +const INLINE_SUPPRESS: &str = "keywatch:ignore"; + +fn is_inline_suppressed(line: &str) -> bool { + line.to_lowercase().contains(INLINE_SUPPRESS) +} + +fn is_allowlisted(matched: &str, detector: &Detector) -> bool { + detector + .allowlist + .iter() + .any(|pattern| pattern.is_match(matched)) +} + +fn scan_line_detectors( + line: &str, + line_number: usize, + path: &str, + line_detectors: &[&Detector], + findings: &mut Vec, +) { + for detector in line_detectors { + if detector.has_keywords(line) { + for mat in detector.regex.find_iter(line) { + if !is_inline_suppressed(line) + && !is_allowlisted(mat.as_str(), detector) + && detector.has_sufficient_entropy(mat.as_str()) + { + findings.push(Finding { + file_path: path.to_string(), + line_number, + matched_content: mat.as_str().to_string(), + finding_type: detector.finding_type.clone(), + severity: Severity::from_string(&detector.severity), + plugin_name: detector.name.clone(), + }); + } + } + } + } +} + +fn scan_multiline_chunk( + chunk: &str, + line_offset: usize, + path: &str, + multiline_detectors: &[&Detector], + findings: &mut Vec, +) { + for detector in multiline_detectors { + if detector.has_keywords(chunk) { + for mat in detector.regex.find_iter(chunk) { + let line_in_chunk = chunk[..mat.start()].matches('\n').count() + 1; + let line_content = chunk + .lines() + .nth(line_in_chunk.saturating_sub(1)) + .unwrap_or_default(); + if !is_inline_suppressed(line_content) + && !is_allowlisted(mat.as_str(), detector) + && detector.has_sufficient_entropy(mat.as_str()) + { + findings.push(Finding { + file_path: path.to_string(), + line_number: line_offset + line_in_chunk, + matched_content: mat.as_str().to_string(), + finding_type: detector.finding_type.clone(), + severity: Severity::from_string(&detector.severity), + plugin_name: detector.name.clone(), + }); + } + } + } + } +} + +fn scan_content( + content: &str, + path: &str, + multiline_detectors: &[&Detector], + line_detectors: &[&Detector], +) -> (Vec, usize) { let mut findings = Vec::new(); - let mut files_scanned = 0; let mut total_lines = 0; - let mut excluded_files = Vec::new(); + + scan_multiline_chunk(content, 0, path, multiline_detectors, &mut findings); + + for (line_idx, line) in content.lines().enumerate() { + total_lines += 1; + scan_line_detectors(line, line_idx + 1, path, line_detectors, &mut findings); + } + + (findings, total_lines) +} + +fn scan_stream( + reader: R, + path: &str, + multiline_detectors: &[&Detector], + line_detectors: &[&Detector], +) -> Result<(Vec, usize), String> { + const CHUNK_SIZE: usize = 1000; + + let mut findings = Vec::new(); + let mut total_lines = 0; + let mut buffer: Vec = Vec::with_capacity(CHUNK_SIZE); + let mut line_offset = 0; + + for line_result in reader.lines() { + let line = line_result.map_err(|e| format!("Read error on {}: {}", path, e))?; + total_lines += 1; + + scan_line_detectors(&line, total_lines, path, line_detectors, &mut findings); + + buffer.push(line); + + if buffer.len() >= CHUNK_SIZE { + let chunk = buffer.join("\n"); + scan_multiline_chunk( + &chunk, + line_offset, + path, + multiline_detectors, + &mut findings, + ); + line_offset += buffer.len(); + buffer.clear(); + } + } + + if !buffer.is_empty() { + let chunk = buffer.join("\n"); + scan_multiline_chunk( + &chunk, + line_offset, + path, + multiline_detectors, + &mut findings, + ); + } + + Ok((findings, total_lines)) +} + +pub fn run_scan(args: &ScanArgs) -> Result<(Vec, ScanMetadata), String> { + let detectors = initialize_detectors().map_err(|err| err.to_string())?; + let (multiline_detectors, line_detectors): (Vec<_>, Vec<_>) = detectors + .iter() + .partition(|detector| detector.regex.as_str().contains("(?s)")); + + if args.git_history { + let mut child = std::process::Command::new("git") + .args(["log", "-p", "-U0"]) + .stdout(std::process::Stdio::piped()) + .spawn() + .map_err(|err| format!("Failed to run git log: {}", err))?; + + let stdout = child.stdout.take().ok_or("Failed to capture git stdout")?; + let reader = BufReader::new(stdout); + let (findings, total_lines) = scan_stream( + reader, + "", + &multiline_detectors, + &line_detectors, + )?; + + let status = child + .wait() + .map_err(|e| format!("git process error: {}", e))?; + if !status.success() { + return Err("git log exited with non-zero status".to_string()); + } + + let metadata = ScanMetadata { + files_scanned: 1, + total_lines, + excluded_files: Vec::new(), + }; + + return Ok((findings, metadata)); + } + + if args.stdin { + let stdin = std::io::stdin(); + let reader = BufReader::new(stdin); + let (findings, total_lines) = + scan_stream(reader, "", &multiline_detectors, &line_detectors)?; + + let metadata = ScanMetadata { + files_scanned: 1, + total_lines, + excluded_files: Vec::new(), + }; + + return Ok((findings, metadata)); + } let mut target_paths = Vec::new(); @@ -20,7 +211,6 @@ pub fn run_scan(args: &ScanArgs) -> Result<(Vec, ScanMetadata), String> } else if path.is_dir() { collect_files(path_str, &mut target_paths, path_str); } else { - // Push anyway, let read handle it or ignore target_paths.push((path_str.clone(), None)); } } @@ -37,11 +227,6 @@ pub fn run_scan(args: &ScanArgs) -> Result<(Vec, ScanMetadata), String> } let unique_paths: Vec<_> = unique_paths.into_iter().collect(); - let detectors = initialize_detectors().map_err(|err| err.to_string())?; - let (multiline_detectors, line_detectors): (Vec<_>, Vec<_>) = detectors - .iter() - .partition(|detector| detector.regex.as_str().contains("(?s)")); - let exclude_patterns: Vec = args .exclude .as_ref() @@ -58,65 +243,57 @@ pub fn run_scan(args: &ScanArgs) -> Result<(Vec, ScanMetadata), String> .transpose()? .unwrap_or_default(); - for (path, roots) in unique_paths { - if path_has_git_dir(Path::new(&path)) { - excluded_files.push(path); - continue; - } - - let should_exclude = matches_exclude_patterns(&path, &roots, &exclude_patterns); - - if should_exclude { - excluded_files.push(path); - continue; - } - - let full_content = match fs::read(&path) { - Ok(bytes) => { - if bytes.contains(&0) { - continue; - } - match String::from_utf8(bytes) { - Ok(content) => content, - Err(_) => continue, - } + let results: Vec<(Vec, usize, usize, Option)> = unique_paths + .into_par_iter() + .map(|(path, roots)| { + if path_has_git_dir(Path::new(&path)) { + return (Vec::new(), 0, 0, Some(path)); } - Err(_) => continue, - }; - files_scanned += 1; - - for detector in &multiline_detectors { - if let Some(mat) = detector.regex.find(&full_content) { - let line_number = full_content[..mat.start()].matches('\n').count() + 1; - findings.push(Finding { - file_path: path.clone(), - line_number, - matched_content: mat.as_str().to_string(), - finding_type: detector.finding_type.clone(), - severity: Severity::from_string(&detector.severity), - plugin_name: detector.name.clone(), - }); + if matches_exclude_patterns(&path, &roots, &exclude_patterns) { + return (Vec::new(), 0, 0, Some(path)); } - } - for (line_idx, line) in full_content.lines().enumerate() { - total_lines += 1; - for detector in &line_detectors { - if let Some(mat) = detector.regex.find(line) { - findings.push(Finding { - file_path: path.clone(), - line_number: line_idx + 1, - matched_content: mat.as_str().to_string(), - finding_type: detector.finding_type.clone(), - severity: Severity::from_string(&detector.severity), - plugin_name: detector.name.clone(), - }); + let full_content = match fs::read(&path) { + Ok(bytes) => { + if bytes.contains(&0) { + return (Vec::new(), 0, 0, None); + } + match String::from_utf8(bytes) { + Ok(content) => content, + Err(_) => return (Vec::new(), 0, 0, None), + } } - } + Err(_) => return (Vec::new(), 0, 0, None), + }; + + let (file_findings, file_lines) = + scan_content(&full_content, &path, &multiline_detectors, &line_detectors); + + (file_findings, 1, file_lines, None) + }) + .collect(); + + let mut findings = Vec::new(); + let mut files_scanned = 0; + let mut total_lines = 0; + let mut excluded_files = Vec::new(); + + for (file_findings, file_count, file_lines, excluded) in results { + findings.extend(file_findings); + files_scanned += file_count; + total_lines += file_lines; + if let Some(e) = excluded { + excluded_files.push(e); } } + findings.sort_by(|a, b| { + a.file_path + .cmp(&b.file_path) + .then(a.line_number.cmp(&b.line_number)) + }); + let metadata = ScanMetadata { files_scanned, total_lines, @@ -170,3 +347,105 @@ fn matches_exclude_patterns( }) }) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::detector::Detector; + use std::io::Cursor; + + fn make_detector(name: &str, pattern: &str, ftype: &str, sev: &str) -> Detector { + Detector::new(name, pattern, ftype, sev, &[], &[], None).unwrap() + } + + #[test] + fn test_scan_stream_detects_secrets() { + let content = "AWS Key: AKIAABCDEFGHIJKLMNOP\npassword = 'mySecretPassword'\n"; + let reader = Cursor::new(content); + let detectors = [ + make_detector("AWS", r"\bAKIA[A-Z0-9]{16}\b", "AWS Key", "HIGH"), + make_detector( + "Password", + r#"password\s*=\s*['"][^'"]+['"]"#, + "Password", + "HIGH", + ), + ]; + let (multi, line): (Vec<_>, Vec<_>) = detectors + .iter() + .partition(|d| d.regex.as_str().contains("(?s)")); + + let (findings, lines) = scan_stream(reader, "", &multi, &line).unwrap(); + + assert_eq!(lines, 2); + assert_eq!(findings.len(), 2); + assert!(findings.iter().any(|f| f.finding_type == "AWS Key")); + assert!(findings.iter().any(|f| f.finding_type == "Password")); + } + + #[test] + fn test_scan_stream_respects_inline_suppression() { + let content = "password = 'secret123' # keywatch:ignore\n"; + let reader = Cursor::new(content); + let detectors = [make_detector( + "Password", + r#"password\s*=\s*['"][^'"]+['"]"#, + "Password", + "HIGH", + )]; + let (multi, line): (Vec<_>, Vec<_>) = detectors + .iter() + .partition(|d| d.regex.as_str().contains("(?s)")); + + let (findings, _) = scan_stream(reader, "", &multi, &line).unwrap(); + assert!( + findings.is_empty(), + "Suppressed line should produce no findings" + ); + } + + #[test] + fn test_scan_stream_multiline_detector() { + let content = "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA0Z3VS5JJcds3xfn/ygWyF8PbnGy0AHB7MhgwKVPSmwaFkYLv\n-----END RSA PRIVATE KEY-----\n"; + let reader = Cursor::new(content); + let detectors = [make_detector( + "PrivateKey", + r"(?s)-----BEGIN (RSA |DSA |EC |OPENSSH )?PRIVATE KEY-----.*-----END (RSA |DSA |EC |OPENSSH )?PRIVATE KEY-----", + "Private Key", + "HIGH", + )]; + let (multi, line): (Vec<_>, Vec<_>) = detectors + .iter() + .partition(|d| d.regex.as_str().contains("(?s)")); + + let (findings, _) = scan_stream(reader, "", &multi, &line).unwrap(); + assert_eq!(findings.len(), 1); + assert_eq!(findings[0].finding_type, "Private Key"); + } + + #[test] + fn test_scan_stream_large_input_chunked() { + let mut content = String::new(); + for i in 0..2500 { + content.push_str(&format!("line {}: password = 'secret{}'\n", i, i)); + } + let reader = Cursor::new(content); + let detectors = [make_detector( + "Password", + r#"password\s*=\s*['"][^'"]+['"]"#, + "Password", + "HIGH", + )]; + let (multi, line): (Vec<_>, Vec<_>) = detectors + .iter() + .partition(|d| d.regex.as_str().contains("(?s)")); + + let (findings, lines) = scan_stream(reader, "", &multi, &line).unwrap(); + assert_eq!(lines, 2500); + assert_eq!( + findings.len(), + 2500, + "Should find all 2500 secrets across chunks" + ); + } +} diff --git a/src/utils.rs b/src/utils.rs index 4c083d4..bb55bc3 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,7 +1,6 @@ use std::fs::File; use std::io::{Result, Write}; -/// write_to_file writes the given content to a file at the specified path. pub fn write_to_file(path: &str, content: &str) -> Result<()> { let mut file = File::create(path)?; file.write_all(content.as_bytes())?; diff --git a/tests/baseline_tests.rs b/tests/baseline_tests.rs new file mode 100644 index 0000000..d2c6051 --- /dev/null +++ b/tests/baseline_tests.rs @@ -0,0 +1,113 @@ +use key_watch::baseline::{Baseline, BaselineEntry}; +use key_watch::report::{Finding, Severity}; +use std::fs; + +fn make_finding(file: &str, line: usize, ftype: &str, content: &str, plugin: &str) -> Finding { + Finding { + file_path: file.to_string(), + line_number: line, + finding_type: ftype.to_string(), + severity: Severity::High, + matched_content: content.to_string(), + plugin_name: plugin.to_string(), + } +} + +#[test] +fn test_baseline_filters_known_findings() { + let known_finding = make_finding( + "test.txt", + 5, + "AWS Key", + "AKIAIOSFODNN7EXAMPLE", + "AWSAccessKeyDetector", + ); + let baseline = Baseline::from_findings(std::slice::from_ref(&known_finding)); + + let findings = vec![ + known_finding, + make_finding( + "other.txt", + 1, + "API Key", + "sk-abc", + "GenericKeyValueDetector", + ), + ]; + + let filtered = baseline.filter_findings(findings); + assert_eq!(filtered.len(), 1); + assert_eq!(filtered[0].file_path, "other.txt"); +} + +#[test] +fn test_baseline_allows_new_findings() { + let baseline = Baseline::default(); + let findings = vec![make_finding( + "test.txt", + 1, + "API Key", + "sk-abc", + "GenericKeyValueDetector", + )]; + let filtered = baseline.filter_findings(findings); + assert_eq!(filtered.len(), 1); +} + +#[test] +fn test_baseline_save_and_load() -> Result<(), String> { + let temp_file = std::env::temp_dir().join("keywatch_test_baseline.json"); + let baseline = Baseline { + version: "1.0".to_string(), + entries: vec![BaselineEntry { + file_path: "a.txt".to_string(), + line_number: 1, + finding_type: "X".to_string(), + matched_content_hash: "hash_secret".to_string(), + plugin_name: "D".to_string(), + }], + }; + + baseline.save(&temp_file)?; + let loaded = Baseline::load(&temp_file)?; + assert_eq!(loaded.entries.len(), 1); + assert_eq!(loaded.entries[0].file_path, "a.txt"); + + let _ = fs::remove_file(&temp_file); + Ok(()) +} + +#[test] +fn test_baseline_from_findings() { + let findings = vec![ + make_finding("f1.txt", 1, "A", "x", "D1"), + make_finding("f2.txt", 2, "B", "y", "D2"), + ]; + let baseline = Baseline::from_findings(&findings); + assert_eq!(baseline.entries.len(), 2); +} + +#[test] +fn test_baseline_update_merges_new_findings() { + let mut baseline = Baseline::from_findings(&[make_finding("old.txt", 1, "X", "old", "D")]); + let new_findings = vec![ + make_finding("old.txt", 1, "X", "old", "D"), + make_finding("new.txt", 2, "Y", "new", "D2"), + ]; + + baseline.update_with_findings(&new_findings); + + assert_eq!(baseline.entries.len(), 2); + assert!(baseline.entries.iter().any(|e| e.file_path == "old.txt")); + assert!(baseline.entries.iter().any(|e| e.file_path == "new.txt")); +} + +#[test] +fn test_baseline_update_preserves_existing() { + let mut baseline = + Baseline::from_findings(&[make_finding("existing.txt", 5, "API", "secret", "D")]); + baseline.update_with_findings(&[]); + + assert_eq!(baseline.entries.len(), 1); + assert_eq!(baseline.entries[0].file_path, "existing.txt"); +} diff --git a/tests/detector_tests.rs b/tests/detector_tests.rs new file mode 100644 index 0000000..79bbd51 --- /dev/null +++ b/tests/detector_tests.rs @@ -0,0 +1,140 @@ +use key_watch::detector::Detector; + +#[test] +fn test_allowlist_suppresses_matched_content() -> Result<(), String> { + let detector = Detector::new( + "TestDetector", + r"\bsecret_\w+\b", + "Test Secret", + "HIGH", + &[r"secret_allowed".to_string()], + &[], + None, + )?; + + assert!(detector.regex.is_match("secret_here")); + assert!( + detector + .allowlist + .iter() + .any(|p| p.is_match("secret_allowed")), + "Allowlist should match 'secret_allowed'" + ); + assert!( + !detector + .allowlist + .iter() + .any(|p| p.is_match("secret_blocked")), + "Allowlist should not match 'secret_blocked'" + ); + Ok(()) +} + +#[test] +fn test_detector_without_allowlist_allows_all() -> Result<(), String> { + let detector = Detector::new( + "TestDetector", + r"\bsecret_\w+\b", + "Test Secret", + "HIGH", + &[], + &[], + None, + )?; + + assert!(detector.allowlist.is_empty()); + assert!(detector.regex.is_match("secret_anything")); + Ok(()) +} + +#[test] +fn test_invalid_allowlist_pattern_returns_error() { + let result = Detector::new( + "TestDetector", + r"\bsecret_\w+\b", + "Test Secret", + "HIGH", + &[r"[invalid".to_string()], + &[], + None, + ); + assert!( + result.is_err(), + "Invalid allowlist pattern should return error" + ); +} + +#[test] +fn test_keywords_prefilter_skips_non_matching_content() -> Result<(), String> { + let detector = Detector::new( + "TestDetector", + r"\bsecret_\w+\b", + "Test Secret", + "HIGH", + &[], + &["apikey".to_string()], + None, + )?; + + assert!(!detector.has_keywords("some random text without the keyword")); + assert!(detector.has_keywords("this text contains apikey in it")); + assert!(detector.has_keywords("APIKEY in uppercase")); + Ok(()) +} + +#[test] +fn test_empty_keywords_allows_all_content() -> Result<(), String> { + let detector = Detector::new( + "TestDetector", + r"\bsecret_\w+\b", + "Test Secret", + "HIGH", + &[], + &[], + None, + )?; + + assert!(detector.has_keywords("any content should pass")); + assert!(detector.has_keywords("")); + Ok(()) +} + +#[test] +fn test_entropy_filters_low_entropy_matches() -> Result<(), String> { + let detector = Detector::new( + "TestDetector", + r"\bsecret_\w+\b", + "Test Secret", + "HIGH", + &[], + &[], + Some(3.0), + )?; + + assert!( + !detector.has_sufficient_entropy("secret_aaaaaaaa"), + "Low entropy string should be rejected" + ); + assert!( + detector.has_sufficient_entropy("secret_a1B2c3D4e5"), + "High entropy string should pass" + ); + Ok(()) +} + +#[test] +fn test_no_entropy_threshold_allows_all() -> Result<(), String> { + let detector = Detector::new( + "TestDetector", + r"\bsecret_\w+\b", + "Test Secret", + "HIGH", + &[], + &[], + None, + )?; + + assert!(detector.has_sufficient_entropy("secret_aaaaaaaa")); + assert!(detector.has_sufficient_entropy("secret_a1B2c3D4e5")); + Ok(()) +} diff --git a/tests/scanner_tests.rs b/tests/scanner_tests.rs index 0dd5bf1..d7ad1f9 100644 --- a/tests/scanner_tests.rs +++ b/tests/scanner_tests.rs @@ -20,6 +20,8 @@ sk-abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWX\n\ let options = ScanArgs { paths: vec![test_file.to_str().unwrap().to_string()], + stdin: false, + git_history: false, output: None, verbose: false, exclude: None, @@ -48,6 +50,8 @@ Stripe: sk_test_51ABCDEF12345678901234567890\n\ let options = ScanArgs { paths: vec![test_file.to_str().unwrap().to_string()], + stdin: false, + git_history: false, output: None, verbose: false, exclude: None, @@ -77,6 +81,8 @@ AZURE_STORAGE=DefaultEndpointsProtocol=https;AccountName=examplestore; let options = ScanArgs { paths: vec![test_file.to_str().unwrap().to_string()], + stdin: false, + git_history: false, output: None, verbose: false, exclude: None, @@ -107,6 +113,8 @@ b3BlbnNzaC1ldi0xLjAAABgQDQD2FGB3V2t4=\n\ let options = ScanArgs { paths: vec![test_file.to_str().unwrap().to_string()], + stdin: false, + git_history: false, output: None, verbose: false, exclude: None, @@ -131,6 +139,8 @@ fn test_multiple_detections_in_line() { let options = ScanArgs { paths: vec![test_file.to_str().unwrap().to_string()], + stdin: false, + git_history: false, output: None, verbose: false, exclude: None, @@ -167,6 +177,8 @@ fn test_directory_scan_with_exclusions() { let options = ScanArgs { paths: vec![test_dir.to_str().unwrap().to_string()], + stdin: false, + git_history: false, output: None, verbose: false, exclude: None, @@ -202,6 +214,8 @@ fn test_exclude_pattern_filtering() { let options = ScanArgs { paths: vec![test_dir.to_str().unwrap().to_string()], + stdin: false, + git_history: false, output: None, verbose: false, exclude: Some("*.log".to_string()), @@ -239,6 +253,8 @@ fn test_dot_github_directory_is_scanned() { let options = ScanArgs { paths: vec![test_dir.to_str().unwrap().to_string()], + stdin: false, + git_history: false, output: None, verbose: false, exclude: None, @@ -262,6 +278,8 @@ fn test_scan_no_secrets() { let options = ScanArgs { paths: vec![temp_file.to_str().unwrap().to_string()], + stdin: false, + git_history: false, output: None, verbose: false, exclude: None, @@ -286,6 +304,8 @@ fn test_non_utf8_file_handling() { let options = ScanArgs { paths: vec![test_file.to_str().unwrap().to_string()], + stdin: false, + git_history: false, output: None, verbose: false, exclude: None, @@ -314,6 +334,8 @@ fn test_multiple_files_scan() { test_file1.to_str().unwrap().to_string(), test_file2.to_str().unwrap().to_string(), ], + stdin: false, + git_history: false, output: None, verbose: false, exclude: None, @@ -343,6 +365,8 @@ fn test_duplicate_paths_are_scanned_once() { temp_file.to_str().unwrap().to_string(), temp_file.to_str().unwrap().to_string(), ], + stdin: false, + git_history: false, output: None, verbose: false, exclude: None, @@ -386,6 +410,8 @@ fn test_mixed_file_and_directory_paths_are_scanned_once() { direct_file.to_str().unwrap().to_string(), test_dir.to_str().unwrap().to_string(), ], + stdin: false, + git_history: false, output: None, verbose: false, exclude: None, @@ -421,6 +447,8 @@ fn test_nonexistent_paths_are_ignored_without_counting_as_scanned() { let options = ScanArgs { paths: vec![missing_path.to_str().unwrap().to_string()], + stdin: false, + git_history: false, output: None, verbose: false, exclude: None, @@ -454,6 +482,8 @@ fn test_detect_aadhaar() { let options = ScanArgs { paths: vec![test_file.to_str().unwrap().to_string()], + stdin: false, + git_history: false, output: None, verbose: false, exclude: None, @@ -485,6 +515,8 @@ fn test_detect_voter_id() { let options = ScanArgs { paths: vec![test_file.to_str().unwrap().to_string()], + stdin: false, + git_history: false, output: None, verbose: false, exclude: None, @@ -513,6 +545,8 @@ fn test_detect_pan_card() { let options = ScanArgs { paths: vec![test_file.to_str().unwrap().to_string()], + stdin: false, + git_history: false, output: None, verbose: false, exclude: None, @@ -541,6 +575,8 @@ fn test_detect_abha() { let options = ScanArgs { paths: vec![test_file.to_str().unwrap().to_string()], + stdin: false, + git_history: false, output: None, verbose: false, exclude: None, @@ -570,6 +606,8 @@ fn test_multiple_indian_ids() { let options = ScanArgs { paths: vec![test_file.to_str().unwrap().to_string()], + stdin: false, + git_history: false, output: None, verbose: false, exclude: None, @@ -619,16 +657,13 @@ fn test_overlapping_scan_roots_with_exclusions() { let test_file = root2.join("secret.txt"); fs::write(&test_file, "password=secret123").expect("Write test file"); - // The exclude pattern "subdir/secret.txt" should match relative to root1, - // or just "secret.txt" should match relative to root2. - // Let's exclude "subdir/secret.txt". It matches relative to root1. - // If root2 is used as the only root, "secret.txt" stripped of root2 is "secret.txt", - // which does NOT match "subdir/secret.txt", so it would NOT be excluded and find the secret! let options = ScanArgs { paths: vec![ root2.to_str().unwrap().to_string(), // Root 2 comes first to try to mess up order root1.to_str().unwrap().to_string(), ], + stdin: false, + git_history: false, output: None, verbose: false, exclude: Some("subdir/secret.txt".to_string()), @@ -639,7 +674,6 @@ fn test_overlapping_scan_roots_with_exclusions() { let (findings, metadata) = run_scan(&options).expect("run_scan should succeed"); - // Because we exclude "subdir/secret.txt", it should be excluded via root1's perspective. assert!( metadata .excluded_files @@ -654,3 +688,141 @@ fn test_overlapping_scan_roots_with_exclusions() { fs::remove_dir_all(root1).expect("Cleanup"); } + +#[test] +fn test_inline_suppression_ignores_marked_lines() -> Result<(), String> { + let temp_dir = temp_dir(); + let test_file = temp_dir.join("key_watch_inline_suppress.txt"); + + let content = "\ +AWS Key: AKIAABCDEFGHIJKLMNOP # keywatch:ignore\npassword = 'mySecretPassword'\nemail = user@example.com // keywatch:ignore\nFirebase: AIzaSy012345678901234567890123456789012\n"; + fs::write(&test_file, content).map_err(|e| format!("Unable to write test file: {}", e))?; + + let path_str = test_file.to_string_lossy().to_string(); + + let options = ScanArgs { + paths: vec![path_str], + stdin: false, + git_history: false, + output: None, + verbose: false, + exclude: None, + exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, + }; + + let (findings, _) = run_scan(&options)?; + + let aws_suppressed = findings + .iter() + .any(|f| f.matched_content.contains("AKIAABCDEFGHIJKLMNOP")); + let email_suppressed = findings + .iter() + .any(|f| f.matched_content.contains("user@example.com")); + let firebase_found = findings.iter().any(|f| { + f.matched_content + .contains("AIzaSy012345678901234567890123456789012") + }); + + assert!( + !aws_suppressed, + "AWS key with # keywatch:ignore should be suppressed" + ); + assert!( + !email_suppressed, + "Email with // keywatch:ignore should be suppressed" + ); + assert!( + firebase_found, + "Firebase key without suppression should still be found" + ); + + fs::remove_file(test_file).map_err(|e| format!("Cleanup failed: {}", e))?; + Ok(()) +} + +#[test] +fn test_stdin_scanning() { + let options = ScanArgs { + paths: vec![], + stdin: true, + git_history: false, + output: None, + verbose: false, + exclude: None, + exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, + }; + + assert!(options.validate().is_ok()); +} + +#[test] +fn test_git_history_scanning() -> Result<(), String> { + use std::process::Command; + + let temp_dir = std::env::temp_dir().join("keywatch_test_git_history"); + let _ = fs::remove_dir_all(&temp_dir); + fs::create_dir_all(&temp_dir).map_err(|e| format!("Create dir: {}", e))?; + + Command::new("git") + .args(["init", "--quiet"]) + .current_dir(&temp_dir) + .output() + .map_err(|e| format!("git init: {}", e))?; + + Command::new("git") + .args(["config", "user.email", "test@test.com"]) + .current_dir(&temp_dir) + .output() + .map_err(|e| format!("git config: {}", e))?; + + Command::new("git") + .args(["config", "user.name", "Test"]) + .current_dir(&temp_dir) + .output() + .map_err(|e| format!("git config: {}", e))?; + + let secret_file = temp_dir.join("secrets.txt"); + fs::write(&secret_file, "AWS Key: AKIAABCDEFGHIJKLMNOP\n") + .map_err(|e| format!("Write file: {}", e))?; + + Command::new("git") + .args(["add", "."]) + .current_dir(&temp_dir) + .output() + .map_err(|e| format!("git add: {}", e))?; + + Command::new("git") + .args(["commit", "-m", "initial", "--quiet"]) + .current_dir(&temp_dir) + .output() + .map_err(|e| format!("git commit: {}", e))?; + + let options = ScanArgs { + paths: vec![], + stdin: false, + git_history: true, + output: None, + verbose: false, + exclude: None, + exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, + }; + + let (findings, metadata) = run_scan(&options)?; + + assert!( + findings + .iter() + .any(|f| f.matched_content.contains("AKIAABCDEFGHIJKLMNOP")), + "Should find AWS key in git history" + ); + assert_eq!(metadata.files_scanned, 1); + + let _ = fs::remove_dir_all(&temp_dir); + Ok(()) +} diff --git a/typos.toml b/typos.toml new file mode 100644 index 0000000..7ca515b --- /dev/null +++ b/typos.toml @@ -0,0 +1,7 @@ +[default] +extend-ignore-identifiers-re = [ + # HashiCorp is a real company name, not a typo for "Hash" + "HashiCorp", + # Fly.io token prefix pattern (fo1_) + "fo1_", +] From 2c421c8e281ccc27206de418cf195088eaae2437 Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Wed, 29 Jul 2026 23:46:47 +0530 Subject: [PATCH 05/17] chore: put omo in gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index fa44e16..c9b70ed 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # AI /.sisyphus +/.omo # Rust builds /target From beee475ad48c2123d6d8ea4059393c0cc15bd7df Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Wed, 29 Jul 2026 23:59:10 +0530 Subject: [PATCH 06/17] fix: scan_stream chunk overlap, salted SHA-256 baseline, real stdin integration test, graceful git-not-found --- Cargo.lock | 72 ++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 1 + src/baseline.rs | 14 +++++--- src/report.rs | 2 +- src/scanner.rs | 9 +++--- tests/scanner_tests.rs | 45 +++++++++++++++++++++++++- 6 files changed, 133 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index edfffc9..6043db4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -61,6 +61,15 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -113,6 +122,15 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "crossbeam-deque" version = "0.8.7" @@ -138,6 +156,26 @@ version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "dirs" version = "6.0.0" @@ -171,6 +209,16 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -233,6 +281,7 @@ dependencies = [ "regex", "serde", "serde_json", + "sha2", "toml", ] @@ -399,6 +448,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "strsim" version = "0.11.1" @@ -475,6 +535,12 @@ version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -487,6 +553,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" diff --git a/Cargo.toml b/Cargo.toml index 12153db..0e04bf5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,3 +20,4 @@ serde_json = "1.0.150" glob = "0.3.3" dirs = "6.0" rayon = "1" +sha2 = "0.10" diff --git a/src/baseline.rs b/src/baseline.rs index b3896c2..c8de070 100644 --- a/src/baseline.rs +++ b/src/baseline.rs @@ -6,10 +6,16 @@ use std::path::Path; use crate::report::Finding; fn hash_content(content: &str) -> String { - use std::hash::{DefaultHasher, Hash, Hasher}; - let mut hasher = DefaultHasher::new(); - content.hash(&mut hasher); - format!("{:016x}", hasher.finish()) + use sha2::{Digest, Sha256}; + let domain_separator = "keywatch-baseline-v1"; + let mut hasher = Sha256::new(); + hasher.update(domain_separator.as_bytes()); + hasher.update(content.as_bytes()); + hasher + .finalize() + .iter() + .map(|b| format!("{:02x}", b)) + .collect() } #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Hash)] diff --git a/src/report.rs b/src/report.rs index 97cafc7..03cd05e 100644 --- a/src/report.rs +++ b/src/report.rs @@ -11,7 +11,7 @@ pub enum Severity { impl Severity { pub fn from_string(s: &str) -> Severity { - match s.to_uppercase().as_str() { + match s.trim().to_uppercase().as_str() { "CRITICAL" => Severity::Critical, "HIGH" => Severity::High, "MEDIUM" => Severity::Medium, diff --git a/src/scanner.rs b/src/scanner.rs index 1f0cd94..269ca68 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -107,10 +107,11 @@ fn scan_stream( line_detectors: &[&Detector], ) -> Result<(Vec, usize), String> { const CHUNK_SIZE: usize = 1000; + const OVERLAP_LINES: usize = 50; let mut findings = Vec::new(); let mut total_lines = 0; - let mut buffer: Vec = Vec::with_capacity(CHUNK_SIZE); + let mut buffer: Vec = Vec::with_capacity(CHUNK_SIZE + OVERLAP_LINES); let mut line_offset = 0; for line_result in reader.lines() { @@ -121,7 +122,7 @@ fn scan_stream( buffer.push(line); - if buffer.len() >= CHUNK_SIZE { + if buffer.len() >= CHUNK_SIZE + OVERLAP_LINES { let chunk = buffer.join("\n"); scan_multiline_chunk( &chunk, @@ -130,8 +131,8 @@ fn scan_stream( multiline_detectors, &mut findings, ); - line_offset += buffer.len(); - buffer.clear(); + line_offset += buffer.len() - OVERLAP_LINES; + buffer.drain(..buffer.len() - OVERLAP_LINES); } } diff --git a/tests/scanner_tests.rs b/tests/scanner_tests.rs index d7ad1f9..5bcd315 100644 --- a/tests/scanner_tests.rs +++ b/tests/scanner_tests.rs @@ -743,7 +743,7 @@ AWS Key: AKIAABCDEFGHIJKLMNOP # keywatch:ignore\npassword = 'mySecretPassword'\n } #[test] -fn test_stdin_scanning() { +fn test_stdin_args_validation() { let options = ScanArgs { paths: vec![], stdin: true, @@ -759,10 +759,53 @@ fn test_stdin_scanning() { assert!(options.validate().is_ok()); } +#[test] +fn test_stdin_scanning_integration() -> Result<(), String> { + use std::io::Write; + use std::process::{Command, Stdio}; + + let bin_path = env!("CARGO_BIN_EXE_key-watch"); + let mut child = Command::new(bin_path) + .args(["scan", "--stdin"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| format!("spawn key-watch --stdin: {}", e))?; + + let mut stdin = child.stdin.take().ok_or("Failed to capture stdin")?; + stdin + .write_all(b"AWS Key: AKIAABCDEFGHIJKLMNOP\npassword = 'secret123'\n") + .map_err(|e| format!("write to stdin: {}", e))?; + drop(stdin); + + let output = child + .wait_with_output() + .map_err(|e| format!("wait: {}", e))?; + + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + + let combined = format!("{}{}", stdout, stderr); + assert!( + combined.contains("3 potential secret(s)"), + "Should detect 3 secrets from stdin input\nstdout:\n{}\nstderr:\n{}", + stdout, + stderr + ); + + Ok(()) +} + #[test] fn test_git_history_scanning() -> Result<(), String> { use std::process::Command; + // Skip if git is not available + if Command::new("git").arg("--version").output().is_err() { + return Ok(()); + } + let temp_dir = std::env::temp_dir().join("keywatch_test_git_history"); let _ = fs::remove_dir_all(&temp_dir); fs::create_dir_all(&temp_dir).map_err(|e| format!("Create dir: {}", e))?; From 6e104bd7f2ff48b89637dcda06cc8ac0c070b0ea Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Fri, 31 Jul 2026 00:14:06 +0530 Subject: [PATCH 07/17] docs(changelog): add PR review fix entries --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b479aa3..61f64ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,10 @@ All notable changes to this project will be documented in this file. - CRITICAL severity was silently downgraded to LOW at runtime - All clippy warnings resolved (`Default` impl, redundant closures, identity maps) - Public API unit tests moved to `tests/` directory (only private API tests remain in `src/`) +- Baseline hash domain separator renamed from `SALT` to `DOMAIN_SEPARATOR` for clarity +- `Severity::from_string()` now trims whitespace from input before parsing +- `scan_stream()` chunk overlap fixed for accurate multiline detection on split chunks +- Graceful error handling when `git` is not installed on the system ### Removed From 8019783696f7fcaa896c27b74c88bd803258ac6c Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Sat, 1 Aug 2026 00:33:40 +0530 Subject: [PATCH 08/17] refactor: use const for baseline version and domain separator, rename severity param --- src/baseline.rs | 18 ++++++++++++++---- src/report.rs | 5 +++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/baseline.rs b/src/baseline.rs index c8de070..e27dc33 100644 --- a/src/baseline.rs +++ b/src/baseline.rs @@ -5,11 +5,21 @@ use std::path::Path; use crate::report::Finding; +/// Version of the baseline file format, not of the application itself. +/// +/// Bump this only when the on-disk format changes incompatibly +/// (e.g. a different hashing scheme or a new required field). +/// Application releases do not invalidate existing baselines. +const BASELINE_VERSION: &str = "1.0"; + +/// Domain-separation prefix for fingerprint hashes so that baseline hashes +/// cannot collide with plain SHA-256 of the matched content. +const HASH_DOMAIN_SEPARATOR: &str = "keywatch-baseline-v1"; + fn hash_content(content: &str) -> String { use sha2::{Digest, Sha256}; - let domain_separator = "keywatch-baseline-v1"; let mut hasher = Sha256::new(); - hasher.update(domain_separator.as_bytes()); + hasher.update(HASH_DOMAIN_SEPARATOR.as_bytes()); hasher.update(content.as_bytes()); hasher .finalize() @@ -36,7 +46,7 @@ pub struct Baseline { impl Baseline { pub fn new() -> Self { Self { - version: "1.0".to_string(), + version: BASELINE_VERSION.to_string(), entries: Vec::new(), } } @@ -94,7 +104,7 @@ impl Baseline { .collect(); Self { - version: "1.0".to_string(), + version: BASELINE_VERSION.to_string(), entries, } } diff --git a/src/report.rs b/src/report.rs index 03cd05e..e64817c 100644 --- a/src/report.rs +++ b/src/report.rs @@ -10,8 +10,9 @@ pub enum Severity { } impl Severity { - pub fn from_string(s: &str) -> Severity { - match s.trim().to_uppercase().as_str() { + /// Parse a severity from a string. Unknown or empty input defaults to `Low`. + pub fn from_string(value: &str) -> Severity { + match value.trim().to_uppercase().as_str() { "CRITICAL" => Severity::Critical, "HIGH" => Severity::High, "MEDIUM" => Severity::Medium, From 9329a92687eb6aabd2576471a7b7b37a4cc6efa4 Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:35:18 +0530 Subject: [PATCH 09/17] fix: stabilize baseline fingerprints --- Cargo.lock | 80 +++++++++++++++++++++++++- Cargo.toml | 3 + src/baseline.rs | 121 ++++++++++++++++++++++++---------------- tests/baseline_tests.rs | 69 ++++++++++++++++------- 4 files changed, 206 insertions(+), 67 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6043db4..7ca8224 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -61,6 +61,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + [[package]] name = "block-buffer" version = "0.10.4" @@ -209,6 +215,22 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + [[package]] name = "generic-array" version = "0.14.7" @@ -230,6 +252,17 @@ dependencies = [ "wasi", ] +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + [[package]] name = "glob" version = "0.3.4" @@ -282,6 +315,7 @@ dependencies = [ "serde", "serde_json", "sha2", + "tempfile", "toml", ] @@ -300,12 +334,24 @@ dependencies = [ "libc", ] +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "memchr" version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + [[package]] name = "once_cell_polyfill" version = "1.70.2" @@ -336,6 +382,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "rayon" version = "1.12.0" @@ -362,7 +414,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ - "getrandom", + "getrandom 0.2.17", "libredox", "thiserror", ] @@ -396,6 +448,19 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + [[package]] name = "serde" version = "1.0.229" @@ -476,6 +541,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys", +] + [[package]] name = "thiserror" version = "2.0.19" diff --git a/Cargo.toml b/Cargo.toml index 0e04bf5..ccb5566 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,3 +21,6 @@ glob = "0.3.3" dirs = "6.0" rayon = "1" sha2 = "0.10" + +[dev-dependencies] +tempfile = "3" diff --git a/src/baseline.rs b/src/baseline.rs index e27dc33..df1e648 100644 --- a/src/baseline.rs +++ b/src/baseline.rs @@ -18,14 +18,46 @@ const HASH_DOMAIN_SEPARATOR: &str = "keywatch-baseline-v1"; fn hash_content(content: &str) -> String { use sha2::{Digest, Sha256}; + use std::fmt::Write as _; + let mut hasher = Sha256::new(); hasher.update(HASH_DOMAIN_SEPARATOR.as_bytes()); hasher.update(content.as_bytes()); - hasher - .finalize() - .iter() - .map(|b| format!("{:02x}", b)) - .collect() + let mut output = String::with_capacity(64); + for byte in hasher.finalize() { + let _ = write!(&mut output, "{:02x}", byte); + } + output +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +struct BaselineFingerprint { + file_path: String, + finding_type: String, + matched_content_hash: String, + plugin_name: String, +} + +impl BaselineFingerprint { + fn from_finding(finding: &Finding) -> Self { + Self { + file_path: finding.file_path.clone(), + finding_type: finding.finding_type.clone(), + matched_content_hash: hash_content(&finding.matched_content), + plugin_name: finding.plugin_name.clone(), + } + } +} + +impl From<&BaselineEntry> for BaselineFingerprint { + fn from(entry: &BaselineEntry) -> Self { + Self { + file_path: entry.file_path.clone(), + finding_type: entry.finding_type.clone(), + matched_content_hash: entry.matched_content_hash.clone(), + plugin_name: entry.plugin_name.clone(), + } + } } #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Hash)] @@ -79,29 +111,38 @@ impl Baseline { Ok(()) } - fn build_fingerprints(&self) -> HashSet { - self.entries.iter().map(fingerprint).collect() + fn build_fingerprints(&self) -> HashSet { + self.entries.iter().map(BaselineFingerprint::from).collect() + } + + fn entry_from_finding(finding: &Finding) -> BaselineEntry { + BaselineEntry { + file_path: finding.file_path.clone(), + line_number: finding.line_number, + finding_type: finding.finding_type.clone(), + matched_content_hash: hash_content(&finding.matched_content), + plugin_name: finding.plugin_name.clone(), + } } pub fn filter_findings(&self, findings: Vec) -> Vec { let fingerprints = self.build_fingerprints(); findings .into_iter() - .filter(|f| !fingerprints.contains(&finding_fingerprint(f))) + .filter(|finding| !fingerprints.contains(&BaselineFingerprint::from_finding(finding))) .collect() } pub fn from_findings(findings: &[Finding]) -> Self { - let entries = findings - .iter() - .map(|f| BaselineEntry { - file_path: f.file_path.clone(), - line_number: f.line_number, - finding_type: f.finding_type.clone(), - matched_content_hash: hash_content(&f.matched_content), - plugin_name: f.plugin_name.clone(), - }) - .collect(); + let mut entries = Vec::new(); + let mut fingerprints = HashSet::new(); + + for finding in findings { + let fingerprint = BaselineFingerprint::from_finding(finding); + if fingerprints.insert(fingerprint) { + entries.push(Self::entry_from_finding(finding)); + } + } Self { version: BASELINE_VERSION.to_string(), @@ -110,17 +151,12 @@ impl Baseline { } pub fn update_with_findings(&mut self, findings: &[Finding]) { - let existing: HashSet = self.entries.iter().map(fingerprint).collect(); + let mut existing: HashSet = + self.entries.iter().map(BaselineFingerprint::from).collect(); for f in findings { - let fp = finding_fingerprint(f); - if !existing.contains(&fp) { - self.entries.push(BaselineEntry { - file_path: f.file_path.clone(), - line_number: f.line_number, - finding_type: f.finding_type.clone(), - matched_content_hash: hash_content(&f.matched_content), - plugin_name: f.plugin_name.clone(), - }); + let fingerprint = BaselineFingerprint::from_finding(f); + if existing.insert(fingerprint) { + self.entries.push(Self::entry_from_finding(f)); } } } @@ -132,24 +168,15 @@ impl Default for Baseline { } } -fn fingerprint(entry: &BaselineEntry) -> String { - format!( - "{}:{}:{}:{}:{}", - entry.file_path, - entry.line_number, - entry.finding_type, - entry.matched_content_hash, - entry.plugin_name - ) -} +#[cfg(test)] +mod tests { + use super::hash_content; -fn finding_fingerprint(finding: &Finding) -> String { - format!( - "{}:{}:{}:{}:{}", - finding.file_path, - finding.line_number, - finding.finding_type, - hash_content(&finding.matched_content), - finding.plugin_name - ) + #[test] + fn hash_content_uses_expected_lowercase_hex() { + assert_eq!( + hash_content(""), + "49969dbf750f1c12188f4646dc9b5ff608ceb35d74e5de79fad99e88ebd445d6" + ); + } } diff --git a/tests/baseline_tests.rs b/tests/baseline_tests.rs index d2c6051..d49c83a 100644 --- a/tests/baseline_tests.rs +++ b/tests/baseline_tests.rs @@ -1,6 +1,6 @@ use key_watch::baseline::{Baseline, BaselineEntry}; use key_watch::report::{Finding, Severity}; -use std::fs; +use tempfile::tempdir; fn make_finding(file: &str, line: usize, ftype: &str, content: &str, plugin: &str) -> Finding { Finding { @@ -14,7 +14,7 @@ fn make_finding(file: &str, line: usize, ftype: &str, content: &str, plugin: &st } #[test] -fn test_baseline_filters_known_findings() { +fn test_baseline_filters_moved_finding_in_same_file() { let known_finding = make_finding( "test.txt", 5, @@ -24,16 +24,36 @@ fn test_baseline_filters_known_findings() { ); let baseline = Baseline::from_findings(std::slice::from_ref(&known_finding)); - let findings = vec![ - known_finding, - make_finding( - "other.txt", - 1, - "API Key", - "sk-abc", - "GenericKeyValueDetector", - ), - ]; + let findings = vec![make_finding( + "test.txt", + 42, + "AWS Key", + "AKIAIOSFODNN7EXAMPLE", + "AWSAccessKeyDetector", + )]; + + let filtered = baseline.filter_findings(findings); + assert!(filtered.is_empty()); +} + +#[test] +fn test_baseline_keeps_same_finding_in_different_file() { + let known_finding = make_finding( + "test.txt", + 5, + "AWS Key", + "AKIAIOSFODNN7EXAMPLE", + "AWSAccessKeyDetector", + ); + let baseline = Baseline::from_findings(std::slice::from_ref(&known_finding)); + + let findings = vec![make_finding( + "other.txt", + 42, + "AWS Key", + "AKIAIOSFODNN7EXAMPLE", + "AWSAccessKeyDetector", + )]; let filtered = baseline.filter_findings(findings); assert_eq!(filtered.len(), 1); @@ -56,7 +76,8 @@ fn test_baseline_allows_new_findings() { #[test] fn test_baseline_save_and_load() -> Result<(), String> { - let temp_file = std::env::temp_dir().join("keywatch_test_baseline.json"); + let temp_dir = tempdir().map_err(|err| err.to_string())?; + let temp_file = temp_dir.path().join("baseline.json"); let baseline = Baseline { version: "1.0".to_string(), entries: vec![BaselineEntry { @@ -72,27 +93,28 @@ fn test_baseline_save_and_load() -> Result<(), String> { let loaded = Baseline::load(&temp_file)?; assert_eq!(loaded.entries.len(), 1); assert_eq!(loaded.entries[0].file_path, "a.txt"); - - let _ = fs::remove_file(&temp_file); + assert_eq!(loaded.entries[0].line_number, 1); Ok(()) } #[test] -fn test_baseline_from_findings() { +fn test_baseline_from_findings_deduplicates_and_keeps_first_metadata() { let findings = vec![ make_finding("f1.txt", 1, "A", "x", "D1"), - make_finding("f2.txt", 2, "B", "y", "D2"), + make_finding("f1.txt", 99, "A", "x", "D1"), ]; let baseline = Baseline::from_findings(&findings); - assert_eq!(baseline.entries.len(), 2); + assert_eq!(baseline.entries.len(), 1); + assert_eq!(baseline.entries[0].line_number, 1); } #[test] -fn test_baseline_update_merges_new_findings() { +fn test_baseline_update_merges_new_findings_once() { let mut baseline = Baseline::from_findings(&[make_finding("old.txt", 1, "X", "old", "D")]); let new_findings = vec![ make_finding("old.txt", 1, "X", "old", "D"), make_finding("new.txt", 2, "Y", "new", "D2"), + make_finding("new.txt", 99, "Y", "new", "D2"), ]; baseline.update_with_findings(&new_findings); @@ -100,6 +122,15 @@ fn test_baseline_update_merges_new_findings() { assert_eq!(baseline.entries.len(), 2); assert!(baseline.entries.iter().any(|e| e.file_path == "old.txt")); assert!(baseline.entries.iter().any(|e| e.file_path == "new.txt")); + assert_eq!( + baseline + .entries + .iter() + .find(|entry| entry.file_path == "new.txt") + .expect("new entry should exist") + .line_number, + 2 + ); } #[test] From 5f5fc986f03352f8515e8ca81bbdef513f627680 Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:35:27 +0530 Subject: [PATCH 10/17] fix: validate detector severity and names --- detectors.toml | 4 +- src/detector.rs | 74 +++++++++++++++++++++++++++++++---- src/report.rs | 58 +++++++++++++++++++++++---- tests/detector_tests.rs | 86 +++++++++++++++++++++++++++++++++++++---- 4 files changed, 197 insertions(+), 25 deletions(-) diff --git a/detectors.toml b/detectors.toml index ba9aa51..3358c59 100644 --- a/detectors.toml +++ b/detectors.toml @@ -736,14 +736,14 @@ severity = "CRITICAL" keywords = ["master"] [[detectors]] -name = "RazorpayKeyDetector" +name = "RazorpayAPIKeyDetector" pattern = "\\b(rzp_(live|test)_[a-zA-Z0-9]{20,})\\b" finding_type = "Razorpay API Key" severity = "HIGH" keywords = ["rzp_live_", "rzp_test_"] [[detectors]] -name = "PaystackKeyDetector" +name = "PaystackSecretKeyDetector" pattern = "\\b(sk_(live|test)_[a-zA-Z0-9]{30,})\\b" finding_type = "Paystack Secret Key" severity = "HIGH" diff --git a/src/detector.rs b/src/detector.rs index f46e309..ae4304f 100644 --- a/src/detector.rs +++ b/src/detector.rs @@ -1,12 +1,53 @@ +use crate::report::{ParseSeverityError, Severity}; use regex::Regex; use serde::Deserialize; +use std::fmt; use std::fs; +use std::str::FromStr; + +#[derive(Debug)] +pub enum DetectorError { + InvalidPattern { + detector: String, + source: regex::Error, + }, + InvalidAllowlistPattern { + detector: String, + source: regex::Error, + }, + InvalidSeverity { + detector: String, + source: ParseSeverityError, + }, +} + +impl fmt::Display for DetectorError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + DetectorError::InvalidPattern { detector, source } => { + write!(f, "invalid pattern in detector '{}': {}", detector, source) + } + DetectorError::InvalidAllowlistPattern { detector, source } => { + write!( + f, + "invalid allowlist pattern in detector '{}': {}", + detector, source + ) + } + DetectorError::InvalidSeverity { detector, source } => { + write!(f, "invalid severity in detector '{}': {}", detector, source) + } + } + } +} + +impl std::error::Error for DetectorError {} pub struct Detector { pub name: String, pub regex: Regex, pub finding_type: String, - pub severity: String, + pub severity: Severity, pub allowlist: Vec, pub keywords: Vec, pub entropy_threshold: Option, @@ -21,15 +62,25 @@ impl Detector { allowlist: &[String], keywords: &[String], entropy_threshold: Option, - ) -> Result { - let regex = Regex::new(pattern) - .map_err(|err| format!("Invalid pattern in detector '{}': {}", name, err))?; + ) -> Result { + let regex = Regex::new(pattern).map_err(|source| DetectorError::InvalidPattern { + detector: name.to_string(), + source, + })?; + + let parsed_severity = + Severity::from_str(severity).map_err(|source| DetectorError::InvalidSeverity { + detector: name.to_string(), + source, + })?; let mut compiled_allowlist = Vec::new(); for pat in allowlist { - let compiled = Regex::new(pat).map_err(|err| { - format!("Invalid allowlist pattern in detector '{}': {}", name, err) - })?; + let compiled = + Regex::new(pat).map_err(|source| DetectorError::InvalidAllowlistPattern { + detector: name.to_string(), + source, + })?; compiled_allowlist.push(compiled); } @@ -37,7 +88,7 @@ impl Detector { name: name.to_string(), regex, finding_type: finding_type.to_string(), - severity: severity.to_string(), + severity: parsed_severity, allowlist: compiled_allowlist, keywords: keywords.to_vec(), entropy_threshold, @@ -124,6 +175,13 @@ pub fn initialize_detectors() -> Result, Box Severity { +/// Error returned when a string cannot be parsed as a [`Severity`]. +#[derive(Debug, PartialEq)] +pub struct ParseSeverityError { + pub input: String, +} + +impl fmt::Display for ParseSeverityError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "invalid severity '{}': expected one of CRITICAL, HIGH, MEDIUM, LOW", + self.input + ) + } +} + +impl std::error::Error for ParseSeverityError {} + +impl FromStr for Severity { + type Err = ParseSeverityError; + + fn from_str(value: &str) -> Result { match value.trim().to_uppercase().as_str() { - "CRITICAL" => Severity::Critical, - "HIGH" => Severity::High, - "MEDIUM" => Severity::Medium, - _ => Severity::Low, + "CRITICAL" => Ok(Severity::Critical), + "HIGH" => Ok(Severity::High), + "MEDIUM" => Ok(Severity::Medium), + "LOW" => Ok(Severity::Low), + _ => Err(ParseSeverityError { + input: value.to_string(), + }), } } } +impl Severity { + /// Return the canonical uppercase string representation. + pub fn as_str(self) -> &'static str { + match self { + Severity::Critical => "CRITICAL", + Severity::High => "HIGH", + Severity::Medium => "MEDIUM", + Severity::Low => "LOW", + } + } +} + +impl fmt::Display for Severity { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + #[derive(Serialize)] #[serde(rename_all = "UPPERCASE")] pub enum ScanStatus { diff --git a/tests/detector_tests.rs b/tests/detector_tests.rs index 79bbd51..ad6bfb6 100644 --- a/tests/detector_tests.rs +++ b/tests/detector_tests.rs @@ -1,7 +1,9 @@ -use key_watch::detector::Detector; +use key_watch::detector::{Detector, DetectorError}; +use key_watch::report::Severity; +use std::str::FromStr; #[test] -fn test_allowlist_suppresses_matched_content() -> Result<(), String> { +fn test_allowlist_suppresses_matched_content() -> Result<(), DetectorError> { let detector = Detector::new( "TestDetector", r"\bsecret_\w+\b", @@ -31,7 +33,7 @@ fn test_allowlist_suppresses_matched_content() -> Result<(), String> { } #[test] -fn test_detector_without_allowlist_allows_all() -> Result<(), String> { +fn test_detector_without_allowlist_allows_all() -> Result<(), DetectorError> { let detector = Detector::new( "TestDetector", r"\bsecret_\w+\b", @@ -65,7 +67,7 @@ fn test_invalid_allowlist_pattern_returns_error() { } #[test] -fn test_keywords_prefilter_skips_non_matching_content() -> Result<(), String> { +fn test_keywords_prefilter_skips_non_matching_content() -> Result<(), DetectorError> { let detector = Detector::new( "TestDetector", r"\bsecret_\w+\b", @@ -83,7 +85,7 @@ fn test_keywords_prefilter_skips_non_matching_content() -> Result<(), String> { } #[test] -fn test_empty_keywords_allows_all_content() -> Result<(), String> { +fn test_empty_keywords_allows_all_content() -> Result<(), DetectorError> { let detector = Detector::new( "TestDetector", r"\bsecret_\w+\b", @@ -100,7 +102,7 @@ fn test_empty_keywords_allows_all_content() -> Result<(), String> { } #[test] -fn test_entropy_filters_low_entropy_matches() -> Result<(), String> { +fn test_entropy_filters_low_entropy_matches() -> Result<(), DetectorError> { let detector = Detector::new( "TestDetector", r"\bsecret_\w+\b", @@ -123,7 +125,7 @@ fn test_entropy_filters_low_entropy_matches() -> Result<(), String> { } #[test] -fn test_no_entropy_threshold_allows_all() -> Result<(), String> { +fn test_no_entropy_threshold_allows_all() -> Result<(), DetectorError> { let detector = Detector::new( "TestDetector", r"\bsecret_\w+\b", @@ -138,3 +140,73 @@ fn test_no_entropy_threshold_allows_all() -> Result<(), String> { assert!(detector.has_sufficient_entropy("secret_a1B2c3D4e5")); Ok(()) } + +#[test] +fn test_severity_from_str_valid_variants() { + assert_eq!(Severity::from_str("CRITICAL").unwrap(), Severity::Critical); + assert_eq!(Severity::from_str("HIGH").unwrap(), Severity::High); + assert_eq!(Severity::from_str("MEDIUM").unwrap(), Severity::Medium); + assert_eq!(Severity::from_str("LOW").unwrap(), Severity::Low); + assert_eq!(Severity::from_str("critical").unwrap(), Severity::Critical); + assert_eq!(Severity::from_str(" High ").unwrap(), Severity::High); +} + +#[test] +fn test_severity_from_str_invalid_returns_error() { + let err = Severity::from_str("UNKNOWN").unwrap_err(); + assert!( + err.to_string().contains("UNKNOWN"), + "Error message should include the offending input" + ); + assert!(Severity::from_str("").is_err()); + assert!(Severity::from_str("warn").is_err()); +} + +#[test] +fn test_severity_as_str_canonical_uppercase() { + assert_eq!(Severity::Critical.as_str(), "CRITICAL"); + assert_eq!(Severity::High.as_str(), "HIGH"); + assert_eq!(Severity::Medium.as_str(), "MEDIUM"); + assert_eq!(Severity::Low.as_str(), "LOW"); +} + +#[test] +fn test_detector_new_invalid_severity_returns_typed_error() { + let result = Detector::new( + "BadSevDetector", + r"\btest\b", + "Test", + "BOGUS", + &[], + &[], + None, + ); + match result { + Err(DetectorError::InvalidSeverity { detector, source }) => { + assert_eq!(detector, "BadSevDetector"); + assert!(source.to_string().contains("BOGUS")); + } + _ => panic!("expected InvalidSeverity error variant"), + } +} + +#[test] +fn test_detector_new_stores_parsed_severity() -> Result<(), DetectorError> { + let detector = Detector::new("SevDetector", r"\btest\b", "Test", "MEDIUM", &[], &[], None)?; + assert_eq!(detector.severity, Severity::Medium); + Ok(()) +} + +#[test] +fn test_initialize_detectors_all_names_unique() { + let detectors = key_watch::detector::initialize_detectors() + .expect("detectors.toml should load without error"); + let mut names = std::collections::HashSet::new(); + for det in &detectors { + assert!( + names.insert(det.name.as_str()), + "duplicate detector name: {}", + det.name + ); + } +} From df0669a552cb87095ceb45e203f586c0b416e699 Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:35:33 +0530 Subject: [PATCH 11/17] fix: harden file and git history scanning --- src/cli.rs | 4 +- src/scanner.rs | 91 ++++++++--- tests/scanner_tests.rs | 360 +++++++++++++++++++++++++++++++++++------ 3 files changed, 381 insertions(+), 74 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index 92966d8..007c9f4 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -80,8 +80,8 @@ impl ScanArgs { if self.stdin { return Err("Cannot specify both --git-history and --stdin".to_string()); } - if !self.paths.is_empty() { - return Err("Cannot specify paths with --git-history".to_string()); + if self.paths.len() > 1 { + return Err("Cannot specify more than one path with --git-history".to_string()); } return Ok(()); } diff --git a/src/scanner.rs b/src/scanner.rs index 269ca68..e7840d6 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -1,6 +1,6 @@ use crate::cli::ScanArgs; use crate::detector::{Detector, initialize_detectors}; -use crate::report::{Finding, ScanMetadata, Severity}; +use crate::report::{Finding, ScanMetadata}; use glob::Pattern; use rayon::prelude::*; use std::fs; @@ -25,13 +25,17 @@ fn scan_line_detectors( line_number: usize, path: &str, line_detectors: &[&Detector], + line_is_suppressed: bool, findings: &mut Vec, ) { + if line_is_suppressed { + return; + } + for detector in line_detectors { if detector.has_keywords(line) { for mat in detector.regex.find_iter(line) { - if !is_inline_suppressed(line) - && !is_allowlisted(mat.as_str(), detector) + if !is_allowlisted(mat.as_str(), detector) && detector.has_sufficient_entropy(mat.as_str()) { findings.push(Finding { @@ -39,7 +43,7 @@ fn scan_line_detectors( line_number, matched_content: mat.as_str().to_string(), finding_type: detector.finding_type.clone(), - severity: Severity::from_string(&detector.severity), + severity: detector.severity, plugin_name: detector.name.clone(), }); } @@ -63,7 +67,9 @@ fn scan_multiline_chunk( .lines() .nth(line_in_chunk.saturating_sub(1)) .unwrap_or_default(); - if !is_inline_suppressed(line_content) + let line_is_suppressed = is_inline_suppressed(line_content); + + if !line_is_suppressed && !is_allowlisted(mat.as_str(), detector) && detector.has_sufficient_entropy(mat.as_str()) { @@ -72,7 +78,7 @@ fn scan_multiline_chunk( line_number: line_offset + line_in_chunk, matched_content: mat.as_str().to_string(), finding_type: detector.finding_type.clone(), - severity: Severity::from_string(&detector.severity), + severity: detector.severity, plugin_name: detector.name.clone(), }); } @@ -94,7 +100,14 @@ fn scan_content( for (line_idx, line) in content.lines().enumerate() { total_lines += 1; - scan_line_detectors(line, line_idx + 1, path, line_detectors, &mut findings); + scan_line_detectors( + line, + line_idx + 1, + path, + line_detectors, + is_inline_suppressed(line), + &mut findings, + ); } (findings, total_lines) @@ -118,7 +131,14 @@ fn scan_stream( let line = line_result.map_err(|e| format!("Read error on {}: {}", path, e))?; total_lines += 1; - scan_line_detectors(&line, total_lines, path, line_detectors, &mut findings); + scan_line_detectors( + &line, + total_lines, + path, + line_detectors, + is_inline_suppressed(&line), + &mut findings, + ); buffer.push(line); @@ -157,8 +177,22 @@ pub fn run_scan(args: &ScanArgs) -> Result<(Vec, ScanMetadata), String> .partition(|detector| detector.regex.as_str().contains("(?s)")); if args.git_history { + let git_root = args + .paths + .first() + .map(Path::new) + .unwrap_or_else(|| Path::new(".")); let mut child = std::process::Command::new("git") - .args(["log", "-p", "-U0"]) + .current_dir(git_root) + .args([ + "-c", + "diff.external=", + "log", + "-p", + "-U0", + "--no-ext-diff", + "--no-textconv", + ]) .stdout(std::process::Stdio::piped()) .spawn() .map_err(|err| format!("Failed to run git log: {}", err))?; @@ -207,12 +241,17 @@ pub fn run_scan(args: &ScanArgs) -> Result<(Vec, ScanMetadata), String> for path_str in &args.paths { let path = Path::new(path_str); - if path.is_file() { + let Ok(metadata) = fs::symlink_metadata(path) else { + continue; + }; + let file_type = metadata.file_type(); + if file_type.is_symlink() { + continue; + } + if file_type.is_file() { target_paths.push((path_str.clone(), None)); - } else if path.is_dir() { + } else if file_type.is_dir() { collect_files(path_str, &mut target_paths, path_str); - } else { - target_paths.push((path_str.clone(), None)); } } @@ -255,6 +294,15 @@ pub fn run_scan(args: &ScanArgs) -> Result<(Vec, ScanMetadata), String> return (Vec::new(), 0, 0, Some(path)); } + let metadata = match fs::symlink_metadata(&path) { + Ok(metadata) => metadata, + Err(_) => return (Vec::new(), 0, 0, None), + }; + let file_type = metadata.file_type(); + if file_type.is_symlink() || !file_type.is_file() { + return (Vec::new(), 0, 0, None); + } + let full_content = match fs::read(&path) { Ok(bytes) => { if bytes.contains(&0) { @@ -307,16 +355,21 @@ pub fn run_scan(args: &ScanArgs) -> Result<(Vec, ScanMetadata), String> fn collect_files(dir_path: &str, files: &mut Vec<(String, Option)>, root: &str) { if let Ok(entries) = fs::read_dir(dir_path) { for entry in entries.flatten() { + let Ok(file_type) = entry.file_type() else { + continue; + }; + if file_type.is_symlink() { + continue; + } let path = entry.path(); - if path.is_file() { + if file_type.is_file() { if let Some(path_str) = path.to_str() { files.push((path_str.to_string(), Some(root.to_string()))); } - } else if path.is_dir() - && path.file_name().is_none_or(|name| name != ".git") - && let Some(path_str) = path.to_str() - { - collect_files(path_str, files, root); + } else if file_type.is_dir() && path.file_name().is_none_or(|name| name != ".git") { + if let Some(path_str) = path.to_str() { + collect_files(path_str, files, root); + } } } } diff --git a/tests/scanner_tests.rs b/tests/scanner_tests.rs index 5bcd315..e7b78e8 100644 --- a/tests/scanner_tests.rs +++ b/tests/scanner_tests.rs @@ -2,6 +2,92 @@ use key_watch::cli::{ExitMode, ScanArgs}; use key_watch::scanner::run_scan; use std::env::temp_dir; use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; +use std::time::{SystemTime, UNIX_EPOCH}; + +fn unique_temp_dir(name: &str) -> PathBuf { + let stamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system time should be after unix epoch") + .as_nanos(); + + temp_dir().join(format!("keywatch_{name}_{stamp}_{}", std::process::id())) +} + +fn git_available() -> bool { + Command::new("git").arg("--version").output().is_ok() +} + +fn init_git_repo(path: &Path) -> Result<(), String> { + fs::create_dir_all(path).map_err(|e| format!("create repo dir: {e}"))?; + + let status = Command::new("git") + .args(["init", "--quiet"]) + .current_dir(path) + .status() + .map_err(|e| format!("git init: {e}"))?; + if !status.success() { + return Err("git init failed".to_string()); + } + + for (key, value) in [("user.email", "test@test.com"), ("user.name", "Test")] { + let status = Command::new("git") + .args(["config", key, value]) + .current_dir(path) + .status() + .map_err(|e| format!("git config {key}: {e}"))?; + if !status.success() { + return Err(format!("git config {key} failed")); + } + } + + Ok(()) +} + +fn commit_file(path: &Path, file_name: &str, contents: &str, message: &str) -> Result<(), String> { + let file_path = path.join(file_name); + fs::write(&file_path, contents).map_err(|e| format!("write file: {e}"))?; + + let status = Command::new("git") + .args(["add", file_name]) + .current_dir(path) + .status() + .map_err(|e| format!("git add: {e}"))?; + if !status.success() { + return Err("git add failed".to_string()); + } + + let status = Command::new("git") + .args(["commit", "-m", message, "--quiet"]) + .current_dir(path) + .status() + .map_err(|e| format!("git commit: {e}"))?; + if !status.success() { + return Err("git commit failed".to_string()); + } + + Ok(()) +} + +fn detectors_config_path() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("detectors.toml") +} + +fn run_git_history_scan(current_dir: &Path, extra_args: &[&str]) -> Result { + Command::new(env!("CARGO_BIN_EXE_key-watch")) + .args(["scan", "--git-history"]) + .args(extra_args) + .env("KEYWATCH_CONFIG_PATH", detectors_config_path()) + .current_dir(current_dir) + .output() + .map_err(|e| format!("run key-watch scan --git-history: {e}")) +} + +#[cfg(unix)] +fn symlink_file(original: &Path, link: &Path) -> Result<(), String> { + std::os::unix::fs::symlink(original, link).map_err(|e| format!("create symlink: {e}")) +} #[test] fn test_find_secrets_in_file() { @@ -472,6 +558,92 @@ fn test_nonexistent_paths_are_ignored_without_counting_as_scanned() { ); } +#[cfg(unix)] +#[test] +fn test_explicit_symlink_path_is_skipped() -> Result<(), String> { + let test_dir = unique_temp_dir("explicit_symlink_skip"); + let outside_file = test_dir.join("outside-secret.txt"); + let link_path = test_dir.join("linked-secret.txt"); + let _ = fs::remove_dir_all(&test_dir); + fs::create_dir_all(&test_dir).map_err(|e| format!("create test dir: {e}"))?; + fs::write(&outside_file, "AWS Key: AKIAABCDEFGHIJKLMNOP\n") + .map_err(|e| format!("write outside secret: {e}"))?; + symlink_file(&outside_file, &link_path)?; + + let options = ScanArgs { + paths: vec![ + link_path + .to_str() + .ok_or("link path should be utf-8")? + .to_string(), + ], + stdin: false, + git_history: false, + output: None, + verbose: false, + exclude: None, + exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, + }; + + let (findings, metadata) = run_scan(&options)?; + + assert!(findings.is_empty(), "Symlink target should not be scanned"); + assert_eq!( + metadata.files_scanned, 0, + "Symlink should not count as scanned" + ); + + fs::remove_dir_all(&test_dir).map_err(|e| format!("cleanup: {e}"))?; + Ok(()) +} + +#[cfg(unix)] +#[test] +fn test_recursive_symlink_path_is_skipped() -> Result<(), String> { + let test_dir = unique_temp_dir("recursive_symlink_skip"); + let outside_file = test_dir.join("outside-secret.txt"); + let scan_root = test_dir.join("scan-root"); + let link_path = scan_root.join("linked-secret.txt"); + let _ = fs::remove_dir_all(&test_dir); + fs::create_dir_all(&scan_root).map_err(|e| format!("create scan root: {e}"))?; + fs::write(&outside_file, "AWS Key: AKIAABCDEFGHIJKLMNOP\n") + .map_err(|e| format!("write outside secret: {e}"))?; + symlink_file(&outside_file, &link_path)?; + + let options = ScanArgs { + paths: vec![ + scan_root + .to_str() + .ok_or("scan root should be utf-8")? + .to_string(), + ], + stdin: false, + git_history: false, + output: None, + verbose: false, + exclude: None, + exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, + }; + + let (findings, metadata) = run_scan(&options)?; + + assert!( + findings.is_empty(), + "Recursive symlink target should not be scanned" + ); + assert_eq!( + metadata.files_scanned, 0, + "Recursive symlink should not count as scanned" + ); + + fs::remove_dir_all(&test_dir).map_err(|e| format!("cleanup: {e}"))?; + Ok(()) +} + #[test] fn test_detect_aadhaar() { let temp_dir = temp_dir(); @@ -798,74 +970,156 @@ fn test_stdin_scanning_integration() -> Result<(), String> { } #[test] -fn test_git_history_scanning() -> Result<(), String> { - use std::process::Command; +fn test_git_history_args_validation_allows_zero_or_one_path() { + let zero_paths = ScanArgs { + paths: vec![], + stdin: false, + git_history: true, + output: None, + verbose: false, + exclude: None, + exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, + }; + let one_path = ScanArgs { + paths: vec!["/tmp/requested-root".to_string()], + stdin: false, + git_history: true, + output: None, + verbose: false, + exclude: None, + exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, + }; + let two_paths = ScanArgs { + paths: vec![ + "/tmp/requested-root".to_string(), + "/tmp/other-root".to_string(), + ], + stdin: false, + git_history: true, + output: None, + verbose: false, + exclude: None, + exit_mode: ExitMode::Strict, + baseline: None, + update_baseline: false, + }; + + assert!(zero_paths.validate().is_ok()); + assert!(one_path.validate().is_ok()); + assert!(two_paths.validate().is_err()); +} - // Skip if git is not available - if Command::new("git").arg("--version").output().is_err() { +#[test] +fn test_git_history_defaults_to_current_directory_when_no_path_is_provided() -> Result<(), String> { + if !git_available() { return Ok(()); } - let temp_dir = std::env::temp_dir().join("keywatch_test_git_history"); - let _ = fs::remove_dir_all(&temp_dir); - fs::create_dir_all(&temp_dir).map_err(|e| format!("Create dir: {}", e))?; + let repo_dir = unique_temp_dir("git_history_default_cwd"); + let _ = fs::remove_dir_all(&repo_dir); + init_git_repo(&repo_dir)?; + commit_file( + &repo_dir, + "secrets.txt", + "AWS Key: AKIAABCDEFGHIJKLMNOP\n", + "initial", + )?; + + let output = run_git_history_scan(&repo_dir, &[])?; + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + let combined = format!("{}{}", stdout, stderr); - Command::new("git") - .args(["init", "--quiet"]) - .current_dir(&temp_dir) - .output() - .map_err(|e| format!("git init: {}", e))?; + assert!( + matches!(output.status.code(), Some(1)), + "default cwd git history scan should report findings\nstdout:\n{}\nstderr:\n{}", + stdout, + stderr + ); + assert!(combined.contains("potential secret(s) detected")); - Command::new("git") - .args(["config", "user.email", "test@test.com"]) - .current_dir(&temp_dir) - .output() - .map_err(|e| format!("git config: {}", e))?; + let _ = fs::remove_dir_all(&repo_dir); + Ok(()) +} - Command::new("git") - .args(["config", "user.name", "Test"]) - .current_dir(&temp_dir) - .output() - .map_err(|e| format!("git config: {}", e))?; +#[test] +fn test_git_history_scans_requested_root_from_a_different_current_directory() -> Result<(), String> +{ + if !git_available() { + return Ok(()); + } - let secret_file = temp_dir.join("secrets.txt"); - fs::write(&secret_file, "AWS Key: AKIAABCDEFGHIJKLMNOP\n") - .map_err(|e| format!("Write file: {}", e))?; + let parent_dir = unique_temp_dir("git_history_requested_root_parent"); + let requested_root = parent_dir.join("requested-repo"); + let _ = fs::remove_dir_all(&parent_dir); + init_git_repo(&requested_root)?; + commit_file( + &requested_root, + "secrets.txt", + "AWS Key: AKIAQRSTUVWXYZABCDEF\n", + "initial", + )?; + + let output = run_git_history_scan(&parent_dir, &[requested_root.to_str().unwrap()])?; + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + let combined = format!("{}{}", stdout, stderr); - Command::new("git") - .args(["add", "."]) - .current_dir(&temp_dir) - .output() - .map_err(|e| format!("git add: {}", e))?; + assert!( + matches!(output.status.code(), Some(1)), + "explicit git root scan should report findings\nstdout:\n{}\nstderr:\n{}", + stdout, + stderr + ); + assert!(combined.contains("potential secret(s) detected")); - Command::new("git") - .args(["commit", "-m", "initial", "--quiet"]) - .current_dir(&temp_dir) - .output() - .map_err(|e| format!("git commit: {}", e))?; + let _ = fs::remove_dir_all(&parent_dir); + Ok(()) +} - let options = ScanArgs { - paths: vec![], - stdin: false, - git_history: true, - output: None, - verbose: false, - exclude: None, - exit_mode: ExitMode::Strict, - baseline: None, - update_baseline: false, - }; +#[test] +fn test_git_history_does_not_execute_textconv_helpers() -> Result<(), String> { + if !git_available() { + return Ok(()); + } - let (findings, metadata) = run_scan(&options)?; + let repo_dir = unique_temp_dir("git_history_no_textconv"); + let marker_path = repo_dir.join("textconv-helper-ran"); + let _ = fs::remove_dir_all(&repo_dir); + init_git_repo(&repo_dir)?; + + let helper = format!( + "sh -c 'printf textconv-ran > \"{}\"; cat \"$1\"' -", + marker_path.display() + ); + let status = Command::new("git") + .env("GIT_MASTER", "1") + .args(["config", "diff.keywatchmarker.textconv", &helper]) + .current_dir(&repo_dir) + .status() + .map_err(|e| format!("git config textconv: {e}"))?; + if !status.success() { + return Err("git config textconv failed".to_string()); + } + + commit_file( + &repo_dir, + ".gitattributes", + "*.kw diff=keywatchmarker\n", + "attrs", + )?; + commit_file(&repo_dir, "sample.kw", "ordinary text\n", "sample")?; + let _output = run_git_history_scan(&repo_dir, &[])?; assert!( - findings - .iter() - .any(|f| f.matched_content.contains("AKIAABCDEFGHIJKLMNOP")), - "Should find AWS key in git history" + !marker_path.exists(), + "git history scan must not execute configured textconv helpers" ); - assert_eq!(metadata.files_scanned, 1); - let _ = fs::remove_dir_all(&temp_dir); + let _ = fs::remove_dir_all(&repo_dir); Ok(()) } From 1113feb90b53cf76e5c205a6bca502fc57f7e90b Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:35:40 +0530 Subject: [PATCH 12/17] fix: harden generated git hooks --- templates/pre-commit.sh | 9 +- templates/pre-push.sh | 85 +++++++++- tests/hooks_tests.rs | 333 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 419 insertions(+), 8 deletions(-) diff --git a/templates/pre-commit.sh b/templates/pre-commit.sh index 460d1ac..95b76cb 100644 --- a/templates/pre-commit.sh +++ b/templates/pre-commit.sh @@ -14,16 +14,19 @@ while IFS= read -r -d '' file; do if [ -z "$file" ]; then continue fi - if [ ! -f "$file" ]; then + if [ -L "$file" ]; then continue fi - if "$KEYWATCH_BIN" scan "$file" --exclude "$EXCLUDE_PATTERNS" 2>/dev/null; then + if [ ! -f "$file" ]; then continue fi + "$KEYWATCH_BIN" scan "$file" --exclude "$EXCLUDE_PATTERNS" >/dev/null 2>&1 EXIT_CODE=$? + if [ $EXIT_CODE -eq 0 ]; then + continue + fi if [ $EXIT_CODE -eq 1 ]; then echo "ERROR: Secret detected in $file" - "$KEYWATCH_BIN" scan "$file" --exclude "$EXCLUDE_PATTERNS" --verbose exit 1 fi echo "Error: key-watch failed on $file (exit code: $EXIT_CODE)" >&2 diff --git a/templates/pre-push.sh b/templates/pre-push.sh index 9eb3ce0..60f0add 100644 --- a/templates/pre-push.sh +++ b/templates/pre-push.sh @@ -9,13 +9,91 @@ if ! command -v "$KEYWATCH_BIN" >/dev/null 2>&1; then exit 1 fi -CURRENT_REMOTE=$(git remote get-url --push origin 2>/dev/null || git remote get-url origin 2>/dev/null || true) +REMOTE_NAME=${1:-origin} +CURRENT_REMOTE=${2:-} +if [ -z "$CURRENT_REMOTE" ]; then + CURRENT_REMOTE=$(git remote get-url --push "$REMOTE_NAME" 2>/dev/null || git remote get-url "$REMOTE_NAME" 2>/dev/null || true) +fi + +trim_repo() { + local value="$1" + value="${value#"${value%%[![:space:]]*}"}" + value="${value%"${value##*[![:space:]]}"}" + printf '%s' "$value" +} + +normalize_repo() { + local repo lower scheme rest authority path host port + repo=$(trim_repo "$1") + lower=$(printf '%s' "$repo" | tr '[:upper:]' '[:lower:]') + + if [[ "$lower" == https://* ]]; then + scheme="https" + rest="${repo:8}" + elif [[ "$lower" == http://* ]]; then + scheme="http" + rest="${repo:7}" + elif [[ "$lower" == ssh://* ]]; then + scheme="ssh" + rest="${repo:6}" + elif [[ "$repo" == *:* && "${repo%%:*}" != */* ]]; then + scheme="scp" + authority="${repo%%:*}" + path="${repo#*:}" + else + scheme="plain" + rest="$repo" + fi + + if [ "$scheme" != "scp" ]; then + [ "$rest" != "${rest#*/}" ] || return 1 + authority="${rest%%/*}" + path="${rest#*/}" + fi + + authority="${authority##*@}" + host="${authority%%:*}" + port="" + if [[ "$authority" == *:* ]]; then + port="${authority#*:}" + [[ "$port" != *:* ]] || return 1 + fi + host=$(printf '%s' "$host" | tr '[:upper:]' '[:lower:]') + + case "$scheme:$port" in + https:443|http:80|ssh:22) port="" ;; + esac + if [ -n "$port" ]; then + host="$host:$port" + fi + + while [[ "$path" == */ ]]; do path="${path%/}"; done + lower=$(printf '%s' "$path" | tr '[:upper:]' '[:lower:]') + if [[ "$lower" == *.git ]]; then path="${path:0:${#path}-4}"; fi + while [[ "$path" == */ ]]; do path="${path%/}"; done + + [ -n "$host" ] && [[ "$path" == */* ]] || return 1 + [ -n "${path%%/*}" ] && [ -n "${path#*/}" ] || return 1 + [[ "${path#*/}" != */* ]] || return 1 + if [ "$host" = "github.com" ]; then + path=$(printf '%s' "$path" | tr '[:upper:]' '[:lower:]') + fi + printf '%s/%s' "$host" "$path" +} + +CURRENT_REPO=$(normalize_repo "$CURRENT_REMOTE" 2>/dev/null || true) + +if [ -n "$CURRENT_REMOTE" ] && [ -z "$CURRENT_REPO" ] && { [ -n "${ALLOWED_REPOS:-}" ] || [ -n "${BLOCKED_REPOS:-}" ]; }; then + echo "Error: unable to validate remote $CURRENT_REMOTE" >&2 + exit 1 +fi if [ -n "$CURRENT_REMOTE" ] && [ -n "${ALLOWED_REPOS:-}" ]; then allowed_match=0 IFS=',' read -r -a allowed_list <<< "$ALLOWED_REPOS" for allowed_repo in "${allowed_list[@]}"; do - if [ -n "$allowed_repo" ] && [[ "$CURRENT_REMOTE" == *"$allowed_repo"* ]]; then + allowed_repo=$(normalize_repo "$allowed_repo") + if [ -n "$allowed_repo" ] && [ "$CURRENT_REPO" = "$allowed_repo" ]; then allowed_match=1 break fi @@ -30,7 +108,8 @@ fi if [ -n "$CURRENT_REMOTE" ] && [ -n "${BLOCKED_REPOS:-}" ]; then IFS=',' read -r -a blocked_list <<< "$BLOCKED_REPOS" for blocked_repo in "${blocked_list[@]}"; do - if [ -n "$blocked_repo" ] && [[ "$CURRENT_REMOTE" == *"$blocked_repo"* ]]; then + blocked_repo=$(normalize_repo "$blocked_repo") + if [ -n "$blocked_repo" ] && [ "$CURRENT_REPO" = "$blocked_repo" ]; then echo "Error: push blocked for remote $CURRENT_REMOTE" >&2 exit 1 fi diff --git a/tests/hooks_tests.rs b/tests/hooks_tests.rs index 8a9118e..34aec02 100644 --- a/tests/hooks_tests.rs +++ b/tests/hooks_tests.rs @@ -1,5 +1,9 @@ use key_watch::cli::{CliOptions, Command, ExitMode, HookAction, HookInstallArgs, HookType, Shell}; use key_watch::hooks::{generate_pre_commit_hook, generate_pre_push_hook}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Output; +use std::time::{SystemTime, UNIX_EPOCH}; fn hook_install_args( hook_type: HookType, @@ -16,6 +20,77 @@ fn hook_install_args( } } +#[cfg(unix)] +fn unique_temp_dir(name: &str) -> PathBuf { + let stamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system time should be after unix epoch") + .as_nanos(); + + std::env::temp_dir().join(format!( + "keywatch_hook_{name}_{stamp}_{}", + std::process::id() + )) +} + +#[cfg(unix)] +fn write_executable(path: &Path, contents: &str) { + use std::os::unix::fs::PermissionsExt; + + fs::write(path, contents).expect("write executable"); + let mut permissions = fs::metadata(path) + .expect("read executable metadata") + .permissions(); + permissions.set_mode(0o755); + fs::set_permissions(path, permissions).expect("mark executable"); +} + +#[cfg(unix)] +fn generated_binary_name() -> String { + std::env::current_exe() + .expect("current test executable should resolve") + .file_name() + .expect("current test executable should have a file name") + .to_string_lossy() + .into_owned() +} + +#[cfg(unix)] +fn run_hook_with_args( + hook: &str, + git_script: &str, + keywatch_script: &str, + cwd: &Path, + hook_args: &[&str], +) -> Output { + let bin_dir = cwd.join("bin"); + fs::create_dir_all(&bin_dir).expect("create fake bin dir"); + write_executable(&bin_dir.join("git"), git_script); + write_executable(&bin_dir.join(generated_binary_name()), keywatch_script); + + let hook_path = cwd.join("hook.sh"); + fs::write(&hook_path, hook).expect("write hook"); + + let path = format!( + "{}:{}", + bin_dir.display(), + std::env::var("PATH").unwrap_or_default() + ); + + std::process::Command::new("bash") + .arg(&hook_path) + .args(hook_args) + .current_dir(cwd) + .env("PATH", path) + .output() + .expect("run hook") +} + +#[cfg(unix)] +fn run_hook(hook: &str, git_script: &str, keywatch_script: &str, cwd: &Path) -> Output { + run_hook_with_args(hook, git_script, keywatch_script, cwd, &[]) +} + #[test] fn test_hook_generation_pre_commit() { let options = hook_install_args(HookType::PreCommit, None, None, Some("*.log,*.tmp")); @@ -36,6 +111,18 @@ fn test_hook_generation_pre_commit() { hook.contains("scan \"$file\""), "Should use scan subcommand" ); + assert!( + hook.contains("[ -L \"$file\" ]"), + "Should skip staged symlinks" + ); + assert!( + hook.contains(">/dev/null 2>&1"), + "Should suppress scanner output before concise failure" + ); + assert!( + !hook.contains("--verbose"), + "Should not rerun verbosely and print matched secrets" + ); } #[test] @@ -54,9 +141,251 @@ fn test_hook_generation_pre_push() { "Should use scan subcommand for pre-push" ); assert!( - hook.contains("CURRENT_REMOTE=$(git remote get-url --push origin"), - "Should enforce repo restrictions" + hook.contains("CURRENT_REMOTE=${2:-}"), + "Should prefer the pushed remote URL from hook argv" + ); + assert!( + hook.contains("REMOTE_NAME=${1:-origin}"), + "Should name the remote explicitly before fallback" + ); + assert!( + hook.contains("git remote get-url --push"), + "Should fall back to the remote name when argv URL is absent" + ); + assert!( + hook.contains("normalize_repo"), + "Should normalize remote and configured repo identities" + ); + assert!( + hook.contains("[ \"$CURRENT_REPO\" = \"$allowed_repo\" ]"), + "Should compare allowed repos by exact identity" + ); + assert!( + !hook.contains("*\"$allowed_repo\"*"), + "Should not use substring allow-list matching" + ); +} + +#[cfg(unix)] +#[test] +fn test_pre_commit_failure_output_does_not_print_matched_secret() { + let temp_dir = unique_temp_dir("pre_commit_secret_output"); + let _ = fs::remove_dir_all(&temp_dir); + fs::create_dir_all(&temp_dir).expect("create temp dir"); + fs::write(temp_dir.join("secret.txt"), "secret").expect("write staged file"); + + let hook = generate_pre_commit_hook(&hook_install_args(HookType::PreCommit, None, None, None)); + let git_script = + "#!/bin/bash\nif [ \"$1\" = \"diff\" ]; then printf 'secret.txt\\0'; exit 0; fi\nexit 1\n"; + let keywatch_script = "#!/bin/bash\nprintf 'MATCHED_SECRET_VALUE\\n'\nprintf 'MATCHED_SECRET_VALUE\\n' >&2\nexit 1\n"; + let output = run_hook(&hook, git_script, keywatch_script, &temp_dir); + let combined = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + + assert_eq!(output.status.code(), Some(1)); + assert!(combined.contains("ERROR: Secret detected in secret.txt")); + assert!( + !combined.contains("MATCHED_SECRET_VALUE"), + "Generated pre-commit hook must not print matched secrets" + ); + + fs::remove_dir_all(&temp_dir).expect("cleanup temp dir"); +} + +#[cfg(unix)] +#[test] +fn test_pre_commit_skips_staged_symlinks() { + let temp_dir = unique_temp_dir("pre_commit_symlink_skip"); + let marker = temp_dir.join("scanner-ran"); + let _ = fs::remove_dir_all(&temp_dir); + fs::create_dir_all(&temp_dir).expect("create temp dir"); + fs::write(temp_dir.join("target.txt"), "secret").expect("write target"); + std::os::unix::fs::symlink("target.txt", temp_dir.join("linked.txt")).expect("create symlink"); + + let hook = generate_pre_commit_hook(&hook_install_args(HookType::PreCommit, None, None, None)); + let git_script = + "#!/bin/bash\nif [ \"$1\" = \"diff\" ]; then printf 'linked.txt\\0'; exit 0; fi\nexit 1\n"; + let keywatch_script = format!("#!/bin/bash\nprintf ran > '{}'\nexit 1\n", marker.display()); + let output = run_hook(&hook, git_script, &keywatch_script, &temp_dir); + + assert!( + output.status.success(), + "symlink-only pre-commit should pass" + ); + assert!( + !marker.exists(), + "scanner should not run for staged symlinks" + ); + + fs::remove_dir_all(&temp_dir).expect("cleanup temp dir"); +} + +#[cfg(unix)] +#[test] +fn test_pre_push_allows_normalized_https_and_scp_equivalent() { + let temp_dir = unique_temp_dir("pre_push_allowed_normalized"); + let _ = fs::remove_dir_all(&temp_dir); + fs::create_dir_all(&temp_dir).expect("create temp dir"); + + let hook = generate_pre_push_hook(&hook_install_args( + HookType::PrePush, + Some(" git@github.com:org/repo.git "), + None, + None, + )); + let git_script = "#!/bin/bash\nif [ \"$1\" = \"remote\" ]; then printf 'https://github.com/org/repo.git/\\n'; exit 0; fi\nexit 1\n"; + let keywatch_script = "#!/bin/bash\nexit 0\n"; + let output = run_hook(&hook, git_script, keywatch_script, &temp_dir); + + assert!( + output.status.success(), + "normalized HTTPS remote should match SCP allow entry" ); + + fs::remove_dir_all(&temp_dir).expect("cleanup temp dir"); +} + +#[cfg(unix)] +#[test] +fn test_pre_push_allows_https_userinfo_case_and_default_port() { + let temp_dir = unique_temp_dir("pre_push_allowed_url_variants"); + let _ = fs::remove_dir_all(&temp_dir); + fs::create_dir_all(&temp_dir).expect("create temp dir"); + + let hook = generate_pre_push_hook(&hook_install_args( + HookType::PrePush, + Some("git@github.com:org/repo.git"), + None, + None, + )); + let git_script = "#!/bin/bash\nif [ \"$1\" = \"remote\" ]; then printf 'https://x-access-token:T@GitHub.COM:443/ORG/REPO.git\\n'; exit 0; fi\nexit 1\n"; + let output = run_hook(&hook, git_script, "#!/bin/bash\nexit 0\n", &temp_dir); + + assert!(output.status.success()); + fs::remove_dir_all(&temp_dir).expect("cleanup temp dir"); +} + +#[cfg(unix)] +#[test] +fn test_pre_push_blocks_https_userinfo_case_and_default_port() { + let temp_dir = unique_temp_dir("pre_push_blocked_url_variants"); + let _ = fs::remove_dir_all(&temp_dir); + fs::create_dir_all(&temp_dir).expect("create temp dir"); + + let hook = generate_pre_push_hook(&hook_install_args( + HookType::PrePush, + None, + Some("ssh://git@github.com/org/repo"), + None, + )); + let git_script = "#!/bin/bash\nif [ \"$1\" = \"remote\" ]; then printf 'https://user@GitHub.COM:443/ORG/REPO.git\\n'; exit 0; fi\nexit 1\n"; + let output = run_hook(&hook, git_script, "#!/bin/bash\nexit 0\n", &temp_dir); + + assert_eq!(output.status.code(), Some(1)); + fs::remove_dir_all(&temp_dir).expect("cleanup temp dir"); +} + +#[cfg(unix)] +#[test] +fn test_pre_push_rejects_spoofed_substring_remote() { + let temp_dir = unique_temp_dir("pre_push_spoofed_remote"); + let _ = fs::remove_dir_all(&temp_dir); + fs::create_dir_all(&temp_dir).expect("create temp dir"); + + let hook = generate_pre_push_hook(&hook_install_args( + HookType::PrePush, + Some("git@evil.example:org/repo.git"), + None, + None, + )); + let git_script = "#!/bin/bash\nif [ \"$1\" = \"remote\" ]; then printf 'https://evil.example/github.com/org/repo.git\\n'; exit 0; fi\nexit 1\n"; + let keywatch_script = "#!/bin/bash\nexit 0\n"; + let output = run_hook(&hook, git_script, keywatch_script, &temp_dir); + + assert_eq!(output.status.code(), Some(1)); + + fs::remove_dir_all(&temp_dir).expect("cleanup temp dir"); +} + +#[cfg(unix)] +#[test] +fn test_pre_push_blocks_when_actual_pushed_remote_is_unallowed() { + let temp_dir = unique_temp_dir("pre_push_blocks_actual_unallowed"); + let _ = fs::remove_dir_all(&temp_dir); + fs::create_dir_all(&temp_dir).expect("create temp dir"); + + let hook = generate_pre_push_hook(&hook_install_args( + HookType::PrePush, + Some("github.com/org/repo"), + None, + None, + )); + let git_script = "#!/bin/bash\nif [ \"$1\" = \"remote\" ]; then printf 'git@github.com:org/repo.git\\n'; exit 0; fi\nexit 1\n"; + let keywatch_script = "#!/bin/bash\nexit 0\n"; + let output = run_hook_with_args( + &hook, + git_script, + keywatch_script, + &temp_dir, + &["mirror", "https://evil.example/org/repo.git"], + ); + + assert_eq!(output.status.code(), Some(1)); + + fs::remove_dir_all(&temp_dir).expect("cleanup temp dir"); +} + +#[cfg(unix)] +#[test] +fn test_pre_push_blocks_when_actual_pushed_remote_is_blocked() { + let temp_dir = unique_temp_dir("pre_push_blocks_actual_blocked"); + let _ = fs::remove_dir_all(&temp_dir); + fs::create_dir_all(&temp_dir).expect("create temp dir"); + + let hook = generate_pre_push_hook(&hook_install_args( + HookType::PrePush, + None, + Some("https://evil.example/org/repo.git"), + None, + )); + let git_script = "#!/bin/bash\nif [ \"$1\" = \"remote\" ]; then printf 'git@github.com:org/repo.git\\n'; exit 0; fi\nexit 1\n"; + let keywatch_script = "#!/bin/bash\nexit 0\n"; + let output = run_hook_with_args( + &hook, + git_script, + keywatch_script, + &temp_dir, + &["origin", "https://evil.example/org/repo.git"], + ); + + assert_eq!(output.status.code(), Some(1)); + + fs::remove_dir_all(&temp_dir).expect("cleanup temp dir"); +} + +#[cfg(unix)] +#[test] +fn test_pre_push_blocks_normalized_https_and_scp_equivalent() { + let temp_dir = unique_temp_dir("pre_push_blocked_normalized"); + let _ = fs::remove_dir_all(&temp_dir); + fs::create_dir_all(&temp_dir).expect("create temp dir"); + + let hook = generate_pre_push_hook(&hook_install_args( + HookType::PrePush, + None, + Some("https://github.com/org/repo"), + None, + )); + let git_script = "#!/bin/bash\nif [ \"$1\" = \"remote\" ]; then printf 'git@github.com:org/repo.git\\n'; exit 0; fi\nexit 1\n"; + let keywatch_script = "#!/bin/bash\nexit 0\n"; + let output = run_hook(&hook, git_script, keywatch_script, &temp_dir); + + assert_eq!(output.status.code(), Some(1)); + + fs::remove_dir_all(&temp_dir).expect("cleanup temp dir"); } #[test] From 3d6d955565c7c72b227452e8fd7456ef69ec2e7d Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:35:59 +0530 Subject: [PATCH 13/17] style: group Rust imports --- src/baseline.rs | 8 ++------ src/detector.rs | 4 +--- src/report.rs | 3 +-- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/src/baseline.rs b/src/baseline.rs index df1e648..07357ec 100644 --- a/src/baseline.rs +++ b/src/baseline.rs @@ -1,7 +1,6 @@ use serde::{Deserialize, Serialize}; -use std::collections::HashSet; -use std::fs; -use std::path::Path; +use sha2::{Digest, Sha256}; +use std::{collections::HashSet, fmt::Write as _, fs, path::Path}; use crate::report::Finding; @@ -17,9 +16,6 @@ const BASELINE_VERSION: &str = "1.0"; const HASH_DOMAIN_SEPARATOR: &str = "keywatch-baseline-v1"; fn hash_content(content: &str) -> String { - use sha2::{Digest, Sha256}; - use std::fmt::Write as _; - let mut hasher = Sha256::new(); hasher.update(HASH_DOMAIN_SEPARATOR.as_bytes()); hasher.update(content.as_bytes()); diff --git a/src/detector.rs b/src/detector.rs index ae4304f..e729813 100644 --- a/src/detector.rs +++ b/src/detector.rs @@ -1,9 +1,7 @@ use crate::report::{ParseSeverityError, Severity}; use regex::Regex; use serde::Deserialize; -use std::fmt; -use std::fs; -use std::str::FromStr; +use std::{fmt, fs, str::FromStr}; #[derive(Debug)] pub enum DetectorError { diff --git a/src/report.rs b/src/report.rs index f15e3a4..adff5a9 100644 --- a/src/report.rs +++ b/src/report.rs @@ -1,6 +1,5 @@ use serde::Serialize; -use std::fmt; -use std::str::FromStr; +use std::{fmt, str::FromStr}; #[derive(Serialize, Clone, PartialEq, Copy, Debug)] #[serde(rename_all = "UPPERCASE")] From eedadcbfc7a901b9eebf8712302aae522df8c52c Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:52:00 +0530 Subject: [PATCH 14/17] fix: guard Unix-only hook test imports --- tests/hooks_tests.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/hooks_tests.rs b/tests/hooks_tests.rs index 34aec02..b5f7250 100644 --- a/tests/hooks_tests.rs +++ b/tests/hooks_tests.rs @@ -1,9 +1,12 @@ use key_watch::cli::{CliOptions, Command, ExitMode, HookAction, HookInstallArgs, HookType, Shell}; use key_watch::hooks::{generate_pre_commit_hook, generate_pre_push_hook}; -use std::fs; -use std::path::{Path, PathBuf}; -use std::process::Output; -use std::time::{SystemTime, UNIX_EPOCH}; +#[cfg(unix)] +use std::{ + fs, + path::{Path, PathBuf}, + process::Output, + time::{SystemTime, UNIX_EPOCH}, +}; fn hook_install_args( hook_type: HookType, From ec06083111937ac472277ca79aea45113a8ae5c1 Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Sun, 2 Aug 2026 09:46:22 +0530 Subject: [PATCH 15/17] refactor: clarify pre-push remote policy shell --- templates/pre-push.sh | 161 ++++++++++++++++++++++++------------------ tests/hooks_tests.rs | 149 +++++++++++++++++++++++++++++++++++--- 2 files changed, 230 insertions(+), 80 deletions(-) diff --git a/templates/pre-push.sh b/templates/pre-push.sh index 60f0add..d7aec3f 100644 --- a/templates/pre-push.sh +++ b/templates/pre-push.sh @@ -4,51 +4,39 @@ {{repo_section}}KEYWATCH_BIN={{binary_name}} -if ! command -v "$KEYWATCH_BIN" >/dev/null 2>&1; then - echo "Error: $KEYWATCH_BIN not found on PATH" >&2 - exit 1 -fi - -REMOTE_NAME=${1:-origin} -CURRENT_REMOTE=${2:-} -if [ -z "$CURRENT_REMOTE" ]; then - CURRENT_REMOTE=$(git remote get-url --push "$REMOTE_NAME" 2>/dev/null || git remote get-url "$REMOTE_NAME" 2>/dev/null || true) -fi - -trim_repo() { - local value="$1" - value="${value#"${value%%[![:space:]]*}"}" - value="${value%"${value##*[![:space:]]}"}" - printf '%s' "$value" +trim_repository() { + local repository="$1" + repository="${repository#"${repository%%[![:space:]]*}"}" + repository="${repository%"${repository##*[![:space:]]}"}" + printf '%s' "$repository" } -normalize_repo() { - local repo lower scheme rest authority path host port - repo=$(trim_repo "$1") - lower=$(printf '%s' "$repo" | tr '[:upper:]' '[:lower:]') +normalize_repository_url() { + local repository lowercase_repository scheme repository_after_scheme authority path host port + repository=$(trim_repository "$1") + lowercase_repository=$(printf '%s' "$repository" | tr '[:upper:]' '[:lower:]') - if [[ "$lower" == https://* ]]; then + if [[ "$lowercase_repository" == https://* ]]; then scheme="https" - rest="${repo:8}" - elif [[ "$lower" == http://* ]]; then + repository_after_scheme="${repository:8}" + elif [[ "$lowercase_repository" == http://* ]]; then scheme="http" - rest="${repo:7}" - elif [[ "$lower" == ssh://* ]]; then + repository_after_scheme="${repository:7}" + elif [[ "$lowercase_repository" == ssh://* ]]; then scheme="ssh" - rest="${repo:6}" - elif [[ "$repo" == *:* && "${repo%%:*}" != */* ]]; then + repository_after_scheme="${repository:6}" + elif [[ "$repository" == *:* && "${repository%%:*}" != */* ]]; then scheme="scp" - authority="${repo%%:*}" - path="${repo#*:}" + authority="${repository%%:*}" + path="${repository#*:}" else scheme="plain" - rest="$repo" + repository_after_scheme="$repository" fi - if [ "$scheme" != "scp" ]; then - [ "$rest" != "${rest#*/}" ] || return 1 - authority="${rest%%/*}" - path="${rest#*/}" + [ "$repository_after_scheme" != "${repository_after_scheme#*/}" ] || return 1 + authority="${repository_after_scheme%%/*}" + path="${repository_after_scheme#*/}" fi authority="${authority##*@}" @@ -59,19 +47,16 @@ normalize_repo() { [[ "$port" != *:* ]] || return 1 fi host=$(printf '%s' "$host" | tr '[:upper:]' '[:lower:]') - case "$scheme:$port" in https:443|http:80|ssh:22) port="" ;; esac if [ -n "$port" ]; then host="$host:$port" fi - while [[ "$path" == */ ]]; do path="${path%/}"; done - lower=$(printf '%s' "$path" | tr '[:upper:]' '[:lower:]') - if [[ "$lower" == *.git ]]; then path="${path:0:${#path}-4}"; fi + lowercase_repository=$(printf '%s' "$path" | tr '[:upper:]' '[:lower:]') + if [[ "$lowercase_repository" == *.git ]]; then path="${path:0:${#path}-4}"; fi while [[ "$path" == */ ]]; do path="${path%/}"; done - [ -n "$host" ] && [[ "$path" == */* ]] || return 1 [ -n "${path%%/*}" ] && [ -n "${path#*/}" ] || return 1 [[ "${path#*/}" != */* ]] || return 1 @@ -81,40 +66,76 @@ normalize_repo() { printf '%s/%s' "$host" "$path" } -CURRENT_REPO=$(normalize_repo "$CURRENT_REMOTE" 2>/dev/null || true) - -if [ -n "$CURRENT_REMOTE" ] && [ -z "$CURRENT_REPO" ] && { [ -n "${ALLOWED_REPOS:-}" ] || [ -n "${BLOCKED_REPOS:-}" ]; }; then - echo "Error: unable to validate remote $CURRENT_REMOTE" >&2 - exit 1 -fi - -if [ -n "$CURRENT_REMOTE" ] && [ -n "${ALLOWED_REPOS:-}" ]; then - allowed_match=0 - IFS=',' read -r -a allowed_list <<< "$ALLOWED_REPOS" - for allowed_repo in "${allowed_list[@]}"; do - allowed_repo=$(normalize_repo "$allowed_repo") - if [ -n "$allowed_repo" ] && [ "$CURRENT_REPO" = "$allowed_repo" ]; then - allowed_match=1 - break +resolve_remote_url() { + local remote_name="$1" + local remote_url="$2" + + if [ -n "$remote_url" ]; then + printf '%s' "$remote_url" + return 0 + fi + git remote get-url --push "$remote_name" 2>/dev/null || git remote get-url "$remote_name" 2>/dev/null || true +} + +repository_list_contains() { + local repositories="$1" + local current_repository="$2" + local configured_repository normalized_repository + local repository_list + + IFS=',' read -r -a repository_list <<< "$repositories" + for configured_repository in "${repository_list[@]}"; do + normalized_repository=$(normalize_repository_url "$configured_repository") + if [ -n "$normalized_repository" ] && [ "$current_repository" = "$normalized_repository" ]; then + return 0 fi done + return 1 +} + +repository_filters_are_configured() { + [ -n "${ALLOWED_REPOS:-}" ] || [ -n "${BLOCKED_REPOS:-}" ] +} + +enforce_repository_policy() { + local remote_url="$1" + local current_repository - if [ "$allowed_match" -eq 0 ]; then - echo "Error: push blocked for remote $CURRENT_REMOTE" >&2 + current_repository=$(normalize_repository_url "$remote_url" 2>/dev/null || true) + if [ -z "$remote_url" ]; then + return 0 + fi + + if [ -z "$current_repository" ] && repository_filters_are_configured; then + echo "Error: unable to validate remote $remote_url" >&2 + return 1 + fi + + if [ -n "${ALLOWED_REPOS:-}" ] && ! repository_list_contains "$ALLOWED_REPOS" "$current_repository"; then + echo "Error: push blocked for remote $remote_url" >&2 + return 1 + fi + + if [ -n "${BLOCKED_REPOS:-}" ] && repository_list_contains "$BLOCKED_REPOS" "$current_repository"; then + echo "Error: push blocked for remote $remote_url" >&2 + return 1 + fi + return 0 +} + +main() { + local remote_name="${1:-origin}" + local remote_url_arg="${2:-}" + local remote_url + + if ! command -v "$KEYWATCH_BIN" >/dev/null 2>&1; then + echo "Error: $KEYWATCH_BIN not found on PATH" >&2 exit 1 fi -fi - -if [ -n "$CURRENT_REMOTE" ] && [ -n "${BLOCKED_REPOS:-}" ]; then - IFS=',' read -r -a blocked_list <<< "$BLOCKED_REPOS" - for blocked_repo in "${blocked_list[@]}"; do - blocked_repo=$(normalize_repo "$blocked_repo") - if [ -n "$blocked_repo" ] && [ "$CURRENT_REPO" = "$blocked_repo" ]; then - echo "Error: push blocked for remote $CURRENT_REMOTE" >&2 - exit 1 - fi - done -fi + remote_url=$(resolve_remote_url "$remote_name" "$remote_url_arg") + enforce_repository_policy "$remote_url" || exit 1 + "$KEYWATCH_BIN" scan . --exit-mode critical + exit $? +} -"$KEYWATCH_BIN" scan . --exit-mode critical -exit $? +main "$@" diff --git a/tests/hooks_tests.rs b/tests/hooks_tests.rs index b5f7250..055c481 100644 --- a/tests/hooks_tests.rs +++ b/tests/hooks_tests.rs @@ -94,6 +94,14 @@ fn run_hook(hook: &str, git_script: &str, keywatch_script: &str, cwd: &Path) -> run_hook_with_args(hook, git_script, keywatch_script, cwd, &[]) } +#[cfg(unix)] +fn keywatch_script_that_records_args(marker: &Path) -> String { + format!( + "#!/bin/bash\nprintf '%s\\n' \"$*\" > '{}'\nexit 0\n", + marker.display() + ) +} + #[test] fn test_hook_generation_pre_commit() { let options = hook_install_args(HookType::PreCommit, None, None, Some("*.log,*.tmp")); @@ -144,24 +152,24 @@ fn test_hook_generation_pre_push() { "Should use scan subcommand for pre-push" ); assert!( - hook.contains("CURRENT_REMOTE=${2:-}"), - "Should prefer the pushed remote URL from hook argv" + hook.matches("scan . --exit-mode critical").count() == 1, + "Should invoke KeyWatch exactly once for scanning" ); assert!( - hook.contains("REMOTE_NAME=${1:-origin}"), - "Should name the remote explicitly before fallback" + hook.contains("resolve_remote_url"), + "Should keep remote resolution readable in the shell hook" ); assert!( - hook.contains("git remote get-url --push"), - "Should fall back to the remote name when argv URL is absent" + hook.contains("normalize_repository_url"), + "Should keep repository canonicalization readable in the shell hook" ); assert!( - hook.contains("normalize_repo"), - "Should normalize remote and configured repo identities" + hook.contains("repository_list_contains"), + "Should keep exact list membership readable in the shell hook" ); assert!( - hook.contains("[ \"$CURRENT_REPO\" = \"$allowed_repo\" ]"), - "Should compare allowed repos by exact identity" + hook.contains("enforce_repository_policy"), + "Should keep repository policy in the shell hook" ); assert!( !hook.contains("*\"$allowed_repo\"*"), @@ -391,6 +399,127 @@ fn test_pre_push_blocks_normalized_https_and_scp_equivalent() { fs::remove_dir_all(&temp_dir).expect("cleanup temp dir"); } +#[cfg(unix)] +#[test] +fn test_pre_push_uses_named_remote_push_url_when_argv_url_is_absent() { + let temp_dir = unique_temp_dir("pre_push_named_push_url"); + let marker = temp_dir.join("keywatch-args"); + fs::create_dir_all(&temp_dir).expect("create temp dir"); + + let hook = generate_pre_push_hook(&hook_install_args( + HookType::PrePush, + Some("https://push.example/org/repo.git"), + None, + None, + )); + let git_script = "#!/bin/bash\nif [ \"$1 $2 $3\" = \"remote get-url --push\" ] && [ \"$4\" = \"mirror\" ]; then printf 'https://push.example/org/repo.git\\n'; exit 0; fi\nif [ \"$1 $2 $3\" = \"remote get-url mirror\" ]; then printf 'https://fetch.example/org/repo.git\\n'; exit 0; fi\nexit 1\n"; + let output = run_hook_with_args( + &hook, + git_script, + &keywatch_script_that_records_args(&marker), + &temp_dir, + &["mirror"], + ); + + assert!(output.status.success()); + assert_eq!( + fs::read_to_string(&marker).expect("read scanner args"), + "scan . --exit-mode critical\n" + ); + fs::remove_dir_all(&temp_dir).expect("cleanup temp dir"); +} + +#[cfg(unix)] +#[test] +fn test_pre_push_falls_back_to_named_remote_fetch_url() { + let temp_dir = unique_temp_dir("pre_push_named_fetch_url"); + fs::create_dir_all(&temp_dir).expect("create temp dir"); + + let hook = generate_pre_push_hook(&hook_install_args( + HookType::PrePush, + Some("https://fetch.example/org/repo.git"), + None, + None, + )); + let git_script = "#!/bin/bash\nif [ \"$1 $2 $3\" = \"remote get-url --push\" ]; then exit 1; fi\nif [ \"$1 $2\" = \"remote get-url\" ] && [ \"$3\" = \"mirror\" ]; then printf 'https://fetch.example/org/repo.git\\n'; exit 0; fi\nexit 1\n"; + let output = run_hook_with_args( + &hook, + git_script, + "#!/bin/bash\nexit 0\n", + &temp_dir, + &["mirror"], + ); + + assert!(output.status.success()); + fs::remove_dir_all(&temp_dir).expect("cleanup temp dir"); +} + +#[cfg(unix)] +#[test] +fn test_pre_push_ignores_invalid_configured_entries() { + let temp_dir = unique_temp_dir("pre_push_invalid_config_entries"); + fs::create_dir_all(&temp_dir).expect("create temp dir"); + + let hook = generate_pre_push_hook(&hook_install_args( + HookType::PrePush, + Some("not-a-repo,https://github.com/org/repo"), + Some("not-a-repo"), + None, + )); + let git_script = "#!/bin/bash\nif [ \"$1\" = \"remote\" ]; then printf 'git@github.com:org/repo.git\\n'; exit 0; fi\nexit 1\n"; + let output = run_hook(&hook, git_script, "#!/bin/bash\nexit 0\n", &temp_dir); + + assert!(output.status.success()); + fs::remove_dir_all(&temp_dir).expect("cleanup temp dir"); +} + +#[cfg(unix)] +#[test] +fn test_pre_push_fails_closed_for_unnormalizable_remote_when_filters_exist() { + let temp_dir = unique_temp_dir("pre_push_invalid_remote_filtered"); + fs::create_dir_all(&temp_dir).expect("create temp dir"); + + let hook = generate_pre_push_hook(&hook_install_args( + HookType::PrePush, + Some("https://github.com/org/repo"), + None, + None, + )); + let git_script = "#!/bin/bash\nif [ \"$1\" = \"remote\" ]; then printf 'not-a-repo\\n'; exit 0; fi\nexit 1\n"; + let output = run_hook(&hook, git_script, "#!/bin/bash\nexit 0\n", &temp_dir); + + assert_eq!(output.status.code(), Some(1)); + assert!( + String::from_utf8_lossy(&output.stderr) + .contains("Error: unable to validate remote not-a-repo") + ); + fs::remove_dir_all(&temp_dir).expect("cleanup temp dir"); +} + +#[cfg(unix)] +#[test] +fn test_pre_push_scans_unnormalizable_remote_when_no_filters_exist() { + let temp_dir = unique_temp_dir("pre_push_invalid_remote_unfiltered"); + let marker = temp_dir.join("keywatch-args"); + fs::create_dir_all(&temp_dir).expect("create temp dir"); + + let hook = generate_pre_push_hook(&hook_install_args(HookType::PrePush, None, None, None)); + let git_script = "#!/bin/bash\nif [ \"$1\" = \"remote\" ]; then printf 'not-a-repo\\n'; exit 0; fi\nexit 1\n"; + let output = run_hook( + &hook, + git_script, + &keywatch_script_that_records_args(&marker), + &temp_dir, + ); + + assert!(output.status.success()); + assert_eq!( + fs::read_to_string(&marker).expect("read scanner args"), + "scan . --exit-mode critical\n" + ); + fs::remove_dir_all(&temp_dir).expect("cleanup temp dir"); +} + #[test] fn test_hook_shell_escaping() { let options = hook_install_args( From 15dc56a812ca0fd7a8b01970e8e6ac35705da60f Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Sun, 2 Aug 2026 09:46:27 +0530 Subject: [PATCH 16/17] refactor: simplify baseline hashing --- src/baseline.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/baseline.rs b/src/baseline.rs index 07357ec..fbd7efe 100644 --- a/src/baseline.rs +++ b/src/baseline.rs @@ -1,6 +1,6 @@ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; -use std::{collections::HashSet, fmt::Write as _, fs, path::Path}; +use std::{collections::HashSet, fs, path::Path}; use crate::report::Finding; @@ -19,11 +19,7 @@ fn hash_content(content: &str) -> String { let mut hasher = Sha256::new(); hasher.update(HASH_DOMAIN_SEPARATOR.as_bytes()); hasher.update(content.as_bytes()); - let mut output = String::with_capacity(64); - for byte in hasher.finalize() { - let _ = write!(&mut output, "{:02x}", byte); - } - output + format!("{:x}", hasher.finalize()) } #[derive(Clone, Debug, PartialEq, Eq, Hash)] From 60d25b091afc3799d0ae3a75f09a387922cc87b4 Mon Sep 17 00:00:00 2001 From: PiX <69745008+pixincreate@users.noreply.github.com> Date: Sun, 2 Aug 2026 09:46:33 +0530 Subject: [PATCH 17/17] style: use descriptive scanner test errors --- tests/scanner_tests.rs | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/tests/scanner_tests.rs b/tests/scanner_tests.rs index e7b78e8..f2ce937 100644 --- a/tests/scanner_tests.rs +++ b/tests/scanner_tests.rs @@ -20,13 +20,13 @@ fn git_available() -> bool { } fn init_git_repo(path: &Path) -> Result<(), String> { - fs::create_dir_all(path).map_err(|e| format!("create repo dir: {e}"))?; + fs::create_dir_all(path).map_err(|error| format!("create repo dir: {error}"))?; let status = Command::new("git") .args(["init", "--quiet"]) .current_dir(path) .status() - .map_err(|e| format!("git init: {e}"))?; + .map_err(|error| format!("git init: {error}"))?; if !status.success() { return Err("git init failed".to_string()); } @@ -36,7 +36,7 @@ fn init_git_repo(path: &Path) -> Result<(), String> { .args(["config", key, value]) .current_dir(path) .status() - .map_err(|e| format!("git config {key}: {e}"))?; + .map_err(|error| format!("git config {key}: {error}"))?; if !status.success() { return Err(format!("git config {key} failed")); } @@ -47,13 +47,13 @@ fn init_git_repo(path: &Path) -> Result<(), String> { fn commit_file(path: &Path, file_name: &str, contents: &str, message: &str) -> Result<(), String> { let file_path = path.join(file_name); - fs::write(&file_path, contents).map_err(|e| format!("write file: {e}"))?; + fs::write(&file_path, contents).map_err(|error| format!("write file: {error}"))?; let status = Command::new("git") .args(["add", file_name]) .current_dir(path) .status() - .map_err(|e| format!("git add: {e}"))?; + .map_err(|error| format!("git add: {error}"))?; if !status.success() { return Err("git add failed".to_string()); } @@ -62,7 +62,7 @@ fn commit_file(path: &Path, file_name: &str, contents: &str, message: &str) -> R .args(["commit", "-m", message, "--quiet"]) .current_dir(path) .status() - .map_err(|e| format!("git commit: {e}"))?; + .map_err(|error| format!("git commit: {error}"))?; if !status.success() { return Err("git commit failed".to_string()); } @@ -81,12 +81,12 @@ fn run_git_history_scan(current_dir: &Path, extra_args: &[&str]) -> Result Result<(), String> { - std::os::unix::fs::symlink(original, link).map_err(|e| format!("create symlink: {e}")) + std::os::unix::fs::symlink(original, link).map_err(|error| format!("create symlink: {error}")) } #[test] @@ -565,9 +565,9 @@ fn test_explicit_symlink_path_is_skipped() -> Result<(), String> { let outside_file = test_dir.join("outside-secret.txt"); let link_path = test_dir.join("linked-secret.txt"); let _ = fs::remove_dir_all(&test_dir); - fs::create_dir_all(&test_dir).map_err(|e| format!("create test dir: {e}"))?; + fs::create_dir_all(&test_dir).map_err(|error| format!("create test dir: {error}"))?; fs::write(&outside_file, "AWS Key: AKIAABCDEFGHIJKLMNOP\n") - .map_err(|e| format!("write outside secret: {e}"))?; + .map_err(|error| format!("write outside secret: {error}"))?; symlink_file(&outside_file, &link_path)?; let options = ScanArgs { @@ -595,7 +595,7 @@ fn test_explicit_symlink_path_is_skipped() -> Result<(), String> { "Symlink should not count as scanned" ); - fs::remove_dir_all(&test_dir).map_err(|e| format!("cleanup: {e}"))?; + fs::remove_dir_all(&test_dir).map_err(|error| format!("cleanup: {error}"))?; Ok(()) } @@ -607,9 +607,9 @@ fn test_recursive_symlink_path_is_skipped() -> Result<(), String> { let scan_root = test_dir.join("scan-root"); let link_path = scan_root.join("linked-secret.txt"); let _ = fs::remove_dir_all(&test_dir); - fs::create_dir_all(&scan_root).map_err(|e| format!("create scan root: {e}"))?; + fs::create_dir_all(&scan_root).map_err(|error| format!("create scan root: {error}"))?; fs::write(&outside_file, "AWS Key: AKIAABCDEFGHIJKLMNOP\n") - .map_err(|e| format!("write outside secret: {e}"))?; + .map_err(|error| format!("write outside secret: {error}"))?; symlink_file(&outside_file, &link_path)?; let options = ScanArgs { @@ -640,7 +640,7 @@ fn test_recursive_symlink_path_is_skipped() -> Result<(), String> { "Recursive symlink should not count as scanned" ); - fs::remove_dir_all(&test_dir).map_err(|e| format!("cleanup: {e}"))?; + fs::remove_dir_all(&test_dir).map_err(|error| format!("cleanup: {error}"))?; Ok(()) } @@ -868,7 +868,8 @@ fn test_inline_suppression_ignores_marked_lines() -> Result<(), String> { let content = "\ AWS Key: AKIAABCDEFGHIJKLMNOP # keywatch:ignore\npassword = 'mySecretPassword'\nemail = user@example.com // keywatch:ignore\nFirebase: AIzaSy012345678901234567890123456789012\n"; - fs::write(&test_file, content).map_err(|e| format!("Unable to write test file: {}", e))?; + fs::write(&test_file, content) + .map_err(|error| format!("Unable to write test file: {error}"))?; let path_str = test_file.to_string_lossy().to_string(); @@ -910,7 +911,7 @@ AWS Key: AKIAABCDEFGHIJKLMNOP # keywatch:ignore\npassword = 'mySecretPassword'\n "Firebase key without suppression should still be found" ); - fs::remove_file(test_file).map_err(|e| format!("Cleanup failed: {}", e))?; + fs::remove_file(test_file).map_err(|error| format!("Cleanup failed: {error}"))?; Ok(()) } @@ -943,17 +944,17 @@ fn test_stdin_scanning_integration() -> Result<(), String> { .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() - .map_err(|e| format!("spawn key-watch --stdin: {}", e))?; + .map_err(|error| format!("spawn key-watch --stdin: {error}"))?; let mut stdin = child.stdin.take().ok_or("Failed to capture stdin")?; stdin .write_all(b"AWS Key: AKIAABCDEFGHIJKLMNOP\npassword = 'secret123'\n") - .map_err(|e| format!("write to stdin: {}", e))?; + .map_err(|error| format!("write to stdin: {error}"))?; drop(stdin); let output = child .wait_with_output() - .map_err(|e| format!("wait: {}", e))?; + .map_err(|error| format!("wait: {error}"))?; let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); @@ -1101,7 +1102,7 @@ fn test_git_history_does_not_execute_textconv_helpers() -> Result<(), String> { .args(["config", "diff.keywatchmarker.textconv", &helper]) .current_dir(&repo_dir) .status() - .map_err(|e| format!("git config textconv: {e}"))?; + .map_err(|error| format!("git config textconv: {error}"))?; if !status.success() { return Err("git config textconv failed".to_string()); }