fix(provenance): stop short secret values from rewriting unrelated log text (#6416)

Author: icecrasher321

apps/sim/executor/utils/resolved-secret-content-projection.ts

@@ -38,6 +38,7 @@ function createResolvedSecretModelMatcher(
 ): ResolvedSecretMatcher | undefined {
   const matcher = createResolvedSecretMatcher(matches, {
     preserveNamedProvenanceLabels: true,
+    mode: 'render',
   })
   if (!matcher) return undefined
 
@@ -74,7 +75,7 @@ function createResolvedSecretModelMatcher(
       })),
       ...opaquePlaceholderMatches,
     ],
-    { preserveNamedProvenanceLabels: true }
+    { preserveNamedProvenanceLabels: true, mode: 'render' }
   )
 }
 

apps/sim/executor/utils/resolved-secret-match-policy.test.ts

@@ -0,0 +1,91 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import {
+  getResolvedSecretMatchPolicy,
+  isWordBoundaryMatch,
+  MIN_UNANCHORED_MATCH_LENGTH,
+} from '@/executor/utils/resolved-secret-match-policy'
+
+describe('getResolvedSecretMatchPolicy', () => {
+  it.each(['test', 'Test', '483920', 'hunter2', 'F', ''])(
+    'restricts short value %s to boundary matches',
+    (value) => {
+      expect(value.length).toBeLessThan(MIN_UNANCHORED_MATCH_LENGTH)
+      expect(getResolvedSecretMatchPolicy(value)).toBe('boundary')
+    }
+  )
+
+  it.each([
+    ['32-char hex', '5f4dcc3b5aa765d61d8327deb882cf99'],
+    ['base64 key', 'sk-proj-Ab3xK9mQ2pLw7nRt5vYc8Zd4'],
+    ['github pat', 'ghp_16C7e42F292c6912E7710c838347Ae178B4a'],
+    ['slack bot token', 'xoxb-2334-4567-abcdefGHIJKL'],
+    ['9-digit value', '123456789'],
+    ['8-char password', 'Passw0rd'],
+  ])('allows unanchored matching for a %s', (_label, value) => {
+    expect(getResolvedSecretMatchPolicy(value)).toBe('anywhere')
+  })
+
+  /**
+   * Every one of these scores below 3.0 bits/char; an entropy floor would have demoted them.
+   * Prefixed shapes are assembled at runtime so the source carries no literal that reads as a
+   * live credential to a secret scanner.
+   */
+  it.each([
+    ['all-f HMAC key', 'f'.repeat(32)],
+    ['test PAN', '4111111111111111'],
+    ['padded AWS key id', `AKIA${'0'.repeat(16)}`],
+    ['repeated-block hex', 'deadbeefdeadbeefdeadbeefdeadbeef'],
+    ['padded stripe-style key', `sk_live_${'0'.repeat(24)}`],
+    ['padded PAT', `ghp_${'a'.repeat(36)}`],
+  ])('keeps unanchored matching for a low-variety full-length %s', (_label, value) => {
+    expect(getResolvedSecretMatchPolicy(value)).toBe('anywhere')
+  })
+})
+
+describe('isWordBoundaryMatch', () => {
+  it.each([
+    ['test', 0, 4, true],
+    ['key=test', 4, 8, true],
+    ['"test"', 1, 5, true],
+    ['{"k":"test"}', 6, 10, true],
+    ['test ok', 0, 4, true],
+    ['latest', 2, 6, false],
+    ['tested', 0, 4, false],
+    ['prefixtest', 6, 10, false],
+  ])('anchors %s at [%i,%i) => %s', (value, start, end, expected) => {
+    expect(isWordBoundaryMatch(value, start, end)).toBe(expected)
+  })
+
+  it.each([
+    ['user_test_id', 5, 9],
+    ['sk_live_test', 8, 12],
+    ['test_suffix', 0, 4],
+  ])('anchors %s across an underscore, the dominant identifier joiner', (value, start, end) => {
+    expect(isWordBoundaryMatch(value, start, end)).toBe(true)
+  })
+
+  it('treats non-ASCII letters as word characters', () => {
+    expect(isWordBoundaryMatch('прtestка', 2, 6)).toBe(false)
+  })
+
+  it('treats astral-plane letters as word characters', () => {
+    expect(isWordBoundaryMatch('\u{1D400}test\u{1D401}', 2, 6)).toBe(false)
+    expect(isWordBoundaryMatch('x\u{20000}test\u{20000}y', 3, 7)).toBe(false)
+  })
+
+  it('keeps a combining mark attached to the word it decorates', () => {
+    expect(isWordBoundaryMatch('test́ing', 0, 4)).toBe(false)
+  })
+
+  it('anchors a match whose own edge characters are not word characters', () => {
+    expect(isWordBoundaryMatch('a!!!!b', 1, 5)).toBe(true)
+  })
+
+  it('reads an out-of-range probe as a non-word character rather than a match', () => {
+    expect(isWordBoundaryMatch('abc', 0, 3)).toBe(true)
+    expect(isWordBoundaryMatch('abc', 3, 3)).toBe(true)
+  })
+})

apps/sim/executor/utils/resolved-secret-match-policy.ts

@@ -0,0 +1,96 @@
+/**
+ * Decides how a known secret literal is allowed to match inside a larger string.
+ *
+ * The matcher knows every secret's exact bytes, so this is not detection — it is the narrower
+ * question of whether a substring hit is distinctive enough to be attributed to the secret rather
+ * than to coincidence. A four-character value such as `test` occurs inside ordinary words; an
+ * eight-character one effectively does not.
+ */
+
+/**
+ * `'anywhere'` substitutes a hit at any offset.
+ *
+ * `'boundary'` substitutes a hit only when it sits on a word boundary, so a short literal can still
+ * be replaced when it stands alone (`test`), is delimited (`key=test`, `"test"`, `user_test`), or is
+ * the whole value, but cannot rewrite the interior of an unrelated token (`latest`).
+ */
+export type ResolvedSecretMatchPolicy = 'anywhere' | 'boundary'
+
+/**
+ * Shortest literal that may be substituted at an arbitrary offset inside surrounding text.
+ *
+ * Length, not randomness, is what makes a coincidental hit implausible. Shannon entropy measured
+ * over a literal's own character distribution answers "is this string internally varied", which is
+ * not the same question and misfires badly on real credentials: an all-`f` 32-character HMAC key
+ * scores 0.00 bits/char, a zero-padded card number scores 0.34, a zero-padded AWS key id scores
+ * 1.02, and a zero-padded `sk_live_` key scores 1.50 — every one of them a full-length secret that
+ * an entropy floor would demote. Sampling confirms the same for genuinely random values, where the
+ * finite-sample bias of a short string drags the estimate down: at a 3.0 bits/char floor, 46% of
+ * random 12-character hex, 74% of random 16-digit numerics, and 99% of 9-digit values fall below it.
+ *
+ * Eight is chosen because every false positive observed in practice came from a value of seven
+ * characters or fewer, and because a literal that short is the only kind that plausibly appears
+ * inside unrelated log text by accident. Values below the floor are still substituted — they just
+ * have to land on a word boundary, which covers standing alone, delimited, and whole-value cases.
+ */
+export const MIN_UNANCHORED_MATCH_LENGTH = 8
+
+/**
+ * Combining marks count so a substitution cannot split a grapheme cluster. `_` deliberately does
+ * NOT: `sk_live_...` and `user_483920_profile` are the dominant way a secret gets joined into an
+ * identifier, and treating `_` as a word character would suppress those hits entirely.
+ */
+const WORD_CHARACTER = /[\p{L}\p{N}\p{M}]/u
+
+/** Classifies one secret literal by whether a hit on it could plausibly be a coincidence. */
+export function getResolvedSecretMatchPolicy(plaintext: string): ResolvedSecretMatchPolicy {
+  return plaintext.length >= MIN_UNANCHORED_MATCH_LENGTH ? 'anywhere' : 'boundary'
+}
+
+/**
+ * Reads the whole code point occupying `index`, including when `index` addresses the trailing half
+ * of a surrogate pair. Returns undefined out of range, which callers treat as "not a word
+ * character" so an out-of-bounds probe widens the match rather than suppressing it.
+ */
+function codePointAt(value: string, index: number): number | undefined {
+  if (index < 0 || index >= value.length) return undefined
+  const code = value.codePointAt(index)
+  if (code !== undefined && code >= 0xdc00 && code <= 0xdfff && index > 0) {
+    const paired = value.codePointAt(index - 1)
+    if (paired !== undefined && paired > 0xffff) return paired
+  }
+  return code
+}
+
+function isWordCharacter(value: string, index: number): boolean {
+  const code = codePointAt(value, index)
+  if (code === undefined) return false
+  if (code < 0x80) {
+    return (code >= 48 && code <= 57) || (code >= 65 && code <= 90) || (code >= 97 && code <= 122)
+  }
+  return WORD_CHARACTER.test(String.fromCodePoint(code))
+}
+
+/**
+ * True when the span `[start, end)` is not spliced into the middle of a surrounding word.
+ *
+ * A boundary exists wherever two adjacent characters are not both word characters, which is the
+ * generalization of a regex `\b` to a span. `key=test` and `"test"` are anchored because `=` and
+ * `"` are not word characters; `latest` is not, because `a` and `t` both are. A whole-value match
+ * is anchored by the string edges, so an exact value is always replaceable regardless of policy.
+ */
+export function isWordBoundaryMatch(value: string, start: number, end: number): boolean {
+  const startsWord = isWordCharacter(value, start - 1) && isWordCharacter(value, start)
+  const endsWord = isWordCharacter(value, end) && isWordCharacter(value, end - 1)
+  return !startsWord && !endsWord
+}
+
+/** True when a hit at `[start, end)` may be substituted under `policy`. Omitted policy is wide. */
+export function satisfiesResolvedSecretMatchPolicy(
+  value: string,
+  start: number,
+  end: number,
+  policy: ResolvedSecretMatchPolicy | undefined
+): boolean {
+  return policy !== 'boundary' || isWordBoundaryMatch(value, start, end)
+}

apps/sim/executor/utils/resolved-secret-matcher.test.ts

@@ -3,9 +3,12 @@
  */
 import { describe, expect, it } from 'vitest'
 import {
+  type CreateResolvedSecretMatcherOptions,
   containsResolvedSecret,
   createResolvedSecretMatcher,
   OPAQUE_RESOLVED_SECRET_REPLACEMENT,
+  type ResolvedSecretMatch,
+  type ResolvedSecretMatcher,
   sanitizeResolvedSecretPrimitive,
   sanitizeResolvedSecretString,
   scanResolvedSecretString,
@@ -192,3 +195,176 @@ describe('resolved secret matcher', () => {
     expect(sanitizeResolvedSecretString('Test', matcher)).toBe('')
   })
 })
+
+describe('resolved secret matcher match policy', () => {
+  const SHORT = [{ plaintext: 'test', replacement: '{{TOKEN}}' }]
+  const API_KEY = 'sk-proj-Ab3xK9mQ2pLw7nRt5vYc8Zd4'
+  const LONG = [{ plaintext: API_KEY, replacement: '{{API_KEY}}' }]
+
+  function build(
+    matches: ResolvedSecretMatch[],
+    options?: CreateResolvedSecretMatcherOptions
+  ): ResolvedSecretMatcher {
+    const matcher = createResolvedSecretMatcher(matches, options)
+    if (!matcher) throw new Error('expected a matcher')
+    return matcher
+  }
+
+  it('matches a short literal anywhere when classifying content', () => {
+    const matcher = build(SHORT)
+
+    expect(containsResolvedSecret('the latest news', matcher)).toBe(true)
+    expect(sanitizeResolvedSecretString('the latest news', matcher)).toBe('the la{{TOKEN}} news')
+  })
+
+  it.each([
+    ['test', '{{TOKEN}}'],
+    ['key=test', 'key={{TOKEN}}'],
+    ['"test"', '"{{TOKEN}}"'],
+    ['{"k":"test"}', '{"k":"{{TOKEN}}"}'],
+    ['test test', '{{TOKEN}} {{TOKEN}}'],
+    ['user_test_id', 'user_{{TOKEN}}_id'],
+  ])('still renders a boundary-anchored short literal in %s', (value, expected) => {
+    const matcher = build(SHORT, { mode: 'render' })
+
+    expect(sanitizeResolvedSecretString(value, matcher)).toBe(expected)
+    expect(containsResolvedSecret(value, matcher)).toBe(true)
+  })
+
+  it.each(['the latest news', 'tested', 'prefixtest'])(
+    'leaves an unanchored short literal in %s untouched when rendering',
+    (value) => {
+      const matcher = build(SHORT, { mode: 'render' })
+
+      expect(sanitizeResolvedSecretString(value, matcher)).toBe(value)
+      expect(containsResolvedSecret(value, matcher)).toBe(false)
+    }
+  )
+
+  it('renders a full-length literal at any offset, including mid-token', () => {
+    const matcher = build(LONG, { mode: 'render' })
+
+    expect(sanitizeResolvedSecretString(`prefix${API_KEY}suffix`, matcher)).toBe(
+      'prefix{{API_KEY}}suffix'
+    )
+    expect(containsResolvedSecret(`prefix${API_KEY}suffix`, matcher)).toBe(true)
+  })
+
+  /** Prefixed shapes are assembled at runtime so no source literal reads as a live credential. */
+  it.each([
+    ['f'.repeat(32), 'all-f HMAC key'],
+    ['4111111111111111', 'test PAN'],
+    [`AKIA${'0'.repeat(16)}`, 'padded AWS key id'],
+    [`sk_live_${'0'.repeat(24)}`, 'padded stripe-style key'],
+  ])('renders low-variety full-length credential (%s) mid-token', (secret) => {
+    const matcher = build([{ plaintext: secret, replacement: '{{KEY}}' }], { mode: 'render' })
+
+    expect(sanitizeResolvedSecretString(`etag_${secret}x`, matcher)).toBe('etag_{{KEY}}x')
+    expect(containsResolvedSecret(`etag_${secret}x`, matcher)).toBe(true)
+  })
+
+  it('settles a boundary that an earlier substitution exposed', () => {
+    const matcher = build(
+      [
+        { plaintext: API_KEY, replacement: '{{API_KEY}}' },
+        { plaintext: 'test', replacement: '{{TOKEN}}' },
+      ],
+      { mode: 'render' }
+    )
+
+    expect(sanitizeResolvedSecretString(`${API_KEY}test`, matcher)).toBe('{{API_KEY}}{{TOKEN}}')
+  })
+
+  it('settles a literal that an empty replacement spliced into existence', () => {
+    const matcher = build([
+      { plaintext: API_KEY, replacement: '' },
+      { plaintext: 'password', replacement: '{{PW}}' },
+    ])
+
+    expect(sanitizeResolvedSecretString(`pass${API_KEY}word`, matcher)).toBe('{{PW}}')
+  })
+
+  it('keeps the substitution pass and its invariant in agreement', () => {
+    const matcher = build(SHORT, { mode: 'render' })
+
+    for (const value of ['the latest news', 'key=test', 'contest testable test']) {
+      const sanitized = sanitizeResolvedSecretString(value, matcher)
+      expect(containsResolvedSecret(sanitized, matcher)).toBe(false)
+    }
+  })
+
+  it('reports a suppressed match to provenance callbacks so detection stays conservative', () => {
+    const matcher = build(SHORT, { mode: 'render' })
+    const matches: string[] = []
+
+    expect(
+      sanitizeResolvedSecretString('the latest news', matcher, undefined, (plaintext) =>
+        matches.push(plaintext)
+      )
+    ).toBe('the latest news')
+    expect(matches).toEqual(['test'])
+  })
+
+  it.each([
+    ['Test', '{{Test}}'],
+    ['{{Test}}', '{{Test}}'],
+    ['Test {{Test}} Test', '{{Test}} {{Test}} {{Test}}'],
+    ['laTest news', 'laTest news'],
+  ])('preserves named provenance labels under the render policy for %s', (value, expected) => {
+    const matcher = build([{ plaintext: 'Test', replacement: '{{Test}}' }], {
+      ...PRESERVE_NAMED_PROVENANCE,
+      mode: 'render',
+    })
+
+    expect(sanitizeResolvedSecretString(value, matcher)).toBe(expected)
+  })
+
+  it('keeps the protected-placeholder behaviours under the options production uses', () => {
+    const composite = build(
+      [
+        { plaintext: 'x{{Test}}y', replacement: '{{COMPOSITE}}' },
+        { plaintext: 'Test', replacement: '{{Test}}' },
+      ],
+      { ...PRESERVE_NAMED_PROVENANCE, mode: 'render' }
+    )
+    expect(sanitizeResolvedSecretString('x{{Test}}y', composite)).toBe('{{COMPOSITE}}')
+
+    const malformed = build([{ plaintext: 'Test', replacement: '{{Test{B}}}' }], {
+      ...PRESERVE_NAMED_PROVENANCE,
+      mode: 'render',
+    })
+    expect(sanitizeResolvedSecretString('Test', malformed)).toBe(OPAQUE_RESOLVED_SECRET_REPLACEMENT)
+
+    const chained = build(
+      [
+        { plaintext: 'Test', replacement: 'visible-Test' },
+        { plaintext: 'REDACTED', replacement: '{{OTHER}}' },
+      ],
+      { mode: 'render' }
+    )
+    expect(sanitizeResolvedSecretString('Test', chained)).toBe('')
+  })
+
+  it('keeps exact replacement available below the length floor', () => {
+    const matcher = build([{ plaintext: '23', replacement: '{{TOKEN}}' }], { mode: 'render' })
+
+    expect(sanitizeResolvedSecretPrimitive('23', matcher)).toBe('{{TOKEN}}')
+    expect(sanitizeResolvedSecretString('23', matcher)).toBe('{{TOKEN}}')
+    expect(sanitizeResolvedSecretString('123', matcher)).toBe('123')
+  })
+
+  it('builds a matcher for an astral-plane literal instead of failing construction', () => {
+    const secret = 'k\u{1F600}ey12345'
+    const matcher = build([{ plaintext: secret, replacement: '{{EMOJI}}' }], { mode: 'render' })
+
+    expect(sanitizeResolvedSecretString(`token ${secret} end`, matcher)).toBe('token {{EMOJI}} end')
+  })
+
+  it('does not rewrite a token interior next to an astral-plane letter', () => {
+    const matcher = build(SHORT, { mode: 'render' })
+
+    expect(sanitizeResolvedSecretString('\u{1D400}test\u{1D401}', matcher)).toBe(
+      '\u{1D400}test\u{1D401}'
+    )
+  })
+})