fix(knowledge): content-address the conversation hash, and keep capped

listings out of deletion reconciliation

Two round-5 findings, one of which reverts my own round-4 change.

Greptile P1: metadata proxies still collide. updatedAt is millisecond-
resolution, and message count plus stored byte size both survive a
same-millisecond replacement that happens to preserve them, so a real
transcript change could still hash identically. Each proxy narrowed the
window without closing it. The hash is now keyed on md5 of the stored JSON,
computed in Postgres so the listing still transfers 32 characters rather
than the payload — the reason data is not selected there. The hash now moves
if and only if the transcript moved.

Cursor Bugbot: my round-4 relaxation was wrong, and Bugbot's own round-4
finding does not hold here. A cap implies descending order (round 2), and a
descending keyset can miss a row updated between pages — it moves ABOVE the
cursor rather than below it, so unlike ascending it is not re-seen. That
makes "we consumed exactly the budget" no proof of "we saw everything", and
treating it as proof let reconciliation hard-delete a source item that still
exists. A capped listing now always blocks reconciliation, with the ordering
interaction written down so the relaxation is not reintroduced. The shared
isListingTruncated helper is removed rather than left unused; the Asana rule
it copied is sound for Asana because Asana lists ascending.

Author: mzxchandra

apps/sim/connectors/sim-conversations/sim-conversations.test.ts

@@ -27,6 +27,7 @@ const BASE_ROW: ConversationRow = {
   updatedAt: new Date('2026-01-02T00:00:00.000Z'),
   messageCount: 6,
   approxBytes: 1024,
+  contentDigest: 'd41d8cd98f00b204e9800998ecf8427e',
 }
 
 const META = {
@@ -229,7 +230,7 @@ describe('conversationToStub', () => {
 
   it('hashes on the update watermark, which moves whenever a message is appended', () => {
     const base = conversationToStub(BASE_ROW).contentHash
-    expect(base).toBe('memory:mem-1:2026-01-02T00:00:00.000Z:6:1024')
+    expect(base).toBe('memory:mem-1:2026-01-02T00:00:00.000Z:d41d8cd98f00b204e9800998ecf8427e')
 
     const appended = conversationToStub({
       ...BASE_ROW,
@@ -256,18 +257,31 @@ describe('conversationToStub', () => {
     const appendedSameMs = conversationToStub({
       ...BASE_ROW,
       messageCount: BASE_ROW.messageCount + 2,
-      approxBytes: BASE_ROW.approxBytes + 180,
+      contentDigest: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
     }).contentHash
 
     expect(appendedSameMs).not.toBe(indexed)
   })
 
-  /** A same-count replacement still moves the byte size. */
-  it('distinguishes a same-millisecond, same-count content replacement', () => {
+  /**
+   * The case metadata proxies could not close: a same-millisecond replacement that
+   * preserves both message count and stored byte size. Only the content digest moves.
+   */
+  it('distinguishes a replacement that preserves count and byte size', () => {
     expect(
-      conversationToStub({ ...BASE_ROW, approxBytes: BASE_ROW.approxBytes + 40 }).contentHash
+      conversationToStub({
+        ...BASE_ROW,
+        contentDigest: 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb',
+      }).contentHash
     ).not.toBe(conversationToStub(BASE_ROW).contentHash)
   })
+
+  /** Metadata churn without a content change must NOT force a re-index. */
+  it('is unchanged when only the byte-size hint moves', () => {
+    expect(
+      conversationToStub({ ...BASE_ROW, approxBytes: BASE_ROW.approxBytes + 40 }).contentHash
+    ).toBe(conversationToStub(BASE_ROW).contentHash)
+  })
 })
 
 describe('cursor', () => {

apps/sim/connectors/sim-conversations/sim-conversations.ts

@@ -11,7 +11,6 @@ import type {
 } from '@/connectors/types'
 import {
   CONNECTOR_MAX_FILE_BYTES,
-  isListingTruncated,
   isSkippedDocument,
   markSkipped,
   parseTagDate,
@@ -39,6 +38,7 @@ export interface ConversationRow {
   updatedAt: Date
   messageCount: number
   approxBytes: number
+  contentDigest: string
 }
 
 interface Cursor {
@@ -169,15 +169,15 @@ export function renderTranscript(
  *
  * Single source of truth for `contentHash`, used by both listing and hydration.
  *
- * `updatedAt` is the primary watermark — `Memory.appendMessage` bumps it on every
- * appended message — but it only has millisecond resolution, so two appends landing
- * in the same millisecond as the previously indexed value would hash identically and
- * `classifyExternalDoc` would call the transcript unchanged, leaving the new messages
- * out of the knowledge base until some later write moved the clock.
+ * Keyed on a digest of the stored JSON rather than on metadata proxies. `updatedAt`
+ * is only millisecond-resolution, and message count plus byte size still collide for
+ * a same-millisecond replacement that happens to preserve both — each proxy narrows
+ * the window without closing it. A content digest closes it outright: the hash moves
+ * if and only if the transcript moved.
  *
- * `messageCount` closes that: an append always increments it. `approxBytes` covers
- * the rarer case of a same-count replacement. Both are already selected for the
- * listing, so neither costs an extra query.
+ * Postgres computes the digest, so the listing transfers 32 characters instead of the
+ * payload — the reason `data` is deliberately not selected here. `updatedAt` stays in
+ * the hash purely so a stored value is legible when debugging.
  */
 export function conversationToStub(row: ConversationRow): ExternalDocument {
   return {
@@ -187,7 +187,7 @@ export function conversationToStub(row: ConversationRow): ExternalDocument {
     contentDeferred: true,
     mimeType: 'text/plain',
     // No sourceUrl: conversations have no page of their own in the app.
-    contentHash: `memory:${row.id}:${row.updatedAt.toISOString()}:${row.messageCount}:${row.approxBytes}`,
+    contentHash: `memory:${row.id}:${row.updatedAt.toISOString()}:${row.contentDigest}`,
     metadata: {
       conversationId: row.key,
       messageCount: row.messageCount,
@@ -218,6 +218,12 @@ const CONVERSATION_ROW_COLUMNS = {
     CASE WHEN jsonb_typeof(${memory.data}) = 'array'
          THEN jsonb_array_length(${memory.data}) ELSE 0 END`,
   approxBytes: sql<number>`pg_column_size(${memory.data})`,
+  /**
+   * Content-addressed change detection, computed in Postgres so the transcript
+   * itself never crosses the wire during listing. `jsonb::text` is canonical
+   * (keys sorted, whitespace normalized), so the digest is stable for a value.
+   */
+  contentDigest: sql<string>`md5(${memory.data}::text)`,
 } as const
 
 function readPrefix(sourceConfig: Record<string, unknown>): string {
@@ -291,15 +297,23 @@ export const simConversationsConnector: ConnectorConfig = {
        * matching note in the files connector. Exhausting the source at exactly
        * `maxConversations` is a complete listing.
        */
-      if (
-        isListingTruncated({
-          capReached,
-          droppedFromPage: documents.length < items.length,
-          morePagesAvailable: pageFilled,
-        })
-      ) {
-        syncContext.listingCapped = true
-      }
+      /**
+       * A capped listing can NEVER certify completeness, so it always blocks deletion
+       * reconciliation — even when the source happened to run out at exactly
+       * `maxConversations` with nothing dropped.
+       *
+       * The reason is the descending order a cap implies: a row updated between pages
+       * moves ABOVE the keyset cursor and is skipped for this run. (Ascending has the
+       * opposite skew — the row moves below the cursor and is re-seen, which the
+       * engine dedupes.) So "we consumed exactly the budget" does not prove "we saw
+       * everything", and treating it as proof lets reconciliation hard-delete a source
+       * item that still exists.
+       *
+       * Do not relax this into a `droppedFromPage || morePagesAvailable` check. That
+       * reads correct in isolation and is how the Asana connector decides truncation,
+       * but Asana lists ascending — the inference does not transfer.
+       */
+      syncContext.listingCapped = true
       return { documents, hasMore: false }
     }
 

apps/sim/connectors/sim-files/sim-files.ts

@@ -23,7 +23,6 @@ import type {
 } from '@/connectors/types'
 import {
   CONNECTOR_MAX_FILE_BYTES,
-  isListingTruncated,
   isSkippedDocument,
   markSkipped,
   parseMultiValue,
@@ -390,15 +389,23 @@ export const simFilesConnector: ConnectorConfig = {
        * source would never leave the knowledge base. Mirrors `decideTaskCap`'s
        * `droppedFromPage || (hitLimit && morePagesAvailable)` in the Asana connector.
        */
-      if (
-        isListingTruncated({
-          capReached,
-          droppedFromPage: documents.length < items.length,
-          morePagesAvailable: pageFilled,
-        })
-      ) {
-        syncContext.listingCapped = true
-      }
+      /**
+       * A capped listing can NEVER certify completeness, so it always blocks deletion
+       * reconciliation — even when the source happened to run out at exactly
+       * `maxFiles` with nothing dropped.
+       *
+       * The reason is the descending order a cap implies: a row updated between pages
+       * moves ABOVE the keyset cursor and is skipped for this run. (Ascending has the
+       * opposite skew — the row moves below the cursor and is re-seen, which the
+       * engine dedupes.) So "we consumed exactly the budget" does not prove "we saw
+       * everything", and treating it as proof lets reconciliation hard-delete a source
+       * item that still exists.
+       *
+       * Do not relax this into a `droppedFromPage || morePagesAvailable` check. That
+       * reads correct in isolation and is how the Asana connector decides truncation,
+       * but Asana lists ascending — the inference does not transfer.
+       */
+      syncContext.listingCapped = true
       return { documents, hasMore: false }
     }
 

apps/sim/connectors/utils-listing-cap.test.ts

@@ -229,25 +229,6 @@ export function takeIndexableWithinCap<T>(
   return { documents, indexableCount, capReached: alreadyIndexed + indexableCount >= max }
 }
 
-/**
- * Whether a capped listing actually stopped short of the source.
- *
- * `takeIndexableWithinCap` reports `capReached` — the budget is spent — which is NOT
- * the same as truncation. A source that runs out at exactly the cap yields a complete
- * listing, and marking it truncated would suppress deletion reconciliation forever,
- * so an item deleted at the source could never leave the knowledge base.
- *
- * Same rule as `decideTaskCap` in the Asana connector: truncated only when this page
- * dropped items, or the budget ran out with more pages still available.
- */
-export function isListingTruncated(args: {
-  capReached: boolean
-  droppedFromPage: boolean
-  morePagesAvailable: boolean
-}): boolean {
-  return args.droppedFromPage || (args.capReached && args.morePagesAvailable)
-}
-
 /**
  * Raised by a connector when a file exceeds its size cap mid-download — i.e. the
  * listing did not report a size, so the limit is only discovered while streaming.