fix(uploads): manual file uploads must ignore provenance stamping (#6314)

* fix(attachments): model egress attachments

* fix(uploads): manual uploads provenance ignore

* further checks on kb

* tags fixes

* fix more stuff

* fix tests

* address comments

* fix

* fix

* fix

* address timestamp concern

Author: icecrasher321

apps/sim/app/api/knowledge/[id]/documents/[documentId]/tag-definitions/route.ts

@@ -11,6 +11,7 @@ import {
   createOrUpdateTagDefinitionsBulk,
   deleteAllTagDefinitions,
   getDocumentTagDefinitions,
+  KnowledgeTagProvenanceConflictError,
 } from '@/lib/knowledge/tags/service'
 import type { BulkTagDefinitionsData } from '@/lib/knowledge/tags/types'
 import { checkDocumentAccess, checkDocumentWriteAccess } from '@/app/api/knowledge/utils'
@@ -198,6 +199,9 @@ export const DELETE = withRouteHandler(
         data: { deleted: deletedCount },
       })
     } catch (error) {
+      if (error instanceof KnowledgeTagProvenanceConflictError) {
+        return NextResponse.json({ error: error.message }, { status: 409 })
+      }
       logger.error(`[${requestId}] Error with tag definitions operation`, error)
       return NextResponse.json({ error: 'Failed to process tag definitions' }, { status: 500 })
     }

apps/sim/app/api/knowledge/[id]/tag-definitions/[tagId]/route.ts

@@ -5,7 +5,10 @@ import { deleteTagDefinitionContract } from '@/lib/api/contracts/knowledge'
 import { parseRequest } from '@/lib/api/server'
 import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
-import { deleteTagDefinition } from '@/lib/knowledge/tags/service'
+import {
+  deleteTagDefinition,
+  KnowledgeTagProvenanceConflictError,
+} from '@/lib/knowledge/tags/service'
 import { checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils'
 
 export const dynamic = 'force-dynamic'
@@ -45,6 +48,9 @@ export const DELETE = withRouteHandler(
         message: `Tag definition "${deletedTag.displayName}" deleted successfully`,
       })
     } catch (error) {
+      if (error instanceof KnowledgeTagProvenanceConflictError) {
+        return NextResponse.json({ error: error.message }, { status: 409 })
+      }
       logger.error(`[${requestId}] Error deleting tag definition`, error)
       return NextResponse.json({ error: 'Failed to delete tag definition' }, { status: 500 })
     }

apps/sim/app/api/knowledge/secret-provenance.test.ts

@@ -10,25 +10,171 @@ import {
   PRIVATE_SECRET_PROVENANCE_FIELD,
   PRIVATE_SECRET_PROVENANCE_HEADER,
 } from '@/lib/execution/private-tool-metadata'
-import { resolveKnowledgeWriteSecretProvenance } from '@/app/api/knowledge/secret-provenance'
+import {
+  resolveKnowledgeDocumentWriteSecretProvenance,
+  resolveKnowledgeWriteSecretProvenance,
+} from '@/app/api/knowledge/secret-provenance'
+
+const PRIVATE_PROVENANCE_SCOPE = {
+  userId: 'user-1',
+  workspaceId: 'workspace-1',
+} as const
 
-function createRequest(payload: Record<string, unknown>): NextRequest {
+function createRequest(
+  payload: Record<string, unknown>,
+  provenanceHeader = PRIVATE_SECRET_PROVENANCE_BUNDLE_V1
+): NextRequest {
+  return new NextRequest('http://localhost/api/knowledge/kb/documents', {
+    method: 'POST',
+    headers: { [PRIVATE_SECRET_PROVENANCE_HEADER]: provenanceHeader },
+    body: JSON.stringify(payload),
+  })
+}
+
+function createHeaderlessRequest(payload: Record<string, unknown>): NextRequest {
   return new NextRequest('http://localhost/api/knowledge/kb/documents', {
     method: 'POST',
-    headers: { [PRIVATE_SECRET_PROVENANCE_HEADER]: PRIVATE_SECRET_PROVENANCE_BUNDLE_V1 },
     body: JSON.stringify(payload),
   })
 }
 
 describe('knowledge write secret provenance', () => {
+  it('classifies a headerless external chunk write as exact-empty', () => {
+    const payload = { content: 'manual content' }
+
+    const result = resolveKnowledgeWriteSecretProvenance({
+      request: createHeaderlessRequest(payload),
+      payload,
+      authType: AuthType.API_KEY,
+      userId: 'user-1',
+      workspaceId: 'workspace-1',
+      selectionKeys: ['chunk-content'],
+    })
+
+    expect(result).toEqual({
+      success: true,
+      provenances: [{ status: 'exact', entries: [] }],
+    })
+  })
+
+  it('classifies a headerless external document write as exact-empty', () => {
+    const payload = {
+      filename: 'manual.txt',
+    }
+
+    const result = resolveKnowledgeDocumentWriteSecretProvenance({
+      request: createHeaderlessRequest(payload),
+      payload,
+      authType: AuthType.SESSION,
+      userId: 'user-1',
+      workspaceId: 'workspace-1',
+      documents: [payload],
+    })
+
+    expect(result).toEqual({
+      success: true,
+      provenances: [
+        {
+          filename: { status: 'exact', entries: [] },
+          content: { status: 'exact', entries: [] },
+          tags: [],
+        },
+      ],
+    })
+  })
+
+  it('does not track durable provenance for a legacy headerless internal write', () => {
+    const payload = { content: 'legacy workflow content' }
+
+    const result = resolveKnowledgeWriteSecretProvenance({
+      request: createHeaderlessRequest(payload),
+      payload,
+      authType: AuthType.INTERNAL_JWT,
+      userId: 'user-1',
+      workspaceId: 'workspace-1',
+      selectionKeys: ['chunk-content'],
+    })
+
+    expect(result).toEqual({ success: true })
+  })
+
+  it('tracks exact-empty provenance only when an internal write supplies a verified envelope', () => {
+    const bundle = {
+      version: 1 as const,
+      complete: true,
+      selections: [
+        {
+          key: 'chunk-content',
+          provenance: {
+            version: 1 as const,
+            complete: true,
+            entries: [],
+            scope: PRIVATE_PROVENANCE_SCOPE,
+          },
+        },
+      ],
+    }
+    const payload = { content: 'workflow content', [PRIVATE_SECRET_PROVENANCE_FIELD]: bundle }
+
+    const result = resolveKnowledgeWriteSecretProvenance({
+      request: createRequest(payload),
+      payload,
+      authType: AuthType.INTERNAL_JWT,
+      userId: 'user-1',
+      workspaceId: 'workspace-1',
+      selectionKeys: ['chunk-content'],
+    })
+
+    expect(result).toEqual({
+      success: true,
+      provenances: [{ status: 'exact', entries: [] }],
+    })
+  })
+
+  it('rejects a private provenance envelope from an external caller', () => {
+    const bundle = {
+      version: 1 as const,
+      complete: true,
+      selections: [
+        {
+          key: 'chunk-content',
+          provenance: {
+            version: 1 as const,
+            complete: true,
+            entries: [],
+            scope: PRIVATE_PROVENANCE_SCOPE,
+          },
+        },
+      ],
+    }
+    const payload = { content: 'external content', [PRIVATE_SECRET_PROVENANCE_FIELD]: bundle }
+
+    const result = resolveKnowledgeWriteSecretProvenance({
+      request: createRequest(payload),
+      payload,
+      authType: AuthType.API_KEY,
+      userId: 'user-1',
+      workspaceId: 'workspace-1',
+      selectionKeys: ['chunk-content'],
+    })
+
+    expect(result.success).toBe(false)
+    if (!result.success) expect(result.response.status).toBe(400)
+  })
+
   it('rejects an unavailable verified selection before a write can start', () => {
     const bundle = {
       version: 1 as const,
       complete: true,
       selections: [
         {
           key: 'document-source:0',
-          provenance: { version: 1 as const, complete: false, entries: [] },
+          provenance: {
+            version: 1 as const,
+            complete: false,
+            entries: [],
+            scope: PRIVATE_PROVENANCE_SCOPE,
+          },
         },
       ],
     }

apps/sim/executor/handlers/agent/agent-handler.test.ts

@@ -18,6 +18,7 @@ import {
   vi,
 } from 'vitest'
 import type { AutoRoutingSignals } from '@/lib/model-router/resolve'
+import * as userFileBase64 from '@/lib/uploads/utils/user-file-base64.server'
 import { getAllBlocks } from '@/blocks'
 import { AGENT, BlockType, isMcpTool } from '@/executor/constants'
 import { AgentBlockHandler } from '@/executor/handlers/agent/agent-handler'
@@ -488,6 +489,47 @@ describe('AgentBlockHandler', () => {
       })
     })
 
+    it('normalizes the persisted workspace-picker shape before provider execution', async () => {
+      const key = 'workspace/ws-1/example.png'
+      const hydrationSpy = vi
+        .spyOn(userFileBase64, 'hydrateUserFilesWithBase64')
+        .mockImplementationOnce(async (files) =>
+          files.map((file) => ({ ...file, base64: 'aW1hZ2U=' }))
+        )
+
+      try {
+        mockGetProviderFromModel.mockReturnValue('openai')
+
+        await handler.execute(mockContext, mockBlock, {
+          model: 'gpt-4o',
+          userPrompt: 'Analyze this file',
+          files: [
+            {
+              name: 'example.png',
+              path: `/api/files/serve/${encodeURIComponent(key)}?context=workspace`,
+              key,
+              size: 128,
+              type: 'image/png',
+            },
+          ],
+          apiKey: 'test-api-key',
+        })
+
+        const normalizedFile = hydrationSpy.mock.calls[0][0][0]
+        expect(normalizedFile).toMatchObject({
+          id: expect.stringMatching(/^file-\d+$/),
+          key,
+          name: 'example.png',
+          type: 'image/png',
+        })
+        expect(mockExecuteProviderRequest.mock.calls[0][1].messages.at(-1)?.files).toEqual([
+          expect.objectContaining({ key, name: 'example.png', base64: 'aW1hZ2U=' }),
+        ])
+      } finally {
+        hydrationSpy.mockRestore()
+      }
+    })
+
     it('should reject files for providers without attachment support', async () => {
       const inputs = {
         model: 'deepseek-chat',

apps/sim/lib/copilot/request/lifecycle/run.test.ts

@@ -496,14 +496,10 @@ describe('runCopilotLifecycle', () => {
     expect(JSON.parse(capturedRequestBody).fileAttachments).toEqual([safe])
   })
 
-  it('continues without attachments when durable provenance cannot be verified', async () => {
+  it('rejects when durable attachment provenance cannot be verified', async () => {
     mockFilterModelSafeWorkspaceFileAttachments.mockRejectedValueOnce(new Error('db unavailable'))
-    let capturedRequestBody = ''
-    mockRunStreamLoop.mockImplementationOnce(async (_url: string, request: RequestInit) => {
-      capturedRequestBody = String(request.body)
-    })
 
-    await runCopilotLifecycle(
+    const result = await runCopilotLifecycle(
       {
         message: 'Continue safely',
         fileAttachments: [{ id: 'wf-file', name: 'file.txt', key: 'workspace/ws-1/file.txt' }],
@@ -517,7 +513,11 @@ describe('runCopilotLifecycle', () => {
       }
     )
 
-    expect(JSON.parse(capturedRequestBody)).not.toHaveProperty('fileAttachments')
+    expect(result).toMatchObject({
+      success: false,
+      error: 'Copilot model input could not be safely projected',
+    })
+    expect(mockRunStreamLoop).not.toHaveBeenCalled()
   })
 
   it.each(['123', 'true'])(

apps/sim/lib/copilot/request/lifecycle/run.ts

@@ -607,11 +607,11 @@ async function omitUnsafeInitialCopilotAttachments(
     try {
       safeAttachments = await filterModelSafeWorkspaceFileAttachments(attachments, { workspaceId })
     } catch (error) {
-      logger.warn('Workspace file secret provenance could not be verified; omitting attachments', {
+      logger.error('Workspace file secret provenance could not be verified', {
         attachmentCount: attachments.length,
         error: toError(error).message,
       })
-      safeAttachments = []
+      throw new CopilotModelContentProjectionError()
     }
 
     if (safeAttachments.length === attachments.length) continue