Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
2cdb896
feat(hosted keys): Implement serper hosted key
Feb 13, 2026
3e6527a
Handle required fields correctly for hosted keys
Feb 13, 2026
e5c8aec
Add rate limiting (3 tries, exponential backoff)
Feb 13, 2026
8a78f80
Add custom pricing, switch to exa as first hosted key
Feb 13, 2026
d174a6a
Add telemetry
Feb 13, 2026
c12e92c
Consolidate byok type definitions
Feb 13, 2026
2a36143
Add warning comment if default calculation is used
Feb 13, 2026
36e6464
Record usage to user stats table
Feb 13, 2026
f237d6f
Fix unit tests, use cost property
Feb 13, 2026
0a002fd
Include more metadata in cost output
Feb 13, 2026
36d49ef
Fix disabled tests
Feb 13, 2026
fbd1cdf
Fix spacing
Feb 14, 2026
dc4c611
Fix lint
Feb 14, 2026
68da290
Move knowledge cost restructuring away from generic block handler
Feb 16, 2026
ce02a30
Migrate knowledge unit tests
Feb 16, 2026
e6d98c6
Lint
Feb 16, 2026
ecdbe29
Fix broken tests
Mar 5, 2026
2325535
Merge branch 'staging' into feat/sim-provided-key
Mar 5, 2026
693a3d3
Add user based hosted key throttling
Mar 5, 2026
242d6e0
Refactor hosted key handling. Add optimistic handling of throttling f…
Mar 5, 2026
7b8e24e
Remove research as hosted key. Recommend BYOK if throtttling occurs
Mar 5, 2026
cd160d3
Make adding api keys adjustable via env vars
Mar 6, 2026
2082bc4
Remove vestigial fields from research
Mar 6, 2026
a90777a
Make billing actor id required for throttling
Mar 6, 2026
d7ea0af
Switch to round robin for api key distribution
Mar 6, 2026
1c5425e
Add helper method for adding hosted key cost
Mar 6, 2026
3832e5c
Strip leading double underscores to avoid breaking change
Mar 6, 2026
34cffdc
Lint fix
Mar 6, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion apps/sim/app/api/workspaces/[id]/byok-keys/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import { getUserEntityPermissions, getWorkspaceById } from '@/lib/workspaces/per

const logger = createLogger('WorkspaceBYOKKeysAPI')

const VALID_PROVIDERS = ['openai', 'anthropic', 'google', 'mistral'] as const
const VALID_PROVIDERS = ['openai', 'anthropic', 'google', 'mistral', 'exa'] as const

const UpsertKeySchema = z.object({
providerId: z.enum(VALID_PROVIDERS),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import {
buildCanonicalIndex,
evaluateSubBlockCondition,
isSubBlockFeatureEnabled,
isSubBlockHiddenByHostedKey,
isSubBlockVisibleForMode,
} from '@/lib/workflows/subblocks/visibility'
import type { BlockConfig, SubBlockConfig, SubBlockType } from '@/blocks/types'
Expand Down Expand Up @@ -108,6 +109,9 @@ export function useEditorSubblockLayout(
// Check required feature if specified - declarative feature gating
if (!isSubBlockFeatureEnabled(block)) return false

// Hide tool API key fields when hosted
if (isSubBlockHiddenByHostedKey(block)) return false

// Special handling for trigger-config type (legacy trigger configuration UI)
if (block.type === ('trigger-config' as SubBlockType)) {
const isPureTriggerBlock = config?.triggers?.enabled && config.category === 'triggers'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import {
evaluateSubBlockCondition,
hasAdvancedValues,
isSubBlockFeatureEnabled,
isSubBlockHiddenByHostedKey,
isSubBlockVisibleForMode,
resolveDependencyValue,
} from '@/lib/workflows/subblocks/visibility'
Expand Down Expand Up @@ -950,6 +951,7 @@ export const WorkflowBlock = memo(function WorkflowBlock({
if (block.hidden) return false
if (block.hideFromPreview) return false
if (!isSubBlockFeatureEnabled(block)) return false
if (isSubBlockHiddenByHostedKey(block)) return false

const isPureTriggerBlock = config?.triggers?.enabled && config.category === 'triggers'

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,15 @@ import {
ModalFooter,
ModalHeader,
} from '@/components/emcn'
import { AnthropicIcon, GeminiIcon, MistralIcon, OpenAIIcon } from '@/components/icons'
import { AnthropicIcon, ExaAIIcon, GeminiIcon, MistralIcon, OpenAIIcon } from '@/components/icons'
import { Skeleton } from '@/components/ui'
import {
type BYOKKey,
type BYOKProviderId,
useBYOKKeys,
useDeleteBYOKKey,
useUpsertBYOKKey,
} from '@/hooks/queries/byok-keys'
import type { BYOKProviderId } from '@/tools/types'

const logger = createLogger('BYOKSettings')

Expand Down Expand Up @@ -60,6 +60,13 @@ const PROVIDERS: {
description: 'LLM calls and Knowledge Base OCR',
placeholder: 'Enter your API key',
},
{
id: 'exa',
name: 'Exa',
icon: ExaAIIcon,
description: 'AI-powered search and research',
placeholder: 'Enter your Exa API key',
},
]

function BYOKKeySkeleton() {
Expand Down
14 changes: 13 additions & 1 deletion apps/sim/blocks/blocks/exa.ts
Original file line number Diff line number Diff line change
Expand Up @@ -309,14 +309,26 @@ export const ExaBlock: BlockConfig<ExaResponse> = {
value: () => 'exa-research',
condition: { field: 'operation', value: 'exa_research' },
},
// API Key (common)
// API Key — hidden when hosted for operations with hosted key support
{
id: 'apiKey',
title: 'API Key',
type: 'short-input',
placeholder: 'Enter your Exa API key',
password: true,
required: true,
hideWhenHosted: true,
condition: { field: 'operation', value: 'exa_research', not: true },
},
// API Key — always visible for research (no hosted key support)
{
id: 'apiKey',
title: 'API Key',
type: 'short-input',
placeholder: 'Enter your Exa API key',
password: true,
required: true,
condition: { field: 'operation', value: 'exa_research' },
},
],
tools: {
Expand Down
1 change: 1 addition & 0 deletions apps/sim/blocks/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,7 @@ export interface SubBlockConfig {
hidden?: boolean
hideFromPreview?: boolean // Hide this subblock from the workflow block preview
requiresFeature?: string // Environment variable name that must be truthy for this subblock to be visible
hideWhenHosted?: boolean // Hide this subblock when running on hosted sim
description?: string
tooltip?: string // Tooltip text displayed via info icon next to the title
value?: (params: Record<string, any>) => string
Expand Down
215 changes: 0 additions & 215 deletions apps/sim/executor/handlers/generic/generic-handler.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -147,219 +147,4 @@ describe('GenericBlockHandler', () => {
'Block execution of Some Custom Tool failed with no error message'
)
})

describe('Knowledge block cost tracking', () => {
beforeEach(() => {
// Set up knowledge block mock
mockBlock = {
...mockBlock,
config: { tool: 'knowledge_search', params: {} },
}

mockTool = {
...mockTool,
id: 'knowledge_search',
name: 'Knowledge Search',
}

mockGetTool.mockImplementation((toolId) => {
if (toolId === 'knowledge_search') {
return mockTool
}
return undefined
})
})

it.concurrent(
'should extract and restructure cost information from knowledge tools',
async () => {
const inputs = { query: 'test query' }
const mockToolResponse = {
success: true,
output: {
results: [],
query: 'test query',
totalResults: 0,
cost: {
input: 0.00001042,
output: 0,
total: 0.00001042,
tokens: {
input: 521,
output: 0,
total: 521,
},
model: 'text-embedding-3-small',
pricing: {
input: 0.02,
output: 0,
updatedAt: '2025-07-10',
},
},
},
}

mockExecuteTool.mockResolvedValue(mockToolResponse)

const result = await handler.execute(mockContext, mockBlock, inputs)

// Verify cost information is restructured correctly for enhanced logging
expect(result).toEqual({
results: [],
query: 'test query',
totalResults: 0,
cost: {
input: 0.00001042,
output: 0,
total: 0.00001042,
},
tokens: {
input: 521,
output: 0,
total: 521,
},
model: 'text-embedding-3-small',
})
}
)

it.concurrent('should handle knowledge_upload_chunk cost information', async () => {
// Update to upload_chunk tool
mockBlock.config.tool = 'knowledge_upload_chunk'
mockTool.id = 'knowledge_upload_chunk'
mockTool.name = 'Knowledge Upload Chunk'

mockGetTool.mockImplementation((toolId) => {
if (toolId === 'knowledge_upload_chunk') {
return mockTool
}
return undefined
})

const inputs = { content: 'test content' }
const mockToolResponse = {
success: true,
output: {
data: {
id: 'chunk-123',
content: 'test content',
chunkIndex: 0,
},
message: 'Successfully uploaded chunk',
documentId: 'doc-123',
cost: {
input: 0.00000521,
output: 0,
total: 0.00000521,
tokens: {
input: 260,
output: 0,
total: 260,
},
model: 'text-embedding-3-small',
pricing: {
input: 0.02,
output: 0,
updatedAt: '2025-07-10',
},
},
},
}

mockExecuteTool.mockResolvedValue(mockToolResponse)

const result = await handler.execute(mockContext, mockBlock, inputs)

// Verify cost information is restructured correctly
expect(result).toEqual({
data: {
id: 'chunk-123',
content: 'test content',
chunkIndex: 0,
},
message: 'Successfully uploaded chunk',
documentId: 'doc-123',
cost: {
input: 0.00000521,
output: 0,
total: 0.00000521,
},
tokens: {
input: 260,
output: 0,
total: 260,
},
model: 'text-embedding-3-small',
})
})

it('should pass through output unchanged for knowledge tools without cost info', async () => {
const inputs = { query: 'test query' }
const mockToolResponse = {
success: true,
output: {
results: [],
query: 'test query',
totalResults: 0,
// No cost information
},
}

mockExecuteTool.mockResolvedValue(mockToolResponse)

const result = await handler.execute(mockContext, mockBlock, inputs)

// Should return original output without cost transformation
expect(result).toEqual({
results: [],
query: 'test query',
totalResults: 0,
})
})

it.concurrent(
'should process cost info for all tools (universal cost extraction)',
async () => {
mockBlock.config.tool = 'some_other_tool'
mockTool.id = 'some_other_tool'

mockGetTool.mockImplementation((toolId) => {
if (toolId === 'some_other_tool') {
return mockTool
}
return undefined
})

const inputs = { param: 'value' }
const mockToolResponse = {
success: true,
output: {
result: 'success',
cost: {
input: 0.001,
output: 0.002,
total: 0.003,
tokens: { input: 100, output: 50, total: 150 },
model: 'some-model',
},
},
}

mockExecuteTool.mockResolvedValue(mockToolResponse)

const result = await handler.execute(mockContext, mockBlock, inputs)

expect(result).toEqual({
result: 'success',
cost: {
input: 0.001,
output: 0.002,
total: 0.003,
},
tokens: { input: 100, output: 50, total: 150 },
model: 'some-model',
})
}
)
})
})
22 changes: 1 addition & 21 deletions apps/sim/executor/handlers/generic/generic-handler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,27 +98,7 @@ export class GenericBlockHandler implements BlockHandler {
throw error
}

const output = result.output
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moved this to knowledge block transformation, so generic handler doesn't need to handle special cases

let cost = null

if (output?.cost) {
cost = output.cost
}

if (cost) {
return {
...output,
cost: {
input: cost.input,
output: cost.output,
total: cost.total,
},
tokens: cost.tokens,
model: cost.model,
}
}

return output
return result.output
} catch (error: any) {
if (!error.message || error.message === 'undefined (undefined)') {
let errorMessage = `Block execution of ${tool?.name || block.config.tool} failed`
Expand Down
3 changes: 1 addition & 2 deletions apps/sim/hooks/queries/byok-keys.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
import { createLogger } from '@sim/logger'
import { keepPreviousData, useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
import { API_ENDPOINTS } from '@/stores/constants'
import type { BYOKProviderId } from '@/tools/types'

const logger = createLogger('BYOKKeysQueries')

export type BYOKProviderId = 'openai' | 'anthropic' | 'google' | 'mistral'

export interface BYOKKey {
id: string
providerId: BYOKProviderId
Expand Down
3 changes: 1 addition & 2 deletions apps/sim/lib/api-key/byok.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,10 @@ import { isHosted } from '@/lib/core/config/feature-flags'
import { decryptSecret } from '@/lib/core/security/encryption'
import { getHostedModels } from '@/providers/models'
import { useProvidersStore } from '@/stores/providers/store'
import type { BYOKProviderId } from '@/tools/types'

const logger = createLogger('BYOKKeys')

export type BYOKProviderId = 'openai' | 'anthropic' | 'google' | 'mistral'

export interface BYOKKeyResult {
apiKey: string
isBYOK: true
Expand Down
Loading