fix(embeddings): project before batching, and keep the sunset block's docs icon

Review round 2.

Projection ran inside callEmbeddingAPI, after batchByTokenLimit had already
measured and truncated the original text. The projector rewrites resolved
secrets to placeholders, which changes length, so batching sized against a
string that was never sent: a lengthening projection then pushed input past
the model's ceiling and the provider rejected it, and a shortening one
discarded document content that would have fit.

Project once up front, then batch the projected text, so truncation measures
what actually goes to the provider. This also keeps projection to exactly one
call per embed(), so no retry can re-project.

Separately, marking the legacy openai block hideFromToolbar dropped it from
the generated docs icon map, which only retains hidden blocks when they are
versioned. integrations/openai.mdx is deliberately kept — docsLink is baked
into every placed instance — so BlockInfoCard lost its icon and fell back to
a text tile. A sunset block keeps its docs page for the same reason a hidden
versioned block does, so the generator now treats it the same way.

The sim-side integrations map still omits it, which is intended: that feeds
the discovery page a sunset block should not appear on, and placed blocks
render from the registry's own icon reference.

Author: mzxchandra

apps/docs/components/ui/icon-mapping.ts

@@ -159,6 +159,7 @@ import {
   ObsidianIcon,
   OktaIcon,
   OnePasswordIcon,
+  OpenAIIcon,
   OutlookIcon,
   PackageSearchIcon,
   PagerDutyIcon,
@@ -439,6 +440,7 @@ export const blockTypeToIconMap: Record<string, IconComponent> = {
   okta: OktaIcon,
   onedrive: MicrosoftOneDriveIcon,
   onepassword: OnePasswordIcon,
+  openai: OpenAIIcon,
   outlook: OutlookIcon,
   pagerduty: PagerDutyIcon,
   parallel_ai: ParallelIcon,

apps/sim/lib/embeddings/client.test.ts

@@ -285,6 +285,29 @@ describe('embed', () => {
       expect(result.totalTokens).toBeLessThan(10)
     })
 
+    /**
+     * Projection changes length, and batching truncates whatever it measures.
+     * Batching the pre-projection text sized against a string that was never
+     * sent: a lengthening projection then exceeded the model's ceiling, and a
+     * shortening one discarded content that would have fit.
+     */
+    it('batches the projected text, not the original', async () => {
+      fetchMock.mockResolvedValue(jsonResponse({ embeddings: [{ values: [1] }] }))
+      // Under Gemini's 2048 ceiling before projection, far over it after.
+      const short = 'secret'
+
+      await embed([short], {
+        model: 'gemini-embedding-001',
+        apiKey: 'g-test',
+        projectInputs: () => ['word '.repeat(8000)],
+      })
+
+      const body = JSON.parse((fetchMock.mock.calls[0][1] as RequestInit).body as string)
+      const sent = body.requests[0].content.parts[0].text
+      // Truncated against the model ceiling, so the lengthened text cannot go out whole.
+      expect(sent.length).toBeLessThan('word '.repeat(8000).length)
+    })
+
     it('projects once even when the request is retried', async () => {
       const projectInputs = vi.fn((values: readonly string[]) => values.map(() => 'projected'))
       fetchMock

apps/sim/lib/embeddings/client.ts

@@ -111,22 +111,16 @@ async function resolveProvider(model: string, options: EmbedOptions): Promise<Re
   }
 }
 
+/** `inputs` are already projected and batched by {@link embed}. */
 async function callEmbeddingAPI(
   inputs: string[],
   provider: ResolvedProvider,
-  taskType: EmbeddingTaskType,
-  projectInputs: ((values: readonly string[]) => string[]) | null
+  taskType: EmbeddingTaskType
 ): Promise<{ embeddings: number[][]; totalTokens: number }> {
-  /**
-   * Projected once, outside the retry loop, so a retry cannot re-project
-   * already-projected content. Token counts are taken from these same values so
-   * usage reflects what was actually sent.
-   */
-  const modelInputs = projectInputs ? projectInputs(inputs) : inputs
   return retryWithExponentialBackoff(
     async () => {
       const request = provider.adapter.buildRequest({
-        inputs: modelInputs,
+        inputs,
         taskType,
         dimensions: provider.requestedDimensions,
       })
@@ -154,7 +148,7 @@ async function callEmbeddingAPI(
       const totalTokens =
         request.parseTokens?.(json) ??
         // Providers that omit usage (e.g. Gemini) get an estimate from their tokenizer
-        modelInputs.reduce(
+        inputs.reduce(
           (sum, text) => sum + estimateTokenCount(text, provider.info.tokenizerProvider).count,
           0
         )
@@ -184,15 +178,27 @@ export async function embed(texts: string[], options: EmbedOptions): Promise<Emb
   const taskType = options.taskType ?? 'document'
   const provider = await resolveProvider(model, options)
 
+  /**
+   * Projected before batching, not after. The projector rewrites resolved-secret
+   * plaintext to placeholders, which changes length, and `batchByTokenLimit`
+   * measures and truncates whatever it is handed. Batching the pre-projection
+   * text would size against a different string than the one actually sent: a
+   * lengthening projection then exceeds the model's ceiling and the provider
+   * rejects it, and a shortening one discards content that would have fit.
+   *
+   * Doing it here also keeps projection to exactly once per call, so no retry
+   * can re-project already-projected content.
+   */
+  const modelInputs = options.projectInputs ? options.projectInputs(texts) : texts
+
   /**
    * Batched against the selected model's own ceiling rather than one shared
-   * constant. `batchByTokenLimit` truncates any single text above the limit, so
-   * a value that is too high sends oversized input the provider rejects, and one
-   * that is too low silently drops content the provider would have accepted.
-   * Using the per-input ceiling as the per-batch budget also keeps every
-   * individual text within it.
+   * constant. A value that is too high sends oversized input the provider
+   * rejects; one that is too low silently drops content the provider would have
+   * accepted. Using the per-input ceiling as the per-batch budget also keeps
+   * every individual text within it.
    */
-  const tokenBatches = batchByTokenLimit(texts, provider.info.maxInputTokens, model)
+  const tokenBatches = batchByTokenLimit(modelInputs, provider.info.maxInputTokens, model)
   const itemLimit = provider.adapter.maxItemsPerRequest ?? provider.info.maxItemsPerRequest
   const batches = itemLimit
     ? tokenBatches.flatMap((batch) => splitByItemLimit(batch, itemLimit))
@@ -203,7 +209,7 @@ export async function embed(texts: string[], options: EmbedOptions): Promise<Emb
     MAX_CONCURRENT_BATCHES,
     async (batch, i) => {
       try {
-        return await callEmbeddingAPI(batch, provider, taskType, options.projectInputs)
+        return await callEmbeddingAPI(batch, provider, taskType)
       } catch (error) {
         logger.error(`Failed to generate embeddings for batch ${i + 1}/${batches.length}:`, error)
         throw error

scripts/generate-docs.ts

@@ -440,7 +440,16 @@ async function generateIconMapping(options: {
           }
 
           const isVersionedBlockType = isVersionedType(blockType)
-          if (!hideFromToolbar || (options.includeHidden && isVersionedBlockType)) {
+          /**
+           * A sunset block keeps its docs page — `docsLink` is baked into every
+           * placed instance — so it still needs an icon there, exactly like a
+           * hidden versioned block. Without this it renders as a text tile.
+           */
+          const isSunsetBlockType = /sunset\s*:\s*\{/.test(stripSourceComments(blockContent))
+          if (
+            !hideFromToolbar ||
+            (options.includeHidden && (isVersionedBlockType || isSunsetBlockType))
+          ) {
             iconMapping[blockType] = {
               name: iconName,
               source: resolveIconSource(fileContent, iconName),