From 73d0c665d157797aa2c8017457019fa3483212a7 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 5 May 2026 17:23:19 -0700 Subject: [PATCH 1/5] improvement(confluence): expand scopes, persist canonical mode toggle --- .../add-connector-modal.tsx | 11 ++-- .../edit-connector-modal.tsx | 63 +++++++++++++++++-- .../[id]/hooks/use-connector-config-fields.ts | 6 +- apps/sim/connectors/confluence/confluence.ts | 16 +++-- apps/sim/tools/confluence/utils.ts | 12 ++-- 5 files changed, 89 insertions(+), 19 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx index bb46a1fd7ad..849b4820a9d 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx @@ -157,10 +157,13 @@ export function AddConnectorModal({ for (const [key, value] of Object.entries(resolveSourceConfig())) { if (value) resolvedConfig[key] = value } - const finalSourceConfig = - disabledTagIds.size > 0 - ? { ...resolvedConfig, disabledTagIds: Array.from(disabledTagIds) } - : resolvedConfig + if (disabledTagIds.size > 0) { + resolvedConfig.disabledTagIds = Array.from(disabledTagIds) + } + if (Object.keys(canonicalModes).length > 0) { + resolvedConfig._canonicalModes = canonicalModes + } + const finalSourceConfig = resolvedConfig createConnector( { diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx index e54846e6fbc..4c0c81d7b26 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx @@ -43,8 +43,33 @@ import type { SelectorKey } from '@/hooks/selectors/types' const logger = createLogger('EditConnectorModal') -/** Keys injected by the sync engine — not user-editable */ -const INTERNAL_CONFIG_KEYS = new Set(['tagSlotMapping', 'disabledTagIds']) +/** Keys injected by the sync engine or modal state — not user-editable */ +const INTERNAL_CONFIG_KEYS = new Set(['tagSlotMapping', 'disabledTagIds', '_canonicalModes']) + +const CANONICAL_MODES_KEY = '_canonicalModes' + +function readPersistedCanonicalModes( + sourceConfig: Record +): Record { + const raw = sourceConfig[CANONICAL_MODES_KEY] + if (!raw || typeof raw !== 'object') return {} + const result: Record = {} + for (const [key, value] of Object.entries(raw as Record)) { + if (value === 'basic' || value === 'advanced') result[key] = value + } + return result +} + +function didCanonicalModesChange( + current: Record, + persisted: Record +): boolean { + const keys = new Set([...Object.keys(persisted), ...Object.keys(current)]) + for (const key of keys) { + if ((current[key] ?? 'basic') !== (persisted[key] ?? 'basic')) return true + } + return false +} interface EditConnectorModalProps { open: boolean @@ -87,6 +112,10 @@ export function EditConnectorModal({ return config }) + const [initialCanonicalModes] = useState>(() => + readPersistedCanonicalModes(connector.sourceConfig) + ) + const { sourceConfig, canonicalModes, @@ -95,7 +124,11 @@ export function EditConnectorModal({ handleFieldChange, toggleCanonicalMode, resolveSourceConfig, - } = useConnectorConfigFields({ connectorConfig, initialSourceConfig }) + } = useConnectorConfigFields({ + connectorConfig, + initialSourceConfig, + initialCanonicalModes, + }) const { mutate: updateConnector, isPending: isSaving } = useUpdateConnector() @@ -105,12 +138,20 @@ export function EditConnectorModal({ const hasChanges = useMemo(() => { if (syncInterval !== connector.syncIntervalMinutes) return true + const persisted = readPersistedCanonicalModes(connector.sourceConfig) + if (didCanonicalModesChange(canonicalModes, persisted)) return true const resolved = resolveSourceConfig() for (const [key, value] of Object.entries(resolved)) { if (String(connector.sourceConfig[key] ?? '') !== value) return true } return false - }, [resolveSourceConfig, syncInterval, connector.syncIntervalMinutes, connector.sourceConfig]) + }, [ + resolveSourceConfig, + syncInterval, + connector.syncIntervalMinutes, + connector.sourceConfig, + canonicalModes, + ]) const handleSave = () => { setError(null) @@ -126,8 +167,18 @@ export function EditConnectorModal({ for (const [key, value] of Object.entries(resolved)) { if (String(connector.sourceConfig[key] ?? '') !== value) changedEntries[key] = value } - if (Object.keys(changedEntries).length > 0) { - updates.sourceConfig = { ...connector.sourceConfig, ...changedEntries } + + const persistedModes = readPersistedCanonicalModes(connector.sourceConfig) + const modesChanged = didCanonicalModesChange(canonicalModes, persistedModes) + + if (Object.keys(changedEntries).length > 0 || modesChanged) { + const next: Record = { ...connector.sourceConfig, ...changedEntries } + if (Object.keys(canonicalModes).length > 0) { + next[CANONICAL_MODES_KEY] = canonicalModes + } else { + delete next[CANONICAL_MODES_KEY] + } + updates.sourceConfig = next } if (Object.keys(updates).length === 0) { diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields.ts b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields.ts index 54ff7c16906..8419b749602 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields.ts +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields.ts @@ -7,6 +7,7 @@ import type { ConnectorConfig, ConnectorConfigField } from '@/connectors/types' export interface UseConnectorConfigFieldsOptions { connectorConfig: ConnectorConfig | null initialSourceConfig?: Record + initialCanonicalModes?: Record } export interface UseConnectorConfigFieldsResult { @@ -34,11 +35,14 @@ export interface UseConnectorConfigFieldsResult { export function useConnectorConfigFields({ connectorConfig, initialSourceConfig, + initialCanonicalModes, }: UseConnectorConfigFieldsOptions): UseConnectorConfigFieldsResult { const [sourceConfig, setSourceConfig] = useState>( () => initialSourceConfig ?? {} ) - const [canonicalModes, setCanonicalModes] = useState>({}) + const [canonicalModes, setCanonicalModes] = useState>( + () => initialCanonicalModes ?? {} + ) const canonicalGroups = useMemo(() => { const groups = new Map() diff --git a/apps/sim/connectors/confluence/confluence.ts b/apps/sim/connectors/confluence/confluence.ts index 56527fdc527..2180f932f08 100644 --- a/apps/sim/connectors/confluence/confluence.ts +++ b/apps/sim/connectors/confluence/confluence.ts @@ -4,7 +4,7 @@ import { ConfluenceIcon } from '@/components/icons' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { htmlToPlainText, joinTagArray, parseTagDate } from '@/connectors/utils' -import { getConfluenceCloudId } from '@/tools/confluence/utils' +import { getConfluenceCloudId, normalizeConfluenceDomainHost } from '@/tools/confluence/utils' const logger = createLogger('ConfluenceConnector') @@ -141,7 +141,15 @@ export const confluenceConnector: ConnectorConfig = { auth: { mode: 'oauth', provider: 'confluence', - requiredScopes: ['read:confluence-content.all', 'read:page:confluence', 'offline_access'], + requiredScopes: [ + 'read:confluence-content.all', + 'read:page:confluence', + 'read:blogpost:confluence', + 'read:space:confluence', + 'read:label:confluence', + 'search:confluence', + 'offline_access', + ], }, configFields: [ @@ -205,7 +213,7 @@ export const confluenceConnector: ConnectorConfig = { cursor?: string, syncContext?: Record ): Promise => { - const domain = sourceConfig.domain as string + const domain = normalizeConfluenceDomainHost(sourceConfig.domain as string) const spaceKey = sourceConfig.spaceKey as string const contentType = (sourceConfig.contentType as string) || 'page' const labelFilter = (sourceConfig.labelFilter as string) || '' @@ -269,7 +277,7 @@ export const confluenceConnector: ConnectorConfig = { externalId: string, syncContext?: Record ): Promise => { - const domain = sourceConfig.domain as string + const domain = normalizeConfluenceDomainHost(sourceConfig.domain as string) let cloudId = syncContext?.cloudId as string | undefined if (!cloudId) { cloudId = await getConfluenceCloudId(domain, accessToken) diff --git a/apps/sim/tools/confluence/utils.ts b/apps/sim/tools/confluence/utils.ts index 99c2fb17d70..1045cb9d322 100644 --- a/apps/sim/tools/confluence/utils.ts +++ b/apps/sim/tools/confluence/utils.ts @@ -1,11 +1,15 @@ import type { RetryOptions } from '@/lib/knowledge/documents/utils' import { fetchWithRetry } from '@/lib/knowledge/documents/utils' -function normalizeDomain(domain: string): string { - return `https://${domain +/** + * Strips protocol and trailing slashes from a Confluence domain to produce + * a bare host (e.g. `yoursite.atlassian.net`). + */ +export function normalizeConfluenceDomainHost(domain: string): string { + return domain .trim() .replace(/^https?:\/\//i, '') - .replace(/\/+$/, '')}`.toLowerCase() + .replace(/\/+$/, '') } export async function getConfluenceCloudId( @@ -31,7 +35,7 @@ export async function getConfluenceCloudId( throw new Error('No Confluence resources found') } - const normalized = normalizeDomain(domain) + const normalized = `https://${normalizeConfluenceDomainHost(domain)}`.toLowerCase() const match = resources.find( (r: { url: string }) => r.url.toLowerCase().replace(/\/+$/, '') === normalized ) From 2b1e2239a95dba2771d8e0e2f7ceab1982098227 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 5 May 2026 17:33:32 -0700 Subject: [PATCH 2/5] improvement(confluence): memoize persisted canonical modes parse --- .../edit-connector-modal/edit-connector-modal.tsx | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx index 4c0c81d7b26..3b6c85d806d 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx @@ -136,10 +136,14 @@ export function EditConnectorModal({ const subscriptionAccess = getSubscriptionAccessState(subscriptionResponse?.data) const hasMaxAccess = !isBillingEnabled || subscriptionAccess.hasUsableMaxAccess + const persistedCanonicalModes = useMemo( + () => readPersistedCanonicalModes(connector.sourceConfig), + [connector.sourceConfig] + ) + const hasChanges = useMemo(() => { if (syncInterval !== connector.syncIntervalMinutes) return true - const persisted = readPersistedCanonicalModes(connector.sourceConfig) - if (didCanonicalModesChange(canonicalModes, persisted)) return true + if (didCanonicalModesChange(canonicalModes, persistedCanonicalModes)) return true const resolved = resolveSourceConfig() for (const [key, value] of Object.entries(resolved)) { if (String(connector.sourceConfig[key] ?? '') !== value) return true @@ -151,6 +155,7 @@ export function EditConnectorModal({ connector.syncIntervalMinutes, connector.sourceConfig, canonicalModes, + persistedCanonicalModes, ]) const handleSave = () => { @@ -168,8 +173,7 @@ export function EditConnectorModal({ if (String(connector.sourceConfig[key] ?? '') !== value) changedEntries[key] = value } - const persistedModes = readPersistedCanonicalModes(connector.sourceConfig) - const modesChanged = didCanonicalModesChange(canonicalModes, persistedModes) + const modesChanged = didCanonicalModesChange(canonicalModes, persistedCanonicalModes) if (Object.keys(changedEntries).length > 0 || modesChanged) { const next: Record = { ...connector.sourceConfig, ...changedEntries } From 1569355b60bc1df805371337f31cac6bd0233df6 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 5 May 2026 17:58:12 -0700 Subject: [PATCH 3/5] fix(confluence): paginate space selector dropdown Confluence v2 spaces endpoint caps at limit=250 per page. The selector endpoint was making one request and silently dropping every space past the first page, which is why some spaces only worked when entered as a manual spaceKey. Now follows _links.next cursor up to 20 pages (5000 spaces). --- .../tools/confluence/selector-spaces/route.ts | 76 +++++++++++++------ 1 file changed, 51 insertions(+), 25 deletions(-) diff --git a/apps/sim/app/api/tools/confluence/selector-spaces/route.ts b/apps/sim/app/api/tools/confluence/selector-spaces/route.ts index 3d7e2cc0ac9..7c9ed4a101c 100644 --- a/apps/sim/app/api/tools/confluence/selector-spaces/route.ts +++ b/apps/sim/app/api/tools/confluence/selector-spaces/route.ts @@ -83,35 +83,61 @@ export const POST = withRouteHandler(async (request: NextRequest) => { return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 }) } - const url = `https://api.atlassian.com/ex/confluence/${cloudIdValidation.sanitized}/wiki/api/v2/spaces?limit=250` - - const response = await fetch(url, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) + const baseUrl = `https://api.atlassian.com/ex/confluence/${cloudIdValidation.sanitized}/wiki/api/v2/spaces` + const PAGE_LIMIT = 250 + const MAX_PAGES = 20 + const spaces: { id: string; name: string; key: string }[] = [] + let cursor: string | undefined + let pageCount = 0 + + while (pageCount < MAX_PAGES) { + const params = new URLSearchParams({ limit: String(PAGE_LIMIT) }) + if (cursor) params.set('cursor', cursor) + const url = `${baseUrl}?${params.toString()}` - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, + const response = await fetch(url, { + method: 'GET', + headers: { + Accept: 'application/json', + Authorization: `Bearer ${accessToken}`, + }, }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) + + if (!response.ok) { + const errorText = await response.text() + logger.error('Confluence API error response:', { + status: response.status, + statusText: response.statusText, + error: errorText, + }) + return NextResponse.json( + { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, + { status: response.status } + ) + } + + const data = await response.json() + for (const space of data.results || []) { + spaces.push({ id: space.id, name: space.name, key: space.key }) + } + + const nextLink = data._links?.next as string | undefined + if (!nextLink) break + try { + cursor = new URL(nextLink, 'https://placeholder').searchParams.get('cursor') || undefined + } catch { + cursor = undefined + } + if (!cursor) break + pageCount += 1 } - const data = await response.json() - const spaces = (data.results || []).map((space: { id: string; name: string; key: string }) => ({ - id: space.id, - name: space.name, - key: space.key, - })) + if (pageCount >= MAX_PAGES) { + logger.warn('Confluence space listing hit pagination cap', { + cap: MAX_PAGES * PAGE_LIMIT, + returned: spaces.length, + }) + } return NextResponse.json({ spaces }) } catch (error) { From 19f5b2b17d1e07da9e6f693d761e8e00e41c8f2f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 5 May 2026 18:02:03 -0700 Subject: [PATCH 4/5] fix(confluence): include archived spaces in selector dropdown Confluence v2 /spaces defaults to status=current and the status param is a single-value enum, so archived spaces never surface. They synced fine when entered manually as a spaceKey because the connector looks up spaces via ?keys= which ignores status. Now fetches current and archived in parallel and tags archived ones in the dropdown label. --- .../tools/confluence/selector-spaces/route.ts | 102 +++++++++++------- .../providers/confluence/selectors.ts | 9 +- .../lib/api/contracts/selectors/confluence.ts | 7 +- 3 files changed, 74 insertions(+), 44 deletions(-) diff --git a/apps/sim/app/api/tools/confluence/selector-spaces/route.ts b/apps/sim/app/api/tools/confluence/selector-spaces/route.ts index 7c9ed4a101c..5a339f9f504 100644 --- a/apps/sim/app/api/tools/confluence/selector-spaces/route.ts +++ b/apps/sim/app/api/tools/confluence/selector-spaces/route.ts @@ -86,59 +86,79 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const baseUrl = `https://api.atlassian.com/ex/confluence/${cloudIdValidation.sanitized}/wiki/api/v2/spaces` const PAGE_LIMIT = 250 const MAX_PAGES = 20 - const spaces: { id: string; name: string; key: string }[] = [] - let cursor: string | undefined - let pageCount = 0 - - while (pageCount < MAX_PAGES) { - const params = new URLSearchParams({ limit: String(PAGE_LIMIT) }) - if (cursor) params.set('cursor', cursor) - const url = `${baseUrl}?${params.toString()}` - - const response = await fetch(url, { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${accessToken}`, - }, - }) - if (!response.ok) { - const errorText = await response.text() - logger.error('Confluence API error response:', { - status: response.status, - statusText: response.statusText, - error: errorText, + /** + * Confluence v2 `/spaces` defaults to `status=current` and treats `status` + * as a single-value enum, so archived spaces never surface from one call. + * Listing both surfaces archived spaces in the dropdown — they would + * otherwise only be reachable by typing the space key manually, even + * though sync works against archived spaces just fine. + */ + async function fetchAllPages(status: 'current' | 'archived'): Promise<{ + spaces: { id: string; name: string; key: string; status: string }[] + capped: boolean + }> { + const collected: { id: string; name: string; key: string; status: string }[] = [] + let cursor: string | undefined + let pageCount = 0 + + while (pageCount < MAX_PAGES) { + const params = new URLSearchParams({ limit: String(PAGE_LIMIT), status }) + if (cursor) params.set('cursor', cursor) + const url = `${baseUrl}?${params.toString()}` + + const response = await fetch(url, { + method: 'GET', + headers: { Accept: 'application/json', Authorization: `Bearer ${accessToken}` }, }) - return NextResponse.json( - { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) }, - { status: response.status } - ) - } - const data = await response.json() - for (const space of data.results || []) { - spaces.push({ id: space.id, name: space.name, key: space.key }) + if (!response.ok) { + const errorText = await response.text() + throw new Error( + parseAtlassianErrorMessage(response.status, response.statusText, errorText) + ) + } + + const data = await response.json() + for (const space of data.results || []) { + collected.push({ id: space.id, name: space.name, key: space.key, status }) + } + + const nextLink = data._links?.next as string | undefined + if (!nextLink) return { spaces: collected, capped: false } + try { + cursor = new URL(nextLink, 'https://placeholder').searchParams.get('cursor') || undefined + } catch { + cursor = undefined + } + if (!cursor) return { spaces: collected, capped: false } + pageCount += 1 } - const nextLink = data._links?.next as string | undefined - if (!nextLink) break - try { - cursor = new URL(nextLink, 'https://placeholder').searchParams.get('cursor') || undefined - } catch { - cursor = undefined - } - if (!cursor) break - pageCount += 1 + return { spaces: collected, capped: true } + } + + let currentResult: Awaited> + let archivedResult: Awaited> + try { + ;[currentResult, archivedResult] = await Promise.all([ + fetchAllPages('current'), + fetchAllPages('archived'), + ]) + } catch (error) { + logger.error('Confluence API error response', { error: (error as Error).message }) + return NextResponse.json({ error: (error as Error).message }, { status: 502 }) } - if (pageCount >= MAX_PAGES) { + if (currentResult.capped || archivedResult.capped) { logger.warn('Confluence space listing hit pagination cap', { cap: MAX_PAGES * PAGE_LIMIT, - returned: spaces.length, + currentCount: currentResult.spaces.length, + archivedCount: archivedResult.spaces.length, }) } + const spaces = [...currentResult.spaces, ...archivedResult.spaces] return NextResponse.json({ spaces }) } catch (error) { logger.error('Error listing Confluence spaces:', error) diff --git a/apps/sim/hooks/selectors/providers/confluence/selectors.ts b/apps/sim/hooks/selectors/providers/confluence/selectors.ts index 6a2f81d581b..156193d69cc 100644 --- a/apps/sim/hooks/selectors/providers/confluence/selectors.ts +++ b/apps/sim/hooks/selectors/providers/confluence/selectors.ts @@ -4,6 +4,11 @@ import { fetchOAuthToken } from '@/hooks/selectors/helpers' import { ensureCredential, ensureDomain, SELECTOR_STALE } from '@/hooks/selectors/providers/shared' import type { SelectorDefinition, SelectorKey, SelectorQueryArgs } from '@/hooks/selectors/types' +function formatConfluenceSpaceLabel(space: { name: string; key: string; status?: string }): string { + const base = `${space.name} (${space.key})` + return space.status === 'archived' ? `${base} — archived` : base +} + export const confluenceSelectors = { 'confluence.spaces': { key: 'confluence.spaces', @@ -29,7 +34,7 @@ export const confluenceSelectors = { }) return (data.spaces || []).map((space) => ({ id: space.id, - label: `${space.name} (${space.key})`, + label: formatConfluenceSpaceLabel(space), })) }, fetchById: async ({ context, detailId, signal }: SelectorQueryArgs) => { @@ -46,7 +51,7 @@ export const confluenceSelectors = { }) const space = (data.spaces || []).find((s) => s.id === detailId) ?? null if (!space) return null - return { id: space.id, label: `${space.name} (${space.key})` } + return { id: space.id, label: formatConfluenceSpaceLabel(space) } }, }, 'confluence.pages': { diff --git a/apps/sim/lib/api/contracts/selectors/confluence.ts b/apps/sim/lib/api/contracts/selectors/confluence.ts index 18b3dd76408..e34b65aff7c 100644 --- a/apps/sim/lib/api/contracts/selectors/confluence.ts +++ b/apps/sim/lib/api/contracts/selectors/confluence.ts @@ -10,7 +10,12 @@ import { defineRouteContract } from '@/lib/api/contracts/types' import { validateAlphanumericId } from '@/lib/core/security/input-validation' const confluenceSpaceSchema = z - .object({ id: z.string(), name: z.string(), key: z.string() }) + .object({ + id: z.string(), + name: z.string(), + key: z.string(), + status: z.string().optional(), + }) .passthrough() export const confluencePagesBodySchema = z.object({ From a677ae851e8877641ab3ed3f46045231f56c31b5 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 5 May 2026 19:58:57 -0700 Subject: [PATCH 5/5] improvement(confluence): stream paginated space selector results Bake pagination support into the selector abstraction via an opt-in fetchPage definition so dropdowns populate progressively instead of blocking on a full page-walk. Confluence spaces now stream current then archived in a single cursor sequence. --- .../tools/confluence/selector-spaces/route.ts | 131 ++++++++---------- .../selector-combobox/selector-combobox.tsx | 2 + .../providers/confluence/selectors.ts | 40 +++++- apps/sim/hooks/selectors/types.ts | 15 ++ .../sim/hooks/selectors/use-selector-query.ts | 92 +++++++++++- .../lib/api/contracts/selectors/confluence.ts | 11 +- 6 files changed, 205 insertions(+), 86 deletions(-) diff --git a/apps/sim/app/api/tools/confluence/selector-spaces/route.ts b/apps/sim/app/api/tools/confluence/selector-spaces/route.ts index 5a339f9f504..e8a8b032480 100644 --- a/apps/sim/app/api/tools/confluence/selector-spaces/route.ts +++ b/apps/sim/app/api/tools/confluence/selector-spaces/route.ts @@ -19,13 +19,33 @@ const logger = createLogger('ConfluenceSelectorSpacesAPI') export const dynamic = 'force-dynamic' +const PAGE_LIMIT = 250 + +type SpaceStatus = 'current' | 'archived' + +/** + * Cursor format: `:`. Empty inner cursor means "first page + * of that status". When current is exhausted we hand back `archived:` so the + * client transparently flips to the archived stream — listing both surfaces + * archived spaces in the dropdown, which would otherwise only be reachable by + * typing the space key manually even though sync works against archived spaces. + */ +function parseCursor(raw: string | undefined): { status: SpaceStatus; inner?: string } { + if (!raw) return { status: 'current' } + const idx = raw.indexOf(':') + if (idx === -1) return { status: 'current' } + const status = raw.slice(0, idx) === 'archived' ? 'archived' : 'current' + const inner = raw.slice(idx + 1) + return { status, inner: inner || undefined } +} + export const POST = withRouteHandler(async (request: NextRequest) => { const requestId = generateRequestId() try { const parsed = await parseRequest(confluenceSpacesSelectorContract, request, {}) if (!parsed.success) return parsed.response - const { credential, workflowId, domain } = parsed.data.body + const { credential, workflowId, domain, cursor } = parsed.data.body if (!credential) { logger.error('Missing credential in request') @@ -44,11 +64,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => { return NextResponse.json({ error: authz.error || 'Unauthorized' }, { status: 403 }) } - // Resolve once so we know whether this is an Atlassian SA credential before - // doing any token / cloudId work. Atlassian SAs short-circuit the entire path: - // the API token IS the access token, and cloudId lives in the encrypted secret — - // so we skip refreshAccessTokenIfNeeded (avoids a redundant resolve+decrypt) and - // skip getConfluenceCloudId (which 401s for scoped SA tokens). const resolved = await resolveOAuthAccountId(credential) const isAtlassianServiceAccount = resolved?.providerId === ATLASSIAN_SERVICE_ACCOUNT_PROVIDER_ID && !!resolved.credentialId @@ -84,82 +99,50 @@ export const POST = withRouteHandler(async (request: NextRequest) => { } const baseUrl = `https://api.atlassian.com/ex/confluence/${cloudIdValidation.sanitized}/wiki/api/v2/spaces` - const PAGE_LIMIT = 250 - const MAX_PAGES = 20 - - /** - * Confluence v2 `/spaces` defaults to `status=current` and treats `status` - * as a single-value enum, so archived spaces never surface from one call. - * Listing both surfaces archived spaces in the dropdown — they would - * otherwise only be reachable by typing the space key manually, even - * though sync works against archived spaces just fine. - */ - async function fetchAllPages(status: 'current' | 'archived'): Promise<{ - spaces: { id: string; name: string; key: string; status: string }[] - capped: boolean - }> { - const collected: { id: string; name: string; key: string; status: string }[] = [] - let cursor: string | undefined - let pageCount = 0 - - while (pageCount < MAX_PAGES) { - const params = new URLSearchParams({ limit: String(PAGE_LIMIT), status }) - if (cursor) params.set('cursor', cursor) - const url = `${baseUrl}?${params.toString()}` - - const response = await fetch(url, { - method: 'GET', - headers: { Accept: 'application/json', Authorization: `Bearer ${accessToken}` }, - }) + const { status, inner } = parseCursor(cursor) - if (!response.ok) { - const errorText = await response.text() - throw new Error( - parseAtlassianErrorMessage(response.status, response.statusText, errorText) - ) - } - - const data = await response.json() - for (const space of data.results || []) { - collected.push({ id: space.id, name: space.name, key: space.key, status }) - } - - const nextLink = data._links?.next as string | undefined - if (!nextLink) return { spaces: collected, capped: false } - try { - cursor = new URL(nextLink, 'https://placeholder').searchParams.get('cursor') || undefined - } catch { - cursor = undefined - } - if (!cursor) return { spaces: collected, capped: false } - pageCount += 1 - } + const params = new URLSearchParams({ limit: String(PAGE_LIMIT), status }) + if (inner) params.set('cursor', inner) + const url = `${baseUrl}?${params.toString()}` - return { spaces: collected, capped: true } + const response = await fetch(url, { + method: 'GET', + headers: { Accept: 'application/json', Authorization: `Bearer ${accessToken}` }, + }) + + if (!response.ok) { + const errorText = await response.text() + const message = parseAtlassianErrorMessage(response.status, response.statusText, errorText) + logger.error('Confluence API error response', { error: message, status: response.status }) + return NextResponse.json({ error: message }, { status: 502 }) } - let currentResult: Awaited> - let archivedResult: Awaited> - try { - ;[currentResult, archivedResult] = await Promise.all([ - fetchAllPages('current'), - fetchAllPages('archived'), - ]) - } catch (error) { - logger.error('Confluence API error response', { error: (error as Error).message }) - return NextResponse.json({ error: (error as Error).message }, { status: 502 }) + const data = await response.json() + const spaces = (data.results || []).map((space: { id: string; name: string; key: string }) => ({ + id: space.id, + name: space.name, + key: space.key, + status, + })) + + let nextInner: string | undefined + const nextLink = data._links?.next as string | undefined + if (nextLink) { + try { + nextInner = new URL(nextLink, 'https://placeholder').searchParams.get('cursor') || undefined + } catch { + nextInner = undefined + } } - if (currentResult.capped || archivedResult.capped) { - logger.warn('Confluence space listing hit pagination cap', { - cap: MAX_PAGES * PAGE_LIMIT, - currentCount: currentResult.spaces.length, - archivedCount: archivedResult.spaces.length, - }) + let nextCursor: string | undefined + if (nextInner) { + nextCursor = `${status}:${nextInner}` + } else if (status === 'current') { + nextCursor = 'archived:' } - const spaces = [...currentResult.spaces, ...archivedResult.spaces] - return NextResponse.json({ spaces }) + return NextResponse.json({ spaces, nextCursor }) } catch (error) { logger.error('Error listing Confluence spaces:', error) return NextResponse.json( diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/selector-combobox/selector-combobox.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/selector-combobox/selector-combobox.tsx index f0445fd7d54..0d2964fa6b6 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/selector-combobox/selector-combobox.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/selector-combobox/selector-combobox.tsx @@ -53,6 +53,7 @@ export function SelectorCombobox({ const { data: options = [], isLoading, + hasMore, error, } = useSelectorOptions(selectorKey, { context: selectorContext, @@ -67,6 +68,7 @@ export function SelectorCombobox({ Boolean(activeValue) && Boolean(missingOptionLabel) && !isLoading && + !hasMore && !optionMap.get(activeValue!) const selectedLabel = activeValue ? hasMissingOption diff --git a/apps/sim/hooks/selectors/providers/confluence/selectors.ts b/apps/sim/hooks/selectors/providers/confluence/selectors.ts index 156193d69cc..84e0e528609 100644 --- a/apps/sim/hooks/selectors/providers/confluence/selectors.ts +++ b/apps/sim/hooks/selectors/providers/confluence/selectors.ts @@ -22,6 +22,28 @@ export const confluenceSelectors = { ], enabled: ({ context }) => Boolean(context.oauthCredential && context.domain), fetchList: async ({ context, signal }: SelectorQueryArgs) => { + const credentialId = ensureCredential(context, 'confluence.spaces') + const domain = ensureDomain(context, 'confluence.spaces') + const collected: { id: string; label: string }[] = [] + let cursor: string | undefined + do { + const data = await requestJson(selectorContracts.confluenceSpacesSelectorContract, { + body: { + credential: credentialId, + workflowId: context.workflowId, + domain, + cursor, + }, + signal, + }) + for (const space of data.spaces || []) { + collected.push({ id: space.id, label: formatConfluenceSpaceLabel(space) }) + } + cursor = data.nextCursor + } while (cursor) + return collected + }, + fetchPage: async ({ context, cursor, signal }) => { const credentialId = ensureCredential(context, 'confluence.spaces') const domain = ensureDomain(context, 'confluence.spaces') const data = await requestJson(selectorContracts.confluenceSpacesSelectorContract, { @@ -29,14 +51,24 @@ export const confluenceSelectors = { credential: credentialId, workflowId: context.workflowId, domain, + cursor, }, signal, }) - return (data.spaces || []).map((space) => ({ - id: space.id, - label: formatConfluenceSpaceLabel(space), - })) + return { + items: (data.spaces || []).map((space) => ({ + id: space.id, + label: formatConfluenceSpaceLabel(space), + })), + nextCursor: data.nextCursor, + } }, + /** + * Resolves a single space label. Hits only the first page — the dropdown's + * `fetchPage` stream populates the options cache for spaces beyond page 1, + * and `useSelectorOptionMap` merges them in. Walking all pages here would + * double API load since the stream is already running in parallel. + */ fetchById: async ({ context, detailId, signal }: SelectorQueryArgs) => { if (!detailId) return null const credentialId = ensureCredential(context, 'confluence.spaces') diff --git a/apps/sim/hooks/selectors/types.ts b/apps/sim/hooks/selectors/types.ts index ef8724cbe43..be96287a791 100644 --- a/apps/sim/hooks/selectors/types.ts +++ b/apps/sim/hooks/selectors/types.ts @@ -100,11 +100,26 @@ export interface SelectorQueryArgs { signal?: AbortSignal } +export interface SelectorPage { + items: SelectorOption[] + nextCursor?: string +} + +export interface SelectorPageArgs extends SelectorQueryArgs { + cursor?: string +} + export interface SelectorDefinition { key: SelectorKey contracts?: readonly AnyApiRouteContract[] getQueryKey: (args: SelectorQueryArgs) => QueryKey fetchList: (args: SelectorQueryArgs) => Promise + /** + * Optional. When defined, the selector hook fetches one page at a time and + * auto-drains remaining pages so the dropdown populates progressively. + * Returns `{ items, nextCursor }`; `nextCursor: undefined` ends the stream. + */ + fetchPage?: (args: SelectorPageArgs) => Promise fetchById?: (args: SelectorQueryArgs) => Promise enabled?: (args: SelectorQueryArgs) => boolean staleTime?: number diff --git a/apps/sim/hooks/selectors/use-selector-query.ts b/apps/sim/hooks/selectors/use-selector-query.ts index a4444b762aa..8eb4755834e 100644 --- a/apps/sim/hooks/selectors/use-selector-query.ts +++ b/apps/sim/hooks/selectors/use-selector-query.ts @@ -1,9 +1,14 @@ -import { useMemo } from 'react' -import { useQuery } from '@tanstack/react-query' +import { useEffect, useMemo } from 'react' +import { useInfiniteQuery, useQuery } from '@tanstack/react-query' import { extractEnvVarName, isEnvVarReference, isReference } from '@/executor/constants' import { usePersonalEnvironment } from '@/hooks/queries/environment' import { getSelectorDefinition, mergeOption } from '@/hooks/selectors/registry' -import type { SelectorKey, SelectorOption, SelectorQueryArgs } from '@/hooks/selectors/types' +import type { + SelectorKey, + SelectorOption, + SelectorPage, + SelectorQueryArgs, +} from '@/hooks/selectors/types' interface SelectorHookArgs extends Omit { search?: string @@ -11,7 +16,29 @@ interface SelectorHookArgs extends Omit { enabled?: boolean } -export function useSelectorOptions(key: SelectorKey, args: SelectorHookArgs) { +export interface SelectorOptionsResult { + data: SelectorOption[] | undefined + isLoading: boolean + isFetching: boolean + /** + * True while paginated selectors are draining remaining pages in the + * background. Always false for non-paginated selectors. + */ + isFetchingMore: boolean + /** + * True when the paginated selector still has more pages queued. Always false + * for non-paginated selectors. + */ + hasMore: boolean + error: Error | null +} + +const EMPTY_PAGE: SelectorPage = { items: [], nextCursor: undefined } + +export function useSelectorOptions( + key: SelectorKey, + args: SelectorHookArgs +): SelectorOptionsResult { const definition = getSelectorDefinition(key) const queryArgs: SelectorQueryArgs = { key, @@ -19,12 +46,65 @@ export function useSelectorOptions(key: SelectorKey, args: SelectorHookArgs) { search: args.search, } const isEnabled = args.enabled ?? (definition.enabled ? definition.enabled(queryArgs) : true) - return useQuery({ + const supportsPagination = Boolean(definition.fetchPage) + + const flatQuery = useQuery({ queryKey: definition.getQueryKey(queryArgs), queryFn: ({ signal }) => definition.fetchList({ ...queryArgs, signal }), - enabled: isEnabled, + enabled: !supportsPagination && isEnabled, + staleTime: definition.staleTime ?? 30_000, + }) + + const pagedQuery = useInfiniteQuery({ + queryKey: [...definition.getQueryKey(queryArgs), 'paged'], + queryFn: ({ pageParam, signal }) => { + if (!definition.fetchPage) return Promise.resolve(EMPTY_PAGE) + return definition.fetchPage({ + ...queryArgs, + cursor: pageParam as string | undefined, + signal, + }) + }, + getNextPageParam: (last) => last.nextCursor, + initialPageParam: undefined as string | undefined, + enabled: supportsPagination && isEnabled, staleTime: definition.staleTime ?? 30_000, }) + + const { hasNextPage, isFetchingNextPage, fetchNextPage, isError } = pagedQuery + useEffect(() => { + if (!supportsPagination) return + if (isError) return + if (hasNextPage && !isFetchingNextPage) { + void fetchNextPage() + } + }, [supportsPagination, hasNextPage, isFetchingNextPage, isError, fetchNextPage]) + + const pagedOptions = useMemo(() => { + if (!supportsPagination) return undefined + if (!pagedQuery.data) return undefined + return pagedQuery.data.pages.flatMap((page) => page.items) + }, [supportsPagination, pagedQuery.data]) + + if (supportsPagination) { + return { + data: pagedOptions, + isLoading: pagedQuery.isLoading, + isFetching: pagedQuery.isFetching, + isFetchingMore: pagedQuery.isFetchingNextPage, + hasMore: pagedQuery.hasNextPage ?? false, + error: (pagedQuery.error as Error | null) ?? null, + } + } + + return { + data: flatQuery.data, + isLoading: flatQuery.isLoading, + isFetching: flatQuery.isFetching, + isFetchingMore: false, + hasMore: false, + error: (flatQuery.error as Error | null) ?? null, + } } export function useSelectorOptionDetail( diff --git a/apps/sim/lib/api/contracts/selectors/confluence.ts b/apps/sim/lib/api/contracts/selectors/confluence.ts index e34b65aff7c..3ed1b6bb8e5 100644 --- a/apps/sim/lib/api/contracts/selectors/confluence.ts +++ b/apps/sim/lib/api/contracts/selectors/confluence.ts @@ -359,10 +359,17 @@ const defineConfluenceGetContract = (path: string, que }, }) +export const confluenceSpacesSelectorBodySchema = credentialWorkflowDomainBodySchema.extend({ + cursor: optionalString, +}) + export const confluenceSpacesSelectorContract = definePostSelector( '/api/tools/confluence/selector-spaces', - credentialWorkflowDomainBodySchema, - z.object({ spaces: z.array(confluenceSpaceSchema) }) + confluenceSpacesSelectorBodySchema, + z.object({ + spaces: z.array(confluenceSpaceSchema), + nextCursor: optionalString, + }) ) export const confluencePagesSelectorContract = definePostSelector(