improvement(confluence): expand scopes, persist canonical mode toggle (#4461)

* improvement(confluence): expand scopes, persist canonical mode toggle

* improvement(confluence): memoize persisted canonical modes parse

* fix(confluence): paginate space selector dropdown

Confluence v2 spaces endpoint caps at limit=250 per page. The selector
endpoint was making one request and silently dropping every space past
the first page, which is why some spaces only worked when entered as a
manual spaceKey. Now follows _links.next cursor up to 20 pages (5000
spaces).

* fix(confluence): include archived spaces in selector dropdown

Confluence v2 /spaces defaults to status=current and the status param
is a single-value enum, so archived spaces never surface. They synced
fine when entered manually as a spaceKey because the connector looks
up spaces via ?keys=<key> which ignores status. Now fetches current
and archived in parallel and tags archived ones in the dropdown label.

* improvement(confluence): stream paginated space selector results

Bake pagination support into the selector abstraction via an opt-in
fetchPage definition so dropdowns populate progressively instead of
blocking on a full page-walk. Confluence spaces now stream current
then archived in a single cursor sequence.

Author: waleedlatif1

apps/sim/app/api/tools/confluence/selector-spaces/route.ts

@@ -19,13 +19,33 @@ const logger = createLogger('ConfluenceSelectorSpacesAPI')
 
 export const dynamic = 'force-dynamic'
 
+const PAGE_LIMIT = 250
+
+type SpaceStatus = 'current' | 'archived'
+
+/**
+ * Cursor format: `<status>:<innerCursor>`. Empty inner cursor means "first page
+ * of that status". When current is exhausted we hand back `archived:` so the
+ * client transparently flips to the archived stream — listing both surfaces
+ * archived spaces in the dropdown, which would otherwise only be reachable by
+ * typing the space key manually even though sync works against archived spaces.
+ */
+function parseCursor(raw: string | undefined): { status: SpaceStatus; inner?: string } {
+  if (!raw) return { status: 'current' }
+  const idx = raw.indexOf(':')
+  if (idx === -1) return { status: 'current' }
+  const status = raw.slice(0, idx) === 'archived' ? 'archived' : 'current'
+  const inner = raw.slice(idx + 1)
+  return { status, inner: inner || undefined }
+}
+
 export const POST = withRouteHandler(async (request: NextRequest) => {
   const requestId = generateRequestId()
   try {
     const parsed = await parseRequest(confluenceSpacesSelectorContract, request, {})
     if (!parsed.success) return parsed.response
 
-    const { credential, workflowId, domain } = parsed.data.body
+    const { credential, workflowId, domain, cursor } = parsed.data.body
 
     if (!credential) {
       logger.error('Missing credential in request')
@@ -44,11 +64,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
       return NextResponse.json({ error: authz.error || 'Unauthorized' }, { status: 403 })
     }
 
-    // Resolve once so we know whether this is an Atlassian SA credential before
-    // doing any token / cloudId work. Atlassian SAs short-circuit the entire path:
-    // the API token IS the access token, and cloudId lives in the encrypted secret —
-    // so we skip refreshAccessTokenIfNeeded (avoids a redundant resolve+decrypt) and
-    // skip getConfluenceCloudId (which 401s for scoped SA tokens).
     const resolved = await resolveOAuthAccountId(credential)
     const isAtlassianServiceAccount =
       resolved?.providerId === ATLASSIAN_SERVICE_ACCOUNT_PROVIDER_ID && !!resolved.credentialId
@@ -83,37 +98,51 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
       return NextResponse.json({ error: cloudIdValidation.error }, { status: 400 })
     }
 
-    const url = `https://api.atlassian.com/ex/confluence/${cloudIdValidation.sanitized}/wiki/api/v2/spaces?limit=250`
+    const baseUrl = `https://api.atlassian.com/ex/confluence/${cloudIdValidation.sanitized}/wiki/api/v2/spaces`
+    const { status, inner } = parseCursor(cursor)
+
+    const params = new URLSearchParams({ limit: String(PAGE_LIMIT), status })
+    if (inner) params.set('cursor', inner)
+    const url = `${baseUrl}?${params.toString()}`
 
     const response = await fetch(url, {
       method: 'GET',
-      headers: {
-        Accept: 'application/json',
-        Authorization: `Bearer ${accessToken}`,
-      },
+      headers: { Accept: 'application/json', Authorization: `Bearer ${accessToken}` },
     })
 
     if (!response.ok) {
       const errorText = await response.text()
-      logger.error('Confluence API error response:', {
-        status: response.status,
-        statusText: response.statusText,
-        error: errorText,
-      })
-      return NextResponse.json(
-        { error: parseAtlassianErrorMessage(response.status, response.statusText, errorText) },
-        { status: response.status }
-      )
+      const message = parseAtlassianErrorMessage(response.status, response.statusText, errorText)
+      logger.error('Confluence API error response', { error: message, status: response.status })
+      return NextResponse.json({ error: message }, { status: 502 })
     }
 
     const data = await response.json()
     const spaces = (data.results || []).map((space: { id: string; name: string; key: string }) => ({
       id: space.id,
       name: space.name,
       key: space.key,
+      status,
     }))
 
-    return NextResponse.json({ spaces })
+    let nextInner: string | undefined
+    const nextLink = data._links?.next as string | undefined
+    if (nextLink) {
+      try {
+        nextInner = new URL(nextLink, 'https://placeholder').searchParams.get('cursor') || undefined
+      } catch {
+        nextInner = undefined
+      }
+    }
+
+    let nextCursor: string | undefined
+    if (nextInner) {
+      nextCursor = `${status}:${nextInner}`
+    } else if (status === 'current') {
+      nextCursor = 'archived:'
+    }
+
+    return NextResponse.json({ spaces, nextCursor })
   } catch (error) {
     logger.error('Error listing Confluence spaces:', error)
     return NextResponse.json(

apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx

@@ -157,10 +157,13 @@ export function AddConnectorModal({
     for (const [key, value] of Object.entries(resolveSourceConfig())) {
       if (value) resolvedConfig[key] = value
     }
-    const finalSourceConfig =
-      disabledTagIds.size > 0
-        ? { ...resolvedConfig, disabledTagIds: Array.from(disabledTagIds) }
-        : resolvedConfig
+    if (disabledTagIds.size > 0) {
+      resolvedConfig.disabledTagIds = Array.from(disabledTagIds)
+    }
+    if (Object.keys(canonicalModes).length > 0) {
+      resolvedConfig._canonicalModes = canonicalModes
+    }
+    const finalSourceConfig = resolvedConfig
 
     createConnector(
       {

apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx

@@ -43,8 +43,33 @@ import type { SelectorKey } from '@/hooks/selectors/types'
 
 const logger = createLogger('EditConnectorModal')
 
-/** Keys injected by the sync engine — not user-editable */
-const INTERNAL_CONFIG_KEYS = new Set(['tagSlotMapping', 'disabledTagIds'])
+/** Keys injected by the sync engine or modal state — not user-editable */
+const INTERNAL_CONFIG_KEYS = new Set(['tagSlotMapping', 'disabledTagIds', '_canonicalModes'])
+
+const CANONICAL_MODES_KEY = '_canonicalModes'
+
+function readPersistedCanonicalModes(
+  sourceConfig: Record<string, unknown>
+): Record<string, 'basic' | 'advanced'> {
+  const raw = sourceConfig[CANONICAL_MODES_KEY]
+  if (!raw || typeof raw !== 'object') return {}
+  const result: Record<string, 'basic' | 'advanced'> = {}
+  for (const [key, value] of Object.entries(raw as Record<string, unknown>)) {
+    if (value === 'basic' || value === 'advanced') result[key] = value
+  }
+  return result
+}
+
+function didCanonicalModesChange(
+  current: Record<string, 'basic' | 'advanced'>,
+  persisted: Record<string, 'basic' | 'advanced'>
+): boolean {
+  const keys = new Set([...Object.keys(persisted), ...Object.keys(current)])
+  for (const key of keys) {
+    if ((current[key] ?? 'basic') !== (persisted[key] ?? 'basic')) return true
+  }
+  return false
+}
 
 interface EditConnectorModalProps {
   open: boolean
@@ -87,6 +112,10 @@ export function EditConnectorModal({
     return config
   })
 
+  const [initialCanonicalModes] = useState<Record<string, 'basic' | 'advanced'>>(() =>
+    readPersistedCanonicalModes(connector.sourceConfig)
+  )
+
   const {
     sourceConfig,
     canonicalModes,
@@ -95,22 +124,39 @@ export function EditConnectorModal({
     handleFieldChange,
     toggleCanonicalMode,
     resolveSourceConfig,
-  } = useConnectorConfigFields({ connectorConfig, initialSourceConfig })
+  } = useConnectorConfigFields({
+    connectorConfig,
+    initialSourceConfig,
+    initialCanonicalModes,
+  })
 
   const { mutate: updateConnector, isPending: isSaving } = useUpdateConnector()
 
   const { data: subscriptionResponse } = useSubscriptionData({ enabled: isBillingEnabled })
   const subscriptionAccess = getSubscriptionAccessState(subscriptionResponse?.data)
   const hasMaxAccess = !isBillingEnabled || subscriptionAccess.hasUsableMaxAccess
 
+  const persistedCanonicalModes = useMemo(
+    () => readPersistedCanonicalModes(connector.sourceConfig),
+    [connector.sourceConfig]
+  )
+
   const hasChanges = useMemo(() => {
     if (syncInterval !== connector.syncIntervalMinutes) return true
+    if (didCanonicalModesChange(canonicalModes, persistedCanonicalModes)) return true
     const resolved = resolveSourceConfig()
     for (const [key, value] of Object.entries(resolved)) {
       if (String(connector.sourceConfig[key] ?? '') !== value) return true
     }
     return false
-  }, [resolveSourceConfig, syncInterval, connector.syncIntervalMinutes, connector.sourceConfig])
+  }, [
+    resolveSourceConfig,
+    syncInterval,
+    connector.syncIntervalMinutes,
+    connector.sourceConfig,
+    canonicalModes,
+    persistedCanonicalModes,
+  ])
 
   const handleSave = () => {
     setError(null)
@@ -126,8 +172,17 @@ export function EditConnectorModal({
     for (const [key, value] of Object.entries(resolved)) {
       if (String(connector.sourceConfig[key] ?? '') !== value) changedEntries[key] = value
     }
-    if (Object.keys(changedEntries).length > 0) {
-      updates.sourceConfig = { ...connector.sourceConfig, ...changedEntries }
+
+    const modesChanged = didCanonicalModesChange(canonicalModes, persistedCanonicalModes)
+
+    if (Object.keys(changedEntries).length > 0 || modesChanged) {
+      const next: Record<string, unknown> = { ...connector.sourceConfig, ...changedEntries }
+      if (Object.keys(canonicalModes).length > 0) {
+        next[CANONICAL_MODES_KEY] = canonicalModes
+      } else {
+        delete next[CANONICAL_MODES_KEY]
+      }
+      updates.sourceConfig = next
     }
 
     if (Object.keys(updates).length === 0) {

apps/sim/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields.ts

@@ -7,6 +7,7 @@ import type { ConnectorConfig, ConnectorConfigField } from '@/connectors/types'
 export interface UseConnectorConfigFieldsOptions {
   connectorConfig: ConnectorConfig | null
   initialSourceConfig?: Record<string, string>
+  initialCanonicalModes?: Record<string, 'basic' | 'advanced'>
 }
 
 export interface UseConnectorConfigFieldsResult {
@@ -34,11 +35,14 @@ export interface UseConnectorConfigFieldsResult {
 export function useConnectorConfigFields({
   connectorConfig,
   initialSourceConfig,
+  initialCanonicalModes,
 }: UseConnectorConfigFieldsOptions): UseConnectorConfigFieldsResult {
   const [sourceConfig, setSourceConfig] = useState<Record<string, string>>(
     () => initialSourceConfig ?? {}
   )
-  const [canonicalModes, setCanonicalModes] = useState<Record<string, 'basic' | 'advanced'>>({})
+  const [canonicalModes, setCanonicalModes] = useState<Record<string, 'basic' | 'advanced'>>(
+    () => initialCanonicalModes ?? {}
+  )
 
   const canonicalGroups = useMemo(() => {
     const groups = new Map<string, ConnectorConfigField[]>()

apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/selector-combobox/selector-combobox.tsx

@@ -53,6 +53,7 @@ export function SelectorCombobox({
   const {
     data: options = [],
     isLoading,
+    hasMore,
     error,
   } = useSelectorOptions(selectorKey, {
     context: selectorContext,
@@ -67,6 +68,7 @@ export function SelectorCombobox({
     Boolean(activeValue) &&
     Boolean(missingOptionLabel) &&
     !isLoading &&
+    !hasMore &&
     !optionMap.get(activeValue!)
   const selectedLabel = activeValue
     ? hasMissingOption

apps/sim/connectors/confluence/confluence.ts

@@ -4,7 +4,7 @@ import { ConfluenceIcon } from '@/components/icons'
 import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
 import { htmlToPlainText, joinTagArray, parseTagDate } from '@/connectors/utils'
-import { getConfluenceCloudId } from '@/tools/confluence/utils'
+import { getConfluenceCloudId, normalizeConfluenceDomainHost } from '@/tools/confluence/utils'
 
 const logger = createLogger('ConfluenceConnector')
 
@@ -141,7 +141,15 @@ export const confluenceConnector: ConnectorConfig = {
   auth: {
     mode: 'oauth',
     provider: 'confluence',
-    requiredScopes: ['read:confluence-content.all', 'read:page:confluence', 'offline_access'],
+    requiredScopes: [
+      'read:confluence-content.all',
+      'read:page:confluence',
+      'read:blogpost:confluence',
+      'read:space:confluence',
+      'read:label:confluence',
+      'search:confluence',
+      'offline_access',
+    ],
   },
 
   configFields: [
@@ -205,7 +213,7 @@ export const confluenceConnector: ConnectorConfig = {
     cursor?: string,
     syncContext?: Record<string, unknown>
   ): Promise<ExternalDocumentList> => {
-    const domain = sourceConfig.domain as string
+    const domain = normalizeConfluenceDomainHost(sourceConfig.domain as string)
     const spaceKey = sourceConfig.spaceKey as string
     const contentType = (sourceConfig.contentType as string) || 'page'
     const labelFilter = (sourceConfig.labelFilter as string) || ''
@@ -269,7 +277,7 @@ export const confluenceConnector: ConnectorConfig = {
     externalId: string,
     syncContext?: Record<string, unknown>
   ): Promise<ExternalDocument | null> => {
-    const domain = sourceConfig.domain as string
+    const domain = normalizeConfluenceDomainHost(sourceConfig.domain as string)
     let cloudId = syncContext?.cloudId as string | undefined
     if (!cloudId) {
       cloudId = await getConfluenceCloudId(domain, accessToken)