Skip to content

Commit d9d3b56

Browse files
waleedlatif1claude
waleedlatif1
and
claude
committed
fix(connectors): harden 10 KB connectors after audit
Validated each issue against provider docs before fixing. - jira: migrate from deprecated /rest/api/3/search (Atlassian sunset May 2025) to /rest/api/3/search/jql with nextPageToken pagination - confluence: unify stub hash across v1 CQL (`when`) and v2 (`createdAt`) paths via shared pageToStub helper using version.number - salesforce: replace hardcoded login.salesforce.com userinfo with host fallback so sandbox-issued tokens (test.salesforce.com) work - servicenow: validate sys_id against /^[a-f0-9]{32}$/ and switch getDocument to path-based /api/now/table/{table}/{sys_id} to close encoded-query injection; reject `^` in kbCategory filter - zendesk: URL-encode Search API query via URLSearchParams; whitelist ticket statuses; encode locale path segment - github: add 10MB cap and /git/blobs/{sha} fallback for files >1MB that /contents/ returns with encoding:"none" - slack: replace SHA-256 over formatted-message window with metadata hash slack:{channelId}:{latestTs}:{count} so list and getDocument agree; cache auth.test team_id on syncContext - obsidian: drop syncRunId from stub hash (Local REST API has no HEAD/Last-Modified per OpenAPI spec); fall back to path-only stub so engine two-stage check short-circuits unchanged notes - evernote: title fallback for attachments-only notes — breaks infinite hydration loop where empty plaintext returned null - google-docs: drop residual `error instanceof Error` pattern Confluence and Slack hash format changes self-heal with a one-time re-sync; no data loss. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
1 parent 31cfb74 commit d9d3b56

10 files changed

Lines changed: 402 additions & 164 deletions

File tree

‎apps/sim/connectors/confluence/confluence.ts‎

Lines changed: 49 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -80,35 +80,57 @@ async function fetchLabelsForPages(
8080
}
8181

8282
/**
83-
* Converts a v1 CQL search result item to a lightweight metadata stub.
83+
* Produces a canonical metadata stub with a deterministic contentHash that
84+
* does not depend on which API surface (v1 CQL or v2) returned the page.
8485
*/
85-
function cqlResultToStub(item: Record<string, unknown>, domain: string): ExternalDocument {
86-
const version = item.version as Record<string, unknown> | undefined
87-
const links = item._links as Record<string, string> | undefined
88-
const metadata = item.metadata as Record<string, unknown> | undefined
89-
const labelsWrapper = metadata?.labels as Record<string, unknown> | undefined
90-
const labelResults = (labelsWrapper?.results || []) as Record<string, unknown>[]
91-
const labels = labelResults.map((l) => l.name as string)
92-
const versionNumber = version?.number
86+
function pageToStub(
87+
page: Record<string, unknown>,
88+
options: {
89+
spaceId?: unknown
90+
labels?: string[]
91+
sourceUrl?: string
92+
} = {}
93+
): ExternalDocument {
94+
const version = page.version as Record<string, unknown> | undefined
95+
const versionNumber = version?.number as number | undefined
96+
const lastModified = (version?.createdAt ?? version?.when ?? '') as string
97+
const versionKey = versionNumber ?? lastModified
9398

9499
return {
95-
externalId: String(item.id),
96-
title: (item.title as string) || 'Untitled',
100+
externalId: String(page.id),
101+
title: (page.title as string) || 'Untitled',
97102
content: '',
98103
contentDeferred: true,
99104
mimeType: 'text/plain',
100-
sourceUrl: links?.webui ? `https://${domain}/wiki${links.webui}` : undefined,
101-
contentHash: `confluence:${item.id}:${versionNumber ?? ''}`,
105+
sourceUrl: options.sourceUrl,
106+
contentHash: `confluence:${page.id}:${versionKey}`,
102107
metadata: {
103-
spaceId: (item.space as Record<string, unknown>)?.key,
104-
status: item.status,
108+
spaceId: options.spaceId,
109+
status: page.status,
105110
version: versionNumber,
106-
labels,
107-
lastModified: version?.when,
111+
labels: options.labels ?? [],
112+
lastModified,
108113
},
109114
}
110115
}
111116

117+
/**
118+
* Converts a v1 CQL search result item to a lightweight metadata stub.
119+
*/
120+
function cqlResultToStub(item: Record<string, unknown>, domain: string): ExternalDocument {
121+
const links = item._links as Record<string, string> | undefined
122+
const metadata = item.metadata as Record<string, unknown> | undefined
123+
const labelsWrapper = metadata?.labels as Record<string, unknown> | undefined
124+
const labelResults = (labelsWrapper?.results || []) as Record<string, unknown>[]
125+
const labels = labelResults.map((l) => l.name as string)
126+
127+
return pageToStub(item, {
128+
spaceId: (item.space as Record<string, unknown>)?.key,
129+
labels,
130+
sourceUrl: links?.webui ? `https://${domain}/wiki${links.webui}` : undefined,
131+
})
132+
}
133+
112134
export const confluenceConnector: ConnectorConfig = {
113135
id: 'confluence',
114136
name: 'Confluence',
@@ -286,7 +308,9 @@ export const confluenceConnector: ConnectorConfig = {
286308

287309
const links = page._links as Record<string, unknown> | undefined
288310
const version = page.version as Record<string, unknown> | undefined
289-
const versionNumber = version?.number
311+
const versionNumber = version?.number as number | undefined
312+
const lastModified = (version?.createdAt ?? version?.when ?? '') as string
313+
const versionKey = versionNumber ?? lastModified
290314

291315
return {
292316
externalId: String(page.id),
@@ -295,13 +319,13 @@ export const confluenceConnector: ConnectorConfig = {
295319
contentDeferred: false,
296320
mimeType: 'text/plain',
297321
sourceUrl: links?.webui ? `https://${domain}/wiki${links.webui}` : undefined,
298-
contentHash: `confluence:${page.id}:${versionNumber ?? ''}`,
322+
contentHash: `confluence:${page.id}:${versionKey}`,
299323
metadata: {
300324
spaceId: page.spaceId,
301325
status: page.status,
302326
version: versionNumber,
303327
labels,
304-
lastModified: version?.createdAt,
328+
lastModified,
305329
},
306330
}
307331
},
@@ -420,28 +444,11 @@ async function listDocumentsV2(
420444
const results = data.results || []
421445

422446
const documents: ExternalDocument[] = results.map((page: Record<string, unknown>) => {
423-
const pageId = String(page.id)
424-
const version = page.version as Record<string, unknown> | undefined
425-
const versionNumber = version?.number
426-
427-
return {
428-
externalId: pageId,
429-
title: (page.title as string) || 'Untitled',
430-
content: '',
431-
contentDeferred: true,
432-
mimeType: 'text/plain',
433-
sourceUrl: (page._links as Record<string, string>)?.webui
434-
? `https://${domain}/wiki${(page._links as Record<string, string>).webui}`
435-
: undefined,
436-
contentHash: `confluence:${pageId}:${versionNumber ?? ''}`,
437-
metadata: {
438-
spaceId: page.spaceId,
439-
status: page.status,
440-
version: versionNumber,
441-
labels: [],
442-
lastModified: version?.createdAt,
443-
},
444-
}
447+
const links = page._links as Record<string, string> | undefined
448+
return pageToStub(page, {
449+
spaceId: page.spaceId,
450+
sourceUrl: links?.webui ? `https://${domain}/wiki${links.webui}` : undefined,
451+
})
445452
})
446453

447454
let nextCursor: string | undefined

‎apps/sim/connectors/evernote/evernote.ts‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -462,7 +462,9 @@ export const evernoteConnector: ConnectorConfig = {
462462
const retryOptions = { maxRetries: 3, initialDelayMs: 500 }
463463
const note = await apiGetNote(accessToken, externalId, retryOptions)
464464
const plainText = htmlToPlainText(note.content)
465-
if (!plainText.trim()) return null
465+
const title = note.title || 'Untitled'
466+
const content = plainText.trim() ? plainText : title
467+
if (!content.trim()) return null
466468

467469
const shardId = extractShardId(accessToken)
468470
const userId = extractUserId(accessToken)
@@ -494,8 +496,8 @@ export const evernoteConnector: ConnectorConfig = {
494496

495497
return {
496498
externalId,
497-
title: note.title || 'Untitled',
498-
content: plainText,
499+
title,
500+
content,
499501
contentDeferred: false,
500502
mimeType: 'text/plain',
501503
sourceUrl: `https://${host}/shard/${shardId}/nl/${userId}/${externalId}/`,
@@ -539,7 +541,7 @@ export const evernoteConnector: ConnectorConfig = {
539541

540542
return { valid: true }
541543
} catch (error) {
542-
const message = error instanceof Error ? error.message : 'Failed to connect to Evernote'
544+
const message = toError(error).message || 'Failed to connect to Evernote'
543545
return { valid: false, error: message }
544546
}
545547
},

‎apps/sim/connectors/github/github.ts‎

Lines changed: 60 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ const logger = createLogger('GitHubConnector')
1010
const GITHUB_API_URL = 'https://api.github.com'
1111
const BATCH_SIZE = 30
1212
const GIT_SHA_PREFIX = 'git-sha:'
13+
const MAX_FILE_SIZE = 10 * 1024 * 1024 // 10 MB
1314

1415
/**
1516
* Parses the repository string into owner and repo.
@@ -90,6 +91,44 @@ async function fetchTree(
9091
return (data.tree || []).filter((item: TreeItem) => item.type === 'blob')
9192
}
9293

94+
/**
95+
* Fetches blob content via the Git Blobs API. Used as a fallback when the
96+
* `/contents/` endpoint cannot return the file body (files larger than 1 MB
97+
* return `content: ""` and `encoding: "none"`). Supports blobs up to 100 MB.
98+
*/
99+
async function fetchBlobContent(
100+
accessToken: string,
101+
owner: string,
102+
repo: string,
103+
sha: string
104+
): Promise<string> {
105+
const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/git/blobs/${encodeURIComponent(sha)}`
106+
const response = await fetchWithRetry(url, {
107+
method: 'GET',
108+
headers: {
109+
Accept: 'application/vnd.github+json',
110+
Authorization: `Bearer ${accessToken}`,
111+
'X-GitHub-Api-Version': '2022-11-28',
112+
},
113+
})
114+
115+
if (!response.ok) {
116+
throw new Error(`Failed to fetch git blob ${sha}: ${response.status}`)
117+
}
118+
119+
const data = await response.json()
120+
const content = (data.content as string) || ''
121+
const encoding = data.encoding as string | undefined
122+
123+
if (encoding === 'base64') {
124+
return Buffer.from(content, 'base64').toString('utf8')
125+
}
126+
if (encoding === 'utf-8') {
127+
return content
128+
}
129+
return ''
130+
}
131+
93132
/**
94133
* Creates a lightweight stub ExternalDocument from a tree item.
95134
* Uses the Git blob SHA as contentHash for change detection, avoiding
@@ -257,10 +296,27 @@ export const githubConnector: ConnectorConfig = {
257296

258297
const lastModifiedHeader = response.headers.get('last-modified') || undefined
259298
const data = await response.json()
260-
const content =
261-
data.encoding === 'base64'
262-
? Buffer.from(data.content as string, 'base64').toString('utf-8')
263-
: (data.content as string) || ''
299+
300+
const size = typeof data.size === 'number' ? data.size : 0
301+
if (size > MAX_FILE_SIZE) {
302+
logger.info('Skipping GitHub file exceeding size limit', {
303+
path,
304+
size,
305+
limit: MAX_FILE_SIZE,
306+
})
307+
return null
308+
}
309+
310+
const rawContent = (data.content as string) || ''
311+
const encoding = data.encoding as string | undefined
312+
let content: string
313+
if (encoding === 'base64' && rawContent.length > 0) {
314+
content = Buffer.from(rawContent, 'base64').toString('utf8')
315+
} else if ((encoding === 'none' || rawContent.length === 0) && data.sha) {
316+
content = await fetchBlobContent(accessToken, owner, repo, data.sha as string)
317+
} else {
318+
content = ''
319+
}
264320

265321
return {
266322
externalId,

‎apps/sim/connectors/google-docs/google-docs.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -349,8 +349,7 @@ export const googleDocsConnector: ConnectorConfig = {
349349

350350
return { valid: true }
351351
} catch (error) {
352-
const message = error instanceof Error ? error.message : 'Failed to validate configuration'
353-
return { valid: false, error: message }
352+
return { valid: false, error: toError(error).message || 'Failed to validate configuration' }
354353
}
355354
},
356355

‎apps/sim/connectors/jira/jira.ts‎

Lines changed: 27 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { createLogger } from '@sim/logger'
2+
import { toError } from '@sim/utils/errors'
23
import { JiraIcon } from '@/components/icons'
34
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
45
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
@@ -164,24 +165,24 @@ export const jiraConnector: ConnectorConfig = {
164165
jql = `project = "${safeKey}" AND (${jqlFilter.trim()}) ORDER BY updated DESC`
165166
}
166167

167-
const startAt = cursor ? Number(cursor) : 0
168+
const collectedSoFar = (syncContext?.collectedCount as number | undefined) ?? 0
169+
const remaining = maxIssues > 0 ? Math.max(0, maxIssues - collectedSoFar) : PAGE_SIZE
170+
if (maxIssues > 0 && remaining === 0) {
171+
return { documents: [], hasMore: false }
172+
}
168173

169174
const params = new URLSearchParams()
170175
params.append('jql', jql)
171-
params.append('startAt', String(startAt))
172-
const remaining = maxIssues > 0 ? Math.max(0, maxIssues - startAt) : PAGE_SIZE
173-
if (remaining === 0) {
174-
return { documents: [], hasMore: false }
175-
}
176176
params.append('maxResults', String(Math.min(PAGE_SIZE, remaining)))
177177
params.append(
178178
'fields',
179179
'summary,issuetype,status,priority,assignee,reporter,project,labels,created,updated'
180180
)
181+
if (cursor) params.append('nextPageToken', cursor)
181182

182-
const url = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/search?${params.toString()}`
183+
const url = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/search/jql?${params.toString()}`
183184

184-
logger.info(`Listing Jira issues for project ${projectKey}`, { startAt })
185+
logger.info(`Listing Jira issues for project ${projectKey}`, { hasCursor: Boolean(cursor) })
185186

186187
const response = await fetchWithRetry(url, {
187188
method: 'GET',
@@ -201,17 +202,25 @@ export const jiraConnector: ConnectorConfig = {
201202
}
202203

203204
const data = await response.json()
204-
const issues = (data.issues || []) as Record<string, unknown>[]
205-
const total = (data.total as number) ?? 0
205+
let issues = (data.issues || []) as Record<string, unknown>[]
206+
const nextPageToken = data.nextPageToken as string | undefined
207+
const isLast = Boolean(data.isLast) || !nextPageToken
208+
209+
if (maxIssues > 0 && issues.length > remaining) {
210+
issues = issues.slice(0, remaining)
211+
}
206212

207213
const documents: ExternalDocument[] = issues.map((issue) => issueToStub(issue, domain))
208214

209-
const nextStart = startAt + issues.length
210-
const hasMore = nextStart < total && (maxIssues <= 0 || nextStart < maxIssues)
215+
const newCollected = collectedSoFar + issues.length
216+
if (syncContext) syncContext.collectedCount = newCollected
217+
218+
const reachedCap = maxIssues > 0 && newCollected >= maxIssues
219+
const hasMore = !isLast && !reachedCap
211220

212221
return {
213222
documents,
214-
nextCursor: hasMore ? String(nextStart) : undefined,
223+
nextCursor: hasMore ? nextPageToken : undefined,
215224
hasMore,
216225
}
217226
},
@@ -278,9 +287,9 @@ export const jiraConnector: ConnectorConfig = {
278287
const params = new URLSearchParams()
279288
const safeKey = projectKey.replace(/\\/g, '\\\\').replace(/"/g, '\\"')
280289
params.append('jql', `project = "${safeKey}"`)
281-
params.append('maxResults', '0')
290+
params.append('maxResults', '1')
282291

283-
const url = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/search?${params.toString()}`
292+
const url = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/search/jql?${params.toString()}`
284293
const response = await fetchWithRetry(
285294
url,
286295
{
@@ -304,9 +313,9 @@ export const jiraConnector: ConnectorConfig = {
304313
if (jqlFilter) {
305314
const filterParams = new URLSearchParams()
306315
filterParams.append('jql', `project = "${safeKey}" AND (${jqlFilter})`)
307-
filterParams.append('maxResults', '0')
316+
filterParams.append('maxResults', '1')
308317

309-
const filterUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/search?${filterParams.toString()}`
318+
const filterUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/search/jql?${filterParams.toString()}`
310319
const filterResponse = await fetchWithRetry(
311320
filterUrl,
312321
{
@@ -326,8 +335,7 @@ export const jiraConnector: ConnectorConfig = {
326335

327336
return { valid: true }
328337
} catch (error) {
329-
const message = error instanceof Error ? error.message : 'Failed to validate configuration'
330-
return { valid: false, error: message }
338+
return { valid: false, error: toError(error).message || 'Failed to validate configuration' }
331339
}
332340
},
333341

‎apps/sim/connectors/obsidian/obsidian.ts‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -215,16 +215,22 @@ export const obsidianConnector: ConnectorConfig = {
215215
const offset = cursor ? Number(cursor) : 0
216216
const pageFiles = allFiles.slice(offset, offset + DOCS_PER_PAGE)
217217

218-
const syncRunId = (syncContext?.syncRunId as string) ?? ''
219-
218+
/**
219+
* Stub hash uses path only because the Obsidian Local REST API directory
220+
* listing returns just `{ files: string[] }` — no `stat`/`mtime` and no
221+
* `HEAD` support to read `Last-Modified`. A path-only stub is stable
222+
* across syncs, so unchanged paths short-circuit re-hydration in the
223+
* sync engine. `getDocument` always re-hashes with `mtime`, which
224+
* triggers a refresh whenever the file actually changes.
225+
*/
220226
const documents: ExternalDocument[] = pageFiles.map((filePath) => ({
221227
externalId: filePath,
222228
title: titleFromPath(filePath),
223229
content: '',
224230
contentDeferred: true,
225231
mimeType: 'text/plain' as const,
226232
sourceUrl: `${baseUrl}/vault/${filePath.split('/').map(encodeURIComponent).join('/')}`,
227-
contentHash: `obsidian:stub:${filePath}:${syncRunId}`,
233+
contentHash: `obsidian:${filePath}`,
228234
metadata: {
229235
folder: filePath.includes('/') ? filePath.substring(0, filePath.lastIndexOf('/')) : '',
230236
},

0 commit comments

Comments
 (0)