diff --git a/docs/install/troubleshooting/truncated-logs.mdx b/docs/install/troubleshooting/truncated-logs.mdx index 25bf54af6e6..fa0b5a3d69f 100644 --- a/docs/install/troubleshooting/truncated-logs.mdx +++ b/docs/install/troubleshooting/truncated-logs.mdx @@ -6,30 +6,38 @@ icon: "file-lines" ## Overview -If you see `(truncated)` in the flow run logs, it means the logs have exceeded the maximum allowed file size. +Flow runs have a maximum log size (default **25 MB**). When a run gets close to this limit, Activepieces tries to keep it under the cap by truncating large step **inputs** — you'll see `(truncated)` in place of the original value. -## How It Works +## What Gets Truncated -There is a current limitation where the log file of a run cannot grow past a certain size. When this limit is reached, the engine automatically removes the largest keys in the JSON output until it fits within the allowed size. +Truncation applies to **step inputs only**. Step **outputs are never truncated**, because downstream steps, subflows, and paused/resumed runs need the original output to continue executing correctly. If outputs were dropped, the next step would receive missing data and fail unpredictably. - -**This does not affect flow execution.** Your flow will continue to run normally even when logs are truncated. - +The engine works like this: + +1. If the total run data fits within the limit, nothing is truncated. +2. Otherwise, step input values are replaced with `(truncated)`, starting from the largest, until the run fits. +3. If the run **still** exceeds the limit after all inputs are truncated — meaning step outputs alone are over the cap — the run fails with `LOG_SIZE_EXCEEDED` and an error like: -## Known Limitation + ``` + Flow run data size exceeded the maximum allowed size of 25 MB + ``` -There is one known issue with truncated logs: +## Why Runs Can Still Fail -If you **pause** a flow, then **resume** it, and the resumed step references data from a truncated step, the flow will fail because the referenced data is no longer available in the logs. +If you see this error on a step that downloads or produces a large file (e.g. a multi-megabyte HTTP response, a base64-encoded binary, or a large API payload), the step's **output** is what's pushing the run over the limit. Since outputs cannot be truncated without breaking subsequent steps, the engine has no choice but to fail the run. ## Solution -You can increase the `AP_MAX_FILE_SIZE_MB` environment variable to a higher value to allow larger log files: +Increase the flow run log size limit by setting the `AP_MAX_FLOW_RUN_LOG_SIZE_MB` environment variable: ```bash -AP_MAX_FILE_SIZE_MB=50 +AP_MAX_FLOW_RUN_LOG_SIZE_MB=50 ``` + +For large file handling, prefer passing files between steps using the built-in file storage (e.g. via `Files` / `File` properties) rather than embedding raw bytes in step outputs. This keeps the run log small and avoids the limit entirely. + + -**Future Improvement:** There is a planned enhancement to change this limit from per-log-file to per-step, which will provide more granular control over log sizes. This feature is currently in the planning phase. +**Future Improvement:** A planned enhancement will move this limit from per-run to per-step, giving more granular control over how much data each step can retain. diff --git a/packages/server/api/src/app/database/migration/postgres/1785000000000-ReplacesSandboxWithVercelAiSdk.ts b/packages/server/api/src/app/database/migration/postgres/1785000000000-ReplacesSandboxWithVercelAiSdk.ts index ecc240b3d85..3d9f0ef93a9 100644 --- a/packages/server/api/src/app/database/migration/postgres/1785000000000-ReplacesSandboxWithVercelAiSdk.ts +++ b/packages/server/api/src/app/database/migration/postgres/1785000000000-ReplacesSandboxWithVercelAiSdk.ts @@ -3,8 +3,8 @@ import { Migration } from '../../migration' export class ReplacesSandboxWithVercelAiSdk1785000000000 implements Migration { name = 'ReplacesSandboxWithVercelAiSdk1785000000000' - breaking = true - release = '0.84.0' + breaking = false + release = '0.82.1' transaction = true public async up(queryRunner: QueryRunner): Promise { diff --git a/packages/server/api/src/app/ee/authentication/saml-authn/saml-attributes.ts b/packages/server/api/src/app/ee/authentication/saml-authn/saml-attributes.ts new file mode 100644 index 00000000000..570d0b6e797 --- /dev/null +++ b/packages/server/api/src/app/ee/authentication/saml-authn/saml-attributes.ts @@ -0,0 +1,99 @@ +import { ActivepiecesError, ErrorCode, isNil, SAMLAttributeMapping } from '@activepieces/shared' + +export const resolveSamlAttributes = ({ rawAttributes, mapping }: ResolveArgs): SamlAttributes => { + const safeAttributes = rawAttributes ?? {} + const email = pickFirstValue({ source: safeAttributes, keys: candidatesFor({ field: 'email', mapping }) }) + const firstName = pickFirstValue({ source: safeAttributes, keys: candidatesFor({ field: 'firstName', mapping }) }) + const lastName = pickFirstValue({ source: safeAttributes, keys: candidatesFor({ field: 'lastName', mapping }) }) + if (isNil(email) || isNil(firstName) || isNil(lastName)) { + throw missingFieldsError({ + resolved: { email, firstName, lastName }, + receivedKeys: Object.keys(safeAttributes), + }) + } + return { email, firstName, lastName } +} + +function candidatesFor({ field, mapping }: CandidatesArgs): string[] { + const override = mapping?.[field]?.trim() + return isNil(override) || override.length === 0 + ? DEFAULT_KEYS[field] + : [override, ...DEFAULT_KEYS[field]] +} + +function pickFirstValue({ source, keys }: PickArgs): string | undefined { + return keys.map((key) => unwrap(source[key])).find(isNonEmptyString) +} + +function unwrap(value: unknown): unknown { + return Array.isArray(value) ? value.find(isNonEmptyString) : value +} + +function isNonEmptyString(value: unknown): value is string { + return typeof value === 'string' && value.length > 0 +} + +function missingFieldsError({ resolved, receivedKeys }: MissingFieldsErrorArgs): ActivepiecesError { + const missing: string[] = [] + if (isNil(resolved.email)) { + missing.push('email') + } + if (isNil(resolved.firstName)) { + missing.push('firstName') + } + if (isNil(resolved.lastName)) { + missing.push('lastName') + } + return new ActivepiecesError({ + code: ErrorCode.INVALID_SAML_RESPONSE, + params: { + message: `Invalid SAML response. Missing required field(s): ${missing.join(', ')}. Received attribute keys: [${receivedKeys.join(', ')}]. Configure attributeMapping in SSO settings if your IdP uses non-standard claim names.`, + }, + }) +} + +const DEFAULT_KEYS: Record = { + email: [ + 'email', + 'emailaddress', + 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress', + ], + firstName: [ + 'firstName', + 'firstname', + 'givenname', + 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname', + ], + lastName: [ + 'lastName', + 'lastname', + 'surname', + 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname', + ], +} + +type ResolveArgs = { + rawAttributes: Record | null | undefined + mapping?: SAMLAttributeMapping +} + +type CandidatesArgs = { + field: keyof SamlAttributes + mapping: SAMLAttributeMapping | undefined +} + +type PickArgs = { + source: Record + keys: string[] +} + +type MissingFieldsErrorArgs = { + resolved: Partial + receivedKeys: string[] +} + +export type SamlAttributes = { + email: string + firstName: string + lastName: string +} diff --git a/packages/server/api/src/app/ee/authentication/saml-authn/saml-client.ts b/packages/server/api/src/app/ee/authentication/saml-authn/saml-client.ts index 4f4fc9cef15..27473c8e0f7 100644 --- a/packages/server/api/src/app/ee/authentication/saml-authn/saml-client.ts +++ b/packages/server/api/src/app/ee/authentication/saml-authn/saml-client.ts @@ -1,58 +1,9 @@ - import { safeHttp } from '@activepieces/server-utils' -import { ActivepiecesError, ErrorCode, SAMLAuthnProviderConfig } from '@activepieces/shared' +import { ActivepiecesError, ErrorCode, SAMLAttributeMapping, SAMLAuthnProviderConfig, tryCatch } from '@activepieces/shared' import * as validator from '@authenio/samlify-node-xmllint' import * as saml from 'samlify' -import { z } from 'zod' import { domainHelper } from '../../custom-domains/domain-helper' - - -const samlResponseValidator = z.object({ - email: z.string(), - firstName: z.string(), - lastName: z.string(), -}) - -class SamlClient { - private static readonly LOGIN_REQUEST_BINDING = 'redirect' - private static readonly LOGIN_RESPONSE_BINDING = 'post' - - constructor( - private readonly idp: saml.IdentityProviderInstance, - private readonly sp: saml.ServiceProviderInstance, - ) {} - - getLoginUrl(): string { - const loginRequest = this.sp.createLoginRequest( - this.idp, - SamlClient.LOGIN_REQUEST_BINDING, - ) - - return loginRequest.context - } - - async parseAndValidateLoginResponse(idpLoginResponse: IdpLoginResponse): Promise { - const loginResult = await this.sp.parseLoginResponse( - this.idp, - SamlClient.LOGIN_RESPONSE_BINDING, - idpLoginResponse, - ) - - const atts = loginResult.extract.attributes - if (!samlResponseValidator.safeParse(atts).success) { - throw new ActivepiecesError({ - code: ErrorCode.INVALID_SAML_RESPONSE, - params: { - message: 'Invalid SAML response, It should contain these firstName, lastName, email fields.', - }, - - }) - } - return atts - } -} - -const instanceCache = new Map() +import { resolveSamlAttributes, SamlAttributes } from './saml-attributes' export const createSamlClient = async (platformId: string, samlProvider: SAMLAuthnProviderConfig): Promise => { const cached = instanceCache.get(platformId) @@ -62,8 +13,8 @@ export const createSamlClient = async (platformId: string, samlProvider: SAMLAut saml.setSchemaValidator(validator) const metadataXml = await resolveIdpMetadata(samlProvider.idpMetadata) const idp = createIdp(metadataXml) - const sp = await createSp(platformId, samlProvider.idpCertificate) - const client = new SamlClient(idp, sp) + const sp = await createSp({ platformId, privateKey: samlProvider.idpCertificate }) + const client = samlClient({ idp, sp, attributeMapping: samlProvider.attributeMapping }) instanceCache.set(platformId, client) return client } @@ -72,6 +23,29 @@ export const invalidateSamlClientCache = (platformId: string): void => { instanceCache.delete(platformId) } +const samlClient = ({ idp, sp, attributeMapping }: SamlClientArgs) => ({ + getLoginUrl(): string { + return sp.createLoginRequest(idp, LOGIN_REQUEST_BINDING).context + }, + async parseAndValidateLoginResponse(idpLoginResponse: IdpLoginResponse): Promise { + const { data: loginResult, error: parseError } = await tryCatch( + () => sp.parseLoginResponse(idp, LOGIN_RESPONSE_BINDING, idpLoginResponse), + ) + if (parseError !== null) { + throw new ActivepiecesError({ + code: ErrorCode.INVALID_SAML_RESPONSE, + params: { + message: `Failed to parse SAML response: ${toErrorMessage(parseError)}`, + }, + }) + } + return resolveSamlAttributes({ + rawAttributes: loginResult.extract?.attributes, + mapping: attributeMapping, + }) + }, +}) + const createIdp = (metadata: string): saml.IdentityProviderInstance => { return saml.IdentityProvider({ metadata, @@ -86,32 +60,34 @@ const resolveIdpMetadata = async (idpMetadata: string): Promise => { if (!/^https?:\/\//i.test(trimmed)) { return idpMetadata } - try { - const response = await safeHttp.axios.get(trimmed, { - responseType: 'text', - timeout: 10_000, - maxContentLength: 5 * 1024 * 1024, - maxBodyLength: 5 * 1024 * 1024, - transformResponse: (data) => data, + const { data: response, error } = await tryCatch(() => safeHttp.axios.get(trimmed, { + responseType: 'text', + timeout: 10_000, + maxContentLength: 5 * 1024 * 1024, + maxBodyLength: 5 * 1024 * 1024, + transformResponse: (data) => data, + })) + if (error !== null) { + throw new ActivepiecesError({ + code: ErrorCode.INVALID_SAML_RESPONSE, + params: { + message: `Failed to fetch IdP metadata from URL: ${toErrorMessage(error)}`, + }, }) - const contentType = String(response.headers['content-type'] ?? '').toLowerCase() - if (contentType !== '' && !contentType.includes('xml') && !contentType.includes('text/plain')) { - throw new Error(`Unexpected content-type "${contentType}" — expected XML.`) - } - return typeof response.data === 'string' ? response.data : String(response.data) } - catch (error) { - const message = error instanceof Error ? error.message : String(error) + const contentType = String(response.headers['content-type'] ?? '').toLowerCase() + if (contentType !== '' && !contentType.includes('xml') && !contentType.includes('text/plain')) { throw new ActivepiecesError({ code: ErrorCode.INVALID_SAML_RESPONSE, params: { - message: `Failed to fetch IdP metadata from URL: ${message}`, + message: `Failed to fetch IdP metadata from URL: Unexpected content-type "${contentType}" — expected XML.`, }, }) } + return typeof response.data === 'string' ? response.data : String(response.data) } -const createSp = async (platformId: string, privateKey: string): Promise => { +const createSp = async ({ platformId, privateKey }: CreateSpArgs): Promise => { const acsUrl = await domainHelper.getPublicUrl({ path: '/api/v1/authn/saml/acs', platformId }) return saml.ServiceProvider({ entityID: 'Activepieces', @@ -129,13 +105,31 @@ const createSp = async (platformId: string, privateKey: string): Promise { + return error instanceof Error ? error.message : String(error) +} + +const LOGIN_REQUEST_BINDING = 'redirect' +const LOGIN_RESPONSE_BINDING = 'post' + +const instanceCache = new Map() + +type SamlClient = ReturnType + +type SamlClientArgs = { + idp: saml.IdentityProviderInstance + sp: saml.ServiceProviderInstance + attributeMapping: SAMLAttributeMapping | undefined +} + +type CreateSpArgs = { + platformId: string + privateKey: string +} + export type IdpLoginResponse = { body: Record query: Record } -export type SamlAttributes = { - email: string - firstName: string - lastName: string -} +export type { SamlAttributes } from './saml-attributes' diff --git a/packages/server/api/test/unit/app/ee/authentication/saml-authn/saml-attributes.test.ts b/packages/server/api/test/unit/app/ee/authentication/saml-authn/saml-attributes.test.ts new file mode 100644 index 00000000000..696eab1db63 --- /dev/null +++ b/packages/server/api/test/unit/app/ee/authentication/saml-authn/saml-attributes.test.ts @@ -0,0 +1,200 @@ +import { ActivepiecesError, ErrorCode } from '@activepieces/shared' +import { resolveSamlAttributes } from '../../../../../../src/app/ee/authentication/saml-authn/saml-attributes' + +describe('resolveSamlAttributes', () => { + describe('default mappings', () => { + it('resolves simple email/firstName/lastName keys', () => { + const result = resolveSamlAttributes({ + rawAttributes: { + email: 'mario@flix.com', + firstName: 'Mario', + lastName: 'Siebeck', + }, + }) + + expect(result).toEqual({ + email: 'mario@flix.com', + firstName: 'Mario', + lastName: 'Siebeck', + }) + }) + + it('resolves Microsoft Entra schema-qualified URIs out of the box', () => { + const result = resolveSamlAttributes({ + rawAttributes: { + 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress': 'mario.siebeck@flix.com', + 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname': 'Mario', + 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname': 'Siebeck', + }, + }) + + expect(result).toEqual({ + email: 'mario.siebeck@flix.com', + firstName: 'Mario', + lastName: 'Siebeck', + }) + }) + + it('ignores unrelated and null-valued claims while resolving the required fields', () => { + const result = resolveSamlAttributes({ + rawAttributes: { + 'http://schemas.microsoft.com/identity/claims/tenantid': 'd8d0ad3e', + 'http://schemas.microsoft.com/identity/claims/objectidentifier': '788ab09e', + 'http://schemas.microsoft.com/ws/2008/06/identity/claims/role': null, + 'http://schemas.microsoft.com/claims/authnmethodsreferences': [null], + 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress': 'mario.siebeck@flix.com', + 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname': 'Mario', + 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname': 'Siebeck', + }, + }) + + expect(result.email).toBe('mario.siebeck@flix.com') + expect(result.firstName).toBe('Mario') + expect(result.lastName).toBe('Siebeck') + }) + + it('flattens single-element string arrays returned by samlify', () => { + const result = resolveSamlAttributes({ + rawAttributes: { + email: ['mario@flix.com'], + firstName: ['Mario'], + lastName: ['Siebeck'], + }, + }) + + expect(result.email).toBe('mario@flix.com') + expect(result.firstName).toBe('Mario') + expect(result.lastName).toBe('Siebeck') + }) + + it('is case-insensitive for the simple lowercase variants exposed by Entra', () => { + const result = resolveSamlAttributes({ + rawAttributes: { + emailaddress: 'mario@flix.com', + givenname: 'Mario', + surname: 'Siebeck', + }, + }) + + expect(result.email).toBe('mario@flix.com') + expect(result.firstName).toBe('Mario') + expect(result.lastName).toBe('Siebeck') + }) + }) + + describe('custom mapping', () => { + it('honours admin-supplied mapping ahead of the defaults', () => { + const result = resolveSamlAttributes({ + rawAttributes: { + 'urn:oid:0.9.2342.19200300.100.1.3': 'mario@flix.com', + 'urn:oid:2.5.4.42': 'Mario', + 'urn:oid:2.5.4.4': 'Siebeck', + email: 'fallback@flix.com', + }, + mapping: { + email: 'urn:oid:0.9.2342.19200300.100.1.3', + firstName: 'urn:oid:2.5.4.42', + lastName: 'urn:oid:2.5.4.4', + }, + }) + + expect(result.email).toBe('mario@flix.com') + expect(result.firstName).toBe('Mario') + expect(result.lastName).toBe('Siebeck') + }) + + it('falls back to default keys when the override key is missing', () => { + const result = resolveSamlAttributes({ + rawAttributes: { + email: 'mario@flix.com', + firstName: 'Mario', + lastName: 'Siebeck', + }, + mapping: { + email: 'custom-email', + firstName: 'custom-first', + lastName: 'custom-last', + }, + }) + + expect(result.email).toBe('mario@flix.com') + expect(result.firstName).toBe('Mario') + expect(result.lastName).toBe('Siebeck') + }) + + it('treats blank override strings as unset', () => { + const result = resolveSamlAttributes({ + rawAttributes: { + email: 'mario@flix.com', + firstName: 'Mario', + lastName: 'Siebeck', + }, + mapping: { + email: ' ', + firstName: '', + lastName: '\t', + }, + }) + + expect(result.email).toBe('mario@flix.com') + expect(result.firstName).toBe('Mario') + expect(result.lastName).toBe('Siebeck') + }) + }) + + describe('error reporting', () => { + it('throws INVALID_SAML_RESPONSE listing missing fields and received keys', () => { + const run = () => resolveSamlAttributes({ + rawAttributes: { + 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress': 'mario@flix.com', + }, + }) + + expect(run).toThrow(ActivepiecesError) + + const error = captureActivepiecesError(run) + expect(error.error.code).toBe(ErrorCode.INVALID_SAML_RESPONSE) + const message = readMessage(error) + expect(message).toContain('firstName') + expect(message).toContain('lastName') + expect(message).toContain('http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress') + expect(message).toContain('attributeMapping') + }) + + it('throws when rawAttributes is null or undefined', () => { + expect(() => resolveSamlAttributes({ rawAttributes: null })).toThrow(ActivepiecesError) + expect(() => resolveSamlAttributes({ rawAttributes: undefined })).toThrow(ActivepiecesError) + }) + + it('treats empty-string and null-valued claims as missing', () => { + expect(() => resolveSamlAttributes({ + rawAttributes: { + email: '', + firstName: null, + lastName: undefined, + }, + })).toThrow(ActivepiecesError) + }) + }) +}) + +function captureActivepiecesError(run: () => unknown): ActivepiecesError { + try { + run() + } + catch (error) { + if (error instanceof ActivepiecesError) { + return error + } + throw error + } + throw new Error('Expected ActivepiecesError to be thrown') +} + +function readMessage(error: ActivepiecesError): string { + const params = error.error.params + if (params && typeof params === 'object' && 'message' in params && typeof params.message === 'string') { + return params.message + } + return '' +} diff --git a/packages/shared/package.json b/packages/shared/package.json index 944cd7f8a57..c7f45ff03f0 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -1,6 +1,6 @@ { "name": "@activepieces/shared", - "version": "0.68.6", + "version": "0.69.0", "type": "commonjs", "sideEffects": false, "main": "./dist/src/index.js", diff --git a/packages/shared/src/lib/core/federated-authn/index.ts b/packages/shared/src/lib/core/federated-authn/index.ts index 7a6ec14b31a..0a3ea4823bd 100644 --- a/packages/shared/src/lib/core/federated-authn/index.ts +++ b/packages/shared/src/lib/core/federated-authn/index.ts @@ -29,9 +29,17 @@ export const GithubAuthnProviderConfig = z.object({ }) export type GithubAuthnProviderConfig = z.infer +export const SAMLAttributeMapping = z.object({ + email: z.string(), + firstName: z.string(), + lastName: z.string(), +}) +export type SAMLAttributeMapping = z.infer + export const SAMLAuthnProviderConfig = z.object({ idpMetadata: z.string(), idpCertificate: z.string(), + attributeMapping: SAMLAttributeMapping.optional(), }) export type SAMLAuthnProviderConfig = z.infer