diff --git a/.fernignore b/.fernignore index c41a27d4..8b52e246 100644 --- a/.fernignore +++ b/.fernignore @@ -6,6 +6,9 @@ cl-config.yml src/wrapper src/index.ts +src/v2 +tests/bun +tests/fixtures .github/workflows/ci.yml .github/workflows/cl-create.yml \ No newline at end of file diff --git a/bunfig.toml b/bunfig.toml new file mode 100644 index 00000000..fad65cce --- /dev/null +++ b/bunfig.toml @@ -0,0 +1,2 @@ +[test] +root = "./tests/bun" diff --git a/src/v2/records/index.ts b/src/v2/records/index.ts new file mode 100644 index 00000000..bfcb1769 --- /dev/null +++ b/src/v2/records/index.ts @@ -0,0 +1,612 @@ +import { Records as FernRecords } from "../../api/resources/records/client/Client"; +import { Flatfile } from "../.."; +import * as core from "../../core"; +import urlJoin from "url-join"; +import { + GetRecordsRequestOptions, + JsonlRecord, + WriteRecordsRequestOptions, + WriteRecordsResponse, + WriteStreamingOptions, +} from "./types"; +import * as environments from "../../environments"; +import * as errors from "../../errors"; +import * as serializers from "../../serialization"; +import { toRawResponse } from "../../core/fetcher/RawResponse"; + +interface RecordsOptions extends FernRecords.Options {} + +export class RecordsV2 { + private readonly options: RecordsOptions; + + constructor(options: RecordsOptions = {}) { + this.options = options; + } + + /** + * Retrieve records from a sheet in raw JSONL format. + * + * This method fetches all records at once and returns them as an array of + * JsonlRecord objects, which contain the raw data structure from the API + * including special fields like __k (record ID), __v (version), etc. + * + * @param sheetId - The ID of the sheet to retrieve records from + * @param options - Optional request parameters for filtering, pagination, etc. + * @param requestOptions - Optional request configuration (headers, timeout, etc.) + * @returns Promise that resolves to an array of JsonlRecord objects + * + * @example + * ```typescript + * const rawRecords = await recordsV2.getRaw('us_sh_123', { + * fields: ['firstName', 'lastName'], + * pageSize: 1000 + * }); + * rawRecords.forEach(record => { + * console.log(`Record ID: ${record.__k}`); + * console.log(`Field values:`, record); + * }); + * ``` + */ + public async getRaw( + sheetId: Flatfile.SheetId, + options: GetRecordsRequestOptions = {}, + requestOptions: FernRecords.RequestOptions, + ): Promise { + const url = await this._buildUrl(`/v2-alpha/records.jsonl`); + + // Add sheetId and options as query parameters + const queryParams = this._buildQueryParams({ ...options, sheetId }); + if (queryParams.length > 0) { + url.search = queryParams; + } + + const headers = await this._prepareHeaders(requestOptions); + const response = await this._executeRequest(url.toString(), "GET", { + headers, + requestOptions, + }); + + // Parse the JSONL response into an array of records + const text = await response.text(); + return this._parseJsonlText(text); + } + + /** + * Stream records from a sheet in raw JSONL format. + * + * This method provides an async generator that yields JsonlRecord objects + * as they are received from the server. This is the most memory-efficient + * way to process large datasets as records are yielded individually rather + * than loading all records into memory at once. + * + * @param sheetId - The ID of the sheet to retrieve records from + * @param options - Optional request parameters for filtering, pagination, etc. + * @param requestOptions - Optional request configuration (headers, timeout, etc.) + * @returns AsyncGenerator that yields JsonlRecord objects + * + * @example + * ```typescript + * for await (const rawRecord of recordsV2.getRawStreaming('us_sh_123', { + * includeTimestamps: true + * })) { + * console.log(`Record ID: ${rawRecord.__k}`); + * console.log(`Updated at: ${rawRecord.__u}`); + * // Process each record as it streams in + * } + * ``` + */ + public async *getRawStreaming( + sheetId: Flatfile.SheetId, + options: GetRecordsRequestOptions = {}, + requestOptions: FernRecords.RequestOptions = {}, + ): AsyncGenerator { + const url = await this._buildUrl(`/v2-alpha/records.jsonl`); + + // Add sheetId and options as query parameters + const queryParams = this._buildQueryParams({ ...options, sheetId }); + if (queryParams.length > 0) { + url.search = queryParams; + } + + const headers = await this._prepareHeaders(requestOptions); + const response = await this._executeRequest(url.toString(), "GET", { + headers, + requestOptions, + }); + + // Check if ReadableStream streaming is supported + if (response.body && typeof response.body.getReader === "function") { + // Use streaming approach for supported browsers + yield* this._streamJsonlResponse(response); + } else { + // Final fallback for browsers without streaming support + yield* this._fallbackJsonlResponse(response); + } + } + + /** + * Stream JSONL response using ReadableStream (modern browsers) + */ + private async *_streamJsonlResponse(response: Response): AsyncGenerator { + if (!response.body) { + throw new errors.FlatfileError({ + message: "Response body is null", + rawResponse: undefined, + }); + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + + try { + while (true) { + const { done, value } = await reader.read(); + + if (done) { + // Process any remaining data in buffer + if (buffer.trim()) { + try { + yield JSON.parse(buffer.trim()) as JsonlRecord; + } catch (error) { + // Skip malformed final line + console.warn("Failed to parse final JSONL line:", buffer.trim()); + } + } + break; + } + + // Decode chunk and add to buffer + buffer += decoder.decode(value, { stream: true }); + + // Process complete lines + const lines = buffer.split("\n"); + buffer = lines.pop() || ""; // Keep incomplete line in buffer + + for (const line of lines) { + const trimmedLine = line.trim(); + if (trimmedLine) { + try { + yield JSON.parse(trimmedLine) as JsonlRecord; + } catch (error) { + // Skip malformed line but continue streaming + console.warn("Failed to parse JSONL line:", trimmedLine); + } + } + } + } + } finally { + reader.releaseLock(); + } + } + + /** + * Parse JSONL text into an array of JsonlRecord objects + */ + private _parseJsonlText(text: string): JsonlRecord[] { + const lines = text.split("\n"); + const records: JsonlRecord[] = []; + + for (const line of lines) { + const trimmedLine = line.trim(); + if (trimmedLine) { + try { + records.push(JSON.parse(trimmedLine) as JsonlRecord); + } catch (error) { + console.warn("Failed to parse JSONL line:", trimmedLine); + } + } + } + + return records; + } + + /** + * Fallback JSONL processing for browsers without streaming support + */ + private async *_fallbackJsonlResponse(response: Response): AsyncGenerator { + const text = await response.text(); + const records = this._parseJsonlText(text); + + for (const record of records) { + yield record; + } + } + + /** + * Write records to a sheet in raw JSONL format. + * + * This method takes an array of JsonlRecord objects and writes them to the specified sheet. + * Records can be inserts (no __k field) or updates (with __k field for existing record ID). + * Supports various write options like truncate, snapshot, and sheet targeting. + * + * @param records - Array of JsonlRecord objects to write + * @param options - Write configuration options + * @param requestOptions - Optional request configuration (headers, timeout, etc.) + * @returns Promise that resolves to WriteRecordsResponse with operation results + * + * @example + * ```typescript + * const records: JsonlRecord[] = [ + * { firstName: 'John', lastName: 'Doe', __s: 'us_sh_123' }, + * { __k: 'us_rc_456', firstName: 'Jane', lastName: 'Smith' } // Update existing + * ]; + * const result = await recordsV2.writeRaw(records, { + * sheetId: 'us_sh_123', + * truncate: false + * }); + * console.log(`Created: ${result.created}, Updated: ${result.updated}`); + * ``` + */ + public async writeRaw( + records: JsonlRecord[], + options: WriteRecordsRequestOptions = {}, + requestOptions: FernRecords.RequestOptions = {}, + ): Promise { + const url = await this._buildUrl(`/v2-alpha/records.jsonl`); + + // For write operations, ensure all records have the sheet ID set if provided in options + const enrichedRecords = records.map((record) => { + // Always ensure sheet ID is present when provided in options + if (options.sheetId) { + return { __s: options.sheetId, ...record }; + } + return record; + }); + + // Add options as query parameters, excluding sheetId since it's in the record body + const { sheetId: _, ...queryOptions } = options; + const queryParams = this._buildQueryParams(queryOptions, false); // Write operation + if (queryParams.length > 0) { + url.search = queryParams; + } + + // Convert records to JSONL format (one record per line) + const jsonlBody = enrichedRecords.map((record) => JSON.stringify(record)).join("\n"); + + const headers = await this._prepareHeaders(requestOptions, "application/jsonl"); + + // Add sheet ID header if provided in options and not already in records + if (options.sheetId) { + headers["X-Sheet-Id"] = options.sheetId; + } + + const response = await this._executeRequest(url.toString(), "POST", { + body: jsonlBody, + contentType: "application/jsonl", + headers, + requestOptions, + }); + + // Parse the response + const responseBody = await response.text(); + try { + return JSON.parse(responseBody) as WriteRecordsResponse; + } catch (error) { + // If response isn't JSON, return a basic success response + return { success: true }; + } + } + + /** + * Stream records to a sheet in raw JSONL format using HTTP body streaming. + * + * This method accepts an async generator/iterator of records and streams them + * directly to the server using a ReadableStream as the HTTP request body. + * This approach is memory efficient for large datasets as records are processed + * and transmitted without loading all data into memory at once. + * + * The operation is atomic - all records are sent in a single HTTP request, + * ensuring consistent write semantics. Records can be new inserts (without __k) + * or updates to existing records (with __k field containing the record ID). + * + * @param recordsStream - Async generator/iterator that yields JsonlRecord objects + * @param options - Write configuration options (sheetId, truncate, etc.) + * @param requestOptions - Optional request configuration (headers, timeout, etc.) + * @returns Promise that resolves to WriteRecordsResponse with operation results + * + * @example + * ```typescript + * async function* generateRecords() { + * for (let i = 0; i < 100000; i++) { + * yield { + * firstName: `User${i}`, + * email: `user${i}@example.com`, + * __s: 'us_sh_123' + * }; + * } + * } + * + * const result = await recordsV2.writeRawStreaming(generateRecords(), { + * sheetId: 'us_sh_123', + * truncate: false + * }); + * console.log(`Created: ${result.created}, Updated: ${result.updated}`); + * ``` + */ + public async writeRawStreaming( + recordsStream: AsyncIterable, + options: WriteStreamingOptions = {}, + requestOptions: FernRecords.RequestOptions = {}, + ): Promise { + const url = await this._buildUrl(`/v2-alpha/records.jsonl`); + + // Add options as query parameters, excluding sheetId since it's in the record body + const { sheetId, ...queryOptions } = options; + const queryParams = this._buildQueryParams(queryOptions, false); // Write operation + if (queryParams.length > 0) { + url.search = queryParams; + } + + // Create ReadableStream that converts records to JSONL + const readableStream = new ReadableStream({ + async start(controller) { + const encoder = new TextEncoder(); + try { + for await (const record of recordsStream) { + // Ensure sheet ID is set if provided in options + const enrichedRecord = sheetId && !record.__s ? { ...record, __s: sheetId } : record; + const jsonlLine = JSON.stringify(enrichedRecord) + "\n"; + controller.enqueue(encoder.encode(jsonlLine)); + } + controller.close(); + } catch (error) { + controller.error(error); + } + }, + }); + + const headers = await this._prepareHeaders(requestOptions, "application/jsonl"); + + // Execute the streaming request + const response = await this._executeRequest(url.toString(), "POST", { + body: readableStream as any, // TypeScript might complain about ReadableStream + contentType: "application/jsonl", + headers, + requestOptions, + }); + + // Parse the response + const responseBody = await response.text(); + try { + return JSON.parse(responseBody) as WriteRecordsResponse; + } catch (error) { + // If response isn't JSON, return a basic success response + return { success: true }; + } + } + + private async _prepareHeaders( + requestOptions?: FernRecords.RequestOptions, + contentType: string = "application/json", + ): Promise> { + const authHeader = await this._getAuthorizationHeader(); + const headers: Record = { + "X-Disable-Hooks": requestOptions?.xDisableHooks ?? this.options?.xDisableHooks ?? "true", + "X-Fern-Language": "JavaScript", + "X-Fern-Runtime": core.RUNTIME.type ?? "node", + "X-Fern-Runtime-Version": core.RUNTIME.version ?? "unknown", + "Content-Type": contentType, + ...requestOptions?.headers, + }; + + // Only add Authorization header if we have a token + if (authHeader) { + headers.Authorization = authHeader; + } + + return headers; + } + + /** + * Get authorization header from options + */ + private async _getAuthorizationHeader(): Promise { + const bearer = await core.Supplier.get(this.options.token); + if (bearer != null) { + return `Bearer ${bearer}`; + } + return undefined; + } + + /** + * Build full URL from path + */ + private async _buildUrl(path: string): Promise { + let baseUrl = + (await core.Supplier.get(this.options.baseUrl)) ?? + (await core.Supplier.get(this.options.environment)) ?? + environments.FlatfileEnvironment.Production; + + // Special handling for v2-alpha endpoints - remove /v1 suffix if present + if (path.startsWith("/v2-alpha") && baseUrl.endsWith("/v1")) { + baseUrl = baseUrl.slice(0, -3); + } + + return new URL(urlJoin(baseUrl, path)); + } + + /** + * Build query parameters string from options object + */ + private _buildQueryParams(params: Record, isReadOperation: boolean = true): string { + const searchParams = new URLSearchParams(); + + // Only include stream=true for read operations (GET requests) + if (isReadOperation) { + searchParams.append("stream", "true"); + } + + for (const [key, value] of Object.entries(params)) { + if (value !== undefined && value !== null) { + if (Array.isArray(value)) { + // Handle array values (like fields) + value.forEach((item) => { + if (item !== undefined && item !== null) { + searchParams.append(key, String(item)); + } + }); + } else { + searchParams.append(key, String(value)); + } + } + } + + return searchParams.toString(); + } + + /** + * Execute HTTP request with retry logic and timeout handling + */ + private async _executeRequest( + url: string, + method: string, + options: { + body?: string | Uint8Array | ReadableStream; + contentType?: string; + headers?: Record; + requestOptions?: FernRecords.RequestOptions; + } = {}, + ): Promise { + const { body, contentType = "application/json", headers: directHeaders, requestOptions } = options; + + // Prepare headers - merge direct headers with those from requestOptions + const preparedHeaders = await this._prepareHeaders(requestOptions, contentType); + const finalHeaders = { ...preparedHeaders, ...directHeaders }; + + // Setup timeout and retry configuration + const timeoutMs = requestOptions?.timeoutInSeconds != null ? requestOptions.timeoutInSeconds * 1000 : 60000; + const maxRetries = requestOptions?.maxRetries ?? 0; + + let lastError: Error | null = null; + + for (let attempt = 0; attempt <= maxRetries; attempt++) { + const controller = new AbortController(); + let timeoutId: NodeJS.Timeout | null = null; + + try { + // Setup external abort signal forwarding + if (requestOptions?.abortSignal) { + requestOptions.abortSignal.addEventListener("abort", () => controller.abort()); + } + + // Setup timeout + if (timeoutMs > 0) { + timeoutId = setTimeout(() => controller.abort(), timeoutMs); + } + + // Prepare fetch options + const fetchOptions: RequestInit = { + method, + headers: finalHeaders, + body, + signal: controller.signal, + }; + + // Add duplex option when body is a ReadableStream (only if ReadableStream is available) + if (typeof ReadableStream !== "undefined" && body instanceof ReadableStream) { + (fetchOptions as any).duplex = "half"; + } + + const response = await fetch(url, fetchOptions); + + if (timeoutId) { + clearTimeout(timeoutId); + } + + // Handle non-success status codes by throwing appropriate errors + if (!response.ok) { + const errorBody = await this._parseErrorBody(response); + this._throwErrorForStatus(response.status, errorBody, response); + } + + return response; + } catch (error) { + if (timeoutId) { + clearTimeout(timeoutId); + } + + // Handle abort/timeout + if (error instanceof Error && error.name === "AbortError") { + throw new errors.FlatfileTimeoutError(`Timeout exceeded when calling ${method} ${url}.`); + } + + // If this is already a Flatfile error, rethrow it + if (error instanceof errors.FlatfileError || error instanceof errors.FlatfileTimeoutError) { + throw error; + } + + lastError = error as Error; + + // If we've exhausted retries, throw the last error + if (attempt === maxRetries) { + throw new errors.FlatfileError({ + message: lastError.message, + rawResponse: undefined, + }); + } + + // Wait before retry (exponential backoff) + if (attempt < maxRetries) { + await new Promise((resolve) => setTimeout(resolve, Math.pow(2, attempt) * 1000)); + } + } + } + + // This should never be reached, but just in case + throw new errors.FlatfileError({ + message: lastError?.message ?? "Unknown error occurred", + rawResponse: undefined, + }); + } + + /** + * Parse error body from response + */ + private async _parseErrorBody(response: Response): Promise { + try { + const text = await response.text(); + return text ? JSON.parse(text) : {}; + } catch { + return {}; + } + } + + /** + * Throw appropriate error based on HTTP status code + */ + private _throwErrorForStatus(status: number, errorBody: any, response: Response): never { + const rawResponse = toRawResponse(response); + + switch (status) { + case 400: + throw new Flatfile.BadRequestError( + serializers.Errors.parseOrThrow(errorBody, { + unrecognizedObjectKeys: "passthrough", + allowUnrecognizedUnionMembers: true, + allowUnrecognizedEnumValues: true, + skipValidation: true, + breadcrumbsPrefix: ["response"], + }), + rawResponse, + ); + case 404: + throw new Flatfile.NotFoundError( + serializers.Errors.parseOrThrow(errorBody, { + unrecognizedObjectKeys: "passthrough", + allowUnrecognizedUnionMembers: true, + allowUnrecognizedEnumValues: true, + skipValidation: true, + breadcrumbsPrefix: ["response"], + }), + rawResponse, + ); + default: + throw new errors.FlatfileError({ + statusCode: status, + body: errorBody, + rawResponse: rawResponse, + }); + } + } +} diff --git a/src/v2/records/types.ts b/src/v2/records/types.ts new file mode 100644 index 00000000..e52a02a7 --- /dev/null +++ b/src/v2/records/types.ts @@ -0,0 +1,162 @@ +import * as Flatfile from "../../api/index"; +import { Sheet } from "../../serialization"; + +interface SheetSearchParams { + filter?: string; + filterField?: string; + searchField?: string; + searchValue?: string; + q?: string; + commitId?: string; + sinceCommitId?: string; + ids?: string; + forEvent?: string; +} + +interface SheetSortParams { + sortField?: string; + sortDirection?: "asc" | "desc"; +} + +interface PaginationParams { + pageNumber?: number; + pageSize?: number; +} + +/** + * Request options for records endpoint query parameters + */ +interface GetRecordsParams { + /** The ID of the Sheet */ + sheetId?: Flatfile.SheetId; + /** The ID of the Workbook */ + workbookId?: Flatfile.WorkbookId; + /** The slug of the Sheet */ + sheetSlug?: string; + /** Whether to include links in the response */ + includeLinks?: boolean; + /** Whether to include messages in the response */ + includeMessages?: boolean; + /** Whether to include metadata in the response */ + includeMetadata?: boolean; + /** Whether to include config in the response */ + includeConfig?: boolean; + /** Whether to include sheet in the response */ + includeSheet?: boolean; + /** Whether to include sheet slug in the response */ + includeSheetSlug?: boolean; + /** Whether to include the updated at timestamps in the response */ + includeTimestamps?: boolean; + /** Whether to exclude context from the response */ + noContext?: boolean; + /** Whether to stream the response */ + isStream?: boolean; + /** Whether to include empty cells/fields in the response */ + includeEmptyCells?: boolean; + /** Specific fields to include in the response */ + fields?: string[]; +} + +export type GetRecordsRequestOptions = GetRecordsParams & SheetSearchParams & SheetSortParams & PaginationParams; + +interface JsonlRecordSpecialParams { + /** + * Record ID + */ + __k?: string; + + /** + * Record ID (for Creation) + */ + __nk?: string; + + /** + * Record version ID + */ + __v?: string; + + /** + * Sheet ID + */ + __s?: string; + + /** + * Sheet slug + */ + __n?: string; + + /** + * Config + */ + __c?: Record; + + /** + * Metadata + */ + __m?: Record; + + /** + * Messages + */ + __i?: Record; + + /** + * Whether the record is deleted + */ + __d?: boolean; + + /** + * Whether the record is valid + */ + __e?: boolean; + + /** + * Links to other records + */ + __l?: JsonlRecord[]; + + /** + * Record-level updated timestamp (when includeTimestamps=true) + */ + __u?: string; +} + +export interface JsonlRecord extends JsonlRecordSpecialParams { + [fieldKey: string]: any; +} + +/** + * Request options for writing records + */ +interface WriteRecordsParams { + /** The ID of the Sheet */ + sheetId?: Flatfile.SheetId; + /** The ID of the Workbook */ + workbookId?: Flatfile.WorkbookId; + /** The slug of the Sheet */ + sheetSlug?: string; + /** Whether to truncate (clear) the sheet before writing */ + truncate?: boolean; + /** Whether to create a snapshot before writing */ + snapshot?: boolean; + /** Whether to suppress hooks during writing */ + silent?: boolean; + /** Event ID for context */ + for?: Flatfile.EventId; +} + +export type WriteRecordsRequestOptions = WriteRecordsParams; + +/** + * Response structure for write operations + */ +export interface WriteRecordsResponse { + success: boolean; + created?: number; + updated?: number; +} + +/** + * Options for streaming write operations + */ +export interface WriteStreamingOptions extends WriteRecordsRequestOptions {} diff --git a/tests/bun/records.test.ts b/tests/bun/records.test.ts new file mode 100644 index 00000000..f75aceb2 --- /dev/null +++ b/tests/bun/records.test.ts @@ -0,0 +1,327 @@ +import fs from "fs"; +import { join } from "path"; +import { RecordsV2 } from "../../src/v2/records"; +import { Flatfile } from "../../src"; + +// Mock the global fetch function +const mockFetch = jest.fn(); +global.fetch = mockFetch; + +// Helper to read the fixture data +const getFixtureData = (): string => { + return fs.readFileSync(join(__dirname, "../fixtures/v2.records.get.jsonl"), "utf-8"); +}; + +// Helper to parse JSONL fixture data into records array +const parseFixtureData = (): any[] => { + const data = getFixtureData(); + return data + .split("\n") + .filter((line) => line.trim()) + .map((line) => JSON.parse(line)); +}; + +describe("RecordsV2", () => { + let recordsV2: RecordsV2; + const sheetId = "dev_sh_jVnmFCKg" as Flatfile.SheetId; + const defaultRequestOptions = {}; + + beforeEach(() => { + recordsV2 = new RecordsV2({ + token: "test-token", + baseUrl: "https://api.flatfile.com/v1", + }); + mockFetch.mockClear(); + mockFetch.mockReset(); + }); + + describe("getRaw", () => { + it("should fetch and return raw JSONL records", async () => { + const fixtureData = getFixtureData(); + + mockFetch.mockResolvedValueOnce({ + ok: true, + status: 200, + text: () => Promise.resolve(fixtureData), + body: null, + }); + + const result = await recordsV2.getRaw(sheetId, {}, defaultRequestOptions); + + expect(result).toHaveLength(21); + + // Check first record structure + const firstRecord = result[0]; + expect(firstRecord).toMatchObject({ + __k: "dev_rc_a5d2afda7dda4149afe51229e2674906", + __s: "dev_sh_jVnmFCKg", + __n: "contacts-pCZHI4", + firstname: "John", + lastname: "Smith [X]", + email: "john.smith@example.com", + }); + }); + + it("should handle empty response", async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + status: 200, + text: () => Promise.resolve(""), + body: null, + }); + + const result = await recordsV2.getRaw(sheetId, {}, defaultRequestOptions); + expect(result).toEqual([]); + }); + + it("should skip malformed JSONL lines", async () => { + const malformedData = `{"__k":"valid1","name":"John"} +invalid json line +{"__k":"valid2","name":"Jane"}`; + + // Mock console.warn to avoid test output noise + const consoleSpy = jest.spyOn(console, "warn").mockImplementation(); + + mockFetch.mockResolvedValueOnce({ + ok: true, + status: 200, + text: () => Promise.resolve(malformedData), + body: null, + }); + + const result = await recordsV2.getRaw(sheetId, {}, defaultRequestOptions); + + expect(result).toHaveLength(2); + expect(result[0].__k).toBe("valid1"); + expect(result[1].__k).toBe("valid2"); + expect(consoleSpy).toHaveBeenCalledWith("Failed to parse JSONL line:", "invalid json line"); + + consoleSpy.mockRestore(); + }); + }); + + describe("getRawStreaming", () => { + it("should stream raw JSONL records with ReadableStream", async () => { + // Test ReadableStream with single record to verify core functionality + const testData = `{"__k":"stream_test_id","name":"StreamTest"}`; + const encoder = new TextEncoder(); + const chunk = encoder.encode(testData); + + const mockStream = new ReadableStream({ + start(controller) { + controller.enqueue(chunk); + controller.close(); + }, + }); + + mockFetch.mockResolvedValueOnce({ + ok: true, + status: 200, + body: mockStream, + }); + + const results: any[] = []; + for await (const record of recordsV2.getRawStreaming(sheetId, {}, defaultRequestOptions)) { + results.push(record); + } + + expect(results).toHaveLength(1); + expect(results[0].__k).toBe("stream_test_id"); + expect(results[0].name).toBe("StreamTest"); + }); + + it("should handle streaming without ReadableStream support", async () => { + const fixtureData = getFixtureData(); + + mockFetch.mockResolvedValueOnce({ + ok: true, + status: 200, + text: () => Promise.resolve(fixtureData), + body: null, // No ReadableStream support + }); + + const results: any[] = []; + for await (const record of recordsV2.getRawStreaming(sheetId, {}, defaultRequestOptions)) { + results.push(record); + } + + expect(results).toHaveLength(21); + expect(results[0].__k).toBe("dev_rc_a5d2afda7dda4149afe51229e2674906"); + }); + }); + + describe("writeRaw", () => { + it("should write JSONL records successfully", async () => { + const mockResponse = { success: true, created: 2, updated: 0 }; + + mockFetch.mockResolvedValueOnce({ + ok: true, + status: 200, + text: () => Promise.resolve(JSON.stringify(mockResponse)), + }); + + const records = [ + { firstName: "John", lastName: "Doe", __s: sheetId }, + { firstName: "Jane", lastName: "Smith", __s: sheetId }, + ]; + + const result = await recordsV2.writeRaw(records, { sheetId }, defaultRequestOptions); + + expect(mockFetch).toHaveBeenCalledTimes(1); + + const fetchCall = mockFetch.mock.calls[0]; + expect(fetchCall[0]).toContain("/v2-alpha/records.jsonl"); + expect(fetchCall[1].method).toBe("POST"); + expect(fetchCall[1].headers).toMatchObject({ + Authorization: "Bearer test-token", + "Content-Type": "application/jsonl", + "X-Disable-Hooks": "true", + "X-Fern-Language": "JavaScript", + }); + + // Check the body content by parsing the JSONL + const bodyLines = fetchCall[1].body.split("\n"); + expect(bodyLines).toHaveLength(2); + + const firstRecord = JSON.parse(bodyLines[0]); + expect(firstRecord).toMatchObject({ + __s: sheetId, + firstName: "John", + lastName: "Doe", + }); + + const secondRecord = JSON.parse(bodyLines[1]); + expect(secondRecord).toMatchObject({ + __s: sheetId, + firstName: "Jane", + lastName: "Smith", + }); + + expect(result).toEqual(mockResponse); + }); + + it("should pass query parameters correctly for writeRaw", async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + status: 200, + text: () => Promise.resolve(JSON.stringify({ success: true })), + }); + + const records = [{ firstName: "John", __s: sheetId }]; + const options = { + sheetId, + truncate: true, + snapshot: true, + silent: true, + for: "event_123" as Flatfile.EventId, + }; + + await recordsV2.writeRaw(records, options, defaultRequestOptions); + + const fetchCall = mockFetch.mock.calls[0]; + const url = new URL(fetchCall[0]); + + // sheetId is excluded from query parameters for write operations (it's in the record body) + expect(url.searchParams.get("sheetId")).toBe(null); + expect(url.searchParams.get("truncate")).toBe("true"); + expect(url.searchParams.get("snapshot")).toBe("true"); + expect(url.searchParams.get("silent")).toBe("true"); + expect(url.searchParams.get("for")).toBe("event_123"); + }); + + it("should handle non-JSON response for writeRaw", async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + status: 200, + text: () => Promise.resolve("OK"), + }); + + const records = [{ firstName: "John", __s: sheetId }]; + const result = await recordsV2.writeRaw(records, { sheetId }, defaultRequestOptions); + + expect(result).toEqual({ success: true }); + }); + + it("should handle writeRaw fetch errors", async () => { + mockFetch.mockResolvedValueOnce({ + ok: false, + status: 400, + text: () => Promise.resolve(JSON.stringify({ error: "Bad request" })), + }); + + const records = [{ firstName: "John", __s: sheetId }]; + + await expect(recordsV2.writeRaw(records, { sheetId }, defaultRequestOptions)).rejects.toThrow(); + }); + + it("should send empty body for empty records array", async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + status: 200, + text: () => Promise.resolve(JSON.stringify({ success: true })), + }); + + await recordsV2.writeRaw([], { sheetId }, defaultRequestOptions); + + const fetchCall = mockFetch.mock.calls[0]; + expect(fetchCall[1].body).toBe(""); + }); + }); + + describe("writeRawStreaming", () => { + it("should stream records using body streaming", async () => { + const mockResponse = { success: true, created: 3, updated: 0 }; + mockFetch.mockResolvedValueOnce({ + ok: true, + status: 200, + text: () => Promise.resolve(JSON.stringify(mockResponse)), + }); + + async function* generateRecords() { + yield { firstName: "User1", __s: sheetId }; + yield { firstName: "User2", __s: sheetId }; + yield { firstName: "User3", __s: sheetId }; + } + + const result = await recordsV2.writeRawStreaming(generateRecords(), { sheetId }, defaultRequestOptions); + + expect(mockFetch).toHaveBeenCalledTimes(1); // Single request with streaming body + expect(result).toEqual(mockResponse); + + // Check that the body is a ReadableStream-like object + const fetchCall = mockFetch.mock.calls[0]; + expect(fetchCall[1].body).toBeDefined(); + }); + + it("should handle streaming errors properly", async () => { + mockFetch.mockRejectedValueOnce(new Error("Network error")); + + async function* generateRecords() { + yield { firstName: "User1", __s: sheetId }; + } + + await expect( + recordsV2.writeRawStreaming(generateRecords(), { sheetId }, defaultRequestOptions), + ).rejects.toThrow("Network error"); + }); + + it("should handle empty async generator", async () => { + const mockResponse = { success: true, created: 0, updated: 0 }; + mockFetch.mockResolvedValueOnce({ + ok: true, + status: 200, + text: () => Promise.resolve(JSON.stringify(mockResponse)), + }); + + async function* generateRecords() { + // Empty generator + return; + } + + const result = await recordsV2.writeRawStreaming(generateRecords(), { sheetId }, defaultRequestOptions); + + expect(mockFetch).toHaveBeenCalledTimes(1); // Always makes a request even for empty stream + expect(result).toEqual(mockResponse); + }); + }); +}); diff --git a/tests/fixtures/v2.records.get.jsonl b/tests/fixtures/v2.records.get.jsonl new file mode 100644 index 00000000..be77dcdd --- /dev/null +++ b/tests/fixtures/v2.records.get.jsonl @@ -0,0 +1,21 @@ +{"__k":"dev_rc_a5d2afda7dda4149afe51229e2674906","__s":"dev_sh_jVnmFCKg","__n":"contacts-pCZHI4","firstname":"John","lastname":"Smith [X]","email":"john.smith@example.com","company":"TechCorp","home_address":"123 Main St, New York, NY, 10001, USA","billing_address":"123 Main St, New York, NY, 10001, USA","is_subscribed":"No","birth_date":"6/15/85","gender":"M","lang":"Englisch","notes":"Top client","tag1":"VIP","tag2":"Priority","contact_method_1_type":"Email","contact_method_1_value":"john.smith@example.com","contact_method_2_type":"Phone","contact_method_2_value":"(312) 555-1234"} +{"__k":"dev_rc_8b2910e275e04f45b48f05f6f1774364","__s":"dev_sh_jVnmFCKg","__n":"contacts-pCZHI4","firstname":"Emily","lastname":"Davis","email":"emily.davis@EXAMple.com","company":"HealthCare Inc","home_address":"456 Elm St, Los Angeles, CA, 90001, USA","billing_address":"456 Elm St, Los Angeles, CA, 90001, USA","is_subscribed":"Yes","birth_date":"8/23/90","gender":"F","lang":"Spanisch","notes":"Opted out of newsletters","tag1":"Prospect","contact_method_1_type":"Email","contact_method_1_value":"emily.davis@EXAMple.com","contact_method_2_type":"Phone","contact_method_2_value":"(312) 555-5678"} +{"__k":"dev_rc_2e63adcdbdca49a18d469c4bace6eec3","__s":"dev_sh_jVnmFCKg","__n":"contacts-pCZHI4","firstname":"Michael","lastname":"Johnson","email":"michael.johnson@example.com","company":"FinancePro","home_address":"789 Pine Ave, Chicago, IL, 60601, USA","billing_address":"789 Pine Ave, Chicago, IL, 60601, USA","is_subscribed":"No","birth_date":"2/11/78","gender":"M","lang":"Französisch","tag1":"High Value","contact_method_1_type":"Email","contact_method_1_value":"michael.johnson@example.com","contact_method_2_type":"Phone","contact_method_2_value":"(312) 555-2345"} +{"__k":"dev_rc_6eded17a9c134053a22a25f2ddd28590","__s":"dev_sh_jVnmFCKg","__n":"contacts-pCZHI4","firstname":"Sarah","lastname":"Brown","email":"sarah.brown@example.com","company":"EduPlus","home_address":"101 Maple Dr, Houston, TX, 77001, USA","is_subscribed":"No","birth_date":"11/4/82","gender":"F","lang":"Englisch","tag1":"Referral","tag2":"Potential","contact_method_1_type":"Email","contact_method_1_value":"sarah.brown@example.com","contact_method_2_type":"Phone","contact_method_2_value":"(312) 555-3456"} +{"__k":"dev_rc_ee47a640c52347aaa38f063b1a798444","__s":"dev_sh_jVnmFCKg","__n":"contacts-pCZHI4","firstname":"David","lastname":"Wilson [X]","email":"david.wilson@example.com","company":"AutoGroup","home_address":"202 Cedar Ln, Phoenix, AZ, 85001, USA","is_subscribed":"Yes","birth_date":"5/12/80","gender":"M","lang":"Deutsch","tag1":"Customer","tag2":"Feedback","contact_method_1_type":"Email","contact_method_1_value":"david.wilson@example.com","contact_method_2_type":"Mobile","contact_method_2_value":"555-4567"} +{"__k":"dev_rc_63d66e65bfbd4cb58b91e4decaeea667","__s":"dev_sh_jVnmFCKg","__n":"contacts-pCZHI4","pref":"Dr.","firstname":"Olivia","lastname":"Miller","email":"olivia.miller@example.com","company":"GreenEnergy","home_address":"303 Birch Rd, Philadelphia, PA, 19101, USA","is_subscribed":"No","birth_date":"3/27/95","gender":"F","lang":"Englisch","tag1":"VIP","contact_method_1_type":"Email","contact_method_1_value":"olivia.miller@example.com","contact_method_2_type":"Mobile","contact_method_2_value":"555-5679"} +{"__k":"dev_rc_deb4e6af3ba5469daf03446468d3a9dc","__s":"dev_sh_jVnmFCKg","__n":"contacts-pCZHI4","pref":"Mr.","firstname":"James","lastname":"Taylor","email":"james.taylor@example.com","company":"BuildCorp","home_address":"404 Willow St, San Diego, CA, 92101, USA","is_subscribed":"No","birth_date":"6/12/86","gender":"M","lang":"Spanisch","notes":"Interested in expansion projects","tag1":"Client","tag2":"Follow-Up","contact_method_1_type":"Email","contact_method_1_value":"james.taylor@example.com","contact_method_2_type":"Mobile","contact_method_2_value":"555-6780"} +{"__k":"dev_rc_b1ae5bfce51c449db41083e2ccea516a","__s":"dev_sh_jVnmFCKg","__n":"contacts-pCZHI4","pref":"Ms.","firstname":"Sophia","lastname":"Moore","email":"sophia.moore@example.com","company":"Medica","home_address":"505 Oak Blvd, San Francisco, CA, 94101, USA","is_subscribed":"Yes","birth_date":"9/14/92","gender":"F","lang":"Französisch","notes":"Needs product demo","tag1":"Cold Lead","contact_method_1_type":"Email","contact_method_1_value":"sophia.moore@example.com","contact_method_2_type":"Mobile","contact_method_2_value":"555-7891"} +{"__k":"dev_rc_33363990118f424c9b18f0dcc83d2665","__s":"dev_sh_jVnmFCKg","__n":"contacts-pCZHI4","firstname":"L","lastname":"White","email":"liam.white@example.com","company":"CityGov","home_address":"606 Chestnut Ave, Austin, TX, 73301, USA","is_subscribed":"No","birth_date":"12/19/84","gender":"M","lang":"Englisch","tag1":"High Value","contact_method_1_type":"Email","contact_method_1_value":"liam.white@example.com","contact_method_2_type":"Mobile","contact_method_2_value":"555-8902"} +{"__k":"dev_rc_34aeec7c4078472cacaad531bc8e0af1","__s":"dev_sh_jVnmFCKg","__n":"contacts-pCZHI4","pref":"Mrs.","firstname":"Emma","lastname":"Harris","email":"emma.harris@example.com","company":"TravelWorld","home_address":"707 Aspen St, Miami, FL, 33101, USA","is_subscribed":"Yes","birth_date":"11/2/93","gender":"F","lang":"Deutsch","notes":"Frequent traveler discounts","tag1":"VIP","contact_method_1_type":"Email","contact_method_1_value":"emma.harris@example.com","contact_method_2_type":"Mobile","contact_method_2_value":"555-9013"} +{"__k":"dev_rc_54c32104b6114087bafaacfa690fecb4","__s":"dev_sh_jVnmFCKg","__n":"contacts-pCZHI4","pref":"Dr.","firstname":"Noah","lastname":"Lewis","email":"noah.lewis@example.com","company":"PharmaPlus","home_address":"808 Spruce Ln, Dallas, TX, 75201, USA","is_subscribed":"No","birth_date":"1/22/88","gender":"M","lang":"Englisch","notes":"Research collaborator","contact_method_1_type":"Email","contact_method_1_value":"noah.lewis@example.com","contact_method_2_type":"Mobile","contact_method_2_value":"555-0124"} +{"__k":"dev_rc_99f1b38061574f9aa79b6b82bfc83725","__s":"dev_sh_jVnmFCKg","__n":"contacts-pCZHI4","pref":"Ms.","firstname":"Ava","lastname":"Clark","email":"ava.clark@example.com","company":"ShopEasy","home_address":"909 Redwood Dr, Seattle, WA, 98101, USA","is_subscribed":"No","birth_date":"10/15/90","gender":"F","lang":"Spanisch","tag1":"High Value","tag2":"Priority","contact_method_1_type":"Email","contact_method_1_value":"ava.clark@example.com","contact_method_2_type":"Phone","contact_method_2_value":"555-1235"} +{"__k":"dev_rc_694cdbdee1a7460eaba8709f1d60ba28","__s":"dev_sh_jVnmFCKg","__n":"contacts-pCZHI4","pref":"Mr.","firstname":"William","lastname":"Walker","email":"william.walker@example.com","company":"RealEstate Inc","home_address":"1010 Pine Ave, Denver, CO, 80201, USA","is_subscribed":"Yes","birth_date":"4/19/75","gender":"M","lang":"Französisch","tag1":"High Value","contact_method_1_type":"Email","contact_method_1_value":"william.walker@example.com","contact_method_2_type":"Phone","contact_method_2_value":"555-2346"} +{"__k":"dev_rc_3c0a8fa7b6cc43e8a79815fcfa0a117f","__s":"dev_sh_jVnmFCKg","__n":"contacts-pCZHI4","firstname":"Mia","lastname":"Young","email":"mia.young@example.com","company":"FashionNow","home_address":"1111 Maple St, Boston, MA, 02101, USA","billing_address":"1111 Maple St, Boston, MA, 02101, USA","is_subscribed":"No","birth_date":"9/28/80","gender":"F","lang":"Englisch","tag1":"Referral","tag2":"Priority","contact_method_1_type":"Email","contact_method_1_value":"mia.young@example.com","contact_method_2_type":"Phone","contact_method_2_value":"555-3457"} +{"__k":"dev_rc_8e61af79aeb84418a7eb7808d0c632d1","__s":"dev_sh_jVnmFCKg","__n":"contacts-pCZHI4","firstname":"Benjamin","lastname":"Hall","email":"benjamin.hall@example.com","company":"InnovateTech","home_address":"1212 Cedar Rd, Atlanta, GA, 30301, USA","billing_address":"1212 Cedar Rd, Atlanta, GA, 30301, USA","is_subscribed":"No","birth_date":"9/28/82","gender":"M","lang":"Deutsch","contact_method_1_type":"Email","contact_method_1_value":"benjamin.hall@example.com","contact_method_2_type":"Phone","contact_method_2_value":"555-4568"} +{"__k":"dev_rc_4cda95427f474e1f9370e34fdb804d5e","__s":"dev_sh_jVnmFCKg","__n":"contacts-pCZHI4","firstname":"Charlotte","lastname":"King","email":"charlotte.king@example.com","company":"FoodPro","home_address":"1313 Birch Ln, Las Vegas, NV, 89101, USA","billing_address":"1313 Birch Ln, Las Vegas, NV, 89101, USA","is_subscribed":"Yes","birth_date":"11/2/93","gender":"F","lang":"Englisch","tag1":"Customer","tag2":"Priority","contact_method_1_type":"Email","contact_method_1_value":"charlotte.king@example.com","contact_method_2_type":"Phone","contact_method_2_value":"555-5679"} +{"__k":"dev_rc_c986ccb6802a4efa98fe44665636ad4c","__s":"dev_sh_jVnmFCKg","__n":"contacts-pCZHI4","pref":"Mr.","firstname":"Lucas","lastname":"Wright","email":"lucas.wright@example.com","company":"NextGen","home_address":"1414 Oak Dr, San Jose, CA, 95101, USA","is_subscribed":"No","birth_date":"6/12/86","gender":"M","lang":"Spanisch","notes":"Technical advisor","tag1":"Client","tag2":"Priority","contact_method_1_type":"Email","contact_method_1_value":"lucas.wright@example.com","contact_method_2_type":"Phone","contact_method_2_value":"555-6789"} +{"__k":"dev_rc_ae8c4a157fbc4dc7bf45a6780e0472eb","__s":"dev_sh_jVnmFCKg","__n":"contacts-pCZHI4","pref":"Mrs.","firstname":"Amelia","lastname":"Green","email":"amelia.green@example.com","company":"HomeLife","home_address":"1515 Willow St, Portland, OR, 97201, USA","is_subscribed":"No","birth_date":"9/14/92","gender":"F","lang":"Französisch","notes":"Home renovation interest","tag1":"Cold Lead","tag2":"Follow-Up","contact_method_1_type":"Email","contact_method_1_value":"amelia.green@example.com","contact_method_2_type":"Phone","contact_method_2_value":"555-7890"} +{"__k":"dev_rc_9ccba1eb954941cb8648561f1ef77dac","__s":"dev_sh_jVnmFCKg","__n":"contacts-pCZHI4","pref":"Dr.","firstname":"Ethan","lastname":"Scott","email":"ethan.scott@example.com","company":"AgriTech","home_address":"1616 Chestnut Ave, Nashville, TN, 37201, USA","is_subscribed":"Yes","birth_date":"12/19/84","gender":"M","lang":"Englisch","contact_method_1_type":"Email","contact_method_1_value":"ethan.scott@example.com","contact_method_2_type":"Phone","contact_method_2_value":"555-8901","contact_method_3_type":"Mobile","contact_method_3_value":"555-8901"} +{"__k":"dev_rc_9fd2cdfa318a4d5b88859d33d40e65dc","__s":"dev_sh_jVnmFCKg","__n":"contacts-pCZHI4","pref":"Ms.","firstname":"Isabella","lastname":"Adams","email":"isabella.adams@example.com","company":"EventPro","home_address":"1717 Aspen St, Orlando, FL, 32801, USA","billing_address":"1717 Aspen St, Orlando, FL, 32801, USA","is_subscribed":"No","birth_date":"3/27/95","gender":"F","lang":"Spanisch","notes":"Event planning services","tag1":"VIP","tag2":"Feedback","contact_method_1_type":"Email","contact_method_1_value":"isabella.adams@example.com","contact_method_2_type":"Mobile","contact_method_2_value":"555-9012","contact_method_3_type":"Phone","contact_method_3_value":"555-9012"} +{"__k":"dev_rc_7a797327a5e74e96b6ebb822632d2092","__s":"dev_sh_jVnmFCKg","__n":"contacts-pCZHI4","is_subscribed":"No","birth_date":"3/27/95","gender":"F","lang":"Spanisch"}