diff --git a/e2e-live/build.test.ts b/e2e-live/build.test.ts index 6f1facd..e1e2783 100644 --- a/e2e-live/build.test.ts +++ b/e2e-live/build.test.ts @@ -59,6 +59,13 @@ async function waitForPipeData( return []; } +function expectBuildSuccess(buildResult: Awaited>): void { + if (!buildResult.success) { + throw new Error(buildResult.error ?? "Build failed without an error message."); + } + expect(buildResult.success).toBe(true); +} + describeLive("E2E Live: build", () => { const config = liveConfig as LiveE2EConfig; @@ -68,7 +75,8 @@ describeLive("E2E Live: build", () => { let workspaceId = ""; let workspaceName = ""; let workspaceToken = ""; - let tinybirdBranchName = ""; + const tinybirdBranchNames: string[] = []; + let testBranchIndex = 0; beforeAll(async () => { ensureDistBuild(); @@ -82,7 +90,10 @@ describeLive("E2E Live: build", () => { beforeEach(() => { tempDir = createTempProjectDir(); originalEnv = { ...process.env }; - process.env.GITHUB_REF_NAME = `live-build/${workspaceName}`; + testBranchIndex += 1; + const testBranchName = `live-build/${workspaceName}/${testBranchIndex}`; + process.env.GITHUB_HEAD_REF = testBranchName; + process.env.GITHUB_REF_NAME = testBranchName; process.env.TINYBIRD_TOKEN = workspaceToken; }); @@ -92,7 +103,7 @@ describeLive("E2E Live: build", () => { }); afterAll(async () => { - if (tinybirdBranchName) { + for (const tinybirdBranchName of tinybirdBranchNames) { try { await deleteBranch( { baseUrl: config.baseUrl, token: workspaceToken }, @@ -122,11 +133,12 @@ describeLive("E2E Live: build", () => { setConfigBaseUrl(tempDir, config.baseUrl); const buildResult = await runBuild({ cwd: tempDir }); - expect(buildResult.success).toBe(true); + expectBuildSuccess(buildResult); expect(buildResult.deploy?.success).toBe(true); expect(buildResult.branchInfo?.tinybirdBranch).toBeTruthy(); - tinybirdBranchName = buildResult.branchInfo?.tinybirdBranch ?? ""; + const tinybirdBranchName = buildResult.branchInfo?.tinybirdBranch ?? ""; + tinybirdBranchNames.push(tinybirdBranchName); const branch = await getBranch( { baseUrl: config.baseUrl, token: workspaceToken }, diff --git a/package.json b/package.json index 1fdfba1..9899810 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tinybirdco/sdk", - "version": "0.0.69", + "version": "0.0.70", "description": "TypeScript SDK for Tinybird Forward - define datasources and pipes as TypeScript", "type": "module", "main": "./dist/index.js", diff --git a/src/generator/datasource.test.ts b/src/generator/datasource.test.ts index 8b8a999..b8f282c 100644 --- a/src/generator/datasource.test.ts +++ b/src/generator/datasource.test.ts @@ -188,6 +188,75 @@ describe('Datasource Generator', () => { expect(result.content).toContain("id UUID `json:$.id` DEFAULT generateUUIDv4()"); }); + it("includes materialized expressions without autogenerated JSON paths", () => { + const ds = defineDatasource("test_ds", { + schema: { + body: t.string(), + level: t + .string() + .lowCardinality() + .materialized("JSONExtractString(body, 'level')"), + }, + }); + + const result = generateDatasource(ds); + expect(result.content).toContain("body String `json:$.body`"); + expect(result.content).toContain( + "level LowCardinality(String) MATERIALIZED JSONExtractString(body, 'level')" + ); + expect(result.content).not.toContain("`json:$.level`"); + }); + + it("generates the full datasource file for a schema with a materialized column", () => { + const ds = defineDatasource("raw_logs", { + description: "Raw logs with extracted severity", + schema: { + timestamp: t.dateTime(), + body: t.string(), + service_name: t.string().jsonPath("$.resource.service.name"), + level: t + .string() + .lowCardinality() + .materialized("JSONExtractString(body, 'level')"), + }, + engine: engine.mergeTree({ + sortingKey: ["service_name", "level", "timestamp"], + partitionKey: "toYYYYMM(timestamp)", + }), + }); + + const result = generateDatasource(ds); + expect(result.content).toBe(`DESCRIPTION > + Raw logs with extracted severity + +SCHEMA > + timestamp DateTime \`json:$.timestamp\`, + body String \`json:$.body\`, + service_name String \`json:$.resource.service.name\`, + level LowCardinality(String) MATERIALIZED JSONExtractString(body, 'level') + +ENGINE "MergeTree" +ENGINE_PARTITION_KEY "toYYYYMM(timestamp)" +ENGINE_SORTING_KEY "service_name, level, timestamp"`); + }); + + it("includes explicit JSON paths on materialized columns", () => { + const ds = defineDatasource("test_ds", { + schema: { + body: t.string(), + level: t + .string() + .materialized("JSONExtractString(body, 'level')") + .jsonPath("$.ignored_level"), + }, + }); + + const result = generateDatasource(ds); + expect(result.content).toContain( + "level String `json:$.ignored_level` MATERIALIZED JSONExtractString(body, 'level')" + ); + }); + it('formats null default values', () => { const ds = defineDatasource('test_ds', { schema: { diff --git a/src/generator/datasource.ts b/src/generator/datasource.ts index a0d8ed4..b351a9b 100644 --- a/src/generator/datasource.ts +++ b/src/generator/datasource.ts @@ -99,16 +99,19 @@ function generateColumnLine( const modifiers = getModifiersFromValidator(validator); const parts: string[] = [` ${columnName} ${tinybirdType}`]; + const hasMaterializedExpression = typeof modifiers.materializedExpression === "string"; // Add JSON path for Events API ingestion support if enabled // Use explicit jsonPath if defined, otherwise default to $.columnName - if (includeJsonPaths) { + if (includeJsonPaths && (!hasMaterializedExpression || jsonPath !== undefined)) { const effectiveJsonPath = jsonPath ?? `$.${columnName}`; parts.push(`\`json:${effectiveJsonPath}\``); } - // Add default value/expression if defined - if (modifiers.hasDefault) { + // Add materialized expression or default value/expression if defined + if (hasMaterializedExpression) { + parts.push(`MATERIALIZED ${modifiers.materializedExpression}`); + } else if (modifiers.hasDefault) { if (typeof modifiers.defaultExpression === "string") { parts.push(`DEFAULT ${modifiers.defaultExpression}`); } else if (modifiers.defaultValue !== undefined) { diff --git a/src/schema/types.test.ts b/src/schema/types.test.ts index 3d97fc3..6ee973e 100644 --- a/src/schema/types.test.ts +++ b/src/schema/types.test.ts @@ -137,6 +137,32 @@ describe("Type Validators (t.*)", () => { }); }); + describe("Materialized expression modifier", () => { + it("stores materialized SQL expression in modifiers", () => { + const type = t.string().materialized("JSONExtractString(body, 'level')"); + expect(type._modifiers.materializedExpression).toBe("JSONExtractString(body, 'level')"); + }); + + it("trims materialized SQL expression", () => { + const type = t.string().materialized(" lower(service_name) "); + expect(type._modifiers.materializedExpression).toBe("lower(service_name)"); + }); + + it("throws on empty materialized SQL expression", () => { + expect(() => t.string().materialized(" ")).toThrow( + "Materialized expression cannot be empty." + ); + }); + + it("clears default modifiers when materialized is chained after a default", () => { + const type = t.string().default("unknown").materialized("lower(name)"); + expect(type._modifiers.materializedExpression).toBe("lower(name)"); + expect(type._modifiers.hasDefault).toBeUndefined(); + expect(type._modifiers.defaultValue).toBeUndefined(); + expect(type._modifiers.defaultExpression).toBeUndefined(); + }); + }); + describe("Codec modifier", () => { it("sets codec in modifiers", () => { const type = t.string().codec("LZ4"); diff --git a/src/schema/types.ts b/src/schema/types.ts index bef7a9a..c7839e8 100644 --- a/src/schema/types.ts +++ b/src/schema/types.ts @@ -52,6 +52,14 @@ export interface TypeValidator< TTinybirdType, TModifiers & { hasDefault: true; defaultExpression: string } >; + /** Set a materialized SQL expression for the column */ + materialized( + expression: string, + ): TypeValidator< + TType, + TTinybirdType, + TModifiers & { materializedExpression: string } + >; /** Set a codec for compression */ codec( codec: string, @@ -68,6 +76,7 @@ export interface TypeModifiers { hasDefault?: boolean; defaultValue?: unknown; defaultExpression?: string; + materializedExpression?: string; codec?: string; jsonPath?: string; } @@ -154,6 +163,7 @@ function createValidator( hasDefault: true, defaultValue: value, defaultExpression: undefined, + materializedExpression: undefined, }) as TypeValidator< TType, TTinybirdType, @@ -171,6 +181,7 @@ function createValidator( hasDefault: true, defaultValue: undefined, defaultExpression: trimmed, + materializedExpression: undefined, }) as TypeValidator< TType, TTinybirdType, @@ -178,6 +189,24 @@ function createValidator( >; }, + materialized(expression: string) { + const trimmed = expression.trim(); + if (!trimmed) { + throw new Error("Materialized expression cannot be empty."); + } + return createValidator(tinybirdType, { + ...modifiers, + hasDefault: undefined, + defaultValue: undefined, + defaultExpression: undefined, + materializedExpression: trimmed, + }) as TypeValidator< + TType, + TTinybirdType, + TypeModifiers & { materializedExpression: string } + >; + }, + codec(codec: string) { return createValidator(tinybirdType, { ...modifiers,