From 205824ef1576f3919f9ef40877781245b07d8423 Mon Sep 17 00:00:00 2001 From: Rafa Moreno Date: Tue, 12 May 2026 09:55:14 +0200 Subject: [PATCH 1/6] Add materialized column support --- e2e-live/build.test.ts | 121 ++++++++++++++++++++++++++++++- src/generator/datasource.test.ts | 36 +++++++++ src/generator/datasource.ts | 9 ++- src/schema/types.test.ts | 26 +++++++ src/schema/types.ts | 29 ++++++++ 5 files changed, 215 insertions(+), 6 deletions(-) diff --git a/e2e-live/build.test.ts b/e2e-live/build.test.ts index 6f1facd..855bb6c 100644 --- a/e2e-live/build.test.ts +++ b/e2e-live/build.test.ts @@ -7,6 +7,8 @@ import { afterEach, afterAll, } from "vitest"; +import * as fs from "node:fs"; +import * as path from "node:path"; import { runInit } from "../src/cli/commands/init.js"; import { runBuild } from "../src/cli/commands/build.js"; import { listDatasources, listPipesV1 } from "../src/api/resources.js"; @@ -33,6 +35,51 @@ function toTinybirdDateTime(value: Date): string { return value.toISOString().slice(0, 19).replace("T", " "); } +function writeMaterializedColumnProject(projectDir: string, baseUrl: string): void { + const sourceDir = path.join(projectDir, "src", "tinybird"); + fs.mkdirSync(sourceDir, { recursive: true }); + + fs.writeFileSync( + path.join(projectDir, "tinybird.config.json"), + JSON.stringify( + { + include: ["src/tinybird/materialized-column.ts"], + token: "${TINYBIRD_TOKEN}", + baseUrl, + devMode: "branch", + }, + null, + 2 + ) + ); + + fs.writeFileSync( + path.join(sourceDir, "materialized-column.ts"), + ` +import { defineDatasource, defineProject, engine, t } from "@tinybirdco/sdk"; + +export const rawLogs = defineDatasource("raw_logs", { + schema: { + timestamp: t.dateTime(), + body: t.string(), + level: t + .string() + .lowCardinality() + .materialized("JSONExtractString(body, 'level')"), + }, + engine: engine.mergeTree({ + sortingKey: ["level", "timestamp"], + }), +}); + +export default defineProject({ + datasources: { rawLogs }, + pipes: {}, +}); +` + ); +} + async function waitForPipeData( branchToken: string, baseUrl: string @@ -59,6 +106,36 @@ async function waitForPipeData( return []; } +async function waitForMaterializedColumnData( + branchToken: string, + baseUrl: string, + traceId: string +): Promise> { + const client = createClient({ + baseUrl, + token: branchToken, + }); + + for (let attempt = 0; attempt < 10; attempt++) { + const result = await client.sql<{ level: string }>( + ` + SELECT level + FROM raw_logs + WHERE body LIKE '%${traceId}%' + LIMIT 1 + ` + ); + + if (result.data.length > 0) { + return result.data; + } + + await new Promise((resolve) => setTimeout(resolve, 1_000)); + } + + return []; +} + describeLive("E2E Live: build", () => { const config = liveConfig as LiveE2EConfig; @@ -68,7 +145,7 @@ describeLive("E2E Live: build", () => { let workspaceId = ""; let workspaceName = ""; let workspaceToken = ""; - let tinybirdBranchName = ""; + const tinybirdBranchNames: string[] = []; beforeAll(async () => { ensureDistBuild(); @@ -92,7 +169,7 @@ describeLive("E2E Live: build", () => { }); afterAll(async () => { - if (tinybirdBranchName) { + for (const tinybirdBranchName of tinybirdBranchNames) { try { await deleteBranch( { baseUrl: config.baseUrl, token: workspaceToken }, @@ -126,7 +203,8 @@ describeLive("E2E Live: build", () => { expect(buildResult.deploy?.success).toBe(true); expect(buildResult.branchInfo?.tinybirdBranch).toBeTruthy(); - tinybirdBranchName = buildResult.branchInfo?.tinybirdBranch ?? ""; + const tinybirdBranchName = buildResult.branchInfo?.tinybirdBranch ?? ""; + tinybirdBranchNames.push(tinybirdBranchName); const branch = await getBranch( { baseUrl: config.baseUrl, token: workspaceToken }, @@ -163,4 +241,41 @@ describeLive("E2E Live: build", () => { const rows = await waitForPipeData(branchToken, config.baseUrl); expect(rows.length).toBeGreaterThan(0); }); + + it("deploys a datasource with a materialized column and computes it on ingest", async () => { + writeMaterializedColumnProject(tempDir, config.baseUrl); + + const buildResult = await runBuild({ cwd: tempDir }); + expect(buildResult.success).toBe(true); + expect(buildResult.deploy?.success).toBe(true); + expect(buildResult.branchInfo?.tinybirdBranch).toBeTruthy(); + + const tinybirdBranchName = buildResult.branchInfo?.tinybirdBranch ?? ""; + tinybirdBranchNames.push(tinybirdBranchName); + + const branch = await getBranch( + { baseUrl: config.baseUrl, token: workspaceToken }, + tinybirdBranchName + ); + expect(branch.token).toBeTruthy(); + + const branchToken = branch.token as string; + const traceId = `materialized-column-${Date.now()}`; + const client = createClient({ + baseUrl: config.baseUrl, + token: branchToken, + }); + + await client.ingest("raw_logs", { + timestamp: toTinybirdDateTime(new Date()), + body: JSON.stringify({ level: "warn", trace_id: traceId }), + }); + + const rows = await waitForMaterializedColumnData( + branchToken, + config.baseUrl, + traceId + ); + expect(rows).toEqual([{ level: "warn" }]); + }); }); diff --git a/src/generator/datasource.test.ts b/src/generator/datasource.test.ts index 8b8a999..3ee3a1a 100644 --- a/src/generator/datasource.test.ts +++ b/src/generator/datasource.test.ts @@ -188,6 +188,42 @@ describe('Datasource Generator', () => { expect(result.content).toContain("id UUID `json:$.id` DEFAULT generateUUIDv4()"); }); + it("includes materialized expressions without autogenerated JSON paths", () => { + const ds = defineDatasource("test_ds", { + schema: { + body: t.string(), + level: t + .string() + .lowCardinality() + .materialized("JSONExtractString(body, 'level')"), + }, + }); + + const result = generateDatasource(ds); + expect(result.content).toContain("body String `json:$.body`"); + expect(result.content).toContain( + "level LowCardinality(String) MATERIALIZED JSONExtractString(body, 'level')" + ); + expect(result.content).not.toContain("`json:$.level`"); + }); + + it("includes explicit JSON paths on materialized columns", () => { + const ds = defineDatasource("test_ds", { + schema: { + body: t.string(), + level: t + .string() + .materialized("JSONExtractString(body, 'level')") + .jsonPath("$.ignored_level"), + }, + }); + + const result = generateDatasource(ds); + expect(result.content).toContain( + "level String `json:$.ignored_level` MATERIALIZED JSONExtractString(body, 'level')" + ); + }); + it('formats null default values', () => { const ds = defineDatasource('test_ds', { schema: { diff --git a/src/generator/datasource.ts b/src/generator/datasource.ts index a0d8ed4..b351a9b 100644 --- a/src/generator/datasource.ts +++ b/src/generator/datasource.ts @@ -99,16 +99,19 @@ function generateColumnLine( const modifiers = getModifiersFromValidator(validator); const parts: string[] = [` ${columnName} ${tinybirdType}`]; + const hasMaterializedExpression = typeof modifiers.materializedExpression === "string"; // Add JSON path for Events API ingestion support if enabled // Use explicit jsonPath if defined, otherwise default to $.columnName - if (includeJsonPaths) { + if (includeJsonPaths && (!hasMaterializedExpression || jsonPath !== undefined)) { const effectiveJsonPath = jsonPath ?? `$.${columnName}`; parts.push(`\`json:${effectiveJsonPath}\``); } - // Add default value/expression if defined - if (modifiers.hasDefault) { + // Add materialized expression or default value/expression if defined + if (hasMaterializedExpression) { + parts.push(`MATERIALIZED ${modifiers.materializedExpression}`); + } else if (modifiers.hasDefault) { if (typeof modifiers.defaultExpression === "string") { parts.push(`DEFAULT ${modifiers.defaultExpression}`); } else if (modifiers.defaultValue !== undefined) { diff --git a/src/schema/types.test.ts b/src/schema/types.test.ts index 3d97fc3..6ee973e 100644 --- a/src/schema/types.test.ts +++ b/src/schema/types.test.ts @@ -137,6 +137,32 @@ describe("Type Validators (t.*)", () => { }); }); + describe("Materialized expression modifier", () => { + it("stores materialized SQL expression in modifiers", () => { + const type = t.string().materialized("JSONExtractString(body, 'level')"); + expect(type._modifiers.materializedExpression).toBe("JSONExtractString(body, 'level')"); + }); + + it("trims materialized SQL expression", () => { + const type = t.string().materialized(" lower(service_name) "); + expect(type._modifiers.materializedExpression).toBe("lower(service_name)"); + }); + + it("throws on empty materialized SQL expression", () => { + expect(() => t.string().materialized(" ")).toThrow( + "Materialized expression cannot be empty." + ); + }); + + it("clears default modifiers when materialized is chained after a default", () => { + const type = t.string().default("unknown").materialized("lower(name)"); + expect(type._modifiers.materializedExpression).toBe("lower(name)"); + expect(type._modifiers.hasDefault).toBeUndefined(); + expect(type._modifiers.defaultValue).toBeUndefined(); + expect(type._modifiers.defaultExpression).toBeUndefined(); + }); + }); + describe("Codec modifier", () => { it("sets codec in modifiers", () => { const type = t.string().codec("LZ4"); diff --git a/src/schema/types.ts b/src/schema/types.ts index bef7a9a..c7839e8 100644 --- a/src/schema/types.ts +++ b/src/schema/types.ts @@ -52,6 +52,14 @@ export interface TypeValidator< TTinybirdType, TModifiers & { hasDefault: true; defaultExpression: string } >; + /** Set a materialized SQL expression for the column */ + materialized( + expression: string, + ): TypeValidator< + TType, + TTinybirdType, + TModifiers & { materializedExpression: string } + >; /** Set a codec for compression */ codec( codec: string, @@ -68,6 +76,7 @@ export interface TypeModifiers { hasDefault?: boolean; defaultValue?: unknown; defaultExpression?: string; + materializedExpression?: string; codec?: string; jsonPath?: string; } @@ -154,6 +163,7 @@ function createValidator( hasDefault: true, defaultValue: value, defaultExpression: undefined, + materializedExpression: undefined, }) as TypeValidator< TType, TTinybirdType, @@ -171,6 +181,7 @@ function createValidator( hasDefault: true, defaultValue: undefined, defaultExpression: trimmed, + materializedExpression: undefined, }) as TypeValidator< TType, TTinybirdType, @@ -178,6 +189,24 @@ function createValidator( >; }, + materialized(expression: string) { + const trimmed = expression.trim(); + if (!trimmed) { + throw new Error("Materialized expression cannot be empty."); + } + return createValidator(tinybirdType, { + ...modifiers, + hasDefault: undefined, + defaultValue: undefined, + defaultExpression: undefined, + materializedExpression: trimmed, + }) as TypeValidator< + TType, + TTinybirdType, + TypeModifiers & { materializedExpression: string } + >; + }, + codec(codec: string) { return createValidator(tinybirdType, { ...modifiers, From b18a25e333d563c3b37876610353c8c38bd954d5 Mon Sep 17 00:00:00 2001 From: Rafa Moreno Date: Tue, 12 May 2026 10:09:16 +0200 Subject: [PATCH 2/6] Isolate live build e2e branches --- e2e-live/build.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/e2e-live/build.test.ts b/e2e-live/build.test.ts index 855bb6c..c8b8411 100644 --- a/e2e-live/build.test.ts +++ b/e2e-live/build.test.ts @@ -146,6 +146,7 @@ describeLive("E2E Live: build", () => { let workspaceName = ""; let workspaceToken = ""; const tinybirdBranchNames: string[] = []; + let testBranchIndex = 0; beforeAll(async () => { ensureDistBuild(); @@ -159,7 +160,10 @@ describeLive("E2E Live: build", () => { beforeEach(() => { tempDir = createTempProjectDir(); originalEnv = { ...process.env }; - process.env.GITHUB_REF_NAME = `live-build/${workspaceName}`; + testBranchIndex += 1; + const testBranchName = `live-build/${workspaceName}/${testBranchIndex}`; + process.env.GITHUB_HEAD_REF = testBranchName; + process.env.GITHUB_REF_NAME = testBranchName; process.env.TINYBIRD_TOKEN = workspaceToken; }); From 7b5a704103e76541edfe4f8ec1aca0bc5fc0afe8 Mon Sep 17 00:00:00 2001 From: Rafa Moreno Date: Tue, 12 May 2026 10:13:52 +0200 Subject: [PATCH 3/6] Show live build e2e failures --- e2e-live/build.test.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/e2e-live/build.test.ts b/e2e-live/build.test.ts index c8b8411..262de32 100644 --- a/e2e-live/build.test.ts +++ b/e2e-live/build.test.ts @@ -136,6 +136,13 @@ async function waitForMaterializedColumnData( return []; } +function expectBuildSuccess(buildResult: Awaited>): void { + if (!buildResult.success) { + throw new Error(buildResult.error ?? "Build failed without an error message."); + } + expect(buildResult.success).toBe(true); +} + describeLive("E2E Live: build", () => { const config = liveConfig as LiveE2EConfig; @@ -203,7 +210,7 @@ describeLive("E2E Live: build", () => { setConfigBaseUrl(tempDir, config.baseUrl); const buildResult = await runBuild({ cwd: tempDir }); - expect(buildResult.success).toBe(true); + expectBuildSuccess(buildResult); expect(buildResult.deploy?.success).toBe(true); expect(buildResult.branchInfo?.tinybirdBranch).toBeTruthy(); @@ -250,7 +257,7 @@ describeLive("E2E Live: build", () => { writeMaterializedColumnProject(tempDir, config.baseUrl); const buildResult = await runBuild({ cwd: tempDir }); - expect(buildResult.success).toBe(true); + expectBuildSuccess(buildResult); expect(buildResult.deploy?.success).toBe(true); expect(buildResult.branchInfo?.tinybirdBranch).toBeTruthy(); From 3b3c3584840697987a27a0e1a8490e4523cac044 Mon Sep 17 00:00:00 2001 From: Rafa Moreno Date: Tue, 12 May 2026 10:18:54 +0200 Subject: [PATCH 4/6] Remove unsupported materialized live e2e --- e2e-live/build.test.ts | 114 ----------------------------------------- 1 file changed, 114 deletions(-) diff --git a/e2e-live/build.test.ts b/e2e-live/build.test.ts index 262de32..e1e2783 100644 --- a/e2e-live/build.test.ts +++ b/e2e-live/build.test.ts @@ -7,8 +7,6 @@ import { afterEach, afterAll, } from "vitest"; -import * as fs from "node:fs"; -import * as path from "node:path"; import { runInit } from "../src/cli/commands/init.js"; import { runBuild } from "../src/cli/commands/build.js"; import { listDatasources, listPipesV1 } from "../src/api/resources.js"; @@ -35,51 +33,6 @@ function toTinybirdDateTime(value: Date): string { return value.toISOString().slice(0, 19).replace("T", " "); } -function writeMaterializedColumnProject(projectDir: string, baseUrl: string): void { - const sourceDir = path.join(projectDir, "src", "tinybird"); - fs.mkdirSync(sourceDir, { recursive: true }); - - fs.writeFileSync( - path.join(projectDir, "tinybird.config.json"), - JSON.stringify( - { - include: ["src/tinybird/materialized-column.ts"], - token: "${TINYBIRD_TOKEN}", - baseUrl, - devMode: "branch", - }, - null, - 2 - ) - ); - - fs.writeFileSync( - path.join(sourceDir, "materialized-column.ts"), - ` -import { defineDatasource, defineProject, engine, t } from "@tinybirdco/sdk"; - -export const rawLogs = defineDatasource("raw_logs", { - schema: { - timestamp: t.dateTime(), - body: t.string(), - level: t - .string() - .lowCardinality() - .materialized("JSONExtractString(body, 'level')"), - }, - engine: engine.mergeTree({ - sortingKey: ["level", "timestamp"], - }), -}); - -export default defineProject({ - datasources: { rawLogs }, - pipes: {}, -}); -` - ); -} - async function waitForPipeData( branchToken: string, baseUrl: string @@ -106,36 +59,6 @@ async function waitForPipeData( return []; } -async function waitForMaterializedColumnData( - branchToken: string, - baseUrl: string, - traceId: string -): Promise> { - const client = createClient({ - baseUrl, - token: branchToken, - }); - - for (let attempt = 0; attempt < 10; attempt++) { - const result = await client.sql<{ level: string }>( - ` - SELECT level - FROM raw_logs - WHERE body LIKE '%${traceId}%' - LIMIT 1 - ` - ); - - if (result.data.length > 0) { - return result.data; - } - - await new Promise((resolve) => setTimeout(resolve, 1_000)); - } - - return []; -} - function expectBuildSuccess(buildResult: Awaited>): void { if (!buildResult.success) { throw new Error(buildResult.error ?? "Build failed without an error message."); @@ -252,41 +175,4 @@ describeLive("E2E Live: build", () => { const rows = await waitForPipeData(branchToken, config.baseUrl); expect(rows.length).toBeGreaterThan(0); }); - - it("deploys a datasource with a materialized column and computes it on ingest", async () => { - writeMaterializedColumnProject(tempDir, config.baseUrl); - - const buildResult = await runBuild({ cwd: tempDir }); - expectBuildSuccess(buildResult); - expect(buildResult.deploy?.success).toBe(true); - expect(buildResult.branchInfo?.tinybirdBranch).toBeTruthy(); - - const tinybirdBranchName = buildResult.branchInfo?.tinybirdBranch ?? ""; - tinybirdBranchNames.push(tinybirdBranchName); - - const branch = await getBranch( - { baseUrl: config.baseUrl, token: workspaceToken }, - tinybirdBranchName - ); - expect(branch.token).toBeTruthy(); - - const branchToken = branch.token as string; - const traceId = `materialized-column-${Date.now()}`; - const client = createClient({ - baseUrl: config.baseUrl, - token: branchToken, - }); - - await client.ingest("raw_logs", { - timestamp: toTinybirdDateTime(new Date()), - body: JSON.stringify({ level: "warn", trace_id: traceId }), - }); - - const rows = await waitForMaterializedColumnData( - branchToken, - config.baseUrl, - traceId - ); - expect(rows).toEqual([{ level: "warn" }]); - }); }); From f554d0eb18ecdcc38cc6e6ed8c833aea52bd15cd Mon Sep 17 00:00:00 2001 From: Rafa Moreno Date: Tue, 12 May 2026 10:27:06 +0200 Subject: [PATCH 5/6] Assert full materialized datasource output --- src/generator/datasource.test.ts | 33 ++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/generator/datasource.test.ts b/src/generator/datasource.test.ts index 3ee3a1a..b8f282c 100644 --- a/src/generator/datasource.test.ts +++ b/src/generator/datasource.test.ts @@ -207,6 +207,39 @@ describe('Datasource Generator', () => { expect(result.content).not.toContain("`json:$.level`"); }); + it("generates the full datasource file for a schema with a materialized column", () => { + const ds = defineDatasource("raw_logs", { + description: "Raw logs with extracted severity", + schema: { + timestamp: t.dateTime(), + body: t.string(), + service_name: t.string().jsonPath("$.resource.service.name"), + level: t + .string() + .lowCardinality() + .materialized("JSONExtractString(body, 'level')"), + }, + engine: engine.mergeTree({ + sortingKey: ["service_name", "level", "timestamp"], + partitionKey: "toYYYYMM(timestamp)", + }), + }); + + const result = generateDatasource(ds); + expect(result.content).toBe(`DESCRIPTION > + Raw logs with extracted severity + +SCHEMA > + timestamp DateTime \`json:$.timestamp\`, + body String \`json:$.body\`, + service_name String \`json:$.resource.service.name\`, + level LowCardinality(String) MATERIALIZED JSONExtractString(body, 'level') + +ENGINE "MergeTree" +ENGINE_PARTITION_KEY "toYYYYMM(timestamp)" +ENGINE_SORTING_KEY "service_name, level, timestamp"`); + }); + it("includes explicit JSON paths on materialized columns", () => { const ds = defineDatasource("test_ds", { schema: { From a6a4378c0cc940ec8ff516e1c6c35b8b525e53cc Mon Sep 17 00:00:00 2001 From: Rafa Moreno Date: Tue, 12 May 2026 10:31:03 +0200 Subject: [PATCH 6/6] Bump package version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1fdfba1..9899810 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tinybirdco/sdk", - "version": "0.0.69", + "version": "0.0.70", "description": "TypeScript SDK for Tinybird Forward - define datasources and pipes as TypeScript", "type": "module", "main": "./dist/index.js",