From be49605f871881eae98b0b0dea55743adde1b9d1 Mon Sep 17 00:00:00 2001 From: fuzmish <71511927+fuzmish@users.noreply.github.com> Date: Sat, 1 Aug 2026 06:29:58 +0900 Subject: [PATCH] Fix incremental table compilation failing on metadata.extraProperties --- common/protos/index.ts | 8 ++-- core/actions/incremental_table.ts | 8 +++- core/actions/incremental_table_test.ts | 63 ++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 4 deletions(-) diff --git a/common/protos/index.ts b/common/protos/index.ts index 2bf2777b2..7d34336bb 100644 --- a/common/protos/index.ts +++ b/common/protos/index.ts @@ -33,8 +33,7 @@ const Struct = google.protobuf.Struct; // Save references to the original generated methods const originalVerify = Struct.verify; -// Monkey Patching Methods -Struct.verify = function (object: any) { +export function verifyStruct(this: any, object: any) { if (object && typeof object === "object" && !("fields" in object)) { const fields: { [key: string]: any } = {}; for (const [k, v] of Object.entries(object)) { @@ -44,7 +43,10 @@ Struct.verify = function (object: any) { object.fields = fields; } return originalVerify.call(this, object); -}; +} + +// Monkey Patching Methods +Struct.verify = verifyStruct; // This is a minimalist Typescript equivalent for the validation part of Profobuf's JsonFormat's // mergeMessage method: diff --git a/core/actions/incremental_table.ts b/core/actions/incremental_table.ts index 3ae9560f8..6fb5ab8c0 100644 --- a/core/actions/incremental_table.ts +++ b/core/actions/incremental_table.ts @@ -1,4 +1,4 @@ -import { verifyObjectMatchesProto, VerifyProtoErrorBehaviour } from "df/common/protos"; +import { verifyObjectMatchesProto, VerifyProtoErrorBehaviour, verifyStruct } from "df/common/protos"; import { ActionBuilder, checkConfigAdditionalOptionsOverlap, @@ -692,6 +692,12 @@ export class IncrementalTable extends ActionBuilder { } } + // Since IncrementalTableConfig.metadata.extraProperties expects IMetadata, + // we will normalize the expression into google.protobuf.IStruct here. + if (unverifiedConfig.metadata?.extraProperties) { + verifyStruct(unverifiedConfig.metadata.extraProperties); + } + const config = verifyObjectMatchesProto( dataform.ActionConfig.IncrementalTableConfig, unverifiedConfig, diff --git a/core/actions/incremental_table_test.ts b/core/actions/incremental_table_test.ts index b80ceab43..c08cd21bc 100644 --- a/core/actions/incremental_table_test.ts +++ b/core/actions/incremental_table_test.ts @@ -211,6 +211,69 @@ SELECT 1` }); }); + test("onSchemaChange combined with metadata.extraProperties as a plain object", () => { + const tableName = "on_schema_change_with_metadata"; + const tableContent = ` +config { + type: "incremental", + onSchemaChange: "IGNORE", + metadata: { + extraProperties: { + myField: "example" + } + } +} + +SELECT 1`; + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, `definitions/${tableName}.sqlx`), + tableContent + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect(asPlainObject(result.compile.compiledGraph.tables)).deep.equals([ + { + target: { + database: "defaultProject", + schema: "defaultDataset", + name: tableName + }, + canonicalTarget: { + database: "defaultProject", + schema: "defaultDataset", + name: tableName + }, + type: "incremental", + disabled: false, + protected: false, + hermeticity: "NON_HERMETIC", + onSchemaChange: "IGNORE", + enumType: "INCREMENTAL", + fileName: `definitions/${tableName}.sqlx`, + query: "\n\n\nSELECT 1", + incrementalQuery: "\n\n\nSELECT 1", + incrementalStrategy: "INCREMENTAL_STRATEGY_UNSPECIFIED", + actionDescriptor: { + metadata: { + extraProperties: { + fields: { + myField: { stringValue: "example" } + } + } + } + } + } + ]); + }); + test("sqlx minimal config", () => { const minimalIncrementalTableName = "minimal_incremental"; const minimalIncrementalTableContent = `