From 879c9afee8b019e94e85e7e209401c34f9854448 Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Fri, 8 May 2026 10:32:10 +0100 Subject: [PATCH 1/5] fix(firestore-bigquery-export): accept ISO 8601 string partition values Restores 0.2.x behavior for Firestore string partition fields. The 0.3.0 refactor of PartitionValueConverter narrowed accepted inputs to Firestore Timestamp / timestamp-like / Date, silently coercing strings (including ISO 8601 dates such as "2026-01-01") to NULL and landing rows in the __NULL__ partition. PartitionValueConverter.convert now parses strings via new Date(value). Unparseable strings still return null and trigger the existing warning. Adds a defensive try/catch around the BigQuery formatter switch so any serialization failure degrades to null + warn rather than crashing the row write. Fixes #2803 --- firestore-bigquery-export/CHANGELOG.md | 4 ++ .../bigquery/partitioning/converter.test.ts | 63 ++++++++++++++++++- .../src/bigquery/partitioning/converter.ts | 24 ++++--- 3 files changed, 82 insertions(+), 9 deletions(-) diff --git a/firestore-bigquery-export/CHANGELOG.md b/firestore-bigquery-export/CHANGELOG.md index 6f6e48a63..38465ee0a 100644 --- a/firestore-bigquery-export/CHANGELOG.md +++ b/firestore-bigquery-export/CHANGELOG.md @@ -1,3 +1,7 @@ +## Version 0.3.2 + +fix: restore acceptance of ISO 8601 date/datetime strings as partition field values, regression introduced in 0.3.0 (#2803) + ## Version 0.3.1 chore: bump dependencies diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts index 19fe6604a..570ea4ab1 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts @@ -42,9 +42,22 @@ describe("PartitionValueConverter", () => { expect(result).toBeNull(); }); - test("returns null for string", () => { + test("converts ISO 8601 datetime string to BigQuery timestamp string", () => { + const result = converter.convert("2024-01-15T10:30:00Z"); + expect(result).toContain("2024-01-15"); + }); + + test("converts ISO 8601 date-only string to BigQuery timestamp string", () => { const result = converter.convert("2024-01-15"); - expect(result).toBeNull(); + expect(result).toContain("2024-01-15"); + }); + + test("returns null for unparseable string", () => { + expect(converter.convert("not-a-date")).toBeNull(); + }); + + test("returns null for empty string", () => { + expect(converter.convert("")).toBeNull(); }); test("returns null for null", () => { @@ -104,6 +117,32 @@ describe("PartitionValueConverter", () => { const result = converter.convert(date); expect(result).toBe("2024-01-15"); }); + + test("converts ISO 8601 date-only string to BigQuery date string", () => { + const result = converter.convert("2024-01-15"); + expect(result).toBe("2024-01-15"); + }); + + test("converts ISO 8601 datetime string to BigQuery date string", () => { + const result = converter.convert("2024-01-15T10:30:00Z"); + expect(result).toBe("2024-01-15"); + }); + + test("uses UTC date component for timezone-suffixed datetime string", () => { + // 2024-01-15T22:00:00-08:00 == 2024-01-16T06:00:00Z. The DATE column + // takes the UTC date component, matching how Firestore Timestamps are + // handled. Pinned so future changes to this contract are explicit. + const result = converter.convert("2024-01-15T22:00:00-08:00"); + expect(result).toBe("2024-01-16"); + }); + + test("returns null for unparseable string", () => { + expect(converter.convert("not-a-date")).toBeNull(); + }); + + test("returns null for empty string", () => { + expect(converter.convert("")).toBeNull(); + }); }); describe("convert with DATETIME type", () => { @@ -134,5 +173,25 @@ describe("PartitionValueConverter", () => { expect(result).toBeDefined(); expect(result).toContain("2024-01-15"); }); + + test("converts ISO 8601 datetime string to BigQuery datetime string", () => { + const result = converter.convert("2024-01-15T10:30:00Z"); + expect(result).toBeDefined(); + expect(result).toContain("2024-01-15"); + }); + + test("converts ISO 8601 date-only string to BigQuery datetime string", () => { + const result = converter.convert("2024-01-15"); + expect(result).toBeDefined(); + expect(result).toContain("2024-01-15"); + }); + + test("returns null for unparseable string", () => { + expect(converter.convert("not-a-date")).toBeNull(); + }); + + test("returns null for empty string", () => { + expect(converter.convert("")).toBeNull(); + }); }); }); diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts index c1a6a4849..8c7b31f94 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts @@ -27,17 +27,27 @@ export class PartitionValueConverter { ).toDate(); } else if (value instanceof Date && !isNaN(value.getTime())) { date = value; + } else if (typeof value === "string") { + const parsed = new Date(value); + if (isNaN(parsed.getTime())) { + return null; + } + date = parsed; } else { return null; } - switch (this.fieldType) { - case "DATETIME": - return BigQuery.datetime(date.toISOString()).value; - case "DATE": - return BigQuery.date(date.toISOString().substring(0, 10)).value; - case "TIMESTAMP": - return BigQuery.timestamp(date).value; + try { + switch (this.fieldType) { + case "DATETIME": + return BigQuery.datetime(date.toISOString()).value; + case "DATE": + return BigQuery.date(date.toISOString().substring(0, 10)).value; + case "TIMESTAMP": + return BigQuery.timestamp(date).value; + } + } catch { + return null; } } } From d0654faf96c90237cb42c8553af70bf7d3b6dcb2 Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Fri, 8 May 2026 10:41:10 +0100 Subject: [PATCH 2/5] fix(firestore-bigquery-export): explicit default in partition value switch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds default: return null to PartitionValueConverter.convert. fieldType is typed as a strict union, so exhaustiveness is already enforced at compile time, but the value comes from external config — a runtime mismatch falls through cleanly to null instead of returning undefined. Per gemini-code-assist review on #2813. --- .../src/bigquery/partitioning/converter.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts index 8c7b31f94..aaed055fc 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts @@ -45,6 +45,8 @@ export class PartitionValueConverter { return BigQuery.date(date.toISOString().substring(0, 10)).value; case "TIMESTAMP": return BigQuery.timestamp(date).value; + default: + return null; } } catch { return null; From 2d62fdbf88facd9aa65d46d24d642919fa72f96f Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Fri, 8 May 2026 10:46:02 +0100 Subject: [PATCH 3/5] chore(firestore-bigquery-export): bump version to 0.3.2 --- firestore-bigquery-export/extension.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/firestore-bigquery-export/extension.yaml b/firestore-bigquery-export/extension.yaml index 0da2359f0..506e4caf5 100644 --- a/firestore-bigquery-export/extension.yaml +++ b/firestore-bigquery-export/extension.yaml @@ -13,7 +13,7 @@ # limitations under the License. name: firestore-bigquery-export -version: 0.3.1 +version: 0.3.2 specVersion: v1beta displayName: Stream Firestore to BigQuery From 4392a4f7d60649f6840295081a32adbe893d5d38 Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Fri, 8 May 2026 11:39:43 +0100 Subject: [PATCH 4/5] fix(firestore-bigquery-export): strict ISO 8601 validation for partition strings JavaScript's Date parser is too permissive for partition value validation: new Date("2024-02-30") -> 2024-03-01 (silent month overflow) new Date("2024-01") -> 2024-01-01 (silent partial-date fill) new Date("1") -> 2001-01-01 (bare numeric as year) new Date("2023-02-29") -> 2023-03-01 (non-leap-year overflow) new Date("2024-01-15T10:30") -> engine-dependent local-time interpretation Replaces the loose new Date() check with a strict YYYY-MM-DD prefix regex that requires an explicit timezone designator (Z or +/-HH:MM) when a time component is present, plus a calendar validator built via setUTCFullYear that rejects calendar-invalid components like Feb 30, non-leap-year Feb 29, month 13, and day 32. Per CorieW review on #2813. --- .../bigquery/partitioning/converter.test.ts | 37 +++++++++++++++++++ .../src/bigquery/partitioning/converter.ts | 26 +++++++++++++ 2 files changed, 63 insertions(+) diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts index 570ea4ab1..7d6e7400f 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts @@ -60,6 +60,43 @@ describe("PartitionValueConverter", () => { expect(converter.convert("")).toBeNull(); }); + test("returns null for partial date (year-month only)", () => { + expect(converter.convert("2024-01")).toBeNull(); + }); + + test("returns null for partial date (year only)", () => { + expect(converter.convert("2024")).toBeNull(); + }); + + test("returns null for bare numeric string", () => { + expect(converter.convert("1")).toBeNull(); + }); + + test("returns null for calendar-invalid date (Feb 30)", () => { + expect(converter.convert("2024-02-30")).toBeNull(); + }); + + test("returns null for non-leap-year Feb 29", () => { + expect(converter.convert("2023-02-29")).toBeNull(); + }); + + test("accepts leap-year Feb 29", () => { + const result = converter.convert("2024-02-29"); + expect(result).toContain("2024-02-29"); + }); + + test("returns null for out-of-range month", () => { + expect(converter.convert("2024-13-01")).toBeNull(); + }); + + test("returns null for out-of-range day", () => { + expect(converter.convert("2024-01-32")).toBeNull(); + }); + + test("returns null for datetime without timezone", () => { + expect(converter.convert("2024-01-15T10:30:00")).toBeNull(); + }); + test("returns null for null", () => { const result = converter.convert(null); expect(result).toBeNull(); diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts index aaed055fc..2124c7a0a 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts @@ -28,6 +28,32 @@ export class PartitionValueConverter { } else if (value instanceof Date && !isNaN(value.getTime())) { date = value; } else if (typeof value === "string") { + // Strict ISO 8601 / RFC 3339: YYYY-MM-DD, optionally followed by T or + // space-separated HH:MM[:SS[.ffffff]] and a required timezone designator + // when the time component is present. JS Date parsing alone is too + // permissive — it silently normalizes invalid inputs (e.g. "2024-02-30" + // → "2024-03-01"), accepts partial dates ("2024-01"), and reads bare + // numerics as years ("1" → "2001-01-01"). Reject all of those. + const m = value.match( + /^(\d{4})-(\d{2})-(\d{2})(?:[T ](\d{2}):(\d{2})(?::(\d{2})(?:\.\d+)?)?(Z|[+-]\d{2}:?\d{2}))?$/ + ); + if (!m) { + return null; + } + const yearN = Number(m[1]); + const monthN = Number(m[2]); + const dayN = Number(m[3]); + // Reject calendar-invalid components (Feb 30, non-leap Feb 29, etc.). + // setUTCFullYear avoids the legacy 2-digit-year quirk of Date.UTC(). + const validator = new Date(0); + validator.setUTCFullYear(yearN, monthN - 1, dayN); + if ( + validator.getUTCFullYear() !== yearN || + validator.getUTCMonth() + 1 !== monthN || + validator.getUTCDate() !== dayN + ) { + return null; + } const parsed = new Date(value); if (isNaN(parsed.getTime())) { return null; From fba4ca3a263107778921428e6e795c6f92ed682a Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Fri, 8 May 2026 11:48:36 +0100 Subject: [PATCH 5/5] fix(firestore-bigquery-export): reject year 0 in partition strings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BigQuery DATE / DATETIME / TIMESTAMP all reject year 0 — the supported range is 0001-01-01 to 9999-12-31. The previous strict validator passed "0000-01-01" through (setUTCFullYear(0, 0, 1) yields year 0, matching input components) but BigQuery rejects it server-side, causing the same NULL-partition symptom this PR is fixing. Reject yearN < 1 client-side so the user gets a clear warning log. --- .../bigquery/partitioning/converter.test.ts | 14 ++++++++++++++ .../src/bigquery/partitioning/converter.ts | 6 ++++++ 2 files changed, 20 insertions(+) diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts index 7d6e7400f..f1a70b1c1 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/__tests__/bigquery/partitioning/converter.test.ts @@ -93,6 +93,20 @@ describe("PartitionValueConverter", () => { expect(converter.convert("2024-01-32")).toBeNull(); }); + test("returns null for year 0 (outside BigQuery DATE range)", () => { + expect(converter.convert("0000-01-01")).toBeNull(); + }); + + test("accepts year 0001 (BigQuery DATE minimum)", () => { + const result = converter.convert("0001-01-01"); + expect(result).toContain("0001-01-01"); + }); + + test("accepts year 9999 (BigQuery DATE maximum)", () => { + const result = converter.convert("9999-12-31"); + expect(result).toContain("9999-12-31"); + }); + test("returns null for datetime without timezone", () => { expect(converter.convert("2024-01-15T10:30:00")).toBeNull(); }); diff --git a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts index 2124c7a0a..2ede51498 100644 --- a/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts +++ b/firestore-bigquery-export/firestore-bigquery-change-tracker/src/bigquery/partitioning/converter.ts @@ -43,6 +43,12 @@ export class PartitionValueConverter { const yearN = Number(m[1]); const monthN = Number(m[2]); const dayN = Number(m[3]); + // BigQuery DATE / DATETIME / TIMESTAMP all reject year 0 — the supported + // range is 0001-01-01 to 9999-12-31. Reject client-side so the row gets + // a clear warning instead of a server-side insert error. + if (yearN < 1) { + return null; + } // Reject calendar-invalid components (Feb 30, non-leap Feb 29, etc.). // setUTCFullYear avoids the legacy 2-digit-year quirk of Date.UTC(). const validator = new Date(0);