From d01798b9dffea15c722e9089a396ff740e93c59e Mon Sep 17 00:00:00 2001 From: Makisuo Date: Thu, 20 Aug 2026 13:15:46 +0200 Subject: [PATCH 1/6] fix(errors): recognise Apple crash frames in the error fingerprint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FRAME_LINE_PATTERN matches stack frames by shape, one alternative per runtime — V8/JVM, Python, Ruby, Firefox/Safari, Go/Rust. It had none for Apple's, so no frame of an iOS crash matched anything, _fpFrames came out empty, and the hash fell through to _msgFallback. That branch redacts hex and long digit runs, which normalises `EXC_BAD_ACCESS at 0x10` and `at 0xdeadbeef` to the same string. The result was one issue per exception type per service, however many distinct bugs were behind it. Measured against a local store: four crashes with different stacks and different messages all produced fingerprint 10540390751701351631, every one with an empty TopFrame. The new alternative keys on frame index, binary name and hex address: ^[0-9]+ +[^ ]+ +0x[0-9a-fA-F]+ There is no source position to key on. An iOS crash arrives unsymbolicated, because the app's symbols live in a dSYM that never leaves the build machine. FRAME_REDACTIONS then erases the address and the (deliberately hex-rendered) offset, so a frame reduces to `index binary +` and grouping keys on the sequence of binaries. Coarse, but stable across releases — raw offsets shift with any code change and would re-split every iOS issue on every build. When dSYM symbolication lands, function names drop into the same slot. Verified end to end against a fresh local store: the same crash site on two builds now shares a fingerprint, a different site separates, and a stack with no frames still falls back to the message. FINGERPRINT_VERSION goes 2 -> 3 in the same change; only iOS hashes rotate, no other runtime's frame matching is touched. Nothing is backfilled, for the reason 0003 and 0016 give — recomputing FingerprintHash would re-bucket every existing issue, and the version bump retires the collapsed iOS ones instead. Migration 0018 recreates both frozen MVs for deployed servers, and the local store gets the matching v7 -> v8 edge. --- apps/cli/src/server/local-schema-history.ts | 7 + apps/cli/src/server/local-schema-version.ts | 2 +- apps/cli/src/server/local-store-migrations.ts | 2 + .../v7-to-v8-apple-crash-frames.ts | 257 +++ apps/cli/src/server/schema-identity.ts | 16 +- apps/cli/src/server/schema/local-inserts.json | 2 +- .../cli/src/server/schema/local-schema-v8.sql | 1842 +++++++++++++++++ apps/cli/src/server/schema/local-schema.sql | 8 +- apps/cli/test/local-store-migrations.test.ts | 27 +- apps/ingest/src/clickhouse_insert_mappings.rs | 2 +- .../migrations/0018_apple_crash_frames.ts | 45 + .../src/clickhouse/migrations/index.test.ts | 35 +- .../domain/src/clickhouse/migrations/index.ts | 2 + .../domain/src/generated/clickhouse-schema.ts | 6 +- .../domain/src/tinybird/fingerprint.test.ts | 68 + packages/domain/src/tinybird/fingerprint.ts | 29 +- 16 files changed, 2323 insertions(+), 27 deletions(-) create mode 100644 apps/cli/src/server/local-store-migrations/v7-to-v8-apple-crash-frames.ts create mode 100644 apps/cli/src/server/schema/local-schema-v8.sql create mode 100644 packages/domain/src/clickhouse/migrations/0018_apple_crash_frames.ts diff --git a/apps/cli/src/server/local-schema-history.ts b/apps/cli/src/server/local-schema-history.ts index 337e192f8..6ffdb0da7 100644 --- a/apps/cli/src/server/local-schema-history.ts +++ b/apps/cli/src/server/local-schema-history.ts @@ -72,4 +72,11 @@ export const LOCAL_SCHEMA_HISTORY: ReadonlyArray = Obje manifestDigest: "2814f51596f9eabcdb32fb992249b9c8378d15f0c2a60dcbc2ccb49f3ad9dae6", projectRevision: "73f8f3249a508cd05598289b67b3773a049e38db0302efa6aa9e45e3501d2182", }), + Object.freeze({ + version: 8, + fingerprint: "d0750014b0f79c57", + digest: "d0750014b0f79c57995c31fcdb091049d3130dc97366c07c003e4d1198aa6660", + manifestDigest: "ce9c98712f66bbdd90e204da021c9dfe57721cc0b514a837b23c4f95ae51bb50", + projectRevision: "e37cc45679299abbc1545d48f0256d5e60c062e171a2d358077a35eb0d072ae6", + }), ] as const) diff --git a/apps/cli/src/server/local-schema-version.ts b/apps/cli/src/server/local-schema-version.ts index e3571c962..8bbe13f03 100644 --- a/apps/cli/src/server/local-schema-version.ts +++ b/apps/cli/src/server/local-schema-version.ts @@ -1,4 +1,4 @@ // Increment this value for every structural change to the generated local // schema. The compatibility manifest and migration registry must be updated in // the same change before a new value can ship. -export const LOCAL_SCHEMA_VERSION = 7 as const +export const LOCAL_SCHEMA_VERSION = 8 as const diff --git a/apps/cli/src/server/local-store-migrations.ts b/apps/cli/src/server/local-store-migrations.ts index 1b9e1ad05..162856db3 100644 --- a/apps/cli/src/server/local-store-migrations.ts +++ b/apps/cli/src/server/local-store-migrations.ts @@ -40,6 +40,7 @@ import { v3ToV4WebEventsModule } from "./local-store-migrations/v3-to-v4-web-eve import { v4ToV5ServiceOverviewMinutelyModule } from "./local-store-migrations/v4-to-v5-service-overview-minutely" import { v5ToV6ErrorEventsFingerprintHygieneModule } from "./local-store-migrations/v5-to-v6-error-events-fingerprint-hygiene" import { v6ToV7ErrorServiceVersionModule } from "./local-store-migrations/v6-to-v7-error-service-version" +import { v7ToV8AppleCrashFramesModule } from "./local-store-migrations/v7-to-v8-apple-crash-frames" import type { AnyLocalStoreMigrationModule, LocalStoreMigration, @@ -126,6 +127,7 @@ export const localStoreMigrations: ReadonlyArray = v4ToV5ServiceOverviewMinutelyModule, v5ToV6ErrorEventsFingerprintHygieneModule, v6ToV7ErrorServiceVersionModule, + v7ToV8AppleCrashFramesModule, ] export const validateMigrationRegistry = ( diff --git a/apps/cli/src/server/local-store-migrations/v7-to-v8-apple-crash-frames.ts b/apps/cli/src/server/local-store-migrations/v7-to-v8-apple-crash-frames.ts new file mode 100644 index 000000000..8677f1563 --- /dev/null +++ b/apps/cli/src/server/local-store-migrations/v7-to-v8-apple-crash-frames.ts @@ -0,0 +1,257 @@ +// SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. +import { cp, mkdir, rm } from "node:fs/promises" +import { dirname, resolve } from "node:path" +import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" +import type { + LocalStoreMigrationModule, + MigrationModuleContext, + MigrationOperation, + StateDispositionEntry, +} from "../local-store-migration-module" +import { withRawTelemetryRetentionFloor } from "../schema-manifest" +import { + LOCAL_SCHEMA_V7, + LOCAL_SCHEMA_V7_MANIFEST, + LOCAL_SCHEMA_V7_SQL, + LOCAL_SCHEMA_V8, + LOCAL_SCHEMA_V8_MANIFEST, + LOCAL_SCHEMA_V8_SQL, +} from "../schema-identity" +import { assertPhysicalSchema } from "../schema-physical" + +const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) + +interface V7ToV8State { + readonly module: "local-0007-to-0008-apple-crash-frames" + readonly version: 1 + readonly rawRows: Readonly> + readonly retentionDays?: number +} + +interface V7ToV8Progress { + readonly installed: true +} + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) + +const decodeCounts = (value: unknown): Readonly> => { + if (!isRecord(value)) throw new Error("v7 -> v8 rawRows must be an object") + const counts: Record = {} + for (const table of RAW_TABLES) { + const count = value[table] + if (typeof count !== "string" || !/^\d+$/.test(count)) + throw new Error(`v7 -> v8 rawRows.${table} must be an unsigned decimal string`) + counts[table] = count + } + if (Object.keys(value).some((table) => !RAW_TABLES.includes(table as (typeof RAW_TABLES)[number]))) + throw new Error("v7 -> v8 rawRows contains an unknown table") + return counts +} + +const decodeState = (value: unknown): V7ToV8State => { + if (!isRecord(value)) throw new Error("v7 -> v8 state must be an object") + const allowed = new Set(["module", "version", "rawRows", "retentionDays"]) + if (Object.keys(value).some((key) => !allowed.has(key))) + throw new Error("v7 -> v8 state contains an unknown field") + if (value.module !== "local-0007-to-0008-apple-crash-frames" || value.version !== 1) + throw new Error("v7 -> v8 state has an unsupported module or version") + if ( + value.retentionDays !== undefined && + (typeof value.retentionDays !== "number" || !Number.isSafeInteger(value.retentionDays)) + ) + throw new Error("v7 -> v8 retentionDays must be an integer") + return { + module: "local-0007-to-0008-apple-crash-frames", + version: 1, + rawRows: decodeCounts(value.rawRows), + ...(!(value.retentionDays === undefined) ? { retentionDays: value.retentionDays } : undefined), + } +} + +const decodeProgress = (value: unknown): V7ToV8Progress | undefined => { + if (value === undefined) return undefined + if (!isRecord(value) || Object.keys(value).some((key) => key !== "installed") || value.installed !== true) + throw new Error("v7 -> v8 progress is invalid") + return { installed: true } +} + +const parseJsonEachRow = (value: string): A[] => + value + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line) as A) + +const rawRowCounts = (db: Chdb): Readonly> => { + const quotedTables = RAW_TABLES.map((table) => `'${table}'`).join(", ") + const rows = parseJsonEachRow<{ table: string; rowCount: string }>( + db.query( + `SELECT table, toString(sum(rows)) AS rowCount FROM system.parts WHERE database = 'default' AND active = 1 AND table IN (${quotedTables}) GROUP BY table`, + ), + ) + const byTable = new Map(rows.map((row) => [row.table, row.rowCount])) + return Object.fromEntries(RAW_TABLES.map((table) => [table, byTable.get(table) ?? "0"])) +} + +const expectedManifest = (manifest: typeof LOCAL_SCHEMA_V7_MANIFEST, retentionDays: number | undefined) => + retentionDays === undefined + ? manifest + : withRawTelemetryRetentionFloor(manifest, RAW_TABLES, retentionDays) + +const preflight = async (context: MigrationModuleContext): Promise => { + await context.ensureCapacity() + const retentionDays = readRawTelemetryRetentionDays(context.dataDir) + const rawRows = await context.openSource( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V7_MANIFEST, retentionDays)) + return rawRowCounts(db) + }, + { schemaSql: LOCAL_SCHEMA_V7_SQL, bootstrapSchema: false }, + ) + return { + module: "local-0007-to-0008-apple-crash-frames", + version: 1, + rawRows, + ...(!(retentionDays === undefined) ? { retentionDays } : undefined), + } +} + +const prepareTarget = async (context: MigrationModuleContext, state: V7ToV8State): Promise => { + await context.closeStores() + const source = resolve(context.sourceDataDir) + const target = resolve(context.targetDataDir) + if (source !== target) { + await rm(target, { recursive: true, force: true }) + await mkdir(dirname(target), { recursive: true, mode: 0o700 }) + await cp(source, target, { recursive: true, preserveTimestamps: true }) + } + return state +} + +/** + * Like v5 -> v7, this edge replaces the body of two existing views rather than + * adding anything. A materialized view's SELECT is frozen at creation and the + * bundled DDL uses `CREATE ... IF NOT EXISTS`, so both views must be dropped + * before the v8 schema can install its versions. Dropping a view never touches + * rows already in its target table. + * + * The new body adds an Apple alternative to the fingerprint's frame matcher. + * Before it, no frame of an iOS crash matched any alternative, so `_fpFrames` + * was empty and the hash fell through to the message signature — which redacts + * hex and long digit runs, collapsing every crash of an exception type in a + * service into a single issue. + * + * Historical `error_events` / `error_events_by_time` rows are left exactly as + * they are: recomputing FingerprintHash would re-bucket every existing local + * issue. Forward-only, converging as the retention window rolls, and matching + * what a deployed cluster gets from ClickHouse migration 0018. + */ +const apply = async (context: MigrationModuleContext): Promise => { + await context.openTarget( + (db) => { + db.exec("DROP TABLE IF EXISTS error_events_mv") + db.exec("DROP TABLE IF EXISTS error_events_by_time_mv") + }, + { schemaSql: LOCAL_SCHEMA_V7_SQL, bootstrapSchema: false }, + ) + return context.openTarget(() => ({ installed: true }), { + schemaSql: LOCAL_SCHEMA_V8_SQL, + bootstrapSchema: true, + }) +} + +const verify = async ( + context: MigrationModuleContext, + state: V7ToV8State, + _progress: V7ToV8Progress, +): Promise => { + await context.openTarget( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V8_MANIFEST, state.retentionDays)) + const targetRows = rawRowCounts(db) + for (const table of RAW_TABLES) { + if (targetRows[table] !== state.rawRows[table]) + throw new Error(`v7 -> v8 raw telemetry verification failed for ${table}`) + } + }, + { schemaSql: LOCAL_SCHEMA_V8_SQL, bootstrapSchema: false }, + ) +} + +const operations: ReadonlyArray = [ + { + id: "clone-v7-store", + description: "Clone the stopped v7 store into the staged migration target", + requiresQuiescence: true, + phase: "target-created", + }, + { + id: "rebuild-error-events-views", + description: "Drop and recreate the error-events views with Apple crash frames in the fingerprint", + requiresQuiescence: true, + phase: "copying", + }, + { + id: "verify-v8-schema", + description: "Verify the v8 physical schema and retained raw telemetry counts", + requiresQuiescence: true, + phase: "copy-verified", + }, +] + +const dispositions: ReadonlyArray = [ + { + name: "local store", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: "The clean stopped v7 store is cloned byte-for-byte before the views are replaced.", + }, + { + name: "traces", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: + "The source of the replaced views is neither read nor rewritten; only the view definitions change.", + }, + { + // Rows already materialized keep their collapsed iOS fingerprint — + // recomputing hashes would re-bucket every existing issue. Forward-only, + // and bounded by the tables' 90-day TTL. + name: "error_events", + classification: "derived", + disposition: "rebuild-within-retention-horizon", + guarantee: + "Existing rows are preserved untouched; the Apple frame matching applies to events materialized after the migration and converges as the retention window rolls.", + preservationInterval: "error retention horizon", + sourceRetentionDays: 90, + targetRetentionDays: 90, + }, + { + name: "error_events_by_time", + classification: "derived", + disposition: "rebuild-within-retention-horizon", + guarantee: + "Same projection as error_events and treated identically: preserved rows, forward-only correction.", + preservationInterval: "error retention horizon", + sourceRetentionDays: 90, + targetRetentionDays: 90, + }, +] + +export const v7ToV8AppleCrashFramesModule: LocalStoreMigrationModule = { + id: "local-0007-to-0008-apple-crash-frames", + moduleVersion: 1, + description: "Rebuild the error-events views so the fingerprint recognises Apple crash frames", + from: LOCAL_SCHEMA_V7, + to: LOCAL_SCHEMA_V8, + operations, + dispositions, + decodeState, + decodeProgress, + preflight, + prepareTarget, + apply, + verify, + recover: async (_context, state, progress) => ({ state, progress }), +} diff --git a/apps/cli/src/server/schema-identity.ts b/apps/cli/src/server/schema-identity.ts index 0fa21edf9..e944973a9 100644 --- a/apps/cli/src/server/schema-identity.ts +++ b/apps/cli/src/server/schema-identity.ts @@ -6,6 +6,7 @@ import schemaV4Sql from "./schema/local-schema-v4.sql" with { type: "text" } import schemaV5Sql from "./schema/local-schema-v5.sql" with { type: "text" } import schemaV6Sql from "./schema/local-schema-v6.sql" with { type: "text" } import schemaV7Sql from "./schema/local-schema-v7.sql" with { type: "text" } +import schemaV8Sql from "./schema/local-schema-v8.sql" with { type: "text" } import { schemaDigest as digestSchema, schemaFingerprint as fingerprintSchema } from "./store-version" import { buildLocalSchemaManifest, type LocalSchemaManifest } from "./schema-manifest" import { LOCAL_SCHEMA_VERSION } from "./local-schema-version" @@ -29,7 +30,7 @@ export const LEGACY_SCHEMA_PROJECT_REVISION = export const LEGACY_SCHEMA_FINGERPRINT = "428701854f9fd30e" export const CURRENT_SCHEMA_PROJECT_REVISION = - "73f8f3249a508cd05598289b67b3773a049e38db0302efa6aa9e45e3501d2182" + "e37cc45679299abbc1545d48f0256d5e60c062e171a2d358077a35eb0d072ae6" /** Revision recorded by the issue-297 recovery report. The refreshed upstream * generator currently emits CURRENT_SCHEMA_PROJECT_REVISION; the structural * fingerprint is the compatibility identity used by the migration. */ @@ -75,6 +76,10 @@ export const LOCAL_SCHEMA_V6_MANIFEST_DIGEST = LOCAL_SCHEMA_V6_MANIFEST.digest export const LOCAL_SCHEMA_V7_SQL = schemaV7Sql export const LOCAL_SCHEMA_V7_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaV7Sql) export const LOCAL_SCHEMA_V7_MANIFEST_DIGEST = LOCAL_SCHEMA_V7_MANIFEST.digest + +export const LOCAL_SCHEMA_V8_SQL = schemaV8Sql +export const LOCAL_SCHEMA_V8_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaV8Sql) +export const LOCAL_SCHEMA_V8_MANIFEST_DIGEST = LOCAL_SCHEMA_V8_MANIFEST.digest export interface LocalSchemaIdentity { readonly version: number readonly fingerprint: string @@ -152,6 +157,15 @@ export const LOCAL_SCHEMA_V7: LocalSchemaIdentity = Object.freeze({ projectRevision: LOCAL_SCHEMA_HISTORY[7]!.projectRevision, }) +export const LOCAL_SCHEMA_V8: LocalSchemaIdentity = Object.freeze({ + version: LOCAL_SCHEMA_HISTORY[8]!.version, + fingerprint: LOCAL_SCHEMA_HISTORY[8]!.fingerprint, + digest: LOCAL_SCHEMA_HISTORY[8]!.digest, + manifestDigest: LOCAL_SCHEMA_HISTORY[8]!.manifestDigest, + chdb: CHDB_VERSION, + projectRevision: LOCAL_SCHEMA_HISTORY[8]!.projectRevision, +}) + export const CURRENT_LOCAL_SCHEMA: LocalSchemaIdentity = Object.freeze({ version: LOCAL_SCHEMA_VERSION, fingerprint: SCHEMA_FINGERPRINT, diff --git a/apps/cli/src/server/schema/local-inserts.json b/apps/cli/src/server/schema/local-inserts.json index 715dd7a8c..361fa2c8a 100644 --- a/apps/cli/src/server/schema/local-inserts.json +++ b/apps/cli/src/server/schema/local-inserts.json @@ -1,5 +1,5 @@ { - "projectRevision": "73f8f3249a508cd05598289b67b3773a049e38db0302efa6aa9e45e3501d2182", + "projectRevision": "e37cc45679299abbc1545d48f0256d5e60c062e171a2d358077a35eb0d072ae6", "orgPlaceholder": "__ORG__", "datasources": { "traces": { diff --git a/apps/cli/src/server/schema/local-schema-v8.sql b/apps/cli/src/server/schema/local-schema-v8.sql new file mode 100644 index 000000000..eab30295f --- /dev/null +++ b/apps/cli/src/server/schema/local-schema-v8.sql @@ -0,0 +1,1842 @@ +-- This file is generated by scripts/generate-clickhouse-schema-sql.ts +-- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. +-- projectRevision: e37cc45679299abbc1545d48f0256d5e60c062e171a2d358077a35eb0d072ae6 +-- localSchemaVersion: 8 + +CREATE TABLE IF NOT EXISTS alert_checks ( + OrgId LowCardinality(String), + RuleId String, + GroupKey String, + Timestamp DateTime64(3), + Status LowCardinality(String), + SignalType LowCardinality(String), + Comparator LowCardinality(String), + Threshold Float64, + ObservedValue Nullable(Float64), + SampleCount UInt32, + WindowMinutes UInt16, + WindowStart DateTime64(3), + WindowEnd DateTime64(3), + ConsecutiveBreaches UInt16, + ConsecutiveHealthy UInt16, + IncidentId Nullable(String), + IncidentTransition LowCardinality(String), + EvaluationDurationMs UInt32, + ErrorMessage Nullable(String), + ErrorCategory LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, RuleId, GroupKey, Timestamp) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS attribute_keys_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, Hour, AttributeKey) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS attribute_values_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeValue String, + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, AttributeKey, Hour, AttributeValue) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String, + ServiceVersion LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, FingerprintHash, Timestamp) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events_by_time ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String, + ServiceVersion LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, FingerprintHash) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_fingerprints_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + FingerprintHash UInt64, + ServiceName SimpleAggregateFunction(anyLast, String), + ExceptionType SimpleAggregateFunction(anyLast, String), + ExceptionMessage SimpleAggregateFunction(anyLast, String), + ErrorLabel SimpleAggregateFunction(anyLast, String), + TopFrame SimpleAggregateFunction(anyLast, String), + OccurrenceCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime), + ServiceVersions SimpleAggregateFunction(groupUniqArrayArray, Array(String)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Minute) +ORDER BY (OrgId, Minute, FingerprintHash) +TTL Minute + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + StatusMessage String, + Duration UInt64, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, Timestamp) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS logs ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TimestampTime DateTime, + TraceId String, + SpanId String, + TraceFlags UInt8, + SeverityText LowCardinality(String), + SeverityNumber UInt8, + ServiceName LowCardinality(String), + Body String, + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + LogAttributes Map(LowCardinality(String), String), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + LogAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(LogAttributes), mapValues(LogAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 +) +ENGINE = MergeTree +PARTITION BY toDate(TimestampTime) +ORDER BY (OrgId, toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) +TTL toDate(TimestampTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS logs_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SeverityText LowCardinality(String), + DeploymentEnv LowCardinality(String), + Count SimpleAggregateFunction(sum, UInt64), + SizeBytes SimpleAggregateFunction(sum, UInt64), + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS metric_catalog ( + OrgId LowCardinality(String), + Hour DateTime, + MetricType LowCardinality(String), + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription SimpleAggregateFunction(anyLast, String), + MetricUnit SimpleAggregateFunction(anyLast, String), + IsMonotonic SimpleAggregateFunction(anyLast, UInt8), + DataPointCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, MetricType, ServiceName, MetricName, Hour) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_exponential_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + Scale Int32, + ZeroCount UInt64, + PositiveOffset Int32, + PositiveBucketCounts Array(UInt64), + NegativeOffset Int32, + NegativeBucketCounts Array(UInt64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_gauge ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + BucketCounts Array(UInt64), + ExplicitBounds Array(Float64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_sum ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + AggregationTemporality Int32, + IsMonotonic Bool +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_address_resolutions_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + ParentServerAddress String, + ResolvedTargetService LowCardinality(String), + DeploymentEnv LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_external_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + TargetType LowCardinality(String), + TargetSystem LowCardinality(String), + TargetName String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, TargetType, TargetSystem, TargetName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_children ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, ParentSpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_query_shapes_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + QueryKey String, + QueryLabel SimpleAggregateFunction(any, String), + SampleStatement SimpleAggregateFunction(any, String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedCount SimpleAggregateFunction(sum, Float64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSumMs SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace, QueryKey) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, TargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly_ingest ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount UInt64, + ErrorCount UInt64, + DurationSumMs Float64, + MaxDurationMs Float64, + SampledSpanCount UInt64, + UnsampledSpanCount UInt64, + SampleRateSum Float64 +) +ENGINE = Null; + +CREATE TABLE IF NOT EXISTS service_map_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Hour, SpanName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Minute, SpanName) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, Hour, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, Minute, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + ServiceName LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String), + CommitSha LowCardinality(String), + SampleRate Float64 DEFAULT 1, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_platforms_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + K8sCluster SimpleAggregateFunction(max, String), + K8sPodName SimpleAggregateFunction(max, String), + K8sDeploymentName SimpleAggregateFunction(max, String), + K8sStatefulSetName SimpleAggregateFunction(max, String), + K8sDaemonSetName SimpleAggregateFunction(max, String), + K8sNamespaceName SimpleAggregateFunction(max, String), + CloudPlatform SimpleAggregateFunction(max, String), + CloudProvider SimpleAggregateFunction(max, String), + FaasName SimpleAggregateFunction(max, String), + MapleSdkType SimpleAggregateFunction(max, String), + ProcessRuntimeName SimpleAggregateFunction(max, String), + SpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_usage ( + OrgId LowCardinality(String), + ServiceName LowCardinality(String), + Hour DateTime, + LogCount UInt64, + LogSizeBytes UInt64, + TraceCount UInt64, + TraceSizeBytes UInt64, + SumMetricCount UInt64, + SumMetricSizeBytes UInt64, + GaugeMetricCount UInt64, + GaugeMetricSizeBytes UInt64, + HistogramMetricCount UInt64, + HistogramMetricSizeBytes UInt64, + ExpHistogramMetricCount UInt64, + ExpHistogramMetricSizeBytes UInt64 +) +ENGINE = SummingMergeTree +ORDER BY (OrgId, ServiceName, Hour) +TTL Hour + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS session_events ( + OrgId LowCardinality(String), + SessionId String, + Timestamp DateTime64(9), + Seq UInt32 DEFAULT 0, + Type LowCardinality(String), + Url String DEFAULT '', + TraceId String DEFAULT '', + Level LowCardinality(String) DEFAULT '', + Message String DEFAULT '', + TargetSelector String DEFAULT '', + TargetText String DEFAULT '', + NetMethod LowCardinality(String) DEFAULT '', + NetUrl String DEFAULT '', + NetStatus UInt16 DEFAULT 0, + NetDurationMs UInt32 DEFAULT 0, + ErrorStack String DEFAULT '', + Attributes Map(String, String), + INDEX idx_type Type TYPE set(16) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, Timestamp, Seq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replay_events ( + OrgId LowCardinality(String), + SessionId String, + ChunkSeq UInt32, + Timestamp DateTime64(9), + DurationMs UInt32 DEFAULT 0, + EventCount UInt32 DEFAULT 0, + ByteSize UInt32 DEFAULT 0, + Events String, + IsCheckpoint UInt8 DEFAULT 0 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, ChunkSeq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replays ( + OrgId LowCardinality(String), + SessionId String, + StartTime DateTime64(9), + EndTime Nullable(DateTime64(9)), + DurationMs Nullable(UInt32), + Status LowCardinality(String), + UserId String, + UrlInitial String, + UserAgent String, + BrowserName LowCardinality(String), + OsName LowCardinality(String), + DeviceType LowCardinality(String), + Country LowCardinality(String) DEFAULT '', + ServiceName LowCardinality(String), + PageViews UInt32 DEFAULT 0, + ClickCount UInt32 DEFAULT 0, + ErrorCount UInt32 DEFAULT 0, + TraceIds Array(String) DEFAULT [], + ResourceAttributes Map(LowCardinality(String), String), + Version UInt32, + VisitorId String DEFAULT '', + VisitorIsNew UInt8 DEFAULT 0, + UserEmail String DEFAULT '', + UserName String DEFAULT '', + GroupId String DEFAULT '', + GroupName String DEFAULT '', + UserTraits Map(String, String) DEFAULT map(), + Referrer String DEFAULT '', + ReferrerHost LowCardinality(String) DEFAULT '', + UtmSource LowCardinality(String) DEFAULT '', + UtmMedium LowCardinality(String) DEFAULT '', + UtmCampaign LowCardinality(String) DEFAULT '', + UtmTerm String DEFAULT '', + UtmContent String DEFAULT '', + Host LowCardinality(String) DEFAULT '', + EntryPath String DEFAULT '', + ExitPath String DEFAULT '', + Language LowCardinality(String) DEFAULT '', + LastActivityAt Nullable(DateTime64(9)) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(StartTime) +ORDER BY (OrgId, SessionId) +TTL toDate(StartTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS span_metrics_calls_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + SpanKind LowCardinality(String), + AttrFingerprint UInt64, + ResourceFingerprint UInt64, + StartTimeUnix DateTime64(9), + LastValue AggregateFunction(argMax, Float64, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS trace_detail_spans ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + ResourceAttributes Map(LowCardinality(String), String), + EventsTimestamp Array(DateTime64(9)), + EventsName Array(LowCardinality(String)), + EventsAttributes Array(Map(LowCardinality(String), String)) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS trace_list_mv ( + OrgId LowCardinality(String), + TraceId String, + Timestamp DateTime, + ServiceName LowCardinality(String), + SpanName String, + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + HttpMethod LowCardinality(String), + HttpRoute String, + HttpStatusCode LowCardinality(String), + DeploymentEnv LowCardinality(String), + HasError UInt8, + TraceState String, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + TraceState String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + EventsTimestamp Array(DateTime64(9)), + EventsName Array(LowCardinality(String)), + EventsAttributes Array(Map(LowCardinality(String), String)), + LinksTraceId Array(String), + LinksSpanId Array(String), + LinksTraceState Array(String), + LinksAttributes Array(Map(LowCardinality(String), String)), + SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0), + IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp)) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + StatusCode LowCardinality(String), + IsEntryPoint UInt8, + DeploymentEnv LowCardinality(String), + WeightedCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSum SimpleAggregateFunction(sum, Float64), + WeightedErrorCount SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32), + DurationMin SimpleAggregateFunction(min, UInt64), + DurationMax SimpleAggregateFunction(max, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS web_events ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + SessionId String, + Seq UInt32, + Kind LowCardinality(String), + EventName String, + Host LowCardinality(String), + PagePath String, + Url String, + Attributes Map(String, String), + INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, SessionId, Seq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arraySlice( + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +[^ ]+ +0x[0-9a-fA-F]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection for the message signature below. + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + 1, 120 + ) + ) AS _msgSig, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel, + ResourceAttributes['service.version'] AS ServiceVersion + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all three hold: 4xx, no exception event, and no + -- exception type. 5xx and anything carrying an exception still count, + -- and SpanKind is deliberately not consulted — these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND _exType = '' + ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arraySlice( + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +[^ ]+ +0x[0-9a-fA-F]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection for the message signature below. + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + 1, 120 + ) + ) AS _msgSig, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel, + ResourceAttributes['service.version'] AS ServiceVersion + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all three hold: 4xx, no exception event, and no + -- exception type. 5xx and anything carrying an exception still count, + -- and SpanKind is deliberately not consulted — these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND _exType = '' + ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS +SELECT + OrgId, + toStartOfMinute(Timestamp) AS Minute, + FingerprintHash, + anyLast(ServiceName) AS ServiceName, + anyLast(ExceptionType) AS ExceptionType, + anyLast(ExceptionMessage) AS ExceptionMessage, + anyLast(ErrorLabel) AS ErrorLabel, + anyLast(TopFrame) AS TopFrame, + count() AS OccurrenceCount, + min(Timestamp) AS FirstSeen, + max(Timestamp) AS LastSeen, + -- Distinct builds, not a sample: see ServiceVersions on the datasource. + groupUniqArray(ServiceVersion) AS ServiceVersions + FROM error_events + GROUP BY OrgId, Minute, FingerprintHash; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_spans_mv TO error_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + StatusMessage, + Duration, + ResourceAttributes['deployment.environment'] AS DeploymentEnv + FROM traces + WHERE StatusCode = 'Error'; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(LogAttributes)) AS AttributeKey, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + WHERE LogAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + ARRAY JOIN + mapKeys(LogAttributes) AS AttributeKey, + mapValues(LogAttributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS logs_aggregates_hourly_mv TO logs_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(TimestampTime) AS Hour, + ServiceName, + SeverityText, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + count() AS Count, + sum(length(Body) + 200) AS SizeBytes, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM logs + GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + arrayJoin(mapKeys(Attributes)) AS AttributeKey, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + WHERE Attributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + AttributeKey, + AttributeValue, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + ARRAY JOIN + mapKeys(Attributes) AS AttributeKey, + mapValues(Attributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_exp_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'exponential_histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_exponential_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_gauge_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'gauge' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_gauge + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_sum_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'sum' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + anyLast(toUInt8(IsMonotonic)) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_sum + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + multiIf( + SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', 'messaging', + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc', + 'http' + ) AS TargetType, + multiIf( + SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'], + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'], + '' + ) AS TargetSystem, + multiIf( + SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', + if(SpanAttributes['messaging.destination'] != '', SpanAttributes['messaging.destination'], SpanAttributes['messaging.system']), + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', + if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']), + if(SpanAttributes['server.address'] != '', + SpanAttributes['server.address'], + if(SpanAttributes['http.host'] != '', + SpanAttributes['http.host'], + SpanAttributes['url.authority'])) + ) AS TargetName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + sum(SampleRate) AS SampleRateSum + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND SpanAttributes['db.system.name'] = '' + AND ServiceName != '' + AND ( + SpanAttributes['server.address'] != '' + OR SpanAttributes['http.host'] != '' + OR SpanAttributes['url.authority'] != '' + OR SpanAttributes['messaging.destination'] != '' + OR SpanAttributes['messaging.system'] != '' + OR SpanAttributes['rpc.service'] != '' + OR SpanAttributes['rpc.system'] != '' + ) + GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv + HAVING TargetName != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + ResourceAttributes['deployment.environment'] AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') + AND ParentSpanId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + countIf(TraceState LIKE '%th:%') AS SampledSpanCount, + countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount, + sum(SampleRate) AS SampleRateSum + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_query_shapes_hourly_mv TO service_map_db_query_shapes_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + coalesce( + nullIf(SpanAttributes['db.query.fingerprint'], ''), + nullIf(SpanAttributes['db.statement.fingerprint'], ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\'[^\']*\'', '?'), '\\bin\\s*\\([^)]*\\)', 'in (?)'), '[0-9]+(\\.[0-9]+)?', '?'), '\\s+', ' '), '^\\s+|\\s+$', ''))), ''), ''), + toString(cityHash64(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +))) +) AS QueryKey, + any(substring(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +), 1, 220)) AS QueryLabel, + any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(SampleRate) AS EstimatedCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv, QueryKey; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_edges_hourly_ingest_mv TO service_map_edges_hourly AS +SELECT + OrgId, + Hour, + SourceService, + TargetService, + DeploymentEnv, + CallCount, + ErrorCount, + DurationSumMs, + MaxDurationMs, + SampledSpanCount, + UnsampledSpanCount, + SampleRateSum + FROM service_map_edges_hourly_ingest; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_spans_mv TO service_map_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + ResourceAttributes['deployment.environment'] AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_hourly_mv TO service_operations_hourly AS +SELECT + OrgId, + toStartOfHour(Minute) AS Hour, + ServiceName, + DeploymentEnv, + SpanName, + sum(SpanCount) AS SpanCount, + sum(EstimatedSpanCount) AS EstimatedSpanCount, + sum(ErrorCount) AS ErrorCount, + sum(EstimatedErrorCount) AS EstimatedErrorCount, + sum(DurationSum) AS DurationSum, + quantilesTDigestMergeState(0.5, 0.95)(DurationQuantiles) AS DurationQuantiles + FROM service_operations_minutely + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_minutely_mv TO service_operations_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles + FROM traces + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_hourly_mv TO service_overview_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_minutely_mv TO service_overview_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_spans_mv TO service_overview_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + Duration, + StatusCode, + TraceState, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + SampleRate, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster, + max(ResourceAttributes['k8s.pod.name']) AS K8sPodName, + max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName, + max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName, + max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName, + max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName, + max(ResourceAttributes['cloud.platform']) AS CloudPlatform, + max(ResourceAttributes['cloud.provider']) AS CloudProvider, + max(ResourceAttributes['faas.name']) AS FaasName, + max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType, + max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName, + count() AS SpanCount + FROM traces + WHERE ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_logs_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(TimestampTime) AS Hour, + count() AS LogCount, + sum(length(Body) + 200) AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM logs + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_exp_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + count() AS ExpHistogramMetricCount, + count() * 300 AS ExpHistogramMetricSizeBytes + FROM metrics_exponential_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_gauge_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + count() AS GaugeMetricCount, + count() * 150 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_gauge + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + count() AS HistogramMetricCount, + count() * 250 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_sum_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + count() AS SumMetricCount, + count() * 150 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_sum + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_traces_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS TraceCount, + sum(length(SpanName) + 300) AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM traces + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS span_metrics_calls_hourly_mv TO span_metrics_calls_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + ServiceName, + MetricName, + Attributes['span.kind'] AS SpanKind, + cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint, + cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint, + StartTimeUnix, + argMaxState(Value, TimeUnix) AS LastValue + FROM metrics_sum + WHERE MetricName IN ('span.metrics.calls', 'calls') AND IsMonotonic + GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_detail_spans_mv TO trace_detail_spans AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanId, + ParentSpanId, + SpanName, + SpanKind, + ServiceName, + Duration, + StatusCode, + StatusMessage, + SpanAttributes, + ResourceAttributes, + EventsTimestamp, + EventsName, + EventsAttributes + FROM traces; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_list_mv_mv TO trace_list_mv AS +SELECT + OrgId, + TraceId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + if( + (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS')) + AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''), + concat( + if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), + ' ', + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path']) + ), + SpanName + ) AS SpanName, + SpanKind, + Duration, + StatusCode, + if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod, + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute, + if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + toUInt8( + StatusCode = 'Error' + OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500) + OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500) + ) AS HasError, + TraceState, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(ResourceAttributes)) AS AttributeKey, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE ResourceAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(ResourceAttributes) AS AttributeKey, + mapValues(ResourceAttributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(SpanAttributes)) AS AttributeKey, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE SpanAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(SpanAttributes) AS AttributeKey, + mapValues(SpanAttributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS traces_aggregates_hourly_mv TO traces_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + SpanName, + SpanKind, + StatusCode, + IsEntryPoint, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + sum(SampleRate) AS WeightedCount, + sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum, + sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount, + quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles, + min(Duration) AS DurationMin, + max(Duration) AS DurationMax + FROM traces + GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS web_events_mv TO web_events AS +SELECT + OrgId, + Timestamp, + SessionId, + Seq, + Type AS Kind, + if(Type = 'navigation', '$pageview', Message) AS EventName, + domain(Url) AS Host, + path(Url) AS PagePath, + Url, + Attributes + FROM session_events + WHERE Type IN ('navigation', 'custom'); diff --git a/apps/cli/src/server/schema/local-schema.sql b/apps/cli/src/server/schema/local-schema.sql index e0b53e65f..eab30295f 100644 --- a/apps/cli/src/server/schema/local-schema.sql +++ b/apps/cli/src/server/schema/local-schema.sql @@ -1,7 +1,7 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: 73f8f3249a508cd05598289b67b3773a049e38db0302efa6aa9e45e3501d2182 --- localSchemaVersion: 7 +-- projectRevision: e37cc45679299abbc1545d48f0256d5e60c062e171a2d358077a35eb0d072ae6 +-- localSchemaVersion: 8 CREATE TABLE IF NOT EXISTS alert_checks ( OrgId LowCardinality(String), @@ -875,7 +875,7 @@ WITH -- the SQL that actually runs, silently. arraySlice( arrayFilter( - line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+'), + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +[^ ]+ +0x[0-9a-fA-F]+'), splitByChar('\n', _exStack) ), 1, 3 @@ -1015,7 +1015,7 @@ WITH -- the SQL that actually runs, silently. arraySlice( arrayFilter( - line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+'), + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +[^ ]+ +0x[0-9a-fA-F]+'), splitByChar('\n', _exStack) ), 1, 3 diff --git a/apps/cli/test/local-store-migrations.test.ts b/apps/cli/test/local-store-migrations.test.ts index 6dcedd88c..e506454ec 100644 --- a/apps/cli/test/local-store-migrations.test.ts +++ b/apps/cli/test/local-store-migrations.test.ts @@ -15,8 +15,10 @@ import { LOCAL_SCHEMA_V4_MANIFEST, LOCAL_SCHEMA_V5, LOCAL_SCHEMA_V5_MANIFEST, + LOCAL_SCHEMA_V7_MANIFEST, LOCAL_SCHEMA_V6, LOCAL_SCHEMA_V7, + LOCAL_SCHEMA_V8, SCHEMA_DIGEST, SCHEMA_FINGERPRINT, } from "../src/server/schema-identity" @@ -58,16 +60,16 @@ import { tmpdir } from "node:os" import { join } from "node:path" describe("current local schema identity", () => { - it("matches the generated v7 revision and keeps the issue-297 identity frozen", () => { - expect(SCHEMA_FINGERPRINT).toBe("bc124f30765c8c56") - expect(SCHEMA_DIGEST).toBe("bc124f30765c8c567daccab1872a0e15afbc8ef2123c264bcb5bcdd8d16b6c3c") + it("matches the generated v8 revision and keeps the issue-297 identity frozen", () => { + expect(SCHEMA_FINGERPRINT).toBe("d0750014b0f79c57") + expect(SCHEMA_DIGEST).toBe("d0750014b0f79c57995c31fcdb091049d3130dc97366c07c003e4d1198aa6660") expect(ISSUE_297_TARGET_SCHEMA_PROJECT_REVISION).toBe( "506bc745f7a7eca202ec905a6403a6815e86413faf0cd3cbbf73881023edce91", ) expect(CURRENT_SCHEMA_PROJECT_REVISION).toMatch(/^[0-9a-f]{64}$/) expect(LOCAL_SCHEMA_MANIFEST.objects.length).toBeGreaterThan(60) - expect(CURRENT_LOCAL_SCHEMA.version).toBe(7) - expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V7) + expect(CURRENT_LOCAL_SCHEMA.version).toBe(8) + expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V8) const logs = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === "logs") expect(logs?.columns.some((column) => column.name.startsWith("idx_"))).toBe(false) expect(logs?.indexes).toContain("idx_lower_body") @@ -149,6 +151,18 @@ describe("current local schema identity", () => { ) expect(v5ErrorEventsView?.definition).not.toContain("_httpStatus") }) + + it("recognises Apple crash frames at v8 but not before", () => { + const applePattern = "^[0-9]+ +[^ ]+ +0x[0-9a-fA-F]+" + for (const name of ["error_events_mv", "error_events_by_time_mv"]) { + const view = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === name) + expect(view?.definition).toContain(applePattern) + } + // v7 matched no Apple frame at all, so every iOS crash fell through to the + // message hash and collapsed into one issue per exception type. + const v7View = LOCAL_SCHEMA_V7_MANIFEST.objects.find((object) => object.name === "error_events_mv") + expect(v7View?.definition).not.toContain(applePattern) + }) }) describe("local migration registry", () => { @@ -162,6 +176,7 @@ describe("local migration registry", () => { "local-0004-to-0005-service-overview-minutely", "local-0005-to-0006-error-events-fingerprint-hygiene", "local-0006-to-0007-error-service-version", + "local-0007-to-0008-apple-crash-frames", ]) expect(chain[0]?.from.fingerprint).toBe(LEGACY_SCHEMA_FINGERPRINT) expect(chain[0]?.to).toEqual(LOCAL_SCHEMA_V1) @@ -208,7 +223,7 @@ describe("local migration registry", () => { // One past the current tip — bump alongside LOCAL_SCHEMA_VERSION, or this // stops testing the future-store guard and starts testing the // unknown-fingerprint one. - { ...CURRENT_LOCAL_SCHEMA, version: 8, fingerprint: "future", digest: SCHEMA_DIGEST }, + { ...CURRENT_LOCAL_SCHEMA, version: 9, fingerprint: "future", digest: SCHEMA_DIGEST }, CURRENT_LOCAL_SCHEMA, ), ).toThrow(/newer than this build/) diff --git a/apps/ingest/src/clickhouse_insert_mappings.rs b/apps/ingest/src/clickhouse_insert_mappings.rs index 84ea83c10..06c5c0079 100644 --- a/apps/ingest/src/clickhouse_insert_mappings.rs +++ b/apps/ingest/src/clickhouse_insert_mappings.rs @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-insert-mappings.ts // Do not edit manually. -pub const PROJECT_REVISION: &str = "73f8f3249a508cd05598289b67b3773a049e38db0302efa6aa9e45e3501d2182"; +pub const PROJECT_REVISION: &str = "e37cc45679299abbc1545d48f0256d5e60c062e171a2d358077a35eb0d072ae6"; // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the // Tinybird-coupled PROJECT_REVISION. Compared against // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse diff --git a/packages/domain/src/clickhouse/migrations/0018_apple_crash_frames.ts b/packages/domain/src/clickhouse/migrations/0018_apple_crash_frames.ts new file mode 100644 index 000000000..1b7373cc2 --- /dev/null +++ b/packages/domain/src/clickhouse/migrations/0018_apple_crash_frames.ts @@ -0,0 +1,45 @@ +/** + * Migration 0018 — recognise Apple crash frames in the error fingerprint. + * + * `FRAME_LINE_PATTERN` matches stack frames by shape, one alternative per + * runtime. It had no alternative for Apple's, so every frame of an iOS crash + * reported by `maple-swift` was skipped, `_fpFrames` came out empty, and the + * hash fell through to `_msgFallback` — which redacts hex and long digit runs, + * collapsing `EXC_BAD_ACCESS at 0x10` and `EXC_BAD_ACCESS at 0xdeadbeef` into + * one signature. The practical result was one issue per exception type per + * service, no matter how many distinct bugs were behind it. + * + * The new alternative, `^[0-9]+ +[^ ]+ +0x[0-9a-fA-F]+`, keys on frame index + + * binary name + hex address. There is no source position to key on: an iOS + * crash arrives unsymbolicated, because the app's symbols live in a dSYM that + * never leaves the build machine. `FRAME_REDACTIONS` then erases the address and + * the (deliberately hex-rendered) offset, so a frame reduces to `index binary +` + * and grouping keys on the sequence of binaries. Coarse, but stable across + * releases — raw offsets shift with any code change and would re-split every iOS + * issue on every build. + * + * `FINGERPRINT_VERSION` goes 2 -> 3 in the same change. Only iOS hashes rotate; + * no other runtime's frame matching is touched. NOTHING IS BACKFILLED, for the + * reason 0003 and 0016 give: recomputing `FingerprintHash` would re-bucket every + * existing issue. iOS crashes already in `error_events` keep their collapsed + * hashes for the rest of their TTL and the version bump retires those issues. + * + * The CREATE statements below are the verbatim DDL as the schema emitter + * produced it at v18. Frozen history: never re-derive them from a later + * snapshot. + * + * `requiredForIngest: false` — nothing writes `error_events` directly, so this + * is a read-path correction, and gating on it would un-ready every + * BYO-ClickHouse org's ingest routing for a change the gateway never sees. + */ +export const migration_0018_apple_crash_frames = { + version: 18, + description: "Recreate the error_events MVs so the fingerprint recognises Apple crash frames", + requiredForIngest: false, + statements: [ + "DROP VIEW IF EXISTS error_events_mv", + "DROP VIEW IF EXISTS error_events_by_time_mv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues \u2014 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +[^ ]+ +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist \u2014 works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '\u2514\u2500') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted \u2014 these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", + "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues \u2014 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +[^ ]+ +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist \u2014 works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '\u2514\u2500') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted \u2014 these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", + ], +} as const diff --git a/packages/domain/src/clickhouse/migrations/index.test.ts b/packages/domain/src/clickhouse/migrations/index.test.ts index ab14553a7..1f491406c 100644 --- a/packages/domain/src/clickhouse/migrations/index.test.ts +++ b/packages/domain/src/clickhouse/migrations/index.test.ts @@ -23,6 +23,7 @@ import { } from "./0015_service_overview_minutely" import { migration_0016_error_events_4xx_and_frame_redaction } from "./0016_error_events_4xx_and_frame_redaction" import { migration_0017_error_service_version_columns } from "./0017_error_service_version_columns" +import { migration_0018_apple_crash_frames } from "./0018_apple_crash_frames" import { clickHouseSchemaVersion, latestMigrationVersion, migrations } from "./index" const backfills = migration_0004_service_namespace_projections.statements.filter( @@ -38,12 +39,12 @@ const renderedSql = migration_0004_service_namespace_projections.statements describe("ClickHouse migrations", () => { it("keeps migrations ordered by version", () => { expect(migrations.map((m) => m.version)).toEqual([ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, ]) - expect(migrations.at(-1)).toBe(migration_0017_error_service_version_columns) - expect(latestMigrationVersion).toBe(17) - // 0010, 0014, 0015, 0016 and 0017 are read-path only, so the ingest-gating - // version skips all five and stays at 13 — nothing writes `web_events`, + expect(migrations.at(-1)).toBe(migration_0018_apple_crash_frames) + expect(latestMigrationVersion).toBe(18) + // 0010 and 0014-0018 are read-path only, so the ingest-gating version skips + // all six and stays at 13 — nothing writes `web_events`, // `service_overview_minutely` or `error_events` directly, and bumping it // would un-ready every BYO-CH org's ingest routing for a read-path change. expect(clickHouseSchemaVersion).toBe("13") @@ -52,6 +53,7 @@ describe("ClickHouse migrations", () => { expect(migration_0015_service_overview_minutely.requiredForIngest).toBe(false) expect(migration_0016_error_events_4xx_and_frame_redaction.requiredForIngest).toBe(false) expect(migration_0017_error_service_version_columns.requiredForIngest).toBe(false) + expect(migration_0018_apple_crash_frames.requiredForIngest).toBe(false) }) it("recreates both error-events MVs with the 4xx guard and the widened frame redaction", () => { @@ -498,3 +500,26 @@ describe("ClickHouse migrations", () => { ) }) }) + +describe("migration 0018 — Apple crash frames", () => { + const creates = migration_0018_apple_crash_frames.statements.filter((stmt) => stmt.startsWith("CREATE")) + + it("recreates both error-events MVs with the Apple frame alternative", () => { + expect(creates).toHaveLength(2) + for (const sql of creates) { + // Frame index, binary name, hex address — an iOS crash has no source + // position to key on, because it arrives unsymbolicated. + expect(sql).toContain("^[0-9]+ +[^ ]+ +0x[0-9a-fA-F]+") + // The other runtimes' alternatives are untouched; only iOS hashes rotate. + expect(sql).toContain('^[ \\\\t]*at |^[ \\\\t]*File "') + } + }) + + it("does not backfill", () => { + // Recomputing FingerprintHash would re-bucket every existing issue. The + // FINGERPRINT_VERSION bump retires the collapsed iOS issues instead. + expect(migration_0018_apple_crash_frames.statements.some((stmt) => stmt.includes("UPDATE"))).toBe( + false, + ) + }) +}) diff --git a/packages/domain/src/clickhouse/migrations/index.ts b/packages/domain/src/clickhouse/migrations/index.ts index 79dc2395b..82835fa4c 100644 --- a/packages/domain/src/clickhouse/migrations/index.ts +++ b/packages/domain/src/clickhouse/migrations/index.ts @@ -16,6 +16,7 @@ import { migration_0014_web_events } from "./0014_web_events" import { migration_0015_service_overview_minutely } from "./0015_service_overview_minutely" import { migration_0016_error_events_4xx_and_frame_redaction } from "./0016_error_events_4xx_and_frame_redaction" import { migration_0017_error_service_version_columns } from "./0017_error_service_version_columns" +import { migration_0018_apple_crash_frames } from "./0018_apple_crash_frames" /** * A migration statement is either a raw SQL string (structural DDL) or a @@ -64,6 +65,7 @@ export const migrations: ReadonlyArray = [ migration_0015_service_overview_minutely, migration_0016_error_events_4xx_and_frame_redaction, migration_0017_error_service_version_columns, + migration_0018_apple_crash_frames, ] as const /** Highest migration `version` bundled — i.e. the schema level a fully-applied diff --git a/packages/domain/src/generated/clickhouse-schema.ts b/packages/domain/src/generated/clickhouse-schema.ts index 0a70c494a..68356bef0 100644 --- a/packages/domain/src/generated/clickhouse-schema.ts +++ b/packages/domain/src/generated/clickhouse-schema.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-schema.ts // Do not edit manually. -export const projectRevision = "73f8f3249a508cd05598289b67b3773a049e38db0302efa6aa9e45e3501d2182" as const +export const projectRevision = "e37cc45679299abbc1545d48f0256d5e60c062e171a2d358077a35eb0d072ae6" as const export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS alert_checks (\n OrgId LowCardinality(String),\n RuleId String,\n GroupKey String,\n Timestamp DateTime64(3),\n Status LowCardinality(String),\n SignalType LowCardinality(String),\n Comparator LowCardinality(String),\n Threshold Float64,\n ObservedValue Nullable(Float64),\n SampleCount UInt32,\n WindowMinutes UInt16,\n WindowStart DateTime64(3),\n WindowEnd DateTime64(3),\n ConsecutiveBreaches UInt16,\n ConsecutiveHealthy UInt16,\n IncidentId Nullable(String),\n IncidentTransition LowCardinality(String),\n EvaluationDurationMs UInt32,\n ErrorMessage Nullable(String),\n ErrorCategory LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, RuleId, GroupKey, Timestamp)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", @@ -42,8 +42,8 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS traces (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n ParentSpanId String,\n TraceState String,\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n ServiceName LowCardinality(String),\n ResourceSchemaUrl String,\n ResourceAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n Duration UInt64 DEFAULT 0,\n StatusCode LowCardinality(String),\n StatusMessage String,\n SpanAttributes Map(LowCardinality(String), String),\n EventsTimestamp Array(DateTime64(9)),\n EventsName Array(LowCardinality(String)),\n EventsAttributes Array(Map(LowCardinality(String), String)),\n LinksTraceId Array(String),\n LinksSpanId Array(String),\n LinksTraceState Array(String),\n LinksAttributes Array(Map(LowCardinality(String), String)),\n SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0),\n IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0),\n ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)),\n ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)),\n SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)),\n INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp))\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS traces_aggregates_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n StatusCode LowCardinality(String),\n IsEntryPoint UInt8,\n DeploymentEnv LowCardinality(String),\n WeightedCount SimpleAggregateFunction(sum, Float64),\n WeightedDurationSum SimpleAggregateFunction(sum, Float64),\n WeightedErrorCount SimpleAggregateFunction(sum, Float64),\n DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32),\n DurationMin SimpleAggregateFunction(min, UInt64),\n DurationMax SimpleAggregateFunction(max, UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv)\nTTL toDate(Hour) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS web_events (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n SessionId String,\n Seq UInt32,\n Kind LowCardinality(String),\n EventName String,\n Host LowCardinality(String),\n PagePath String,\n Url String,\n Attributes Map(String, String),\n INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, SessionId, Seq)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", - "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", - "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", + "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +[^ ]+ +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", + "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +[^ ]+ +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", "CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS\nSELECT\n OrgId,\n toStartOfMinute(Timestamp) AS Minute,\n FingerprintHash,\n anyLast(ServiceName) AS ServiceName,\n anyLast(ExceptionType) AS ExceptionType,\n anyLast(ExceptionMessage) AS ExceptionMessage,\n anyLast(ErrorLabel) AS ErrorLabel,\n anyLast(TopFrame) AS TopFrame,\n count() AS OccurrenceCount,\n min(Timestamp) AS FirstSeen,\n max(Timestamp) AS LastSeen,\n -- Distinct builds, not a sample: see ServiceVersions on the datasource.\n groupUniqArray(ServiceVersion) AS ServiceVersions\n FROM error_events\n GROUP BY OrgId, Minute, FingerprintHash", "CREATE MATERIALIZED VIEW IF NOT EXISTS error_spans_mv TO error_spans AS\nSELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n StatusMessage,\n Duration,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv\n FROM traces\n WHERE StatusCode = 'Error'", "CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n arrayJoin(mapKeys(LogAttributes)) AS AttributeKey,\n 'log' AS AttributeScope,\n count() AS UsageCount\n FROM logs\n WHERE LogAttributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope", diff --git a/packages/domain/src/tinybird/fingerprint.test.ts b/packages/domain/src/tinybird/fingerprint.test.ts index 1d36f26e9..f77b79b2f 100644 --- a/packages/domain/src/tinybird/fingerprint.test.ts +++ b/packages/domain/src/tinybird/fingerprint.test.ts @@ -625,3 +625,71 @@ describe("SQL parity", () => { } }) }) + +describe("Apple crash frames", () => { + // The shape `maple-swift`'s CrashReport.stacktrace renders: frame index, binary + // name, hex address, hex offset. Unsymbolicated — MetricKit has no symbols. + const appleStack = (frames: ReadonlyArray<[string, string, string]>) => + frames + .map(([binary, address, offset], i) => `${i} ${binary} 0x${address} +0x${offset}`) + .join("\n") + + it("matches Apple frames and strips address and offset", () => { + const result = computeFingerprintInputs({ + exceptionType: "EXC_BAD_ACCESS", + exceptionStacktrace: appleStack([ + ["MyApp", "104112600", "1d0f0"], + ["MyApp", "1041125a0", "a112"], + ["UIKitCore", "1a2b3c4d0", "44"], + ["libdyld.dylib", "700000000", "1000"], + ]), + statusMessage: "EXC_BAD_ACCESS (SIGSEGV)", + }) + + expect(result.topFrame).toBe("0 MyApp +") + expect(result.fpFrames.split("\n")).toHaveLength(MAX_FINGERPRINT_FRAMES) + expect(result.fpFrames).toContain("UIKitCore") + expect(result.fpFrames).not.toContain("libdyld") + }) + + it("groups the same crash site across rebuilds", () => { + // An offset moves with any code change above it. If it survived redaction every + // release would re-split every iOS issue. + const before = computeFingerprintInputs({ + exceptionType: "EXC_BAD_ACCESS", + exceptionStacktrace: appleStack([["MyApp", "104112600", "1d0f0"]]), + statusMessage: "", + }) + const after = computeFingerprintInputs({ + exceptionType: "EXC_BAD_ACCESS", + exceptionStacktrace: appleStack([["MyApp", "104999999", "2f4a1"]]), + statusMessage: "", + }) + expect(after.fpFrames).toBe(before.fpFrames) + }) + + it("separates crash sites that differ by binary", () => { + const inApp = computeFingerprintInputs({ + exceptionType: "EXC_BAD_ACCESS", + exceptionStacktrace: appleStack([["MyApp", "104112600", "1d0f0"]]), + statusMessage: "", + }) + const inUIKit = computeFingerprintInputs({ + exceptionType: "EXC_BAD_ACCESS", + exceptionStacktrace: appleStack([["UIKitCore", "1a2b3c4d0", "1d0f0"]]), + statusMessage: "", + }) + expect(inUIKit.fpFrames).not.toBe(inApp.fpFrames) + }) + + it("does not swallow other runtimes' lines", () => { + // The alternative is anchored on a leading frame index, so a message that merely + // mentions an address must not be read as a frame. + const result = computeFingerprintInputs({ + exceptionType: "Error", + exceptionStacktrace: "Error: mapping failed at 0x1f into region", + statusMessage: "", + }) + expect(result.fpFrames).toBe("") + }) +}) diff --git a/packages/domain/src/tinybird/fingerprint.ts b/packages/domain/src/tinybird/fingerprint.ts index 4b03cc16c..b128bed24 100644 --- a/packages/domain/src/tinybird/fingerprint.ts +++ b/packages/domain/src/tinybird/fingerprint.ts @@ -73,9 +73,20 @@ export const MSG_SIGNATURE_CHARS = 120 * Ruby: ` from /app/user.rb:12:in 'find'` * Firefox/Safari: `getUser@https://app/assets/index.js:42:18` * Go/Rust: ` /app/main.go:42 +0x1d` + * Apple: `0 MyApp 0x104a2c1f0 +0x1d0f0` + * + * The Apple alternative has no source position to key on, because an iOS crash arrives + * unsymbolicated — the app's symbols live in a dSYM that never leaves the build machine. + * It keys on the shape instead: frame index, binary name, hex address. The offset is hex + * (the SDK renders it that way deliberately) so `FRAME_REDACTIONS` erases it along with + * the address, leaving `index binaryName +`. That is coarse — grouping by the sequence of + * binaries rather than of functions — but it is *stable across releases*, which the raw + * offsets are not: any code change shifts every offset below it and would re-split every + * issue on every build. When dSYM symbolication lands, function names drop into the same + * slot and grouping sharpens with one more version bump. */ export const FRAME_LINE_PATTERN = - '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+' + '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +[^ ]+ +0x[0-9a-fA-F]+' /** An ordered `[pattern, replacement]` list, applied outermost-first. */ export type Redactions = ReadonlyArray @@ -309,8 +320,16 @@ export function computeFingerprintInputs(args: { * cutover forward. That is expected, not a regression — there is no backfill * short of replaying the raw traces. * - * v1 → v2 (this change): frame lines are matched by shape rather than by - * "contains a colon-digit", and the message signature is always folded in and - * additionally strips quoted values and query strings. + * v1 → v2: frame lines are matched by shape rather than by "contains a + * colon-digit", and the message signature is always folded in and additionally + * strips quoted values and query strings. + * + * v2 → v3 (this change): an Apple alternative joins the frame shapes, so iOS + * crashes reported by `maple-swift` group on their stack. Before it, no Apple + * frame matched any alternative, `_fpFrames` was always empty, and every crash + * fell through to the message hash — which redacts hex and digit runs, so + * `EXC_BAD_ACCESS at 0x10` and `at 0xdeadbeef` were one issue. Every crash of a + * given type in a service collapsed into a single row. Only iOS hashes rotate; + * no other runtime's frame matching changed. */ -export const FINGERPRINT_VERSION = 2 +export const FINGERPRINT_VERSION = 3 From 126d6986f9ae720e9dc44c7232bd2b40e7faa46a Mon Sep 17 00:00:00 2001 From: Makisuo Date: Thu, 20 Aug 2026 13:29:21 +0200 Subject: [PATCH 2/6] fix(errors): do not bump the fingerprint version, and match spaced binary names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two defects found reviewing the previous commit. The FINGERPRINT_VERSION bump would have archived every error issue in every org, permanently, for every runtime. ErrorsService archives every kind:"error" issue whose fingerprintVersion is below the constant. The premise for that, stated on the column itself, is that a row on an older version can never receive another occurrence — true only when a bump rotates every hash. This change rotates iOS hashes alone, so every Node, Python, Go and browser hash is byte-identical across the two versions. Their issues would be archived by the next retention phase, and nothing would bring them back: the tick's upsert conflicts on (orgId, fingerprintHash), which is unique, and its SET never clears archivedAt. The rows would keep accumulating occurrences while invisible in /errors. So the version stays at 2. The cost is bounded and small: a collapsed iOS issue stops receiving occurrences the moment its hash changes and retires through the ordinary resolved window instead of on sight. Retiring by hash rather than by version is what would let a partial-rotation change bump this safely, and that belongs in the retention model, not here. The frame alternative also required the binary name to be one space-free token, so it never matched an app whose Mach-O image name contains a space. That name is the target's PRODUCT_NAME, and "My App" is an ordinary thing to call an app — those apps stayed collapsed on the message hash, which is the exact failure the alternative exists to fix. The name is matched as `\S.*` now. Fixing it here rather than in the SDK's renderer is deliberate: 0.3.0 is already tagged, and old SDK versions stay in the wild for a long time. A backend fix covers the crashes they are already sending. Verified against a fresh local store: "MyApp" and "My App" each group their own two builds of one crash site together and separate a different site. --- apps/cli/src/server/local-schema-history.ts | 8 ++-- .../v7-to-v8-apple-crash-frames.ts | 2 +- apps/cli/src/server/schema-identity.ts | 2 +- apps/cli/src/server/schema/local-inserts.json | 2 +- .../cli/src/server/schema/local-schema-v8.sql | 6 +-- apps/cli/src/server/schema/local-schema.sql | 6 +-- apps/cli/test/local-store-migrations.test.ts | 6 +-- apps/ingest/src/clickhouse_insert_mappings.rs | 2 +- .../migrations/0018_apple_crash_frames.ts | 22 +++++++--- .../src/clickhouse/migrations/index.test.ts | 2 +- .../domain/src/generated/clickhouse-schema.ts | 6 +-- .../domain/src/tinybird/fingerprint.test.ts | 36 ++++++++++++++++ packages/domain/src/tinybird/fingerprint.ts | 43 +++++++++++++------ 13 files changed, 101 insertions(+), 42 deletions(-) diff --git a/apps/cli/src/server/local-schema-history.ts b/apps/cli/src/server/local-schema-history.ts index 6ffdb0da7..01f47a706 100644 --- a/apps/cli/src/server/local-schema-history.ts +++ b/apps/cli/src/server/local-schema-history.ts @@ -74,9 +74,9 @@ export const LOCAL_SCHEMA_HISTORY: ReadonlyArray = Obje }), Object.freeze({ version: 8, - fingerprint: "d0750014b0f79c57", - digest: "d0750014b0f79c57995c31fcdb091049d3130dc97366c07c003e4d1198aa6660", - manifestDigest: "ce9c98712f66bbdd90e204da021c9dfe57721cc0b514a837b23c4f95ae51bb50", - projectRevision: "e37cc45679299abbc1545d48f0256d5e60c062e171a2d358077a35eb0d072ae6", + fingerprint: "51081e951066442a", + digest: "51081e951066442a8e5b53df2c4bdda933edd20fc89132a54ed9b4dbb7e55a05", + manifestDigest: "60908c2e8307e24885227d4553916eef64df7f9b23abec23b5697cfea0d84d94", + projectRevision: "bb7da950a3a65af75fcf627bf4ed0436308c98fee86906048b05b6d40d9f7534", }), ] as const) diff --git a/apps/cli/src/server/local-store-migrations/v7-to-v8-apple-crash-frames.ts b/apps/cli/src/server/local-store-migrations/v7-to-v8-apple-crash-frames.ts index 8677f1563..c3664e6ff 100644 --- a/apps/cli/src/server/local-store-migrations/v7-to-v8-apple-crash-frames.ts +++ b/apps/cli/src/server/local-store-migrations/v7-to-v8-apple-crash-frames.ts @@ -130,7 +130,7 @@ const prepareTarget = async (context: MigrationModuleContext, state: V7ToV8State } /** - * Like v5 -> v7, this edge replaces the body of two existing views rather than + * Like v5 -> v6, this edge replaces the body of two existing views rather than * adding anything. A materialized view's SELECT is frozen at creation and the * bundled DDL uses `CREATE ... IF NOT EXISTS`, so both views must be dropped * before the v8 schema can install its versions. Dropping a view never touches diff --git a/apps/cli/src/server/schema-identity.ts b/apps/cli/src/server/schema-identity.ts index e944973a9..125aacd6a 100644 --- a/apps/cli/src/server/schema-identity.ts +++ b/apps/cli/src/server/schema-identity.ts @@ -30,7 +30,7 @@ export const LEGACY_SCHEMA_PROJECT_REVISION = export const LEGACY_SCHEMA_FINGERPRINT = "428701854f9fd30e" export const CURRENT_SCHEMA_PROJECT_REVISION = - "e37cc45679299abbc1545d48f0256d5e60c062e171a2d358077a35eb0d072ae6" + "bb7da950a3a65af75fcf627bf4ed0436308c98fee86906048b05b6d40d9f7534" /** Revision recorded by the issue-297 recovery report. The refreshed upstream * generator currently emits CURRENT_SCHEMA_PROJECT_REVISION; the structural * fingerprint is the compatibility identity used by the migration. */ diff --git a/apps/cli/src/server/schema/local-inserts.json b/apps/cli/src/server/schema/local-inserts.json index 361fa2c8a..a4ab5e71b 100644 --- a/apps/cli/src/server/schema/local-inserts.json +++ b/apps/cli/src/server/schema/local-inserts.json @@ -1,5 +1,5 @@ { - "projectRevision": "e37cc45679299abbc1545d48f0256d5e60c062e171a2d358077a35eb0d072ae6", + "projectRevision": "bb7da950a3a65af75fcf627bf4ed0436308c98fee86906048b05b6d40d9f7534", "orgPlaceholder": "__ORG__", "datasources": { "traces": { diff --git a/apps/cli/src/server/schema/local-schema-v8.sql b/apps/cli/src/server/schema/local-schema-v8.sql index eab30295f..60ba1b4b1 100644 --- a/apps/cli/src/server/schema/local-schema-v8.sql +++ b/apps/cli/src/server/schema/local-schema-v8.sql @@ -1,6 +1,6 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: e37cc45679299abbc1545d48f0256d5e60c062e171a2d358077a35eb0d072ae6 +-- projectRevision: bb7da950a3a65af75fcf627bf4ed0436308c98fee86906048b05b6d40d9f7534 -- localSchemaVersion: 8 CREATE TABLE IF NOT EXISTS alert_checks ( @@ -875,7 +875,7 @@ WITH -- the SQL that actually runs, silently. arraySlice( arrayFilter( - line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +[^ ]+ +0x[0-9a-fA-F]+'), + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+'), splitByChar('\n', _exStack) ), 1, 3 @@ -1015,7 +1015,7 @@ WITH -- the SQL that actually runs, silently. arraySlice( arrayFilter( - line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +[^ ]+ +0x[0-9a-fA-F]+'), + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+'), splitByChar('\n', _exStack) ), 1, 3 diff --git a/apps/cli/src/server/schema/local-schema.sql b/apps/cli/src/server/schema/local-schema.sql index eab30295f..60ba1b4b1 100644 --- a/apps/cli/src/server/schema/local-schema.sql +++ b/apps/cli/src/server/schema/local-schema.sql @@ -1,6 +1,6 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: e37cc45679299abbc1545d48f0256d5e60c062e171a2d358077a35eb0d072ae6 +-- projectRevision: bb7da950a3a65af75fcf627bf4ed0436308c98fee86906048b05b6d40d9f7534 -- localSchemaVersion: 8 CREATE TABLE IF NOT EXISTS alert_checks ( @@ -875,7 +875,7 @@ WITH -- the SQL that actually runs, silently. arraySlice( arrayFilter( - line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +[^ ]+ +0x[0-9a-fA-F]+'), + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+'), splitByChar('\n', _exStack) ), 1, 3 @@ -1015,7 +1015,7 @@ WITH -- the SQL that actually runs, silently. arraySlice( arrayFilter( - line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +[^ ]+ +0x[0-9a-fA-F]+'), + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+'), splitByChar('\n', _exStack) ), 1, 3 diff --git a/apps/cli/test/local-store-migrations.test.ts b/apps/cli/test/local-store-migrations.test.ts index e506454ec..d9405043f 100644 --- a/apps/cli/test/local-store-migrations.test.ts +++ b/apps/cli/test/local-store-migrations.test.ts @@ -61,8 +61,8 @@ import { join } from "node:path" describe("current local schema identity", () => { it("matches the generated v8 revision and keeps the issue-297 identity frozen", () => { - expect(SCHEMA_FINGERPRINT).toBe("d0750014b0f79c57") - expect(SCHEMA_DIGEST).toBe("d0750014b0f79c57995c31fcdb091049d3130dc97366c07c003e4d1198aa6660") + expect(SCHEMA_FINGERPRINT).toBe("51081e951066442a") + expect(SCHEMA_DIGEST).toBe("51081e951066442a8e5b53df2c4bdda933edd20fc89132a54ed9b4dbb7e55a05") expect(ISSUE_297_TARGET_SCHEMA_PROJECT_REVISION).toBe( "506bc745f7a7eca202ec905a6403a6815e86413faf0cd3cbbf73881023edce91", ) @@ -153,7 +153,7 @@ describe("current local schema identity", () => { }) it("recognises Apple crash frames at v8 but not before", () => { - const applePattern = "^[0-9]+ +[^ ]+ +0x[0-9a-fA-F]+" + const applePattern = "^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+" for (const name of ["error_events_mv", "error_events_by_time_mv"]) { const view = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === name) expect(view?.definition).toContain(applePattern) diff --git a/apps/ingest/src/clickhouse_insert_mappings.rs b/apps/ingest/src/clickhouse_insert_mappings.rs index 06c5c0079..e3eae1755 100644 --- a/apps/ingest/src/clickhouse_insert_mappings.rs +++ b/apps/ingest/src/clickhouse_insert_mappings.rs @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-insert-mappings.ts // Do not edit manually. -pub const PROJECT_REVISION: &str = "e37cc45679299abbc1545d48f0256d5e60c062e171a2d358077a35eb0d072ae6"; +pub const PROJECT_REVISION: &str = "bb7da950a3a65af75fcf627bf4ed0436308c98fee86906048b05b6d40d9f7534"; // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the // Tinybird-coupled PROJECT_REVISION. Compared against // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse diff --git a/packages/domain/src/clickhouse/migrations/0018_apple_crash_frames.ts b/packages/domain/src/clickhouse/migrations/0018_apple_crash_frames.ts index 1b7373cc2..6174bcbd4 100644 --- a/packages/domain/src/clickhouse/migrations/0018_apple_crash_frames.ts +++ b/packages/domain/src/clickhouse/migrations/0018_apple_crash_frames.ts @@ -18,11 +18,19 @@ * releases — raw offsets shift with any code change and would re-split every iOS * issue on every build. * - * `FINGERPRINT_VERSION` goes 2 -> 3 in the same change. Only iOS hashes rotate; - * no other runtime's frame matching is touched. NOTHING IS BACKFILLED, for the - * reason 0003 and 0016 give: recomputing `FingerprintHash` would re-bucket every - * existing issue. iOS crashes already in `error_events` keep their collapsed - * hashes for the rest of their TTL and the version bump retires those issues. + * `FINGERPRINT_VERSION` is deliberately NOT bumped, even though iOS hashes + * rotate. That constant drives a version-keyed sweep in `ErrorsService` which + * archives every `kind: "error"` issue below the current version, on the premise + * that an older-version row can never receive another occurrence — true only + * when a bump rotates every hash. This change rotates iOS hashes alone, so a + * bump would archive every issue of every other runtime in every org, and + * nothing would un-archive them. See the note on `FINGERPRINT_VERSION`. + * + * NOTHING IS BACKFILLED either, for the reason 0003 and 0016 give: recomputing + * `FingerprintHash` would re-bucket every existing issue. iOS crashes already in + * `error_events` keep their collapsed hashes for the rest of their TTL, and the + * collapsed issues retire through the ordinary resolved window once their hash + * changes and they stop receiving occurrences. * * The CREATE statements below are the verbatim DDL as the schema emitter * produced it at v18. Frozen history: never re-derive them from a later @@ -39,7 +47,7 @@ export const migration_0018_apple_crash_frames = { statements: [ "DROP VIEW IF EXISTS error_events_mv", "DROP VIEW IF EXISTS error_events_by_time_mv", - "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues \u2014 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +[^ ]+ +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist \u2014 works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '\u2514\u2500') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted \u2014 these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", - "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues \u2014 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +[^ ]+ +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist \u2014 works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '\u2514\u2500') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted \u2014 these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", + "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues \u2014 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist \u2014 works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '\u2514\u2500') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted \u2014 these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", + "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues \u2014 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist \u2014 works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '\u2514\u2500') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted \u2014 these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", ], } as const diff --git a/packages/domain/src/clickhouse/migrations/index.test.ts b/packages/domain/src/clickhouse/migrations/index.test.ts index 1f491406c..714722487 100644 --- a/packages/domain/src/clickhouse/migrations/index.test.ts +++ b/packages/domain/src/clickhouse/migrations/index.test.ts @@ -509,7 +509,7 @@ describe("migration 0018 — Apple crash frames", () => { for (const sql of creates) { // Frame index, binary name, hex address — an iOS crash has no source // position to key on, because it arrives unsymbolicated. - expect(sql).toContain("^[0-9]+ +[^ ]+ +0x[0-9a-fA-F]+") + expect(sql).toContain("^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+") // The other runtimes' alternatives are untouched; only iOS hashes rotate. expect(sql).toContain('^[ \\\\t]*at |^[ \\\\t]*File "') } diff --git a/packages/domain/src/generated/clickhouse-schema.ts b/packages/domain/src/generated/clickhouse-schema.ts index 68356bef0..8600820d3 100644 --- a/packages/domain/src/generated/clickhouse-schema.ts +++ b/packages/domain/src/generated/clickhouse-schema.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-schema.ts // Do not edit manually. -export const projectRevision = "e37cc45679299abbc1545d48f0256d5e60c062e171a2d358077a35eb0d072ae6" as const +export const projectRevision = "bb7da950a3a65af75fcf627bf4ed0436308c98fee86906048b05b6d40d9f7534" as const export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS alert_checks (\n OrgId LowCardinality(String),\n RuleId String,\n GroupKey String,\n Timestamp DateTime64(3),\n Status LowCardinality(String),\n SignalType LowCardinality(String),\n Comparator LowCardinality(String),\n Threshold Float64,\n ObservedValue Nullable(Float64),\n SampleCount UInt32,\n WindowMinutes UInt16,\n WindowStart DateTime64(3),\n WindowEnd DateTime64(3),\n ConsecutiveBreaches UInt16,\n ConsecutiveHealthy UInt16,\n IncidentId Nullable(String),\n IncidentTransition LowCardinality(String),\n EvaluationDurationMs UInt32,\n ErrorMessage Nullable(String),\n ErrorCategory LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, RuleId, GroupKey, Timestamp)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", @@ -42,8 +42,8 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS traces (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n ParentSpanId String,\n TraceState String,\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n ServiceName LowCardinality(String),\n ResourceSchemaUrl String,\n ResourceAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n Duration UInt64 DEFAULT 0,\n StatusCode LowCardinality(String),\n StatusMessage String,\n SpanAttributes Map(LowCardinality(String), String),\n EventsTimestamp Array(DateTime64(9)),\n EventsName Array(LowCardinality(String)),\n EventsAttributes Array(Map(LowCardinality(String), String)),\n LinksTraceId Array(String),\n LinksSpanId Array(String),\n LinksTraceState Array(String),\n LinksAttributes Array(Map(LowCardinality(String), String)),\n SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0),\n IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0),\n ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)),\n ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)),\n SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)),\n INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp))\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS traces_aggregates_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n StatusCode LowCardinality(String),\n IsEntryPoint UInt8,\n DeploymentEnv LowCardinality(String),\n WeightedCount SimpleAggregateFunction(sum, Float64),\n WeightedDurationSum SimpleAggregateFunction(sum, Float64),\n WeightedErrorCount SimpleAggregateFunction(sum, Float64),\n DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32),\n DurationMin SimpleAggregateFunction(min, UInt64),\n DurationMax SimpleAggregateFunction(max, UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv)\nTTL toDate(Hour) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS web_events (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n SessionId String,\n Seq UInt32,\n Kind LowCardinality(String),\n EventName String,\n Host LowCardinality(String),\n PagePath String,\n Url String,\n Attributes Map(String, String),\n INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, SessionId, Seq)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", - "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +[^ ]+ +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", - "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +[^ ]+ +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", + "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", + "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", "CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS\nSELECT\n OrgId,\n toStartOfMinute(Timestamp) AS Minute,\n FingerprintHash,\n anyLast(ServiceName) AS ServiceName,\n anyLast(ExceptionType) AS ExceptionType,\n anyLast(ExceptionMessage) AS ExceptionMessage,\n anyLast(ErrorLabel) AS ErrorLabel,\n anyLast(TopFrame) AS TopFrame,\n count() AS OccurrenceCount,\n min(Timestamp) AS FirstSeen,\n max(Timestamp) AS LastSeen,\n -- Distinct builds, not a sample: see ServiceVersions on the datasource.\n groupUniqArray(ServiceVersion) AS ServiceVersions\n FROM error_events\n GROUP BY OrgId, Minute, FingerprintHash", "CREATE MATERIALIZED VIEW IF NOT EXISTS error_spans_mv TO error_spans AS\nSELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n StatusMessage,\n Duration,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv\n FROM traces\n WHERE StatusCode = 'Error'", "CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n arrayJoin(mapKeys(LogAttributes)) AS AttributeKey,\n 'log' AS AttributeScope,\n count() AS UsageCount\n FROM logs\n WHERE LogAttributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope", diff --git a/packages/domain/src/tinybird/fingerprint.test.ts b/packages/domain/src/tinybird/fingerprint.test.ts index f77b79b2f..d61dced36 100644 --- a/packages/domain/src/tinybird/fingerprint.test.ts +++ b/packages/domain/src/tinybird/fingerprint.test.ts @@ -682,6 +682,24 @@ describe("Apple crash frames", () => { expect(inUIKit.fpFrames).not.toBe(inApp.fpFrames) }) + it("matches a binary name containing spaces", () => { + // A Mach-O image name is the target's PRODUCT_NAME, and "My App" is an + // ordinary thing to call an app. Requiring one space-free token silently + // excluded those apps from frame matching and left them collapsed on the + // message hash — the exact failure this alternative exists to fix. + const result = computeFingerprintInputs({ + exceptionType: "EXC_BAD_ACCESS", + exceptionStacktrace: [ + "0 My App 0x104112600 +0x1d0f0", + "1 My App 0x1041125a0 +0xa112", + ].join("\n"), + statusMessage: "", + }) + expect(result.fpFrames.split("\n")).toHaveLength(2) + expect(result.topFrame).toContain("My App") + expect(result.msgSignature).toBe("") + }) + it("does not swallow other runtimes' lines", () => { // The alternative is anchored on a leading frame index, so a message that merely // mentions an address must not be read as a frame. @@ -692,4 +710,22 @@ describe("Apple crash frames", () => { }) expect(result.fpFrames).toBe("") }) + + it("does not match other runtimes' address-bearing frames", () => { + // The alternative is anchored on a leading frame index with no punctuation + // after it, which is what keeps these out. + for (const line of [ + " 1: 0xb09bc0 node::Abort() [node]", + "1: 0xb09bc0 node::Abort() [node]", + " #00 pc 0000000000045e7c /system/lib/libc.so", + " 0: rust_begin_unwind", + ]) { + const result = computeFingerprintInputs({ + exceptionType: "Error", + exceptionStacktrace: line, + statusMessage: "", + }) + expect(result.fpFrames, line).toBe("") + } + }) }) diff --git a/packages/domain/src/tinybird/fingerprint.ts b/packages/domain/src/tinybird/fingerprint.ts index b128bed24..f6af87776 100644 --- a/packages/domain/src/tinybird/fingerprint.ts +++ b/packages/domain/src/tinybird/fingerprint.ts @@ -73,7 +73,7 @@ export const MSG_SIGNATURE_CHARS = 120 * Ruby: ` from /app/user.rb:12:in 'find'` * Firefox/Safari: `getUser@https://app/assets/index.js:42:18` * Go/Rust: ` /app/main.go:42 +0x1d` - * Apple: `0 MyApp 0x104a2c1f0 +0x1d0f0` + * Apple: `0 My App 0x104a2c1f0 +0x1d0f0` * * The Apple alternative has no source position to key on, because an iOS crash arrives * unsymbolicated — the app's symbols live in a dSYM that never leaves the build machine. @@ -83,10 +83,15 @@ export const MSG_SIGNATURE_CHARS = 120 * binaries rather than of functions — but it is *stable across releases*, which the raw * offsets are not: any code change shifts every offset below it and would re-split every * issue on every build. When dSYM symbolication lands, function names drop into the same - * slot and grouping sharpens with one more version bump. + * slot. + * + * The binary name is matched as `\\S.*`, not as one space-free token: a Mach-O image name + * is the target's PRODUCT_NAME, and `My App` is an ordinary thing to call an app. Keying + * on a single token silently excluded every such app from frame matching and left it + * collapsed on the message hash — the exact failure this alternative exists to fix. */ export const FRAME_LINE_PATTERN = - '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +[^ ]+ +0x[0-9a-fA-F]+' + '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+' /** An ordered `[pattern, replacement]` list, applied outermost-first. */ export type Redactions = ReadonlyArray @@ -320,16 +325,26 @@ export function computeFingerprintInputs(args: { * cutover forward. That is expected, not a regression — there is no backfill * short of replaying the raw traces. * - * v1 → v2: frame lines are matched by shape rather than by "contains a - * colon-digit", and the message signature is always folded in and additionally - * strips quoted values and query strings. + * v1 → v2 (this change): frame lines are matched by shape rather than by + * "contains a colon-digit", and the message signature is always folded in and + * additionally strips quoted values and query strings. + * + * **Adding the Apple frame alternative deliberately did NOT bump this**, even + * though it rotates hashes for iOS crashes that are still occurring, which is + * what the rule above otherwise calls for. The retirement this version drives is + * version-keyed, not hash-keyed: `ErrorsService` archives every `kind: "error"` + * issue whose `fingerprintVersion` is below this constant, on the premise stated + * in `error_issues.fingerprintVersion` that a row on an older version can never + * receive another occurrence. That premise holds only when a bump rotates + * *every* hash. The Apple change rotates iOS hashes alone, so a bump would + * archive every Node, Python, Go and browser issue in every org — and nothing + * un-archives them, because the tick's upsert conflicts on + * `(orgId, fingerprintHash)` and never clears `archivedAt`. * - * v2 → v3 (this change): an Apple alternative joins the frame shapes, so iOS - * crashes reported by `maple-swift` group on their stack. Before it, no Apple - * frame matched any alternative, `_fpFrames` was always empty, and every crash - * fell through to the message hash — which redacts hex and digit runs, so - * `EXC_BAD_ACCESS at 0x10` and `at 0xdeadbeef` were one issue. Every crash of a - * given type in a service collapsed into a single row. Only iOS hashes rotate; - * no other runtime's frame matching changed. + * The cost of not bumping is small and bounded: the collapsed iOS issues stop + * receiving occurrences the moment their hash changes, and retire through the + * ordinary resolved window instead of on sight. Teaching the sweep to retire by + * hash rather than by version is what would let a partial-rotation change bump + * this safely. */ -export const FINGERPRINT_VERSION = 3 +export const FINGERPRINT_VERSION = 2 From df8664b9f9adf30dabe8a04373b30764116168b1 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Thu, 20 Aug 2026 13:56:02 +0200 Subject: [PATCH 3/6] test(cli): pin the native migration probe to the v8 head identity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The probe asserts the marker the native migration promotes to, and it is pinned to the head identity on purpose: that is what proves the migration walks all the way to head rather than merely running. Its own comment says it has to move in lockstep with LOCAL_SCHEMA_VERSION, and I left it on v7. The chain itself was fine — all eight steps verified in CI, including v7 -> v8 — so the failure was the probe's expectation, not the migration. --- apps/cli/test/native-local-store-migration.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/cli/test/native-local-store-migration.sh b/apps/cli/test/native-local-store-migration.sh index 3e79c7a36..0c6444e38 100755 --- a/apps/cli/test/native-local-store-migration.sh +++ b/apps/cli/test/native-local-store-migration.sh @@ -142,7 +142,7 @@ grep -q "local store migrated" "$ROOT/migrate.out" || fail "native migration did # must be bumped in lockstep with LOCAL_SCHEMA_VERSION and the matching # LOCAL_SCHEMA_V.fingerprint in apps/cli/src/server/schema-identity.ts; # leaving it on the previous version is what makes this step fail after a bump. -jq -e '.formatVersion == 2 and .activation == "active" and .schemaVersion == 7 and .schema == "bc124f30765c8c56"' \ +jq -e '.formatVersion == 2 and .activation == "active" and .schemaVersion == 8 and .schema == "51081e951066442a"' \ "$ROOT/maple-store-version.json" >/dev/null || fail "native migration wrote the wrong active identity" step "reopening promoted store in a fresh server" From c7b839017fb0974bb66dbdcaaef91d3c4b58142a Mon Sep 17 00:00:00 2001 From: Makisuo Date: Thu, 20 Aug 2026 14:24:21 +0200 Subject: [PATCH 4/6] refactor(cli): decode the v7 -> v8 journal state with Effect Schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five bespoke `throw new Error` branches and a hand-rolled `isRecord`, copied wholesale from the v5 -> v6 module, replaced by the schema they were approximating. Effect Schema is what the repo uses for this, and `decodeChildMetrics` in apps/cli/src/commands/archive.ts already establishes the exact idiom — `decodeUnknownSync` with `onExcessProperty: "error"`. The signature stays synchronous and throwing. `decodeState` is part of `LocalStoreMigrationModule`, shared with seven sibling modules and called from the plain-TS runner that drives them; making this one return an Effect would mean refactoring all of it, which is not this change. The schema is the part that matters, and `decodeUnknownEffect` is a one-liner away if that runner ever moves. Behaviour is preserved exactly, and now has tests rather than being asserted by reading: unknown fields rejected, another module's state rejected, row counts that are not unsigned decimal strings rejected, a missing or unknown raw table rejected, a non-integer retention floor rejected, absent progress distinguished from invalid progress. `retentionDays` is `optionalKey` rather than `optional`: the journal is JSON, where an absent floor is an absent key, not a present `undefined`. The literal-returning branch in `preflight` keeps it that way, and satisfies no-conditional-empty-object-spread — which is what the original's `!(x === undefined) ? … : undefined` was working around, obscurely. --- .../v7-to-v8-apple-crash-frames.ts | 114 +++++++++--------- apps/cli/test/local-store-migrations.test.ts | 68 +++++++++++ 2 files changed, 126 insertions(+), 56 deletions(-) diff --git a/apps/cli/src/server/local-store-migrations/v7-to-v8-apple-crash-frames.ts b/apps/cli/src/server/local-store-migrations/v7-to-v8-apple-crash-frames.ts index c3664e6ff..7cd473bfe 100644 --- a/apps/cli/src/server/local-store-migrations/v7-to-v8-apple-crash-frames.ts +++ b/apps/cli/src/server/local-store-migrations/v7-to-v8-apple-crash-frames.ts @@ -1,6 +1,7 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. import { cp, mkdir, rm } from "node:fs/promises" import { dirname, resolve } from "node:path" +import { Schema } from "effect" import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" import type { LocalStoreMigrationModule, @@ -21,60 +22,61 @@ import { assertPhysicalSchema } from "../schema-physical" const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) -interface V7ToV8State { - readonly module: "local-0007-to-0008-apple-crash-frames" - readonly version: 1 - readonly rawRows: Readonly> - readonly retentionDays?: number -} +/** Stamped into the journal and matched on the way back out. */ +const MODULE_ID = "local-0007-to-0008-apple-crash-frames" as const -interface V7ToV8Progress { - readonly installed: true -} +/** + * Unsigned decimal string, because a raw-telemetry row count can exceed + * `Number.MAX_SAFE_INTEGER` and the journal has to round-trip it exactly. + */ +const RowCount = Schema.String.check(Schema.isPattern(/^\d+$/)) -const isRecord = (value: unknown): value is Record => - typeof value === "object" && value !== null && !Array.isArray(value) +/** + * Exactly the raw tables, each required. + * + * Built from `RAW_TABLES` rather than written out, so a table added to + * `RAW_TELEMETRY_TTL_COLUMNS` is covered here without this file changing. + * Combined with `onExcessProperty: "error"` at the decode site, this rejects a + * missing table and an unknown one alike. + */ +const RawRowsSchema = Schema.Struct(Object.fromEntries(RAW_TABLES.map((table) => [table, RowCount]))) -const decodeCounts = (value: unknown): Readonly> => { - if (!isRecord(value)) throw new Error("v7 -> v8 rawRows must be an object") - const counts: Record = {} - for (const table of RAW_TABLES) { - const count = value[table] - if (typeof count !== "string" || !/^\d+$/.test(count)) - throw new Error(`v7 -> v8 rawRows.${table} must be an unsigned decimal string`) - counts[table] = count - } - if (Object.keys(value).some((table) => !RAW_TABLES.includes(table as (typeof RAW_TABLES)[number]))) - throw new Error("v7 -> v8 rawRows contains an unknown table") - return counts -} +/** + * The journal state this edge round-trips. + * + * `retentionDays` is `optionalKey`, not `optional`: the journal is JSON, where + * an absent retention floor is an absent key rather than a present `undefined`. + */ +const V7ToV8StateSchema = Schema.Struct({ + module: Schema.Literal(MODULE_ID), + version: Schema.Literal(1), + rawRows: RawRowsSchema, + retentionDays: Schema.optionalKey(Schema.Int), +}) -const decodeState = (value: unknown): V7ToV8State => { - if (!isRecord(value)) throw new Error("v7 -> v8 state must be an object") - const allowed = new Set(["module", "version", "rawRows", "retentionDays"]) - if (Object.keys(value).some((key) => !allowed.has(key))) - throw new Error("v7 -> v8 state contains an unknown field") - if (value.module !== "local-0007-to-0008-apple-crash-frames" || value.version !== 1) - throw new Error("v7 -> v8 state has an unsupported module or version") - if ( - value.retentionDays !== undefined && - (typeof value.retentionDays !== "number" || !Number.isSafeInteger(value.retentionDays)) - ) - throw new Error("v7 -> v8 retentionDays must be an integer") - return { - module: "local-0007-to-0008-apple-crash-frames", - version: 1, - rawRows: decodeCounts(value.rawRows), - ...(!(value.retentionDays === undefined) ? { retentionDays: value.retentionDays } : undefined), - } -} +const V7ToV8ProgressSchema = Schema.Struct({ installed: Schema.Literal(true) }) -const decodeProgress = (value: unknown): V7ToV8Progress | undefined => { - if (value === undefined) return undefined - if (!isRecord(value) || Object.keys(value).some((key) => key !== "installed") || value.installed !== true) - throw new Error("v7 -> v8 progress is invalid") - return { installed: true } -} +type V7ToV8State = typeof V7ToV8StateSchema.Type +type V7ToV8Progress = typeof V7ToV8ProgressSchema.Type + +/** + * `decodeUnknownSync` because `LocalStoreMigrationModule.decodeState` is a + * synchronous throwing contract shared with seven sibling modules and the + * plain-TS runner that drives them. The schema is the declarative part; + * swapping in `decodeUnknownEffect` is a one-liner if that runner ever moves + * into Effect. + * + * `onExcessProperty: "error"` is not decoration. A journal carrying a field + * this build does not know about was written by a different build, and + * silently dropping it would resume someone else's migration under our + * assumptions. + */ +const decodeState = Schema.decodeUnknownSync(V7ToV8StateSchema, { onExcessProperty: "error" }) + +const decodeProgress = (value: unknown): V7ToV8Progress | undefined => + value === undefined + ? undefined + : Schema.decodeUnknownSync(V7ToV8ProgressSchema, { onExcessProperty: "error" })(value) const parseJsonEachRow = (value: string): A[] => value @@ -109,12 +111,12 @@ const preflight = async (context: MigrationModuleContext): Promise }, { schemaSql: LOCAL_SCHEMA_V7_SQL, bootstrapSchema: false }, ) - return { - module: "local-0007-to-0008-apple-crash-frames", - version: 1, - rawRows, - ...(!(retentionDays === undefined) ? { retentionDays } : undefined), - } + // Two literals rather than a conditional spread: `retentionDays` is an + // `optionalKey`, so an absent floor has to be an absent key, not a present + // `undefined`. + return retentionDays === undefined + ? { module: MODULE_ID, version: 1, rawRows } + : { module: MODULE_ID, version: 1, rawRows, retentionDays } } const prepareTarget = async (context: MigrationModuleContext, state: V7ToV8State): Promise => { @@ -240,7 +242,7 @@ const dispositions: ReadonlyArray = [ ] export const v7ToV8AppleCrashFramesModule: LocalStoreMigrationModule = { - id: "local-0007-to-0008-apple-crash-frames", + id: MODULE_ID, moduleVersion: 1, description: "Rebuild the error-events views so the fingerprint recognises Apple crash frames", from: LOCAL_SCHEMA_V7, diff --git a/apps/cli/test/local-store-migrations.test.ts b/apps/cli/test/local-store-migrations.test.ts index d9405043f..591e23d22 100644 --- a/apps/cli/test/local-store-migrations.test.ts +++ b/apps/cli/test/local-store-migrations.test.ts @@ -1,3 +1,4 @@ +import { v7ToV8AppleCrashFramesModule } from "../src/server/local-store-migrations/v7-to-v8-apple-crash-frames" import { describe, expect, it } from "vitest" import { CURRENT_LOCAL_SCHEMA, @@ -1015,3 +1016,70 @@ describe("legacy raw replay cursor", () => { expect(() => legacyToCurrentModule.decodeProgress(progress)).toThrow(/lastHash/) }) }) + +describe("v7 -> v8 journal state decoding", () => { + const RAW_TABLES = [ + "logs", + "traces", + "metrics_sum", + "metrics_gauge", + "metrics_histogram", + "metrics_exponential_histogram", + ] + const rawRows = Object.fromEntries(RAW_TABLES.map((table) => [table, "12"])) + const state = { module: "local-0007-to-0008-apple-crash-frames", version: 1, rawRows } + + it("round-trips a valid state, with and without a retention floor", () => { + expect(v7ToV8AppleCrashFramesModule.decodeState(state)).toEqual(state) + expect(v7ToV8AppleCrashFramesModule.decodeState({ ...state, retentionDays: 90 })).toEqual({ + ...state, + retentionDays: 90, + }) + }) + + it("rejects a field this build does not know about", () => { + // A journal carrying an unknown field was written by a different build. + // Dropping it silently would resume someone else's migration under our + // assumptions. + expect(() => v7ToV8AppleCrashFramesModule.decodeState({ ...state, somethingElse: 1 })).toThrow() + }) + + it("rejects another module's state", () => { + expect(() => + v7ToV8AppleCrashFramesModule.decodeState({ + ...state, + module: "local-0006-to-0007-error-service-version", + }), + ).toThrow() + expect(() => v7ToV8AppleCrashFramesModule.decodeState({ ...state, version: 2 })).toThrow() + }) + + it("rejects row counts that are not unsigned decimal strings", () => { + // They are strings precisely because a count can exceed + // Number.MAX_SAFE_INTEGER, so anything lossy has to fail loudly. + for (const bad of [12, "-1", "1.5", "1e3", ""]) { + expect(() => + v7ToV8AppleCrashFramesModule.decodeState({ ...state, rawRows: { ...rawRows, logs: bad } }), + ).toThrow() + } + }) + + it("rejects a missing or unknown raw table", () => { + const { logs: _dropped, ...missing } = rawRows + expect(() => v7ToV8AppleCrashFramesModule.decodeState({ ...state, rawRows: missing })).toThrow() + expect(() => + v7ToV8AppleCrashFramesModule.decodeState({ ...state, rawRows: { ...rawRows, not_a_table: "1" } }), + ).toThrow() + }) + + it("rejects a non-integer retention floor", () => { + expect(() => v7ToV8AppleCrashFramesModule.decodeState({ ...state, retentionDays: 1.5 })).toThrow() + }) + + it("decodes progress, and treats absent progress as absent", () => { + expect(v7ToV8AppleCrashFramesModule.decodeProgress(undefined)).toBeUndefined() + expect(v7ToV8AppleCrashFramesModule.decodeProgress({ installed: true })).toEqual({ installed: true }) + expect(() => v7ToV8AppleCrashFramesModule.decodeProgress({ installed: false })).toThrow() + expect(() => v7ToV8AppleCrashFramesModule.decodeProgress({})).toThrow() + }) +}) From 01072f0fb7cd4725c171bd92df68d811439722f6 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Thu, 20 Aug 2026 14:35:50 +0200 Subject: [PATCH 5/6] refactor(cli): decode every migration journal with Effect Schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sweep the v7 -> v8 module started. All seven remaining edges carried hand-rolled validation — six of them byte-identical copies of the same `isRecord` + `decodeCounts` loop + five `throw new Error` branches, differing only in the strings. Those six now share `journal-codecs.ts`: one `RawRowsSchema` built from RAW_TELEMETRY_TTL_COLUMNS, so a table added there is covered without touching any edge, and one `makeRawRowsState(moduleId)` for the state shape they all round-trip. legacy-to-current carried its own parser combinator library — `record`, `nonEmptyString`, `nullableString`, `uint64String`, `nonNegativeInteger`, `exactKeys` — under four nested decoders. Those are now schemas, with the two genuine invariants left as explicit checks after the decode, because both are about the map *keys* rather than the values: a table key this build does not know, and an inventory whose own `table` field disagrees with the key it is stored under. The second one matters — if those disagree the journal describes a copy of one table under another's cursor. Only one test touched that decoder beforehand, which is far too thin a net for rewriting the validation guarding a resumable copy out of a pre-v1 store, so its accept/reject boundary is characterized first (9 cases, written against the old implementation and passing before anything changed). That immediately paid for itself. `Schema.Record` keyed on a literal union requires *every* key to be present, but `copied` legitimately holds a subset — it starts empty and fills one table at a time as the replay progresses. The strict keying rejected the journal of every partially-progressed migration. The maps are keyed by `Schema.String` with the table names checked after, which is what the hand-rolled loop did. One pre-existing assertion moves from the old decoder's exact wording to the field it fails on; the behaviour it guards is unchanged. 555 lines deleted, 302 added. --- .../local-store-migrations/journal-codecs.ts | 84 +++++++ .../legacy-to-current.ts | 236 +++++++----------- .../v1-to-v2-error-rollup.ts | 87 ++----- .../v2-to-v3-service-map-ingest-bridge.ts | 73 ++---- .../v3-to-v4-web-events.ts | 73 ++---- .../v4-to-v5-service-overview-minutely.ts | 73 ++---- ...-to-v6-error-events-fingerprint-hygiene.ts | 73 ++---- .../v6-to-v7-error-service-version.ts | 73 ++---- .../v7-to-v8-apple-crash-frames.ts | 57 +---- apps/cli/test/local-store-migrations.test.ts | 112 ++++++++- 10 files changed, 386 insertions(+), 555 deletions(-) create mode 100644 apps/cli/src/server/local-store-migrations/journal-codecs.ts diff --git a/apps/cli/src/server/local-store-migrations/journal-codecs.ts b/apps/cli/src/server/local-store-migrations/journal-codecs.ts new file mode 100644 index 000000000..7db5b4825 --- /dev/null +++ b/apps/cli/src/server/local-store-migrations/journal-codecs.ts @@ -0,0 +1,84 @@ +// Shared journal codecs for the local-store migration edges. +// +// Every versioned edge from v1 onward round-trips the same journal state: which +// module wrote it, the raw-telemetry row counts it must preserve, and the +// retention floor in force at the time. Each edge used to carry its own copy of +// `isRecord` + a `decodeCounts` loop + five `throw new Error` branches — six +// identical copies, drifting only in the strings. +// +// The decoders here are synchronous and throwing because +// `LocalStoreMigrationModule.decodeState` is: the runner that drives these +// modules is plain TypeScript. The schemas are the declarative part, and +// `Schema.decodeUnknownEffect` is a one-line swap if that runner ever moves +// into Effect. +import { Schema } from "effect" +import { RAW_TELEMETRY_TTL_COLUMNS } from "../chdb" + +const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) + +/** + * Unsigned decimal string. + * + * Row counts and ClickHouse UInt64 cursors are carried as text because they can + * exceed `Number.MAX_SAFE_INTEGER`, and the journal has to round-trip them + * exactly. The pattern is not cosmetic: these values are interpolated into SQL + * comparisons, so anything that could change their meaning has to fail here. + */ +export const UnsignedDecimal = Schema.String.check(Schema.isPattern(/^\d+$/)) + +/** + * Exactly the raw tables, each required. + * + * Built from `RAW_TELEMETRY_TTL_COLUMNS` rather than written out, so a table + * added there is covered without touching any edge. With + * `onExcessProperty: "error"` at the decode site this rejects a missing table + * and an unknown one alike, which is what the hand-rolled loop did in two + * separate passes. + */ +export const RawRowsSchema = Schema.Struct( + Object.fromEntries(RAW_TABLES.map((table) => [table, UnsignedDecimal])), +) + +/** + * Rejecting unknown fields is not tidiness. A journal carrying a field this + * build does not know about was written by a different build, and silently + * dropping it would resume someone else's migration under our assumptions. + */ +const strict = { onExcessProperty: "error" } as const + +/** + * The journal state shared by every versioned edge. + * + * `retentionDays` is `optionalKey`, not `optional`: the journal is JSON, where + * an absent retention floor is an absent key rather than a present `undefined`. + */ +export const makeRawRowsState = (moduleId: Id) => { + const schema = Schema.Struct({ + module: Schema.Literal(moduleId), + version: Schema.Literal(1), + rawRows: RawRowsSchema, + retentionDays: Schema.optionalKey(Schema.Int), + }) + return { schema, decode: Schema.decodeUnknownSync(schema, strict) } +} + +/** Progress for an edge whose apply step is a single idempotent install. */ +export const InstalledProgressSchema = Schema.Struct({ installed: Schema.Literal(true) }) + +export type InstalledProgress = typeof InstalledProgressSchema.Type + +const decodeInstalled = Schema.decodeUnknownSync(InstalledProgressSchema, strict) + +/** + * Absent progress means the step has not run; it is not the same as invalid + * progress, which means the journal disagrees with this build. + */ +export const decodeInstalledProgress = (value: unknown): InstalledProgress | undefined => + value === undefined ? undefined : decodeInstalled(value) + +/** + * `decodeUnknownSync` with the strict excess-property policy, for an edge whose + * progress is not the plain `installed` flag. + */ +export const strictDecoder = >(schema: S) => + Schema.decodeUnknownSync(schema, strict) diff --git a/apps/cli/src/server/local-store-migrations/legacy-to-current.ts b/apps/cli/src/server/local-store-migrations/legacy-to-current.ts index 02a20b732..d33e8e15f 100644 --- a/apps/cli/src/server/local-store-migrations/legacy-to-current.ts +++ b/apps/cli/src/server/local-store-migrations/legacy-to-current.ts @@ -1,5 +1,6 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. import { createHash } from "node:crypto" +import { Schema } from "effect" import { LOCAL_SCHEMA_V1_MANIFEST, LOCAL_SCHEMA_V1_SQL, @@ -7,6 +8,7 @@ import { LOCAL_SCHEMA_V1, } from "../schema-identity" import { assertPhysicalSchema } from "../schema-physical" +import { strictDecoder, UnsignedDecimal } from "./journal-codecs" import type { LocalSchemaColumn } from "../schema-manifest" import type { LocalStoreMigrationModule, @@ -135,164 +137,102 @@ const emptyCopyProgress = (): CopyProgress => ({ const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null && !Array.isArray(value) -const record = (value: unknown, label: string): Record => { - if (!isRecord(value)) throw new Error(`${label} must be an object`) - return value -} - -const nonEmptyString = (value: unknown, label: string): string => { - if (typeof value !== "string" || value.length === 0) - throw new Error(`${label} must be a non-empty string`) - return value -} - -const nullableString = (value: unknown, label: string): string | null => { - if (value !== null && typeof value !== "string") throw new Error(`${label} must be a string or null`) - return value -} - -/** ClickHouse emits UInt64 cursor values as decimal strings in JSONEachRow. - * Keep the journal representation textual, but reject anything that could - * change the meaning of the numeric SQL comparisons when interpolated. */ -const uint64String = (value: unknown, label: string): string | null => { - if (value === null) return null - if (typeof value !== "string" || !/^\d+$/.test(value)) - throw new Error(`${label} must be an unsigned decimal string or null`) - return value -} +const NonEmptyString = Schema.String.check(Schema.isMinLength(1)) +const NullableString = Schema.NullOr(Schema.String) +const NonNegativeInt = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)) + +/** + * A ClickHouse UInt64 cursor, or `null` for "no cursor yet". + * + * Textual because a UInt64 does not survive a JS number, and pattern-checked + * because these values are interpolated into the numeric SQL comparisons that + * drive the resumable copy — anything that could change their meaning has to + * fail here rather than there. + */ +const Uint64Cursor = Schema.NullOr(UnsignedDecimal) + +/** Only tables this build knows how to replay. */ +const RawTable = Schema.Literals([...RAW_TABLE_NAMES]) + +const Sha256Hex = Schema.String.check(Schema.isPattern(/^[0-9a-f]{64}$/i)) + +const TableInventorySchema = Schema.Struct({ + table: NonEmptyString, + rowCount: NonEmptyString, + retentionStartAt: NonEmptyString, + minTime: NullableString, + maxTime: NullableString, + hashSum: NonEmptyString, + hashXor: NonEmptyString, +}) -const nonNegativeInteger = (value: unknown, label: string): number => { - if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) - throw new Error(`${label} must be a non-negative integer`) - return value -} +const CopyProgressSchema = Schema.Struct({ + rows: NonNegativeInt, + bytes: NonNegativeInt, + lastTimestamp: NullableString, + lastHash: Uint64Cursor, + lastTieBreak: Uint64Cursor, + /** Cumulative ordinal consumed within the final composite-key group. */ + duplicateCount: NonNegativeInt, + duplicateGroupExhausted: Schema.Boolean, +}) -const exactKeys = (value: Record, allowed: ReadonlyArray, label: string): void => { - const allowedKeys = new Set(allowed) - for (const key of Object.keys(value)) { - if (!allowedKeys.has(key)) throw new Error(`${label} contains unknown field ${key}`) - } -} +const PendingBatchSchema = Schema.Struct({ + table: RawTable, + // Positive, not merely non-negative: an empty batch would commit nothing + // while advancing the cursor past the rows it claims to cover. + rowCount: Schema.Int.check(Schema.isGreaterThanOrEqualTo(1)), + byteLength: NonNegativeInt, + firstTimestamp: NullableString, + firstHash: Uint64Cursor, + firstTieBreak: Uint64Cursor, + lastTimestamp: NullableString, + lastHash: Uint64Cursor, + lastTieBreak: Uint64Cursor, + lastKeyCount: NonNegativeInt, + lastKeyExhausted: Schema.Boolean, + signature: Sha256Hex, +}) -const decodeTableInventory = (value: unknown, label: string): TableInventory => { - const inventory = record(value, label) - exactKeys( - inventory, - ["table", "rowCount", "retentionStartAt", "minTime", "maxTime", "hashSum", "hashXor"], - label, - ) - return { - table: nonEmptyString(inventory.table, `${label}.table`), - rowCount: nonEmptyString(inventory.rowCount, `${label}.rowCount`), - retentionStartAt: nonEmptyString(inventory.retentionStartAt, `${label}.retentionStartAt`), - minTime: nullableString(inventory.minTime, `${label}.minTime`), - maxTime: nullableString(inventory.maxTime, `${label}.maxTime`), - hashSum: nonEmptyString(inventory.hashSum, `${label}.hashSum`), - hashXor: nonEmptyString(inventory.hashXor, `${label}.hashXor`), - } -} +/** + * Keyed by `Schema.String`, not by `RawTable`, and the keys are checked below. + * + * A record keyed on the literal union requires *every* table to be present, but + * both maps legitimately hold a subset: `copied` starts empty and fills one + * table at a time as the replay progresses. Keying it strictly would reject the + * journal of every partially-progressed migration and strand it. + */ +const RawReplayProgressSchema = Schema.Struct({ + sourceInventory: Schema.Record(Schema.String, TableInventorySchema), + copied: Schema.Record(Schema.String, CopyProgressSchema), + pendingBatch: Schema.optionalKey(PendingBatchSchema), +}) -const decodeCopyProgress = (value: unknown, label: string): CopyProgress => { - const progress = record(value, label) - exactKeys( - progress, - [ - "rows", - "bytes", - "lastTimestamp", - "lastHash", - "lastTieBreak", - "duplicateCount", - "duplicateGroupExhausted", - ], - label, - ) - if (typeof progress.duplicateGroupExhausted !== "boolean") - throw new Error(`${label}.duplicateGroupExhausted must be a boolean`) - return { - rows: nonNegativeInteger(progress.rows, `${label}.rows`), - bytes: nonNegativeInteger(progress.bytes, `${label}.bytes`), - lastTimestamp: nullableString(progress.lastTimestamp, `${label}.lastTimestamp`), - lastHash: uint64String(progress.lastHash, `${label}.lastHash`), - lastTieBreak: uint64String(progress.lastTieBreak, `${label}.lastTieBreak`), - duplicateCount: nonNegativeInteger(progress.duplicateCount, `${label}.duplicateCount`), - duplicateGroupExhausted: progress.duplicateGroupExhausted, - } -} +const LegacyStateSchema = Schema.Struct({ + module: Schema.Literal("local-0000-to-0001-raw-replay"), + version: Schema.Literal(1), +}) -const decodePendingBatch = (value: unknown, label: string): PendingBatch => { - const pending = record(value, label) - exactKeys( - pending, - [ - "table", - "rowCount", - "byteLength", - "firstTimestamp", - "firstHash", - "firstTieBreak", - "lastTimestamp", - "lastHash", - "lastTieBreak", - "lastKeyCount", - "lastKeyExhausted", - "signature", - ], - label, - ) - const table = nonEmptyString(pending.table, `${label}.table`) - if (!RAW_TABLE_NAMES.has(table)) throw new Error(`${label}.table is not a registered raw table`) - if (typeof pending.lastKeyExhausted !== "boolean") - throw new Error(`${label}.lastKeyExhausted must be a boolean`) - if (typeof pending.signature !== "string" || !/^[0-9a-f]{64}$/i.test(pending.signature)) - throw new Error(`${label}.signature must be a SHA-256 hex digest`) - const rowCount = nonNegativeInteger(pending.rowCount, `${label}.rowCount`) - if (rowCount === 0) throw new Error(`${label}.rowCount must be positive`) - return { - table, - rowCount, - byteLength: nonNegativeInteger(pending.byteLength, `${label}.byteLength`), - firstTimestamp: nullableString(pending.firstTimestamp, `${label}.firstTimestamp`), - firstHash: uint64String(pending.firstHash, `${label}.firstHash`), - firstTieBreak: uint64String(pending.firstTieBreak, `${label}.firstTieBreak`), - lastTimestamp: nullableString(pending.lastTimestamp, `${label}.lastTimestamp`), - lastHash: uint64String(pending.lastHash, `${label}.lastHash`), - lastTieBreak: uint64String(pending.lastTieBreak, `${label}.lastTieBreak`), - lastKeyCount: nonNegativeInteger(pending.lastKeyCount, `${label}.lastKeyCount`), - lastKeyExhausted: pending.lastKeyExhausted, - signature: pending.signature, - } -} +const decodeProgressStrict = strictDecoder(RawReplayProgressSchema) const decodeRawReplayProgress = (value: unknown): RawReplayProgress => { - const progress = record(value, "legacy raw replay progress") - exactKeys(progress, ["sourceInventory", "copied", "pendingBatch"], "legacy raw replay progress") - const sourceInventoryRecord = record(progress.sourceInventory, "legacy raw replay sourceInventory") - const copiedRecord = record(progress.copied, "legacy raw replay copied") - const sourceInventory: Record = {} - for (const [table, inventory] of Object.entries(sourceInventoryRecord)) { + const progress = decodeProgressStrict(value) + // Two invariants the schema is not the right place to state, both about the + // map *keys* rather than the values. + for (const [table, inventory] of Object.entries(progress.sourceInventory)) { if (!RAW_TABLE_NAMES.has(table)) throw new Error(`legacy raw replay sourceInventory has unknown table ${table}`) - const decoded = decodeTableInventory(inventory, `legacy raw replay sourceInventory.${table}`) - if (decoded.table !== table) + // An inventory is keyed by table and also carries its own `table` field. + // If those disagree the journal describes a copy of one table under + // another's cursor. + if (inventory.table !== table) throw new Error(`legacy raw replay sourceInventory.${table}.table does not match its key`) - sourceInventory[table] = decoded } - const copied: Record = {} - for (const [table, progressValue] of Object.entries(copiedRecord)) { + for (const table of Object.keys(progress.copied)) { if (!RAW_TABLE_NAMES.has(table)) throw new Error(`legacy raw replay copied has unknown table ${table}`) - copied[table] = decodeCopyProgress(progressValue, `legacy raw replay copied.${table}`) - } - return { - sourceInventory, - copied, - ...(!(progress.pendingBatch === undefined) - ? { - pendingBatch: decodePendingBatch(progress.pendingBatch, "legacy raw replay pendingBatch"), - } - : undefined), } + return progress } const asProgress = (value: RawReplayProgress | undefined): RawReplayProgress => @@ -882,13 +822,7 @@ const legacyPreflight = async (context: MigrationModuleContext): Promise { - const state = record(value, "legacy raw replay state") - exactKeys(state, ["module", "version"], "legacy raw replay state") - if (state.module !== "local-0000-to-0001-raw-replay" || state.version !== 1) - throw new Error("legacy raw replay state has an unsupported module or version") - return { module: "local-0000-to-0001-raw-replay", version: 1 } -} +const decodeLegacyState = strictDecoder(LegacyStateSchema) const decodeLegacyProgress = (value: unknown): RawReplayProgress | undefined => value === undefined ? undefined : decodeRawReplayProgress(value) diff --git a/apps/cli/src/server/local-store-migrations/v1-to-v2-error-rollup.ts b/apps/cli/src/server/local-store-migrations/v1-to-v2-error-rollup.ts index 9cccc374e..6295fb3bc 100644 --- a/apps/cli/src/server/local-store-migrations/v1-to-v2-error-rollup.ts +++ b/apps/cli/src/server/local-store-migrations/v1-to-v2-error-rollup.ts @@ -1,6 +1,8 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. import { cp, mkdir, rm } from "node:fs/promises" import { dirname, resolve } from "node:path" +import { Schema } from "effect" +import { makeRawRowsState, strictDecoder, UnsignedDecimal } from "./journal-codecs" import { applyRawTelemetryRetentionFloor, RAW_TELEMETRY_TTL_COLUMNS, @@ -26,65 +28,26 @@ import { assertPhysicalSchema } from "../schema-physical" const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) -interface V1ToV2State { - readonly module: "local-0001-to-0002-error-rollup" - readonly version: 1 - readonly rawRows: Readonly> - readonly retentionDays?: number -} +/** Stamped into the journal and matched on the way back out. */ +const MODULE_ID = "local-0001-to-0002-error-rollup" as const -interface V1ToV2Progress { - readonly backfilledErrorEvents: string -} +const V1ToV2StateCodec = makeRawRowsState(MODULE_ID) -const isRecord = (value: unknown): value is Record => - typeof value === "object" && value !== null && !Array.isArray(value) +/** + * Unlike its siblings this edge backfills, so its progress is a resumable + * cursor rather than an installed flag: how many `error_events` rows the + * backfill has written so far, as an unsigned decimal because the count is a + * ClickHouse UInt64. + */ +const V1ToV2ProgressSchema = Schema.Struct({ backfilledErrorEvents: UnsignedDecimal }) -const decodeCounts = (value: unknown): Readonly> => { - if (!isRecord(value)) throw new Error("v1 -> v2 rawRows must be an object") - const counts: Record = {} - for (const table of RAW_TABLES) { - const count = value[table] - if (typeof count !== "string" || !/^\d+$/.test(count)) - throw new Error(`v1 -> v2 rawRows.${table} must be an unsigned decimal string`) - counts[table] = count - } - if (Object.keys(value).some((table) => !RAW_TABLES.includes(table as (typeof RAW_TABLES)[number]))) - throw new Error("v1 -> v2 rawRows contains an unknown table") - return counts -} +type V1ToV2State = typeof V1ToV2StateCodec.schema.Type +type V1ToV2Progress = typeof V1ToV2ProgressSchema.Type -const decodeState = (value: unknown): V1ToV2State => { - if (!isRecord(value)) throw new Error("v1 -> v2 state must be an object") - const allowed = new Set(["module", "version", "rawRows", "retentionDays"]) - if (Object.keys(value).some((key) => !allowed.has(key))) - throw new Error("v1 -> v2 state contains an unknown field") - if (value.module !== "local-0001-to-0002-error-rollup" || value.version !== 1) - throw new Error("v1 -> v2 state has an unsupported module or version") - if ( - value.retentionDays !== undefined && - (typeof value.retentionDays !== "number" || !Number.isSafeInteger(value.retentionDays)) - ) - throw new Error("v1 -> v2 retentionDays must be an integer") - return { - module: "local-0001-to-0002-error-rollup", - version: 1, - rawRows: decodeCounts(value.rawRows), - ...(!(value.retentionDays === undefined) ? { retentionDays: value.retentionDays } : undefined), - } -} - -const decodeProgress = (value: unknown): V1ToV2Progress | undefined => { - if (value === undefined) return undefined - if ( - !isRecord(value) || - Object.keys(value).some((key) => key !== "backfilledErrorEvents") || - typeof value.backfilledErrorEvents !== "string" || - !/^\d+$/.test(value.backfilledErrorEvents) - ) - throw new Error("v1 -> v2 progress is invalid") - return { backfilledErrorEvents: value.backfilledErrorEvents } -} +const decodeState = V1ToV2StateCodec.decode +const decodeV1ToV2Progress = strictDecoder(V1ToV2ProgressSchema) +const decodeProgress = (value: unknown): V1ToV2Progress | undefined => + value === undefined ? undefined : decodeV1ToV2Progress(value) const parseJsonEachRow = (value: string): A[] => value @@ -119,12 +82,12 @@ const preflight = async (context: MigrationModuleContext): Promise }, { schemaSql: LOCAL_SCHEMA_V1_SQL, bootstrapSchema: false }, ) - return { - module: "local-0001-to-0002-error-rollup", - version: 1, - rawRows, - ...(!(retentionDays === undefined) ? { retentionDays } : undefined), - } + // Two literals rather than a conditional spread: `retentionDays` is an + // `optionalKey`, so an absent floor has to be an absent key, not a present + // `undefined`. + return retentionDays === undefined + ? { module: MODULE_ID, version: 1, rawRows } + : { module: MODULE_ID, version: 1, rawRows, retentionDays } } const prepareTarget = async (context: MigrationModuleContext, state: V1ToV2State): Promise => { @@ -246,7 +209,7 @@ const dispositions: ReadonlyArray = [ ] export const v1ToV2ErrorRollupModule: LocalStoreMigrationModule = { - id: "local-0001-to-0002-error-rollup", + id: MODULE_ID, moduleVersion: 1, description: "Add the durable minutely error-fingerprint rollup to a v1 local store", from: LOCAL_SCHEMA_V1, diff --git a/apps/cli/src/server/local-store-migrations/v2-to-v3-service-map-ingest-bridge.ts b/apps/cli/src/server/local-store-migrations/v2-to-v3-service-map-ingest-bridge.ts index 719cc767d..7e91f67eb 100644 --- a/apps/cli/src/server/local-store-migrations/v2-to-v3-service-map-ingest-bridge.ts +++ b/apps/cli/src/server/local-store-migrations/v2-to-v3-service-map-ingest-bridge.ts @@ -1,6 +1,7 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. import { cp, mkdir, rm } from "node:fs/promises" import { dirname, resolve } from "node:path" +import { decodeInstalledProgress, makeRawRowsState, type InstalledProgress } from "./journal-codecs" import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" import type { LocalStoreMigrationModule, @@ -21,60 +22,16 @@ import { assertPhysicalSchema } from "../schema-physical" const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) -interface V2ToV3State { - readonly module: "local-0002-to-0003-service-map-ingest-bridge" - readonly version: 1 - readonly rawRows: Readonly> - readonly retentionDays?: number -} +/** Stamped into the journal and matched on the way back out. */ +const MODULE_ID = "local-0002-to-0003-service-map-ingest-bridge" as const -interface V2ToV3Progress { - readonly installed: true -} +const V2ToV3StateCodec = makeRawRowsState(MODULE_ID) -const isRecord = (value: unknown): value is Record => - typeof value === "object" && value !== null && !Array.isArray(value) +type V2ToV3State = typeof V2ToV3StateCodec.schema.Type +type V2ToV3Progress = InstalledProgress -const decodeCounts = (value: unknown): Readonly> => { - if (!isRecord(value)) throw new Error("v2 -> v3 rawRows must be an object") - const counts: Record = {} - for (const table of RAW_TABLES) { - const count = value[table] - if (typeof count !== "string" || !/^\d+$/.test(count)) - throw new Error(`v2 -> v3 rawRows.${table} must be an unsigned decimal string`) - counts[table] = count - } - if (Object.keys(value).some((table) => !RAW_TABLES.includes(table as (typeof RAW_TABLES)[number]))) - throw new Error("v2 -> v3 rawRows contains an unknown table") - return counts -} - -const decodeState = (value: unknown): V2ToV3State => { - if (!isRecord(value)) throw new Error("v2 -> v3 state must be an object") - const allowed = new Set(["module", "version", "rawRows", "retentionDays"]) - if (Object.keys(value).some((key) => !allowed.has(key))) - throw new Error("v2 -> v3 state contains an unknown field") - if (value.module !== "local-0002-to-0003-service-map-ingest-bridge" || value.version !== 1) - throw new Error("v2 -> v3 state has an unsupported module or version") - if ( - value.retentionDays !== undefined && - (typeof value.retentionDays !== "number" || !Number.isSafeInteger(value.retentionDays)) - ) - throw new Error("v2 -> v3 retentionDays must be an integer") - return { - module: "local-0002-to-0003-service-map-ingest-bridge", - version: 1, - rawRows: decodeCounts(value.rawRows), - ...(!(value.retentionDays === undefined) ? { retentionDays: value.retentionDays } : undefined), - } -} - -const decodeProgress = (value: unknown): V2ToV3Progress | undefined => { - if (value === undefined) return undefined - if (!isRecord(value) || Object.keys(value).some((key) => key !== "installed") || value.installed !== true) - throw new Error("v2 -> v3 progress is invalid") - return { installed: true } -} +const decodeState = V2ToV3StateCodec.decode +const decodeProgress = decodeInstalledProgress const parseJsonEachRow = (value: string): A[] => value @@ -109,12 +66,12 @@ const preflight = async (context: MigrationModuleContext): Promise }, { schemaSql: LOCAL_SCHEMA_V2_SQL, bootstrapSchema: false }, ) - return { - module: "local-0002-to-0003-service-map-ingest-bridge", - version: 1, - rawRows, - ...(!(retentionDays === undefined) ? { retentionDays } : undefined), - } + // Two literals rather than a conditional spread: `retentionDays` is an + // `optionalKey`, so an absent floor has to be an absent key, not a present + // `undefined`. + return retentionDays === undefined + ? { module: MODULE_ID, version: 1, rawRows } + : { module: MODULE_ID, version: 1, rawRows, retentionDays } } const prepareTarget = async (context: MigrationModuleContext, state: V2ToV3State): Promise => { @@ -207,7 +164,7 @@ const dispositions: ReadonlyArray = [ ] export const v2ToV3ServiceMapIngestBridgeModule: LocalStoreMigrationModule = { - id: "local-0002-to-0003-service-map-ingest-bridge", + id: MODULE_ID, moduleVersion: 1, description: "Restore the deployment-safe error view trigger and add the service-map ingress bridge to v2", diff --git a/apps/cli/src/server/local-store-migrations/v3-to-v4-web-events.ts b/apps/cli/src/server/local-store-migrations/v3-to-v4-web-events.ts index 863361f02..a3c19bd11 100644 --- a/apps/cli/src/server/local-store-migrations/v3-to-v4-web-events.ts +++ b/apps/cli/src/server/local-store-migrations/v3-to-v4-web-events.ts @@ -1,6 +1,7 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. import { cp, mkdir, rm } from "node:fs/promises" import { dirname, resolve } from "node:path" +import { decodeInstalledProgress, makeRawRowsState, type InstalledProgress } from "./journal-codecs" import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" import type { LocalStoreMigrationModule, @@ -21,60 +22,16 @@ import { assertPhysicalSchema } from "../schema-physical" const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) -interface V3ToV4State { - readonly module: "local-0003-to-0004-web-events" - readonly version: 1 - readonly rawRows: Readonly> - readonly retentionDays?: number -} +/** Stamped into the journal and matched on the way back out. */ +const MODULE_ID = "local-0003-to-0004-web-events" as const -interface V3ToV4Progress { - readonly installed: true -} +const V3ToV4StateCodec = makeRawRowsState(MODULE_ID) -const isRecord = (value: unknown): value is Record => - typeof value === "object" && value !== null && !Array.isArray(value) +type V3ToV4State = typeof V3ToV4StateCodec.schema.Type +type V3ToV4Progress = InstalledProgress -const decodeCounts = (value: unknown): Readonly> => { - if (!isRecord(value)) throw new Error("v3 -> v4 rawRows must be an object") - const counts: Record = {} - for (const table of RAW_TABLES) { - const count = value[table] - if (typeof count !== "string" || !/^\d+$/.test(count)) - throw new Error(`v3 -> v4 rawRows.${table} must be an unsigned decimal string`) - counts[table] = count - } - if (Object.keys(value).some((table) => !RAW_TABLES.includes(table as (typeof RAW_TABLES)[number]))) - throw new Error("v3 -> v4 rawRows contains an unknown table") - return counts -} - -const decodeState = (value: unknown): V3ToV4State => { - if (!isRecord(value)) throw new Error("v3 -> v4 state must be an object") - const allowed = new Set(["module", "version", "rawRows", "retentionDays"]) - if (Object.keys(value).some((key) => !allowed.has(key))) - throw new Error("v3 -> v4 state contains an unknown field") - if (value.module !== "local-0003-to-0004-web-events" || value.version !== 1) - throw new Error("v3 -> v4 state has an unsupported module or version") - if ( - value.retentionDays !== undefined && - (typeof value.retentionDays !== "number" || !Number.isSafeInteger(value.retentionDays)) - ) - throw new Error("v3 -> v4 retentionDays must be an integer") - return { - module: "local-0003-to-0004-web-events", - version: 1, - rawRows: decodeCounts(value.rawRows), - ...(!(value.retentionDays === undefined) ? { retentionDays: value.retentionDays } : undefined), - } -} - -const decodeProgress = (value: unknown): V3ToV4Progress | undefined => { - if (value === undefined) return undefined - if (!isRecord(value) || Object.keys(value).some((key) => key !== "installed") || value.installed !== true) - throw new Error("v3 -> v4 progress is invalid") - return { installed: true } -} +const decodeState = V3ToV4StateCodec.decode +const decodeProgress = decodeInstalledProgress const parseJsonEachRow = (value: string): A[] => value @@ -109,12 +66,12 @@ const preflight = async (context: MigrationModuleContext): Promise }, { schemaSql: LOCAL_SCHEMA_V3_SQL, bootstrapSchema: false }, ) - return { - module: "local-0003-to-0004-web-events", - version: 1, - rawRows, - ...(!(retentionDays === undefined) ? { retentionDays } : undefined), - } + // Two literals rather than a conditional spread: `retentionDays` is an + // `optionalKey`, so an absent floor has to be an absent key, not a present + // `undefined`. + return retentionDays === undefined + ? { module: MODULE_ID, version: 1, rawRows } + : { module: MODULE_ID, version: 1, rawRows, retentionDays } } const prepareTarget = async (context: MigrationModuleContext, state: V3ToV4State): Promise => { @@ -220,7 +177,7 @@ const dispositions: ReadonlyArray = [ ] export const v3ToV4WebEventsModule: LocalStoreMigrationModule = { - id: "local-0003-to-0004-web-events", + id: MODULE_ID, moduleVersion: 1, description: "Add the web_events analytics fact table and its materialized view to v3", from: LOCAL_SCHEMA_V3, diff --git a/apps/cli/src/server/local-store-migrations/v4-to-v5-service-overview-minutely.ts b/apps/cli/src/server/local-store-migrations/v4-to-v5-service-overview-minutely.ts index da8a4c560..9d523831d 100644 --- a/apps/cli/src/server/local-store-migrations/v4-to-v5-service-overview-minutely.ts +++ b/apps/cli/src/server/local-store-migrations/v4-to-v5-service-overview-minutely.ts @@ -1,6 +1,7 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. import { cp, mkdir, rm } from "node:fs/promises" import { dirname, resolve } from "node:path" +import { decodeInstalledProgress, makeRawRowsState, type InstalledProgress } from "./journal-codecs" import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" import type { LocalStoreMigrationModule, @@ -21,60 +22,16 @@ import { assertPhysicalSchema } from "../schema-physical" const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) -interface V4ToV5State { - readonly module: "local-0004-to-0005-service-overview-minutely" - readonly version: 1 - readonly rawRows: Readonly> - readonly retentionDays?: number -} +/** Stamped into the journal and matched on the way back out. */ +const MODULE_ID = "local-0004-to-0005-service-overview-minutely" as const -interface V4ToV5Progress { - readonly installed: true -} +const V4ToV5StateCodec = makeRawRowsState(MODULE_ID) -const isRecord = (value: unknown): value is Record => - typeof value === "object" && value !== null && !Array.isArray(value) +type V4ToV5State = typeof V4ToV5StateCodec.schema.Type +type V4ToV5Progress = InstalledProgress -const decodeCounts = (value: unknown): Readonly> => { - if (!isRecord(value)) throw new Error("v4 -> v5 rawRows must be an object") - const counts: Record = {} - for (const table of RAW_TABLES) { - const count = value[table] - if (typeof count !== "string" || !/^\d+$/.test(count)) - throw new Error(`v4 -> v5 rawRows.${table} must be an unsigned decimal string`) - counts[table] = count - } - if (Object.keys(value).some((table) => !RAW_TABLES.includes(table as (typeof RAW_TABLES)[number]))) - throw new Error("v4 -> v5 rawRows contains an unknown table") - return counts -} - -const decodeState = (value: unknown): V4ToV5State => { - if (!isRecord(value)) throw new Error("v4 -> v5 state must be an object") - const allowed = new Set(["module", "version", "rawRows", "retentionDays"]) - if (Object.keys(value).some((key) => !allowed.has(key))) - throw new Error("v4 -> v5 state contains an unknown field") - if (value.module !== "local-0004-to-0005-service-overview-minutely" || value.version !== 1) - throw new Error("v4 -> v5 state has an unsupported module or version") - if ( - value.retentionDays !== undefined && - (typeof value.retentionDays !== "number" || !Number.isSafeInteger(value.retentionDays)) - ) - throw new Error("v4 -> v5 retentionDays must be an integer") - return { - module: "local-0004-to-0005-service-overview-minutely", - version: 1, - rawRows: decodeCounts(value.rawRows), - ...(!(value.retentionDays === undefined) ? { retentionDays: value.retentionDays } : undefined), - } -} - -const decodeProgress = (value: unknown): V4ToV5Progress | undefined => { - if (value === undefined) return undefined - if (!isRecord(value) || Object.keys(value).some((key) => key !== "installed") || value.installed !== true) - throw new Error("v4 -> v5 progress is invalid") - return { installed: true } -} +const decodeState = V4ToV5StateCodec.decode +const decodeProgress = decodeInstalledProgress const parseJsonEachRow = (value: string): A[] => value @@ -109,12 +66,12 @@ const preflight = async (context: MigrationModuleContext): Promise }, { schemaSql: LOCAL_SCHEMA_V4_SQL, bootstrapSchema: false }, ) - return { - module: "local-0004-to-0005-service-overview-minutely", - version: 1, - rawRows, - ...(!(retentionDays === undefined) ? { retentionDays } : undefined), - } + // Two literals rather than a conditional spread: `retentionDays` is an + // `optionalKey`, so an absent floor has to be an absent key, not a present + // `undefined`. + return retentionDays === undefined + ? { module: MODULE_ID, version: 1, rawRows } + : { module: MODULE_ID, version: 1, rawRows, retentionDays } } const prepareTarget = async (context: MigrationModuleContext, state: V4ToV5State): Promise => { @@ -219,7 +176,7 @@ const dispositions: ReadonlyArray = [ ] export const v4ToV5ServiceOverviewMinutelyModule: LocalStoreMigrationModule = { - id: "local-0004-to-0005-service-overview-minutely", + id: MODULE_ID, moduleVersion: 1, description: "Add the service_overview_minutely rollup and its materialized view to v4", from: LOCAL_SCHEMA_V4, diff --git a/apps/cli/src/server/local-store-migrations/v5-to-v6-error-events-fingerprint-hygiene.ts b/apps/cli/src/server/local-store-migrations/v5-to-v6-error-events-fingerprint-hygiene.ts index 53e5c5e1c..1d1191b0b 100644 --- a/apps/cli/src/server/local-store-migrations/v5-to-v6-error-events-fingerprint-hygiene.ts +++ b/apps/cli/src/server/local-store-migrations/v5-to-v6-error-events-fingerprint-hygiene.ts @@ -1,6 +1,7 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. import { cp, mkdir, rm } from "node:fs/promises" import { dirname, resolve } from "node:path" +import { decodeInstalledProgress, makeRawRowsState, type InstalledProgress } from "./journal-codecs" import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" import type { LocalStoreMigrationModule, @@ -21,60 +22,16 @@ import { assertPhysicalSchema } from "../schema-physical" const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) -interface V5ToV6State { - readonly module: "local-0005-to-0006-error-events-fingerprint-hygiene" - readonly version: 1 - readonly rawRows: Readonly> - readonly retentionDays?: number -} +/** Stamped into the journal and matched on the way back out. */ +const MODULE_ID = "local-0005-to-0006-error-events-fingerprint-hygiene" as const -interface V5ToV6Progress { - readonly installed: true -} +const V5ToV6StateCodec = makeRawRowsState(MODULE_ID) -const isRecord = (value: unknown): value is Record => - typeof value === "object" && value !== null && !Array.isArray(value) +type V5ToV6State = typeof V5ToV6StateCodec.schema.Type +type V5ToV6Progress = InstalledProgress -const decodeCounts = (value: unknown): Readonly> => { - if (!isRecord(value)) throw new Error("v5 -> v6 rawRows must be an object") - const counts: Record = {} - for (const table of RAW_TABLES) { - const count = value[table] - if (typeof count !== "string" || !/^\d+$/.test(count)) - throw new Error(`v5 -> v6 rawRows.${table} must be an unsigned decimal string`) - counts[table] = count - } - if (Object.keys(value).some((table) => !RAW_TABLES.includes(table as (typeof RAW_TABLES)[number]))) - throw new Error("v5 -> v6 rawRows contains an unknown table") - return counts -} - -const decodeState = (value: unknown): V5ToV6State => { - if (!isRecord(value)) throw new Error("v5 -> v6 state must be an object") - const allowed = new Set(["module", "version", "rawRows", "retentionDays"]) - if (Object.keys(value).some((key) => !allowed.has(key))) - throw new Error("v5 -> v6 state contains an unknown field") - if (value.module !== "local-0005-to-0006-error-events-fingerprint-hygiene" || value.version !== 1) - throw new Error("v5 -> v6 state has an unsupported module or version") - if ( - value.retentionDays !== undefined && - (typeof value.retentionDays !== "number" || !Number.isSafeInteger(value.retentionDays)) - ) - throw new Error("v5 -> v6 retentionDays must be an integer") - return { - module: "local-0005-to-0006-error-events-fingerprint-hygiene", - version: 1, - rawRows: decodeCounts(value.rawRows), - ...(!(value.retentionDays === undefined) ? { retentionDays: value.retentionDays } : undefined), - } -} - -const decodeProgress = (value: unknown): V5ToV6Progress | undefined => { - if (value === undefined) return undefined - if (!isRecord(value) || Object.keys(value).some((key) => key !== "installed") || value.installed !== true) - throw new Error("v5 -> v6 progress is invalid") - return { installed: true } -} +const decodeState = V5ToV6StateCodec.decode +const decodeProgress = decodeInstalledProgress const parseJsonEachRow = (value: string): A[] => value @@ -109,12 +66,12 @@ const preflight = async (context: MigrationModuleContext): Promise }, { schemaSql: LOCAL_SCHEMA_V5_SQL, bootstrapSchema: false }, ) - return { - module: "local-0005-to-0006-error-events-fingerprint-hygiene", - version: 1, - rawRows, - ...(!(retentionDays === undefined) ? { retentionDays } : undefined), - } + // Two literals rather than a conditional spread: `retentionDays` is an + // `optionalKey`, so an absent floor has to be an absent key, not a present + // `undefined`. + return retentionDays === undefined + ? { module: MODULE_ID, version: 1, rawRows } + : { module: MODULE_ID, version: 1, rawRows, retentionDays } } const prepareTarget = async (context: MigrationModuleContext, state: V5ToV6State): Promise => { @@ -246,7 +203,7 @@ export const v5ToV6ErrorEventsFingerprintHygieneModule: LocalStoreMigrationModul V5ToV6State, V5ToV6Progress > = { - id: "local-0005-to-0006-error-events-fingerprint-hygiene", + id: MODULE_ID, moduleVersion: 1, description: "Rebuild the error-events views: exclude exception-less 4xx client spans and redact ids from fingerprint frames", diff --git a/apps/cli/src/server/local-store-migrations/v6-to-v7-error-service-version.ts b/apps/cli/src/server/local-store-migrations/v6-to-v7-error-service-version.ts index 658fe29c4..2bfc1fbb8 100644 --- a/apps/cli/src/server/local-store-migrations/v6-to-v7-error-service-version.ts +++ b/apps/cli/src/server/local-store-migrations/v6-to-v7-error-service-version.ts @@ -1,6 +1,7 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. import { cp, mkdir, rm } from "node:fs/promises" import { dirname, resolve } from "node:path" +import { decodeInstalledProgress, makeRawRowsState, type InstalledProgress } from "./journal-codecs" import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" import type { LocalStoreMigrationModule, @@ -40,60 +41,16 @@ const SERVICE_VERSION_COLUMNS = [ ], ] as const -interface V6ToV7State { - readonly module: "local-0006-to-0007-error-service-version" - readonly version: 1 - readonly rawRows: Readonly> - readonly retentionDays?: number -} +/** Stamped into the journal and matched on the way back out. */ +const MODULE_ID = "local-0006-to-0007-error-service-version" as const -interface V6ToV7Progress { - readonly installed: true -} +const V6ToV7StateCodec = makeRawRowsState(MODULE_ID) -const isRecord = (value: unknown): value is Record => - typeof value === "object" && value !== null && !Array.isArray(value) +type V6ToV7State = typeof V6ToV7StateCodec.schema.Type +type V6ToV7Progress = InstalledProgress -const decodeCounts = (value: unknown): Readonly> => { - if (!isRecord(value)) throw new Error("v6 -> v7 rawRows must be an object") - const counts: Record = {} - for (const table of RAW_TABLES) { - const count = value[table] - if (typeof count !== "string" || !/^\d+$/.test(count)) - throw new Error(`v6 -> v7 rawRows.${table} must be an unsigned decimal string`) - counts[table] = count - } - if (Object.keys(value).some((table) => !RAW_TABLES.includes(table as (typeof RAW_TABLES)[number]))) - throw new Error("v6 -> v7 rawRows contains an unknown table") - return counts -} - -const decodeState = (value: unknown): V6ToV7State => { - if (!isRecord(value)) throw new Error("v6 -> v7 state must be an object") - const allowed = new Set(["module", "version", "rawRows", "retentionDays"]) - if (Object.keys(value).some((key) => !allowed.has(key))) - throw new Error("v6 -> v7 state contains an unknown field") - if (value.module !== "local-0006-to-0007-error-service-version" || value.version !== 1) - throw new Error("v6 -> v7 state has an unsupported module or version") - if ( - value.retentionDays !== undefined && - (typeof value.retentionDays !== "number" || !Number.isSafeInteger(value.retentionDays)) - ) - throw new Error("v6 -> v7 retentionDays must be an integer") - return { - module: "local-0006-to-0007-error-service-version", - version: 1, - rawRows: decodeCounts(value.rawRows), - ...(!(value.retentionDays === undefined) ? { retentionDays: value.retentionDays } : undefined), - } -} - -const decodeProgress = (value: unknown): V6ToV7Progress | undefined => { - if (value === undefined) return undefined - if (!isRecord(value) || Object.keys(value).some((key) => key !== "installed") || value.installed !== true) - throw new Error("v6 -> v7 progress is invalid") - return { installed: true } -} +const decodeState = V6ToV7StateCodec.decode +const decodeProgress = decodeInstalledProgress const parseJsonEachRow = (value: string): A[] => value @@ -128,12 +85,12 @@ const preflight = async (context: MigrationModuleContext): Promise }, { schemaSql: LOCAL_SCHEMA_V6_SQL, bootstrapSchema: false }, ) - return { - module: "local-0006-to-0007-error-service-version", - version: 1, - rawRows, - ...(!(retentionDays === undefined) ? { retentionDays } : undefined), - } + // Two literals rather than a conditional spread: `retentionDays` is an + // `optionalKey`, so an absent floor has to be an absent key, not a present + // `undefined`. + return retentionDays === undefined + ? { module: MODULE_ID, version: 1, rawRows } + : { module: MODULE_ID, version: 1, rawRows, retentionDays } } const prepareTarget = async (context: MigrationModuleContext, state: V6ToV7State): Promise => { @@ -280,7 +237,7 @@ const dispositions: ReadonlyArray = [ ] export const v6ToV7ErrorServiceVersionModule: LocalStoreMigrationModule = { - id: "local-0006-to-0007-error-service-version", + id: MODULE_ID, moduleVersion: 1, description: "Add ServiceVersion to the error-events tables and rebuild the error-events views on fingerprint v2", diff --git a/apps/cli/src/server/local-store-migrations/v7-to-v8-apple-crash-frames.ts b/apps/cli/src/server/local-store-migrations/v7-to-v8-apple-crash-frames.ts index 7cd473bfe..6966242b8 100644 --- a/apps/cli/src/server/local-store-migrations/v7-to-v8-apple-crash-frames.ts +++ b/apps/cli/src/server/local-store-migrations/v7-to-v8-apple-crash-frames.ts @@ -1,7 +1,7 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. import { cp, mkdir, rm } from "node:fs/promises" import { dirname, resolve } from "node:path" -import { Schema } from "effect" +import { decodeInstalledProgress, makeRawRowsState, type InstalledProgress } from "./journal-codecs" import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" import type { LocalStoreMigrationModule, @@ -25,58 +25,13 @@ const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) /** Stamped into the journal and matched on the way back out. */ const MODULE_ID = "local-0007-to-0008-apple-crash-frames" as const -/** - * Unsigned decimal string, because a raw-telemetry row count can exceed - * `Number.MAX_SAFE_INTEGER` and the journal has to round-trip it exactly. - */ -const RowCount = Schema.String.check(Schema.isPattern(/^\d+$/)) - -/** - * Exactly the raw tables, each required. - * - * Built from `RAW_TABLES` rather than written out, so a table added to - * `RAW_TELEMETRY_TTL_COLUMNS` is covered here without this file changing. - * Combined with `onExcessProperty: "error"` at the decode site, this rejects a - * missing table and an unknown one alike. - */ -const RawRowsSchema = Schema.Struct(Object.fromEntries(RAW_TABLES.map((table) => [table, RowCount]))) +const V7ToV8StateCodec = makeRawRowsState(MODULE_ID) -/** - * The journal state this edge round-trips. - * - * `retentionDays` is `optionalKey`, not `optional`: the journal is JSON, where - * an absent retention floor is an absent key rather than a present `undefined`. - */ -const V7ToV8StateSchema = Schema.Struct({ - module: Schema.Literal(MODULE_ID), - version: Schema.Literal(1), - rawRows: RawRowsSchema, - retentionDays: Schema.optionalKey(Schema.Int), -}) - -const V7ToV8ProgressSchema = Schema.Struct({ installed: Schema.Literal(true) }) - -type V7ToV8State = typeof V7ToV8StateSchema.Type -type V7ToV8Progress = typeof V7ToV8ProgressSchema.Type - -/** - * `decodeUnknownSync` because `LocalStoreMigrationModule.decodeState` is a - * synchronous throwing contract shared with seven sibling modules and the - * plain-TS runner that drives them. The schema is the declarative part; - * swapping in `decodeUnknownEffect` is a one-liner if that runner ever moves - * into Effect. - * - * `onExcessProperty: "error"` is not decoration. A journal carrying a field - * this build does not know about was written by a different build, and - * silently dropping it would resume someone else's migration under our - * assumptions. - */ -const decodeState = Schema.decodeUnknownSync(V7ToV8StateSchema, { onExcessProperty: "error" }) +type V7ToV8State = typeof V7ToV8StateCodec.schema.Type +type V7ToV8Progress = InstalledProgress -const decodeProgress = (value: unknown): V7ToV8Progress | undefined => - value === undefined - ? undefined - : Schema.decodeUnknownSync(V7ToV8ProgressSchema, { onExcessProperty: "error" })(value) +const decodeState = V7ToV8StateCodec.decode +const decodeProgress = decodeInstalledProgress const parseJsonEachRow = (value: string): A[] => value diff --git a/apps/cli/test/local-store-migrations.test.ts b/apps/cli/test/local-store-migrations.test.ts index 591e23d22..fba79310a 100644 --- a/apps/cli/test/local-store-migrations.test.ts +++ b/apps/cli/test/local-store-migrations.test.ts @@ -436,7 +436,9 @@ describe("durable migration recovery", () => { ...base, chain: [{ ...base.chain[0]!, progress: { sourceInventory: [], copied: {} } }], }) - await expect(readMigrationJournal(dataDir)).rejects.toThrow(/sourceInventory must be an object/) + // The message is the schema's, so this asserts the failing field rather + // than the phrasing: an array where the inventory map belongs. + await expect(readMigrationJournal(dataDir)).rejects.toThrow(/sourceInventory/) } finally { await rm(root, { recursive: true, force: true }) } @@ -1083,3 +1085,111 @@ describe("v7 -> v8 journal state decoding", () => { expect(() => v7ToV8AppleCrashFramesModule.decodeProgress({})).toThrow() }) }) + +/** + * Characterization of the legacy raw-replay journal decoder. + * + * It guards a resumable copy out of a pre-v1 store: a journal it wrongly + * accepts resumes someone else's copy under this build's assumptions, and one + * it wrongly rejects strands a user mid-migration. Only one case was pinned + * before this, so these lock the accept/reject boundary in place. + */ +describe("legacy raw replay progress decoding", () => { + const inventory = { + table: "logs", + rowCount: "10", + retentionStartAt: "2026-01-01 00:00:00", + minTime: null, + maxTime: null, + hashSum: "1", + hashXor: "2", + } + const copied = { + rows: 1, + bytes: 2, + lastTimestamp: null, + lastHash: "12", + lastTieBreak: "13", + duplicateCount: 0, + duplicateGroupExhausted: false, + } + const pendingBatch = { + table: "logs", + rowCount: 1, + byteLength: 2, + firstTimestamp: null, + firstHash: "1", + firstTieBreak: "2", + lastTimestamp: null, + lastHash: "3", + lastTieBreak: "4", + lastKeyCount: 1, + lastKeyExhausted: false, + signature: "a".repeat(64), + } + const progress = { sourceInventory: { logs: inventory }, copied: { logs: copied } } + const decode = (value: unknown) => legacyToCurrentModule.decodeProgress(value) + + it("accepts a well-formed progress, with and without a pending batch", () => { + expect(decode(progress)).toEqual(progress) + expect(decode({ ...progress, pendingBatch })).toEqual({ ...progress, pendingBatch }) + // Absent progress is "not started", which is not the same as invalid. + expect(decode(undefined)).toBeUndefined() + }) + + it("rejects a non-object, and unknown top-level fields", () => { + for (const bad of [null, [], "x", 1]) expect(() => decode(bad)).toThrow() + expect(() => decode({ ...progress, somethingElse: 1 })).toThrow() + }) + + it("rejects tables that are not registered raw tables", () => { + expect(() => decode({ ...progress, sourceInventory: { not_a_table: inventory } })).toThrow() + expect(() => decode({ ...progress, copied: { not_a_table: copied } })).toThrow() + expect(() => + decode({ ...progress, pendingBatch: { ...pendingBatch, table: "not_a_table" } }), + ).toThrow() + }) + + it("rejects an inventory whose table disagrees with its key", () => { + expect(() => + decode({ ...progress, sourceInventory: { logs: { ...inventory, table: "traces" } } }), + ).toThrow() + }) + + it("rejects cursors that are not unsigned decimal strings", () => { + // These are interpolated into numeric SQL comparisons, so anything that + // could change their meaning has to fail here rather than there. + for (const bad of ["-1", "1.5", "0x10", "", 12]) { + expect(() => decode({ ...progress, copied: { logs: { ...copied, lastHash: bad } } })).toThrow() + } + // null is a legitimate "no cursor yet". + expect(decode({ ...progress, copied: { logs: { ...copied, lastHash: null } } })).toBeDefined() + }) + + it("rejects counters that are not non-negative safe integers", () => { + for (const bad of [-1, 1.5, Number.MAX_SAFE_INTEGER + 2, "1", null]) { + expect(() => decode({ ...progress, copied: { logs: { ...copied, rows: bad } } })).toThrow() + } + }) + + it("rejects a non-boolean exhaustion flag", () => { + expect(() => + decode({ ...progress, copied: { logs: { ...copied, duplicateGroupExhausted: "no" } } }), + ).toThrow() + }) + + it("rejects a pending batch that is empty or wrongly signed", () => { + // An empty batch would commit nothing while advancing the cursor past it. + expect(() => decode({ ...progress, pendingBatch: { ...pendingBatch, rowCount: 0 } })).toThrow() + for (const bad of ["a".repeat(63), "z".repeat(64), ""]) { + expect(() => decode({ ...progress, pendingBatch: { ...pendingBatch, signature: bad } })).toThrow() + } + }) + + it("rejects missing fields anywhere in the tree", () => { + const { hashSum: _h, ...shortInventory } = inventory + expect(() => decode({ ...progress, sourceInventory: { logs: shortInventory } })).toThrow() + const { rows: _r, ...shortCopied } = copied + expect(() => decode({ ...progress, copied: { logs: shortCopied } })).toThrow() + }) +}) From 8c259166ac4650fe48581052ec43ce015097a9cf Mon Sep 17 00:00:00 2001 From: Makisuo Date: Thu, 20 Aug 2026 17:29:04 +0200 Subject: [PATCH 6/6] refactor(cli): decode the local store's JSON artifacts with Effect Schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The migration journal, store marker, retention config, and archive journal are all untrusted JSON written by some build of this CLI, not necessarily this one. Each was decoded by hand: isRecord guards, typeof chains, an `as string` on every field read, and validation duplicated between the read path and the write path. Declare the shapes instead: - The coordinator journal envelope moves to journal-schema.ts, taking local-store-migrations.ts from 23 casts to one. MigrationPhase, MigrationStepStatus, and MigrationStepJournal now derive from the schema, so the persisted form and the in-memory type cannot drift. The chain invariants stay imperative — they relate phase, currentStepIndex, and per-step status, which is a state machine rather than a struct. - makeStoreMarker constructs through the schema readMarkerState decodes with, so the 16-hex and 64-hex rules are stated once instead of four times. Provenance fields stay lenient on purpose: a store with a garbled `maple` version is still an openable store, and refusing to start over it would be a worse failure than recording "unknown". - parseJsonEachRow was copy-pasted nine times, each ending in `JSON.parse(line) as A` at the point where chDB hands us bytes. One helper now, with the row shapes declared. - The seven versioned edges shed their identical RAW_TABLES, rawRowCounts, and expectedManifest copies. - schema-identity.ts's eight repeated identity blocks and SQL/manifest triples collapse to a derived snapshot table, and the eight copy-pasted checks in the schema gate become a loop. That closes a gap: the copies stopped at v7, so v8 was never verified. A non-vacuity guard keeps the loop honest. - In archives/, only the genuinely duplicated shape rules move — the non-negative safe integer checked at four sites and the sha256 pattern at six. The rest of those parsers are domain invariants with individually tested messages, and a struct declaration would state them worse. Behaviour-preserving throughout: no on-disk format changes, and every schema fingerprint, digest, and manifest digest is byte-identical. Verified with the full CLI suite, the schema gate, and the native probes for migration, retention, checkpoints, and archives. --- apps/cli/src/server/archives/journal.ts | 124 ++++++----- apps/cli/src/server/archives/manifest.ts | 18 +- apps/cli/src/server/archives/schemas.ts | 36 ++++ apps/cli/src/server/chdb-rows.ts | 61 ++++++ apps/cli/src/server/chdb.ts | 46 ++-- apps/cli/src/server/identity-schema.ts | 29 +++ .../server/local-store-migration-module.ts | 31 +-- apps/cli/src/server/local-store-migrations.ts | 163 ++------------ .../local-store-migrations/journal-codecs.ts | 44 +++- .../local-store-migrations/journal-schema.ts | 128 +++++++++++ .../legacy-to-current.ts | 75 ++++--- .../v1-to-v2-error-rollup.ts | 47 +--- .../v2-to-v3-service-map-ingest-bridge.ts | 37 +--- .../v3-to-v4-web-events.ts | 37 +--- .../v4-to-v5-service-overview-minutely.ts | 37 +--- ...-to-v6-error-events-fingerprint-hygiene.ts | 37 +--- .../v6-to-v7-error-service-version.ts | 37 +--- .../v7-to-v8-apple-crash-frames.ts | 37 +--- apps/cli/src/server/schema-identity.ts | 202 ++++++++---------- apps/cli/src/server/schema-physical.ts | 61 +++--- apps/cli/src/server/store-version.ts | 170 ++++++++------- apps/cli/test/journal-schema.test.ts | 143 +++++++++++++ apps/cli/test/store-marker-schema.test.ts | 121 +++++++++++ scripts/check-local-schema-manifest.ts | 125 +++-------- 24 files changed, 1034 insertions(+), 812 deletions(-) create mode 100644 apps/cli/src/server/archives/schemas.ts create mode 100644 apps/cli/src/server/chdb-rows.ts create mode 100644 apps/cli/src/server/identity-schema.ts create mode 100644 apps/cli/src/server/local-store-migrations/journal-schema.ts create mode 100644 apps/cli/test/journal-schema.test.ts create mode 100644 apps/cli/test/store-marker-schema.test.ts diff --git a/apps/cli/src/server/archives/journal.ts b/apps/cli/src/server/archives/journal.ts index 7c481b4f2..45c0fe83d 100644 --- a/apps/cli/src/server/archives/journal.ts +++ b/apps/cli/src/server/archives/journal.ts @@ -12,6 +12,8 @@ import { validateArchiveId, validateRangeDate, } from "./paths" +import { Schema } from "effect" +import { NonEmptyString, NonNegativeSafeInt, Sha256Lower } from "./schemas" import { parseArchiveActivePointer } from "./manifest" import { archiveSignal } from "./signals" @@ -177,6 +179,19 @@ const intentPath = (archiveDir: string, operationId: string): string => const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null +const decodeCountValue = Schema.decodeUnknownSync(NonNegativeSafeInt) + +/** Same rule as the GC targets' byte counts, with the field named in the error. */ +const decodeCount = (value: unknown, field: string): number => { + try { + return decodeCountValue(value) + } catch { + throw new Error(`archive operation gc intent has invalid ${field}: ${String(value)}`) + } +} + +const decodeSha256 = Schema.decodeUnknownSync(Sha256Lower) + const requiredString = (value: unknown, field: string): string => { if (typeof value !== "string" || value.length === 0) throw new Error(`journal field ${field} missing or not a string`) @@ -289,9 +304,10 @@ const parseCreateIntent = ( throw new Error(`archive operation pin purpose does not match generation: ${pinPurpose}`) } const manifestSha256Raw = raw.manifestSha256 - const manifestSha256 = - manifestSha256Raw === null ? null : requiredString(manifestSha256Raw, "manifestSha256").toLowerCase() - if (manifestSha256 !== null && !/^[0-9a-f]{64}$/.test(manifestSha256)) { + let manifestSha256: string | null + try { + manifestSha256 = manifestSha256Raw === null ? null : decodeSha256(manifestSha256Raw) + } catch { throw new Error("invalid archive operation manifestSha256") } if (phaseRequiresManifest(phase) !== (manifestSha256 !== null)) { @@ -337,20 +353,8 @@ const parseGcIntent = ( createdAt: string, updatedAt: string, ): GcOperationIntent => { - const keepRaw = raw.keep - if (typeof keepRaw !== "number" || !Number.isSafeInteger(keepRaw) || keepRaw < 0) { - throw new Error(`archive operation gc intent has invalid keep: ${String(keepRaw)}`) - } - const completedTargetsRaw = raw.completedTargets - if ( - typeof completedTargetsRaw !== "number" || - !Number.isSafeInteger(completedTargetsRaw) || - completedTargetsRaw < 0 - ) { - throw new Error( - `archive operation gc intent has invalid completedTargets: ${String(completedTargetsRaw)}`, - ) - } + const keepRaw = decodeCount(raw.keep, "keep") + const completedTargetsRaw = decodeCount(raw.completedTargets, "completedTargets") const targetsRaw = raw.targets if (!Array.isArray(targetsRaw)) { throw new Error("archive operation gc intent targets is not an array") @@ -402,50 +406,54 @@ const parseGcIntent = ( } } +/** + * One GC target exactly as the journal recorded it. + * + * Only the recorded fields are decoded here. The identities are re-validated + * through the same + * canonicalizing validators the writer used, because a hand-edited journal must + * not be able to point collection at an arbitrary path. + */ +const RecordedGcTarget = Schema.Struct({ + signal: NonEmptyString, + rangeStart: NonEmptyString, + generationId: NonEmptyString, + createdAt: NonEmptyString, + manifestSha256: Sha256Lower, + bytes: NonNegativeSafeInt, + recordedActiveGenerationId: NonEmptyString, + shards: Schema.Array( + Schema.Struct({ + name: NonEmptyString, + bytes: NonNegativeSafeInt, + sha256: Sha256Lower, + }), + ), +}) + +const decodeRecordedGcTarget = Schema.decodeUnknownSync(RecordedGcTarget) + const parseGcTarget = (raw: unknown, index: number): GcTarget => { - if (!isRecord(raw)) throw new Error(`archive gc target ${index} is not a record`) - const signal = archiveSignal(requiredString(raw.signal, "signal")).name - const rangeStart = validateRangeDate(requiredString(raw.rangeStart, "rangeStart")) - const generationId = validateArchiveId(requiredString(raw.generationId, "generationId"), "generation") - const createdAt = requiredString(raw.createdAt, "createdAt") - const manifestSha256 = requiredString(raw.manifestSha256, "manifestSha256").toLowerCase() - if (!/^[0-9a-f]{64}$/.test(manifestSha256)) { - throw new Error(`archive gc target ${index} has invalid manifestSha256`) - } - const bytesRaw = raw.bytes - if (typeof bytesRaw !== "number" || !Number.isSafeInteger(bytesRaw) || bytesRaw < 0) { - throw new Error(`archive gc target ${index} has invalid bytes: ${String(bytesRaw)}`) + let recorded: typeof RecordedGcTarget.Type + try { + recorded = decodeRecordedGcTarget(raw) + } catch (error) { + throw new Error( + `archive gc target ${index} is malformed: ${error instanceof Error ? error.message : String(error)}`, + ) } - const recordedActiveGenerationId = validateArchiveId( - requiredString(raw.recordedActiveGenerationId, "recordedActiveGenerationId"), - "active generation", - ) - const shardsRaw = raw.shards - if (!Array.isArray(shardsRaw)) { - throw new Error(`archive gc target ${index} shards is not an array`) - } - const shards = shardsRaw.map((s, j) => { - if (!isRecord(s)) throw new Error(`archive gc target ${index} shard ${j} is not a record`) - const name = requiredString(s.name, "name") - const bytes = s.bytes - if (typeof bytes !== "number" || !Number.isSafeInteger(bytes) || bytes < 0) { - throw new Error(`archive gc target ${index} shard ${j} invalid bytes`) - } - const sha256 = requiredString(s.sha256, "sha256").toLowerCase() - if (!/^[0-9a-f]{64}$/.test(sha256)) { - throw new Error(`archive gc target ${index} shard ${j} invalid sha256`) - } - return { name, bytes, sha256 } - }) return { - signal, - rangeStart, - generationId, - createdAt, - manifestSha256, - bytes: bytesRaw, - shards, - recordedActiveGenerationId, + signal: archiveSignal(recorded.signal).name, + rangeStart: validateRangeDate(recorded.rangeStart), + generationId: validateArchiveId(recorded.generationId, "generation"), + createdAt: recorded.createdAt, + manifestSha256: recorded.manifestSha256, + bytes: recorded.bytes, + shards: recorded.shards, + recordedActiveGenerationId: validateArchiveId( + recorded.recordedActiveGenerationId, + "active generation", + ), } } diff --git a/apps/cli/src/server/archives/manifest.ts b/apps/cli/src/server/archives/manifest.ts index cb61b70c1..dd684db0a 100644 --- a/apps/cli/src/server/archives/manifest.ts +++ b/apps/cli/src/server/archives/manifest.ts @@ -1,11 +1,13 @@ import { readFileSync } from "node:fs" import { join } from "node:path" +import { Schema } from "effect" import { type ArchiveTuningRecord, type TuningConfigIdentity, LEGACY_TUNING_CONFIG_FORMAT_VERSION, TUNING_CONFIG_FORMAT_VERSION, } from "./config" +import { NonNegativeSafeInt, Sha256Lower } from "./schemas" import { KNOWN_COMPLEX_DIGEST_ALGORITHMS } from "./export" import { assertNoSymlinkSync, @@ -150,12 +152,14 @@ const requiredString = (record: Record, key: string): string => return value } +const decodeCount = Schema.decodeUnknownSync(NonNegativeSafeInt) + const requiredCount = (record: Record, key: string): number => { - const value = record[key] - if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) { + try { + return decodeCount(record[key]) + } catch { throw new Error(`invalid archive manifest field: ${key} (must be a safe non-negative integer)`) } - return value } const requiredPositiveInteger = (record: Record, key: string): number => { @@ -166,7 +170,9 @@ const requiredPositiveInteger = (record: Record, key: string): return value } -const SHA256_HEX = /^[0-9a-f]{64}$/ +/** The same 64-hex rule the journal enforces, stated once in ./schemas. */ +const isSha256Hex = (value: string): boolean => + Schema.decodeUnknownResult(Sha256Lower)(value)._tag === "Success" /** A safe logical config name (no path separators, no traversal). */ const SAFE_CONFIG_NAME = /^[A-Za-z0-9._-]+$/ @@ -208,7 +214,7 @@ const parseTuningConfig = (value: unknown): TuningConfigIdentity | null => { throw new Error(`invalid archive manifest tuningConfig.configName (unsafe name): ${configName}`) } const sha256 = requiredString(value, "sha256") - if (!SHA256_HEX.test(sha256)) { + if (!isSha256Hex(sha256)) { throw new Error(`invalid archive manifest tuningConfig.sha256 (must be 64 hex chars): ${sha256}`) } return { formatVersion, configName, sha256 } @@ -297,7 +303,7 @@ const parseShardRecord = ( return c }) const sha256 = requiredString(value, "sha256") - if (!SHA256_HEX.test(sha256)) + if (!isSha256Hex(sha256)) throw new Error(`invalid archive shard sha256 (must be 64 hex chars): ${sha256}`) const rowCount = requiredCount(value, "rowCount") const minNano = requiredNanoDecimal(value, "minEventTimeUnixNano") diff --git a/apps/cli/src/server/archives/schemas.ts b/apps/cli/src/server/archives/schemas.ts new file mode 100644 index 000000000..77c5a554f --- /dev/null +++ b/apps/cli/src/server/archives/schemas.ts @@ -0,0 +1,36 @@ +// Shape primitives shared by the archive journal and manifest readers. +// +// These are the rules that were genuinely copy-pasted: a non-negative safe +// integer was re-checked at four sites in the GC intent alone, and the sha256 +// pattern at six across the two files. The rest of the archive parsers are +// domain invariants — path containment, phase-vs-kind compatibility, cursor +// consistency — each with its own tested message, and those stay where they +// are rather than being flattened into a struct declaration. +import { Schema, SchemaGetter } from "effect" + +/** Byte counts and cursors: never negative, never beyond exact integer range. */ +export const NonNegativeSafeInt = Schema.Number.check( + Schema.makeFilter((value: number) => + Number.isSafeInteger(value) && value >= 0 ? undefined : "must be a safe non-negative integer", + ), +) + +/** + * A sha256 digest, normalized to lower case before it is compared. + * + * Case normalization is part of the rule, not a courtesy: these digests gate + * whether a generation may be deleted, and an upper-case digest that failed to + * compare equal would look like corruption of data that is in fact intact. + */ +export const Sha256Lower = Schema.String.pipe( + Schema.decodeTo(Schema.String, { + decode: SchemaGetter.transform((value: string) => value.toLowerCase()), + encode: SchemaGetter.passthrough(), + }), +).check( + Schema.makeFilter((value: string) => + /^[0-9a-f]{64}$/.test(value) ? undefined : "must be 64 hex characters", + ), +) + +export const NonEmptyString = Schema.String.check(Schema.isMinLength(1)) diff --git a/apps/cli/src/server/chdb-rows.ts b/apps/cli/src/server/chdb-rows.ts new file mode 100644 index 000000000..d7580e4f3 --- /dev/null +++ b/apps/cli/src/server/chdb-rows.ts @@ -0,0 +1,61 @@ +// Decoding rows out of chDB's JSONEachRow output. +// +// Nine copies of a `split("\n") … JSON.parse(line) as A` helper used to live +// across the migration coordinator, the physical-schema inspector, and every +// versioned edge — each one ending in an unchecked cast at the point where an +// external process hands us bytes. The shape is declared once here and the +// rows are decoded, not asserted. +import { Schema } from "effect" + +/** + * Decode chDB JSONEachRow output. + * + * A row that does not match the schema is a query returning something other + * than what the caller asked for, which is a bug in the SQL rather than data to + * be tolerated — so this throws instead of skipping the row. + */ +export const decodeJsonEachRow = >( + rowSchema: S, +): ((value: string) => Array) => { + const decodeRow = Schema.decodeUnknownSync(rowSchema) + return (value) => + value + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => decodeRow(JSON.parse(line))) +} + +/** + * `sum(rows)` per table, as returned by the raw-telemetry inventory query. + * + * The count is `toString()`-wrapped in SQL because it is a ClickHouse UInt64: + * above 2^53 a JS number silently loses the low bits, and these values are + * compared for exact equality when a migration verifies that no row was lost. + */ +export const TableRowCountSchema = Schema.Struct({ + table: Schema.String, + rowCount: Schema.String.check(Schema.isPattern(/^\d+$/)), +}) + +export const decodeTableRowCounts = decodeJsonEachRow(TableRowCountSchema) + +/** A single `toString(count())`-style scalar, for the same UInt64 reason. */ +export const RowCountSchema = Schema.Struct({ + rowCount: Schema.String.check(Schema.isPattern(/^\d+$/)), +}) + +export const decodeRowCounts = decodeJsonEachRow(RowCountSchema) + +/** + * Rows whose columns are not known ahead of time. + * + * The v0 -> v1 raw replay copies whatever columns the source table happens to + * have, so there is no field list to declare. Requiring each line to be a JSON + * object is the only claim that can honestly be made about it, and it is still + * a claim worth making: a bare scalar or array here means the query returned + * something other than rows. + */ +const OpaqueRowSchema = Schema.Record(Schema.String, Schema.Unknown) + +export const decodeJsonObjectRows = decodeJsonEachRow(OpaqueRowSchema) diff --git a/apps/cli/src/server/chdb.ts b/apps/cli/src/server/chdb.ts index 59c7f43fe..92089e3a8 100644 --- a/apps/cli/src/server/chdb.ts +++ b/apps/cli/src/server/chdb.ts @@ -104,32 +104,34 @@ export const RAW_TELEMETRY_TTL_COLUMNS = [ export const MINIMUM_RAW_TELEMETRY_RETENTION_DAYS = 90 export const MAXIMUM_RAW_TELEMETRY_RETENTION_DAYS = 3_650 -interface RawTelemetryRetentionConfig { - readonly formatVersion: 1 - readonly minimumDays: number -} +/** + * The retention floor an operator has pinned for this store. + * + * Unknown fields are rejected rather than ignored: a config carrying a field + * this build does not understand was written by a different build, and reading + * only the half we recognise would silently apply a policy nobody chose. + */ +const RawTelemetryRetentionConfigSchema = Schema.Struct({ + formatVersion: Schema.Literal(1), + minimumDays: Schema.Int.check( + Schema.makeFilter((days: number) => + days >= MINIMUM_RAW_TELEMETRY_RETENTION_DAYS && days <= MAXIMUM_RAW_TELEMETRY_RETENTION_DAYS + ? undefined + : `raw telemetry retention minimum must be an integer from ${MINIMUM_RAW_TELEMETRY_RETENTION_DAYS} through ${MAXIMUM_RAW_TELEMETRY_RETENTION_DAYS} days`, + ), + ), +}) + +type RawTelemetryRetentionConfig = typeof RawTelemetryRetentionConfigSchema.Type export const rawTelemetryRetentionConfigPath = (dataDir: string): string => `${resolve(dataDir)}.raw-telemetry-retention.json` -const parseRawTelemetryRetentionDays = (value: unknown): number => { - if (typeof value !== "object" || value === null || Array.isArray(value)) - throw new Error("raw telemetry retention config must be a record") - const record = value as Record - if (Object.keys(record).sort().join(",") !== "formatVersion,minimumDays" || record.formatVersion !== 1) - throw new Error("unsupported or malformed raw telemetry retention config") - const days = record.minimumDays - if ( - typeof days !== "number" || - !Number.isSafeInteger(days) || - days < MINIMUM_RAW_TELEMETRY_RETENTION_DAYS || - days > MAXIMUM_RAW_TELEMETRY_RETENTION_DAYS - ) - throw new Error( - `raw telemetry retention minimum must be an integer from ${MINIMUM_RAW_TELEMETRY_RETENTION_DAYS} through ${MAXIMUM_RAW_TELEMETRY_RETENTION_DAYS} days`, - ) - return days -} +const decodeRetentionConfig = Schema.decodeUnknownSync(RawTelemetryRetentionConfigSchema, { + onExcessProperty: "error", +}) + +const parseRawTelemetryRetentionDays = (value: unknown): number => decodeRetentionConfig(value).minimumDays export const readRawTelemetryRetentionDays = (dataDir: string): number | undefined => { const path = rawTelemetryRetentionConfigPath(dataDir) diff --git a/apps/cli/src/server/identity-schema.ts b/apps/cli/src/server/identity-schema.ts new file mode 100644 index 000000000..32c892361 --- /dev/null +++ b/apps/cli/src/server/identity-schema.ts @@ -0,0 +1,29 @@ +// Shared identity primitives for the local store. +// +// The 16-character bundle fingerprint and the 64-character sha256 digest used +// to be re-checked with an inline regex at every site that read or wrote one: +// twice in the marker reader, twice more in the marker constructor, and once +// again inside the v0 -> v1 migration. Four copies of the same rule is four +// chances to relax one of them. +import { Schema } from "effect" + +/** Legacy 16-character bundled-schema fingerprint. */ +export const Fingerprint16 = Schema.String.check(Schema.isPattern(/^[0-9a-f]{16}$/i)) + +/** Full sha256 hex digest. */ +export const Digest64 = Schema.String.check(Schema.isPattern(/^[0-9a-f]{64}$/i)) + +/** + * An ISO timestamp, or the literal `"unknown"`. + * + * Provenance timestamps on a store predate the versioned marker, so a store + * created by an old build genuinely has no creation time. That is recorded + * rather than guessed. + */ +export const IsoOrUnknown = Schema.String.check( + Schema.makeFilter((value: string) => + value === "unknown" || Number.isFinite(Date.parse(value)) + ? undefined + : 'must be an ISO timestamp or "unknown"', + ), +) diff --git a/apps/cli/src/server/local-store-migration-module.ts b/apps/cli/src/server/local-store-migration-module.ts index f0d2f2193..1e47a0394 100644 --- a/apps/cli/src/server/local-store-migration-module.ts +++ b/apps/cli/src/server/local-store-migration-module.ts @@ -1,20 +1,17 @@ // BOUNDARY: This module owns unparsed external values and narrows them before domain use. import type { Chdb } from "./chdb" import type { LocalSchemaIdentity } from "./schema-identity" +import type { + MigrationPhase, + MigrationStepJournalSchema, + MigrationStepStatus, +} from "./local-store-migrations/journal-schema" /** Coordinator-owned transaction phases. Modules may report progress, but - * only the coordinator advances this top-level state machine. */ -export type MigrationPhase = - | "planned" - | "preflight-complete" - | "target-created" - | "copying" - | "copy-verified" - | "promotion-started" - | "promoted" - | "failed" - -export type MigrationStepStatus = "pending" | "running" | "verified" | "completed" + * only the coordinator advances this top-level state machine. Both unions are + * derived from the journal schema so the persisted form and the in-memory type + * cannot drift. */ +export type { MigrationPhase, MigrationStepStatus } export type StateDisposition = | "preserve-exact" @@ -48,15 +45,7 @@ export interface StateDispositionEntry { /** The persisted binding for one edge in a migration chain. The identities * are copied into the journal so a later build cannot reinterpret an old * step through its current-schema constant. */ -export interface MigrationStepJournal { - readonly id: string - readonly moduleVersion: number - readonly from: LocalSchemaIdentity - readonly to: LocalSchemaIdentity - readonly status: MigrationStepStatus - readonly state?: unknown - readonly progress?: unknown -} +export type MigrationStepJournal = typeof MigrationStepJournalSchema.Type export interface MigrationDbOptions { readonly schemaSql?: string diff --git a/apps/cli/src/server/local-store-migrations.ts b/apps/cli/src/server/local-store-migrations.ts index 162856db3..f41dc937c 100644 --- a/apps/cli/src/server/local-store-migrations.ts +++ b/apps/cli/src/server/local-store-migrations.ts @@ -9,7 +9,9 @@ import { randomUUID } from "node:crypto" import { existsSync, readFileSync, statfsSync } from "node:fs" import { lstat, readdir, readFile, stat } from "node:fs/promises" import { dirname, join, relative, resolve } from "node:path" +import { Schema } from "effect" import { Chdb } from "./chdb" +import { decodeJsonEachRow } from "./chdb-rows" import { CURRENT_LOCAL_SCHEMA, LEGACY_LOCAL_SCHEMA, @@ -33,6 +35,7 @@ import { } from "./store-version" import { durableJson, durableRename, ensurePrivateDirectory } from "./durable-files" import { MAPLE_VERSION } from "../version" +import { decodeMigrationJournal, type MigrationJournalSchema } from "./local-store-migrations/journal-schema" import { legacyToCurrentModule } from "./local-store-migrations/legacy-to-current" import { v1ToV2ErrorRollupModule } from "./local-store-migrations/v1-to-v2-error-rollup" import { v2ToV3ServiceMapIngestBridgeModule } from "./local-store-migrations/v2-to-v3-service-map-ingest-bridge" @@ -48,7 +51,6 @@ import type { MigrationModuleContext, MigrationPhase, MigrationStepJournal, - MigrationStepStatus, } from "./local-store-migration-module" export { @@ -84,28 +86,7 @@ export interface MigrationPlan { readonly checkpointDisposition: string } -export interface MigrationJournal { - readonly formatVersion: 2 - readonly migrationId: string - readonly phase: MigrationPhase - readonly chain: ReadonlyArray - readonly currentStepIndex: number - readonly sourceDataDir: string - readonly sourceStoreId: string - readonly sourceChdb: string - readonly sourceFingerprint: string - readonly sourceDigest: string - readonly sourceVersion: number - readonly targetDataDir: string - readonly targetStoreId: string - readonly targetChdb: string - readonly targetFingerprint: string - readonly targetDigest: string - readonly targetVersion: number - readonly cutoffAt: string - readonly createdAt: string - readonly failure?: string -} +export type MigrationJournal = typeof MigrationJournalSchema.Type export interface MigrationResult { readonly migrationId: string @@ -323,8 +304,6 @@ export const migrationRootPath = (dataDir: string, migrationId: string): string export const migrationHistoryPath = (dataDir: string, migrationId: string): string => join(migrationRootPath(dataDir, migrationId), "journal.json") -const migrationIdPattern = /^[A-Za-z0-9._-]+$/ - const safeMigrationPath = (path: string, root: string, label: string): string => { const absolute = resolve(path) const relativePath = relative(resolve(root), absolute) @@ -362,73 +341,6 @@ const assertJournalPaths = (dataDir: string, journal: MigrationJournal): void => throw new Error("local migration journal final target does not match its chain") } -const parsePhase = (value: unknown): MigrationPhase => { - if ( - value !== "planned" && - value !== "preflight-complete" && - value !== "target-created" && - value !== "copying" && - value !== "copy-verified" && - value !== "promotion-started" && - value !== "promoted" && - value !== "failed" - ) - throw new Error(`invalid local migration phase: ${String(value)}`) - return value -} - -const parseIdentity = (value: unknown, label: string): LocalSchemaIdentity => { - if (typeof value !== "object" || value === null || Array.isArray(value)) - throw new Error(`migration journal ${label} identity is invalid`) - const identity = value as Record - if ( - !Number.isInteger(identity.version) || - (identity.version as number) < 0 || - typeof identity.fingerprint !== "string" || - identity.fingerprint.length === 0 || - typeof identity.digest !== "string" || - typeof identity.chdb !== "string" || - identity.chdb.length === 0 - ) - throw new Error(`migration journal ${label} identity is invalid`) - return { - version: identity.version as number, - fingerprint: identity.fingerprint, - digest: identity.digest, - chdb: identity.chdb, - ...(typeof identity.manifestDigest === "string" - ? { manifestDigest: identity.manifestDigest } - : undefined), - ...(typeof identity.projectRevision === "string" - ? { projectRevision: identity.projectRevision } - : undefined), - } -} - -const parseStep = (value: unknown, index: number): MigrationStepJournal => { - if (typeof value !== "object" || value === null || Array.isArray(value)) - throw new Error(`migration journal step ${index} is invalid`) - const step = value as Record - const status = step.status - if ( - typeof step.id !== "string" || - step.id.length === 0 || - !Number.isInteger(step.moduleVersion) || - (step.moduleVersion as number) < 1 || - (status !== "pending" && status !== "running" && status !== "verified" && status !== "completed") - ) - throw new Error(`migration journal step ${index} is invalid`) - return { - id: step.id, - moduleVersion: step.moduleVersion as number, - from: parseIdentity(step.from, `step ${index} from`), - to: parseIdentity(step.to, `step ${index} to`), - status: status as MigrationStepStatus, - ...(!(step.state === undefined) ? { state: step.state } : undefined), - ...(!(step.progress === undefined) ? { progress: step.progress } : undefined), - } -} - const sameJournalIdentity = (a: LocalSchemaIdentity, b: LocalSchemaIdentity): boolean => a.version === b.version && a.fingerprint === b.fingerprint && a.digest === b.digest && a.chdb === b.chdb @@ -503,59 +415,9 @@ const assertJournalChainInvariants = (journal: MigrationJournal): void => { } const parseJournal = (value: unknown): MigrationJournal => { - if (typeof value !== "object" || value === null || Array.isArray(value)) - throw new Error("migration journal is not an object") - const record = value as Record - const requiredStrings = [ - "migrationId", - "sourceDataDir", - "sourceStoreId", - "sourceChdb", - "sourceFingerprint", - "targetDataDir", - "targetStoreId", - "targetChdb", - "targetFingerprint", - "targetDigest", - "cutoffAt", - "createdAt", - ] as const - for (const key of requiredStrings) - if (typeof record[key] !== "string" || record[key] === "") - throw new Error(`migration journal ${key} is invalid`) - for (const key of ["sourceVersion", "targetVersion", "currentStepIndex"] as const) - if (!Number.isInteger(record[key])) throw new Error(`migration journal ${key} is invalid`) - if (record.formatVersion !== 2) - throw new Error(`unsupported migration journal format ${String(record.formatVersion)}`) - if (!migrationIdPattern.test(record.migrationId as string)) - throw new Error("migration journal id is unsafe") - if (!Array.isArray(record.chain) || record.chain.length === 0) - throw new Error("migration journal chain is invalid") - const chain = record.chain.map(parseStep) - if ((record.currentStepIndex as number) < 0 || (record.currentStepIndex as number) > chain.length) + const journal = decodeMigrationJournal(value) + if (journal.currentStepIndex > journal.chain.length) throw new Error("migration journal currentStepIndex is invalid") - const journal: MigrationJournal = { - formatVersion: 2, - migrationId: record.migrationId as string, - phase: parsePhase(record.phase), - chain, - currentStepIndex: record.currentStepIndex as number, - sourceDataDir: resolve(record.sourceDataDir as string), - sourceStoreId: record.sourceStoreId as string, - sourceChdb: record.sourceChdb as string, - sourceFingerprint: record.sourceFingerprint as string, - sourceDigest: typeof record.sourceDigest === "string" ? record.sourceDigest : "", - sourceVersion: record.sourceVersion as number, - targetDataDir: resolve(record.targetDataDir as string), - targetStoreId: record.targetStoreId as string, - targetChdb: record.targetChdb as string, - targetFingerprint: record.targetFingerprint as string, - targetDigest: record.targetDigest as string, - targetVersion: record.targetVersion as number, - cutoffAt: record.cutoffAt as string, - createdAt: record.createdAt as string, - ...(!(record.failure === undefined) ? { failure: String(record.failure) } : undefined), - } assertJournalChainInvariants(journal) for (const [index, step] of journal.chain.entries()) { if (step.status === "verified" || step.status === "completed") { @@ -662,12 +524,11 @@ const assertNoLiveServer = (dataDir: string): void => { } } -const parseJsonEachRow = (value: string): A[] => - value - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length > 0) - .map((line) => JSON.parse(line) as A) +/** Total bytes on disk. A `string | number` because the setting that unquotes + * 64-bit integers is not in force for every libchdb build this CLI supports. */ +const DiskUsageRow = Schema.Struct({ bytes: Schema.Union([Schema.String, Schema.Number]) }) + +const decodeDiskUsageRows = decodeJsonEachRow(DiskUsageRow) const MIN_MIGRATION_FREE_BYTES = 128 * 1024 * 1024 @@ -754,7 +615,7 @@ const ensureMigrationCapacity = async (dataDir: string, session: MigrationDbSess const rows = await session.use( dataDir, (db) => - parseJsonEachRow<{ bytes: string | number }>( + decodeDiskUsageRows( db.query( "SELECT coalesce(sum(bytes_on_disk), 0) AS bytes FROM system.parts WHERE database = 'default' AND active = 1", ), diff --git a/apps/cli/src/server/local-store-migrations/journal-codecs.ts b/apps/cli/src/server/local-store-migrations/journal-codecs.ts index 7db5b4825..b2615af33 100644 --- a/apps/cli/src/server/local-store-migrations/journal-codecs.ts +++ b/apps/cli/src/server/local-store-migrations/journal-codecs.ts @@ -12,9 +12,11 @@ // `Schema.decodeUnknownEffect` is a one-line swap if that runner ever moves // into Effect. import { Schema } from "effect" -import { RAW_TELEMETRY_TTL_COLUMNS } from "../chdb" +import { RAW_TELEMETRY_TTL_COLUMNS, type Chdb } from "../chdb" +import { decodeTableRowCounts } from "../chdb-rows" +import { withRawTelemetryRetentionFloor, type LocalSchemaManifest } from "../schema-manifest" -const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) +const RAW_TABLES_INTERNAL = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) /** * Unsigned decimal string. @@ -36,7 +38,7 @@ export const UnsignedDecimal = Schema.String.check(Schema.isPattern(/^\d+$/)) * separate passes. */ export const RawRowsSchema = Schema.Struct( - Object.fromEntries(RAW_TABLES.map((table) => [table, UnsignedDecimal])), + Object.fromEntries(RAW_TABLES_INTERNAL.map((table) => [table, UnsignedDecimal])), ) /** @@ -82,3 +84,39 @@ export const decodeInstalledProgress = (value: unknown): InstalledProgress | und */ export const strictDecoder = >(schema: S) => Schema.decodeUnknownSync(schema, strict) + +/** Exactly the raw telemetry tables a migration must preserve, in a stable order. */ +export const RAW_TABLES: ReadonlyArray = RAW_TABLES_INTERNAL + +/** + * Row counts per raw table, straight from `system.parts`. + * + * Every versioned edge carried a byte-identical copy of this. It is the input + * to the only guarantee those edges make — that a structural DDL change moves + * no telemetry — so it belongs in one place where that query can be reasoned + * about once. + */ +export const rawRowCounts = (db: Chdb): Readonly> => { + const quotedTables = RAW_TABLES_INTERNAL.map((table) => `'${table}'`).join(", ") + const rows = decodeTableRowCounts( + db.query( + `SELECT table, toString(sum(rows)) AS rowCount FROM system.parts WHERE database = 'default' AND active = 1 AND table IN (${quotedTables}) GROUP BY table`, + ), + ) + const byTable = new Map(rows.map((row) => [row.table, row.rowCount])) + return Object.fromEntries(RAW_TABLES_INTERNAL.map((table) => [table, byTable.get(table) ?? "0"])) +} + +/** + * The manifest an edge should expect to find, given the retention floor an + * operator pinned for this store. A pinned floor rewrites the raw tables' TTL + * intervals, so comparing against the bundled manifest verbatim would report a + * drift the operator asked for. + */ +export const expectedManifest = ( + manifest: LocalSchemaManifest, + retentionDays: number | undefined, +): LocalSchemaManifest => + retentionDays === undefined + ? manifest + : withRawTelemetryRetentionFloor(manifest, RAW_TABLES_INTERNAL, retentionDays) diff --git a/apps/cli/src/server/local-store-migrations/journal-schema.ts b/apps/cli/src/server/local-store-migrations/journal-schema.ts new file mode 100644 index 000000000..9b613547e --- /dev/null +++ b/apps/cli/src/server/local-store-migrations/journal-schema.ts @@ -0,0 +1,128 @@ +// Declarative shape of the coordinator-owned migration journal. +// +// The journal is untrusted JSON: it was written by some build of this CLI, not +// necessarily this one. Everything below used to be a hand-rolled parser in +// `local-store-migrations.ts` — `typeof` chains, `Number.isInteger` guards, and +// an `as string` on every field read. The shape is now declared once and the +// casts are gone; what remains imperative in that file are the cross-field +// invariants (phase x currentStepIndex x per-step status), which are a state +// machine rather than a struct. +// +// Decoding is synchronous and throwing to match the coordinator, which is plain +// TypeScript throw/catch code. `Schema.decodeUnknownEffect` is a one-line swap +// if that ever moves into Effect. +import { Effect, Schema, SchemaGetter } from "effect" +import { resolve } from "node:path" + +/** + * Rejecting unknown fields is not tidiness. A journal carrying a field this + * build does not know about was written by a different build, and silently + * dropping it would resume someone else's migration under our assumptions. + */ +const strict = { onExcessProperty: "error" } as const + +const NonEmptyString = Schema.String.check(Schema.isMinLength(1)) + +/** A migration id is interpolated into filesystem paths; keep it path-safe. */ +const migrationIdPattern = /^[A-Za-z0-9._-]+$/ + +/** + * Paths are normalized on the way in so a journal written with a relative or + * unnormalized path compares equal to the configured data directory. `resolve` + * is idempotent on an absolute path, so the encoding side is a passthrough. + */ +const ResolvedPath = NonEmptyString.pipe( + Schema.decodeTo(Schema.String, { + decode: SchemaGetter.transform(resolve), + encode: SchemaGetter.passthrough(), + }), +) + +/** Coordinator-owned transaction phases. */ +export const MigrationPhaseSchema = Schema.Literals([ + "planned", + "preflight-complete", + "target-created", + "copying", + "copy-verified", + "promotion-started", + "promoted", + "failed", +]) + +export type MigrationPhase = typeof MigrationPhaseSchema.Type + +export const MigrationStepStatusSchema = Schema.Literals(["pending", "running", "verified", "completed"]) + +export type MigrationStepStatus = typeof MigrationStepStatusSchema.Type + +/** + * A schema identity as persisted in a journal. + * + * `manifestDigest` and `projectRevision` are `optionalKey`, not `optional`: the + * journal is JSON, where an absent field is an absent key rather than a present + * `undefined`. That is also what lets the coordinator write them back without a + * conditional spread. + */ +export const LocalSchemaIdentitySchema = Schema.Struct({ + version: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), + fingerprint: NonEmptyString, + digest: Schema.String, + manifestDigest: Schema.optionalKey(Schema.String), + chdb: NonEmptyString, + projectRevision: Schema.optionalKey(Schema.String), +}) + +/** + * One edge in a migration chain. + * + * `state` and `progress` stay `Unknown` on purpose. The coordinator does not + * interpret them; each module decodes its own through `decodeState` / + * `decodeProgress`. Keeping that split is what lets a staged target be + * abandoned when a module's persisted state is corrupt or its code is no + * longer bindable. + */ +export const MigrationStepJournalSchema = Schema.Struct({ + id: NonEmptyString, + moduleVersion: Schema.Int.check(Schema.isGreaterThanOrEqualTo(1)), + from: LocalSchemaIdentitySchema, + to: LocalSchemaIdentitySchema, + status: MigrationStepStatusSchema, + state: Schema.optionalKey(Schema.Unknown), + progress: Schema.optionalKey(Schema.Unknown), +}) + +export const MigrationJournalSchema = Schema.Struct({ + formatVersion: Schema.Literal(2), + migrationId: Schema.String.check(Schema.isPattern(migrationIdPattern)), + phase: MigrationPhaseSchema, + chain: Schema.Array(MigrationStepJournalSchema).check(Schema.isMinLength(1)), + currentStepIndex: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), + sourceDataDir: ResolvedPath, + sourceStoreId: NonEmptyString, + sourceChdb: NonEmptyString, + sourceFingerprint: NonEmptyString, + // Asymmetric with `targetDigest` on purpose: the v0 legacy identity has no + // digest at all (see LOCAL_SCHEMA_HISTORY), so a journal migrating away from + // it carries an empty source digest. A target is always a known schema. + sourceDigest: Schema.String.pipe(Schema.withDecodingDefaultKey(Effect.succeed(""))), + sourceVersion: Schema.Int, + targetDataDir: ResolvedPath, + targetStoreId: NonEmptyString, + targetChdb: NonEmptyString, + targetFingerprint: NonEmptyString, + targetDigest: NonEmptyString, + targetVersion: Schema.Int, + cutoffAt: NonEmptyString, + createdAt: NonEmptyString, + failure: Schema.optionalKey(Schema.String), +}) + +const decodeJournal = Schema.decodeUnknownSync(MigrationJournalSchema, strict) + +/** + * Decode the journal envelope. Cross-field chain invariants are asserted by the + * coordinator after this returns; this only establishes the shape. + */ +export const decodeMigrationJournal = (value: unknown): typeof MigrationJournalSchema.Type => + decodeJournal(value) diff --git a/apps/cli/src/server/local-store-migrations/legacy-to-current.ts b/apps/cli/src/server/local-store-migrations/legacy-to-current.ts index d33e8e15f..fddec8fd4 100644 --- a/apps/cli/src/server/local-store-migrations/legacy-to-current.ts +++ b/apps/cli/src/server/local-store-migrations/legacy-to-current.ts @@ -1,6 +1,7 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. import { createHash } from "node:crypto" import { Schema } from "effect" +import { decodeJsonEachRow, decodeJsonObjectRows } from "../chdb-rows" import { LOCAL_SCHEMA_V1_MANIFEST, LOCAL_SCHEMA_V1_SQL, @@ -238,12 +239,41 @@ const decodeRawReplayProgress = (value: unknown): RawReplayProgress => { const asProgress = (value: RawReplayProgress | undefined): RawReplayProgress => value ?? { sourceInventory: {}, copied: {} } -const parseJsonEachRow = (value: string): A[] => - value - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length > 0) - .map((line) => JSON.parse(line) as A) +/** + * Row shapes read back out of chDB. + * + * `rowCount` and the hash aggregates are `string | number` on purpose: a UInt64 + * arrives quoted or unquoted depending on the libchdb build, and `numberString` + * normalizes it. What must not happen is a `Number` decode above 2^53, which is + * why nothing here is `Schema.Number` alone. + */ +const Uint64Wire = Schema.Union([Schema.String, Schema.Number]) + +const ColumnRowSchema = Schema.Struct({ + name: Schema.String, + type: Schema.String, + position: Schema.Number, + default_kind: Schema.String, + default_expression: Schema.String, + compression_codec: Schema.String, +}) + +const NameRow = Schema.Struct({ name: Schema.String }) + +const InventoryRow = Schema.Struct({ + rowCount: Uint64Wire, + minTime: Schema.NullOr(Schema.String), + maxTime: Schema.NullOr(Schema.String), + hashSum: Uint64Wire, + hashXor: Uint64Wire, +}) + +const RowCountRow = Schema.Struct({ rowCount: Uint64Wire }) + +const decodeColumnRows = decodeJsonEachRow(ColumnRowSchema) +const decodeNameRows = decodeJsonEachRow(NameRow) +const decodeInventoryRows = decodeJsonEachRow(InventoryRow) +const decodeRowCountRows = decodeJsonEachRow(RowCountRow) const identifier = (value: string): string => { if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(value)) throw new Error(`unsafe ClickHouse identifier: ${value}`) @@ -286,22 +316,13 @@ const queryTarget = (context: MigrationModuleContext, sql: string): Promise => context.openTarget((db) => db.exec(sql), { schemaSql: LOCAL_SCHEMA_V1_SQL, bootstrapSchema: false }) -interface ColumnRow { - name: string - type: string - position: number - default_kind: string - default_expression: string - compression_codec: string -} - const tableColumns = async ( context: MigrationModuleContext, side: "source" | "target", table: string, ): Promise> => { const query = `SELECT name, type, position, default_kind, default_expression, compression_codec FROM system.columns WHERE database = 'default' AND table = ${sqlString(table)} ORDER BY position` - const rows = parseJsonEachRow( + const rows = decodeColumnRows( await (side === "source" ? querySource(context, query) : queryTarget(context, query)), ) return rows.map((row) => ({ @@ -318,7 +339,7 @@ const tableColumns = async ( } const tableExists = async (context: MigrationModuleContext, side: "source" | "target", table: string) => { - const rows = parseJsonEachRow<{ name: string }>( + const rows = decodeNameRows( await (side === "source" ? querySource( context, @@ -334,7 +355,7 @@ const tableExists = async (context: MigrationModuleContext, side: "source" | "ta const assertKnownSourceObjects = async (context: MigrationModuleContext): Promise => { const known = new Set(LOCAL_SCHEMA_V1_MANIFEST.objects.map((object) => object.name)) - const rows = parseJsonEachRow<{ name: string }>( + const rows = decodeNameRows( await querySource(context, "SELECT name FROM system.tables WHERE database = 'default' ORDER BY name"), ) for (const row of rows) { @@ -389,13 +410,9 @@ const inventory = async ( // UInt64 aggregates leave SQL as strings: chDB emits 64-bit integers as // JSON numbers, and JS Number rounding above 2^53 corrupts them. const sql = `SELECT count() AS rowCount, min(${identifier(table.timeColumn)}) AS minTime, max(${identifier(table.timeColumn)}) AS maxTime, toString(sum(${hash})) AS hashSum, toString(groupBitXor(${hash})) AS hashXor FROM ${identifier(table.name)} WHERE ${identifier(table.timeColumn)} >= ${timestampLiteral(lowerBound)} AND ${identifier(table.timeColumn)} <= ${timestampLiteral(cutoffAt)}` - const rows = parseJsonEachRow<{ - rowCount: string | number - minTime: string | null - maxTime: string | null - hashSum: string | number - hashXor: string | number - }>(side === "source" ? await querySource(context, sql) : await queryTarget(context, sql)) + const rows = decodeInventoryRows( + side === "source" ? await querySource(context, sql) : await queryTarget(context, sql), + ) const row = rows[0] if (!row) throw new Error(`inventory query returned no row for ${table.name}`) return { @@ -424,7 +441,7 @@ const tableTotalRowCount = async ( table: string, ): Promise => { const sql = `SELECT count() AS rowCount FROM ${identifier(table)}` - const rows = parseJsonEachRow<{ rowCount: string | number }>( + const rows = decodeRowCountRows( side === "source" ? await querySource(context, sql) : await queryTarget(context, sql), ) if (!rows[0]) throw new Error(`row-count query returned no row for ${table}`) @@ -507,7 +524,7 @@ const duplicateGroupCount = async ( const hash = `cityHash64(toString(tuple(${names})))` const tie = `sipHash64(toString(tuple(${names})))` const sql = `SELECT count() AS rowCount FROM ${identifier(table.name)} WHERE ${nsExpression(identifier(table.timeColumn))} = ${uint64Literal(progress.lastTimestamp, "lastTimestamp")} AND ${hash} = ${uint64Literal(progress.lastHash, "lastHash")} AND ${tie} = ${uint64Literal(progress.lastTieBreak, "lastTieBreak")}` - const rows = parseJsonEachRow<{ rowCount: string | number }>(await querySource(context, sql)) + const rows = decodeRowCountRows(await querySource(context, sql)) return rows[0] === undefined ? 0 : Number(rows[0].rowCount) } @@ -548,7 +565,7 @@ const copyTable = async ( context, `SELECT ${columnList}, toString(${timeNs}) AS __maple_timestamp, toString(${hashExpression}) AS __maple_hash, toString(${tieBreakExpression}) AS __maple_tie_break FROM ${identifier(table.name)} WHERE ${identifier(table.timeColumn)} >= ${timestampLiteral(retentionStartAt(context.cutoffAt, table.retentionDays))} AND ${identifier(table.timeColumn)} <= ${timestampLiteral(context.cutoffAt)} ${cursor} ORDER BY ${timeNs}, ${hashExpression}, ${tieBreakExpression} LIMIT ${table.batchRows}${offset}`, ) - const rawRows = parseJsonEachRow>(output) + const rawRows = decodeJsonObjectRows(output) let rows = rawRows if (rows.length === 0) break const candidates = rows.map((row) => { @@ -642,7 +659,7 @@ const recoverPendingBatch = async ( ? "" : `AND (${timeNs} > ${uint64Literal(previous.lastTimestamp, "lastTimestamp")} OR (${timeNs} = ${uint64Literal(previous.lastTimestamp, "lastTimestamp")} AND (${hashExpression} > ${uint64Literal(previous.lastHash, "lastHash")} OR (${hashExpression} = ${uint64Literal(previous.lastHash, "lastHash")} AND ${tieBreakExpression} ${continuation.comparison} ${uint64Literal(previous.lastTieBreak, "lastTieBreak")}))))` const offset = continuation.offset === 0 ? "" : ` OFFSET ${continuation.offset}` - const inserted = parseJsonEachRow>( + const inserted = decodeJsonObjectRows( await queryTarget( context, `SELECT ${columnList}, toString(${timeNs}) AS __maple_timestamp, toString(${hashExpression}) AS __maple_hash, toString(${tieBreakExpression}) AS __maple_tie_break FROM ${identifier(table.name)} WHERE ${identifier(table.timeColumn)} >= ${timestampLiteral(retentionStartAt(context.cutoffAt, table.retentionDays))} AND ${identifier(table.timeColumn)} <= ${timestampLiteral(context.cutoffAt)} ${cursor} ORDER BY ${timeNs}, ${hashExpression}, ${tieBreakExpression} LIMIT ${pending.rowCount}${offset}`, diff --git a/apps/cli/src/server/local-store-migrations/v1-to-v2-error-rollup.ts b/apps/cli/src/server/local-store-migrations/v1-to-v2-error-rollup.ts index 6295fb3bc..9f22845dc 100644 --- a/apps/cli/src/server/local-store-migrations/v1-to-v2-error-rollup.ts +++ b/apps/cli/src/server/local-store-migrations/v1-to-v2-error-rollup.ts @@ -2,20 +2,22 @@ import { cp, mkdir, rm } from "node:fs/promises" import { dirname, resolve } from "node:path" import { Schema } from "effect" -import { makeRawRowsState, strictDecoder, UnsignedDecimal } from "./journal-codecs" +import { decodeRowCounts } from "../chdb-rows" import { - applyRawTelemetryRetentionFloor, - RAW_TELEMETRY_TTL_COLUMNS, - readRawTelemetryRetentionDays, - type Chdb, -} from "../chdb" + UnsignedDecimal, + makeRawRowsState, + strictDecoder, + RAW_TABLES, + rawRowCounts, + expectedManifest, +} from "./journal-codecs" +import { applyRawTelemetryRetentionFloor, readRawTelemetryRetentionDays } from "../chdb" import type { LocalStoreMigrationModule, MigrationModuleContext, MigrationOperation, StateDispositionEntry, } from "../local-store-migration-module" -import { withRawTelemetryRetentionFloor } from "../schema-manifest" import { LOCAL_SCHEMA_V1, LOCAL_SCHEMA_V1_MANIFEST, @@ -26,8 +28,6 @@ import { } from "../schema-identity" import { assertPhysicalSchema } from "../schema-physical" -const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) - /** Stamped into the journal and matched on the way back out. */ const MODULE_ID = "local-0001-to-0002-error-rollup" as const @@ -49,29 +49,6 @@ const decodeV1ToV2Progress = strictDecoder(V1ToV2ProgressSchema) const decodeProgress = (value: unknown): V1ToV2Progress | undefined => value === undefined ? undefined : decodeV1ToV2Progress(value) -const parseJsonEachRow = (value: string): A[] => - value - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length > 0) - .map((line) => JSON.parse(line) as A) - -const rawRowCounts = (db: Chdb): Readonly> => { - const quotedTables = RAW_TABLES.map((table) => `'${table}'`).join(", ") - const rows = parseJsonEachRow<{ table: string; rowCount: string }>( - db.query( - `SELECT table, toString(sum(rows)) AS rowCount FROM system.parts WHERE database = 'default' AND active = 1 AND table IN (${quotedTables}) GROUP BY table`, - ), - ) - const byTable = new Map(rows.map((row) => [row.table, row.rowCount])) - return Object.fromEntries(RAW_TABLES.map((table) => [table, byTable.get(table) ?? "0"])) -} - -const expectedManifest = (manifest: typeof LOCAL_SCHEMA_V1_MANIFEST, retentionDays: number | undefined) => - retentionDays === undefined - ? manifest - : withRawTelemetryRetentionFloor(manifest, RAW_TABLES, retentionDays) - const preflight = async (context: MigrationModuleContext): Promise => { await context.ensureCapacity() const retentionDays = readRawTelemetryRetentionDays(context.dataDir) @@ -135,7 +112,7 @@ const apply = async ( (db) => { if (state.retentionDays !== undefined) applyRawTelemetryRetentionFloor(db, state.retentionDays) db.exec(backfillSql) - const [row] = parseJsonEachRow<{ rowCount: string }>( + const [row] = decodeRowCounts( db.query( "SELECT toString(sum(OccurrenceCount)) AS rowCount FROM error_fingerprints_minutely", ), @@ -159,9 +136,7 @@ const verify = async ( if (targetRows[table] !== state.rawRows[table]) throw new Error(`v1 -> v2 raw telemetry verification failed for ${table}`) } - const [row] = parseJsonEachRow<{ rowCount: string }>( - db.query("SELECT toString(count()) AS rowCount FROM error_events"), - ) + const [row] = decodeRowCounts(db.query("SELECT toString(count()) AS rowCount FROM error_events")) if ((row?.rowCount ?? "0") !== progress.backfilledErrorEvents) throw new Error("v1 -> v2 error rollup backfill verification failed") }, diff --git a/apps/cli/src/server/local-store-migrations/v2-to-v3-service-map-ingest-bridge.ts b/apps/cli/src/server/local-store-migrations/v2-to-v3-service-map-ingest-bridge.ts index 7e91f67eb..10e4e60ac 100644 --- a/apps/cli/src/server/local-store-migrations/v2-to-v3-service-map-ingest-bridge.ts +++ b/apps/cli/src/server/local-store-migrations/v2-to-v3-service-map-ingest-bridge.ts @@ -1,15 +1,21 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. import { cp, mkdir, rm } from "node:fs/promises" import { dirname, resolve } from "node:path" -import { decodeInstalledProgress, makeRawRowsState, type InstalledProgress } from "./journal-codecs" -import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" +import { + decodeInstalledProgress, + makeRawRowsState, + type InstalledProgress, + RAW_TABLES, + rawRowCounts, + expectedManifest, +} from "./journal-codecs" +import { readRawTelemetryRetentionDays } from "../chdb" import type { LocalStoreMigrationModule, MigrationModuleContext, MigrationOperation, StateDispositionEntry, } from "../local-store-migration-module" -import { withRawTelemetryRetentionFloor } from "../schema-manifest" import { LOCAL_SCHEMA_V2, LOCAL_SCHEMA_V2_MANIFEST, @@ -20,8 +26,6 @@ import { } from "../schema-identity" import { assertPhysicalSchema } from "../schema-physical" -const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) - /** Stamped into the journal and matched on the way back out. */ const MODULE_ID = "local-0002-to-0003-service-map-ingest-bridge" as const @@ -33,29 +37,6 @@ type V2ToV3Progress = InstalledProgress const decodeState = V2ToV3StateCodec.decode const decodeProgress = decodeInstalledProgress -const parseJsonEachRow = (value: string): A[] => - value - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length > 0) - .map((line) => JSON.parse(line) as A) - -const rawRowCounts = (db: Chdb): Readonly> => { - const quotedTables = RAW_TABLES.map((table) => `'${table}'`).join(", ") - const rows = parseJsonEachRow<{ table: string; rowCount: string }>( - db.query( - `SELECT table, toString(sum(rows)) AS rowCount FROM system.parts WHERE database = 'default' AND active = 1 AND table IN (${quotedTables}) GROUP BY table`, - ), - ) - const byTable = new Map(rows.map((row) => [row.table, row.rowCount])) - return Object.fromEntries(RAW_TABLES.map((table) => [table, byTable.get(table) ?? "0"])) -} - -const expectedManifest = (manifest: typeof LOCAL_SCHEMA_V2_MANIFEST, retentionDays: number | undefined) => - retentionDays === undefined - ? manifest - : withRawTelemetryRetentionFloor(manifest, RAW_TABLES, retentionDays) - const preflight = async (context: MigrationModuleContext): Promise => { await context.ensureCapacity() const retentionDays = readRawTelemetryRetentionDays(context.dataDir) diff --git a/apps/cli/src/server/local-store-migrations/v3-to-v4-web-events.ts b/apps/cli/src/server/local-store-migrations/v3-to-v4-web-events.ts index a3c19bd11..3a9af3138 100644 --- a/apps/cli/src/server/local-store-migrations/v3-to-v4-web-events.ts +++ b/apps/cli/src/server/local-store-migrations/v3-to-v4-web-events.ts @@ -1,15 +1,21 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. import { cp, mkdir, rm } from "node:fs/promises" import { dirname, resolve } from "node:path" -import { decodeInstalledProgress, makeRawRowsState, type InstalledProgress } from "./journal-codecs" -import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" +import { + decodeInstalledProgress, + makeRawRowsState, + type InstalledProgress, + RAW_TABLES, + rawRowCounts, + expectedManifest, +} from "./journal-codecs" +import { readRawTelemetryRetentionDays } from "../chdb" import type { LocalStoreMigrationModule, MigrationModuleContext, MigrationOperation, StateDispositionEntry, } from "../local-store-migration-module" -import { withRawTelemetryRetentionFloor } from "../schema-manifest" import { LOCAL_SCHEMA_V3, LOCAL_SCHEMA_V3_MANIFEST, @@ -20,8 +26,6 @@ import { } from "../schema-identity" import { assertPhysicalSchema } from "../schema-physical" -const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) - /** Stamped into the journal and matched on the way back out. */ const MODULE_ID = "local-0003-to-0004-web-events" as const @@ -33,29 +37,6 @@ type V3ToV4Progress = InstalledProgress const decodeState = V3ToV4StateCodec.decode const decodeProgress = decodeInstalledProgress -const parseJsonEachRow = (value: string): A[] => - value - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length > 0) - .map((line) => JSON.parse(line) as A) - -const rawRowCounts = (db: Chdb): Readonly> => { - const quotedTables = RAW_TABLES.map((table) => `'${table}'`).join(", ") - const rows = parseJsonEachRow<{ table: string; rowCount: string }>( - db.query( - `SELECT table, toString(sum(rows)) AS rowCount FROM system.parts WHERE database = 'default' AND active = 1 AND table IN (${quotedTables}) GROUP BY table`, - ), - ) - const byTable = new Map(rows.map((row) => [row.table, row.rowCount])) - return Object.fromEntries(RAW_TABLES.map((table) => [table, byTable.get(table) ?? "0"])) -} - -const expectedManifest = (manifest: typeof LOCAL_SCHEMA_V3_MANIFEST, retentionDays: number | undefined) => - retentionDays === undefined - ? manifest - : withRawTelemetryRetentionFloor(manifest, RAW_TABLES, retentionDays) - const preflight = async (context: MigrationModuleContext): Promise => { await context.ensureCapacity() const retentionDays = readRawTelemetryRetentionDays(context.dataDir) diff --git a/apps/cli/src/server/local-store-migrations/v4-to-v5-service-overview-minutely.ts b/apps/cli/src/server/local-store-migrations/v4-to-v5-service-overview-minutely.ts index 9d523831d..e0429be07 100644 --- a/apps/cli/src/server/local-store-migrations/v4-to-v5-service-overview-minutely.ts +++ b/apps/cli/src/server/local-store-migrations/v4-to-v5-service-overview-minutely.ts @@ -1,15 +1,21 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. import { cp, mkdir, rm } from "node:fs/promises" import { dirname, resolve } from "node:path" -import { decodeInstalledProgress, makeRawRowsState, type InstalledProgress } from "./journal-codecs" -import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" +import { + decodeInstalledProgress, + makeRawRowsState, + type InstalledProgress, + RAW_TABLES, + rawRowCounts, + expectedManifest, +} from "./journal-codecs" +import { readRawTelemetryRetentionDays } from "../chdb" import type { LocalStoreMigrationModule, MigrationModuleContext, MigrationOperation, StateDispositionEntry, } from "../local-store-migration-module" -import { withRawTelemetryRetentionFloor } from "../schema-manifest" import { LOCAL_SCHEMA_V4, LOCAL_SCHEMA_V4_MANIFEST, @@ -20,8 +26,6 @@ import { } from "../schema-identity" import { assertPhysicalSchema } from "../schema-physical" -const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) - /** Stamped into the journal and matched on the way back out. */ const MODULE_ID = "local-0004-to-0005-service-overview-minutely" as const @@ -33,29 +37,6 @@ type V4ToV5Progress = InstalledProgress const decodeState = V4ToV5StateCodec.decode const decodeProgress = decodeInstalledProgress -const parseJsonEachRow = (value: string): A[] => - value - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length > 0) - .map((line) => JSON.parse(line) as A) - -const rawRowCounts = (db: Chdb): Readonly> => { - const quotedTables = RAW_TABLES.map((table) => `'${table}'`).join(", ") - const rows = parseJsonEachRow<{ table: string; rowCount: string }>( - db.query( - `SELECT table, toString(sum(rows)) AS rowCount FROM system.parts WHERE database = 'default' AND active = 1 AND table IN (${quotedTables}) GROUP BY table`, - ), - ) - const byTable = new Map(rows.map((row) => [row.table, row.rowCount])) - return Object.fromEntries(RAW_TABLES.map((table) => [table, byTable.get(table) ?? "0"])) -} - -const expectedManifest = (manifest: typeof LOCAL_SCHEMA_V4_MANIFEST, retentionDays: number | undefined) => - retentionDays === undefined - ? manifest - : withRawTelemetryRetentionFloor(manifest, RAW_TABLES, retentionDays) - const preflight = async (context: MigrationModuleContext): Promise => { await context.ensureCapacity() const retentionDays = readRawTelemetryRetentionDays(context.dataDir) diff --git a/apps/cli/src/server/local-store-migrations/v5-to-v6-error-events-fingerprint-hygiene.ts b/apps/cli/src/server/local-store-migrations/v5-to-v6-error-events-fingerprint-hygiene.ts index 1d1191b0b..4818e3f45 100644 --- a/apps/cli/src/server/local-store-migrations/v5-to-v6-error-events-fingerprint-hygiene.ts +++ b/apps/cli/src/server/local-store-migrations/v5-to-v6-error-events-fingerprint-hygiene.ts @@ -1,15 +1,21 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. import { cp, mkdir, rm } from "node:fs/promises" import { dirname, resolve } from "node:path" -import { decodeInstalledProgress, makeRawRowsState, type InstalledProgress } from "./journal-codecs" -import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" +import { + decodeInstalledProgress, + makeRawRowsState, + type InstalledProgress, + RAW_TABLES, + rawRowCounts, + expectedManifest, +} from "./journal-codecs" +import { readRawTelemetryRetentionDays } from "../chdb" import type { LocalStoreMigrationModule, MigrationModuleContext, MigrationOperation, StateDispositionEntry, } from "../local-store-migration-module" -import { withRawTelemetryRetentionFloor } from "../schema-manifest" import { LOCAL_SCHEMA_V5, LOCAL_SCHEMA_V5_MANIFEST, @@ -20,8 +26,6 @@ import { } from "../schema-identity" import { assertPhysicalSchema } from "../schema-physical" -const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) - /** Stamped into the journal and matched on the way back out. */ const MODULE_ID = "local-0005-to-0006-error-events-fingerprint-hygiene" as const @@ -33,29 +37,6 @@ type V5ToV6Progress = InstalledProgress const decodeState = V5ToV6StateCodec.decode const decodeProgress = decodeInstalledProgress -const parseJsonEachRow = (value: string): A[] => - value - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length > 0) - .map((line) => JSON.parse(line) as A) - -const rawRowCounts = (db: Chdb): Readonly> => { - const quotedTables = RAW_TABLES.map((table) => `'${table}'`).join(", ") - const rows = parseJsonEachRow<{ table: string; rowCount: string }>( - db.query( - `SELECT table, toString(sum(rows)) AS rowCount FROM system.parts WHERE database = 'default' AND active = 1 AND table IN (${quotedTables}) GROUP BY table`, - ), - ) - const byTable = new Map(rows.map((row) => [row.table, row.rowCount])) - return Object.fromEntries(RAW_TABLES.map((table) => [table, byTable.get(table) ?? "0"])) -} - -const expectedManifest = (manifest: typeof LOCAL_SCHEMA_V5_MANIFEST, retentionDays: number | undefined) => - retentionDays === undefined - ? manifest - : withRawTelemetryRetentionFloor(manifest, RAW_TABLES, retentionDays) - const preflight = async (context: MigrationModuleContext): Promise => { await context.ensureCapacity() const retentionDays = readRawTelemetryRetentionDays(context.dataDir) diff --git a/apps/cli/src/server/local-store-migrations/v6-to-v7-error-service-version.ts b/apps/cli/src/server/local-store-migrations/v6-to-v7-error-service-version.ts index 2bfc1fbb8..4045f5d6a 100644 --- a/apps/cli/src/server/local-store-migrations/v6-to-v7-error-service-version.ts +++ b/apps/cli/src/server/local-store-migrations/v6-to-v7-error-service-version.ts @@ -1,15 +1,21 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. import { cp, mkdir, rm } from "node:fs/promises" import { dirname, resolve } from "node:path" -import { decodeInstalledProgress, makeRawRowsState, type InstalledProgress } from "./journal-codecs" -import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" +import { + decodeInstalledProgress, + makeRawRowsState, + type InstalledProgress, + RAW_TABLES, + rawRowCounts, + expectedManifest, +} from "./journal-codecs" +import { readRawTelemetryRetentionDays } from "../chdb" import type { LocalStoreMigrationModule, MigrationModuleContext, MigrationOperation, StateDispositionEntry, } from "../local-store-migration-module" -import { withRawTelemetryRetentionFloor } from "../schema-manifest" import { LOCAL_SCHEMA_V6, LOCAL_SCHEMA_V6_MANIFEST, @@ -20,8 +26,6 @@ import { } from "../schema-identity" import { assertPhysicalSchema } from "../schema-physical" -const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) - /** Error-family views replaced by this edge, dropped before the v7 DDL runs. */ const ERROR_VIEWS = ["error_events_mv", "error_events_by_time_mv", "error_fingerprints_minutely_mv"] as const @@ -52,29 +56,6 @@ type V6ToV7Progress = InstalledProgress const decodeState = V6ToV7StateCodec.decode const decodeProgress = decodeInstalledProgress -const parseJsonEachRow = (value: string): A[] => - value - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length > 0) - .map((line) => JSON.parse(line) as A) - -const rawRowCounts = (db: Chdb): Readonly> => { - const quotedTables = RAW_TABLES.map((table) => `'${table}'`).join(", ") - const rows = parseJsonEachRow<{ table: string; rowCount: string }>( - db.query( - `SELECT table, toString(sum(rows)) AS rowCount FROM system.parts WHERE database = 'default' AND active = 1 AND table IN (${quotedTables}) GROUP BY table`, - ), - ) - const byTable = new Map(rows.map((row) => [row.table, row.rowCount])) - return Object.fromEntries(RAW_TABLES.map((table) => [table, byTable.get(table) ?? "0"])) -} - -const expectedManifest = (manifest: typeof LOCAL_SCHEMA_V6_MANIFEST, retentionDays: number | undefined) => - retentionDays === undefined - ? manifest - : withRawTelemetryRetentionFloor(manifest, RAW_TABLES, retentionDays) - const preflight = async (context: MigrationModuleContext): Promise => { await context.ensureCapacity() const retentionDays = readRawTelemetryRetentionDays(context.dataDir) diff --git a/apps/cli/src/server/local-store-migrations/v7-to-v8-apple-crash-frames.ts b/apps/cli/src/server/local-store-migrations/v7-to-v8-apple-crash-frames.ts index 6966242b8..197dffb88 100644 --- a/apps/cli/src/server/local-store-migrations/v7-to-v8-apple-crash-frames.ts +++ b/apps/cli/src/server/local-store-migrations/v7-to-v8-apple-crash-frames.ts @@ -1,15 +1,21 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. import { cp, mkdir, rm } from "node:fs/promises" import { dirname, resolve } from "node:path" -import { decodeInstalledProgress, makeRawRowsState, type InstalledProgress } from "./journal-codecs" -import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" +import { + decodeInstalledProgress, + makeRawRowsState, + type InstalledProgress, + RAW_TABLES, + rawRowCounts, + expectedManifest, +} from "./journal-codecs" +import { readRawTelemetryRetentionDays } from "../chdb" import type { LocalStoreMigrationModule, MigrationModuleContext, MigrationOperation, StateDispositionEntry, } from "../local-store-migration-module" -import { withRawTelemetryRetentionFloor } from "../schema-manifest" import { LOCAL_SCHEMA_V7, LOCAL_SCHEMA_V7_MANIFEST, @@ -20,8 +26,6 @@ import { } from "../schema-identity" import { assertPhysicalSchema } from "../schema-physical" -const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) - /** Stamped into the journal and matched on the way back out. */ const MODULE_ID = "local-0007-to-0008-apple-crash-frames" as const @@ -33,29 +37,6 @@ type V7ToV8Progress = InstalledProgress const decodeState = V7ToV8StateCodec.decode const decodeProgress = decodeInstalledProgress -const parseJsonEachRow = (value: string): A[] => - value - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length > 0) - .map((line) => JSON.parse(line) as A) - -const rawRowCounts = (db: Chdb): Readonly> => { - const quotedTables = RAW_TABLES.map((table) => `'${table}'`).join(", ") - const rows = parseJsonEachRow<{ table: string; rowCount: string }>( - db.query( - `SELECT table, toString(sum(rows)) AS rowCount FROM system.parts WHERE database = 'default' AND active = 1 AND table IN (${quotedTables}) GROUP BY table`, - ), - ) - const byTable = new Map(rows.map((row) => [row.table, row.rowCount])) - return Object.fromEntries(RAW_TABLES.map((table) => [table, byTable.get(table) ?? "0"])) -} - -const expectedManifest = (manifest: typeof LOCAL_SCHEMA_V7_MANIFEST, retentionDays: number | undefined) => - retentionDays === undefined - ? manifest - : withRawTelemetryRetentionFloor(manifest, RAW_TABLES, retentionDays) - const preflight = async (context: MigrationModuleContext): Promise => { await context.ensureCapacity() const retentionDays = readRawTelemetryRetentionDays(context.dataDir) diff --git a/apps/cli/src/server/schema-identity.ts b/apps/cli/src/server/schema-identity.ts index 125aacd6a..dcf566f1b 100644 --- a/apps/cli/src/server/schema-identity.ts +++ b/apps/cli/src/server/schema-identity.ts @@ -41,45 +41,73 @@ export const SCHEMA_FINGERPRINT = fingerprintSchema(schemaSql) export const SCHEMA_DIGEST = digestSchema(schemaSql) export const LOCAL_SCHEMA_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaSql) export const LOCAL_SCHEMA_MANIFEST_DIGEST = LOCAL_SCHEMA_MANIFEST.digest -/** Immutable v1 DDL/manifest snapshot used by the v0 -> v1 module even after - * the generated current schema advances. */ -export const LOCAL_SCHEMA_V1_SQL = schemaV1Sql -export const LOCAL_SCHEMA_V1_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaV1Sql) -export const LOCAL_SCHEMA_V1_MANIFEST_DIGEST = LOCAL_SCHEMA_V1_MANIFEST.digest -/** Immutable v2 DDL/manifest snapshot used by the v1 -> v2 module even after - * the generated current schema advances. */ -export const LOCAL_SCHEMA_V2_SQL = schemaV2Sql -export const LOCAL_SCHEMA_V2_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaV2Sql) -export const LOCAL_SCHEMA_V2_MANIFEST_DIGEST = LOCAL_SCHEMA_V2_MANIFEST.digest -/** Immutable v3 DDL/manifest snapshot used by the v2 -> v3 module after the - * generated current schema advances. */ -export const LOCAL_SCHEMA_V3_SQL = schemaV3Sql -export const LOCAL_SCHEMA_V3_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaV3Sql) -export const LOCAL_SCHEMA_V3_MANIFEST_DIGEST = LOCAL_SCHEMA_V3_MANIFEST.digest -/** Immutable v4 DDL/manifest snapshot used by the v3 -> v4 module after the - * generated current schema advances. */ -export const LOCAL_SCHEMA_V4_SQL = schemaV4Sql -export const LOCAL_SCHEMA_V4_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaV4Sql) -export const LOCAL_SCHEMA_V4_MANIFEST_DIGEST = LOCAL_SCHEMA_V4_MANIFEST.digest -/** Immutable v5 DDL/manifest snapshot used by the v4 -> v5 module after the - * generated current schema advances. */ -export const LOCAL_SCHEMA_V5_SQL = schemaV5Sql -export const LOCAL_SCHEMA_V5_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaV5Sql) -export const LOCAL_SCHEMA_V5_MANIFEST_DIGEST = LOCAL_SCHEMA_V5_MANIFEST.digest -/** Immutable v6 DDL/manifest snapshot used by the v5 -> v6 module after the - * generated current schema advances. */ -export const LOCAL_SCHEMA_V6_SQL = schemaV6Sql -export const LOCAL_SCHEMA_V6_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaV6Sql) -export const LOCAL_SCHEMA_V6_MANIFEST_DIGEST = LOCAL_SCHEMA_V6_MANIFEST.digest -/** Immutable v7 DDL/manifest snapshot used by the v6 -> v7 module after the - * generated current schema advances. */ -export const LOCAL_SCHEMA_V7_SQL = schemaV7Sql -export const LOCAL_SCHEMA_V7_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaV7Sql) -export const LOCAL_SCHEMA_V7_MANIFEST_DIGEST = LOCAL_SCHEMA_V7_MANIFEST.digest +/** + * Immutable per-version DDL and manifest snapshots. + * + * A historical edge must keep constructing and verifying the schema it was + * written for: when v9 ships, v7 -> v8 must still produce v8 rather than + * silently retargeting whatever the generator currently emits. The SQL is + * imported literally because Bun resolves text imports statically; everything + * derived from it is built once, here. + */ +const SNAPSHOT_SQL: ReadonlyArray = [ + schemaV1Sql, + schemaV2Sql, + schemaV3Sql, + schemaV4Sql, + schemaV5Sql, + schemaV6Sql, + schemaV7Sql, + schemaV8Sql, +] + +export interface LocalSchemaSnapshot { + readonly version: number + readonly sql: string + readonly manifest: LocalSchemaManifest + readonly manifestDigest: string +} + +/** Indexed by schema version; index 0 is the fingerprint-only legacy store, which has no DDL. */ +export const LOCAL_SCHEMA_SNAPSHOTS: ReadonlyArray = Object.freeze([ + undefined, + ...SNAPSHOT_SQL.map((sql, index) => { + const manifest = buildLocalSchemaManifest(sql) + return Object.freeze({ version: index + 1, sql, manifest, manifestDigest: manifest.digest }) + }), +]) + +const snapshotAt = (version: number): LocalSchemaSnapshot => { + const snapshot = LOCAL_SCHEMA_SNAPSHOTS[version] + if (!snapshot) throw new Error(`no bundled DDL snapshot for local schema version ${version}`) + return snapshot +} + +export const LOCAL_SCHEMA_V1_SQL = snapshotAt(1).sql +export const LOCAL_SCHEMA_V1_MANIFEST = snapshotAt(1).manifest +export const LOCAL_SCHEMA_V1_MANIFEST_DIGEST = snapshotAt(1).manifestDigest +export const LOCAL_SCHEMA_V2_SQL = snapshotAt(2).sql +export const LOCAL_SCHEMA_V2_MANIFEST = snapshotAt(2).manifest +export const LOCAL_SCHEMA_V2_MANIFEST_DIGEST = snapshotAt(2).manifestDigest +export const LOCAL_SCHEMA_V3_SQL = snapshotAt(3).sql +export const LOCAL_SCHEMA_V3_MANIFEST = snapshotAt(3).manifest +export const LOCAL_SCHEMA_V3_MANIFEST_DIGEST = snapshotAt(3).manifestDigest +export const LOCAL_SCHEMA_V4_SQL = snapshotAt(4).sql +export const LOCAL_SCHEMA_V4_MANIFEST = snapshotAt(4).manifest +export const LOCAL_SCHEMA_V4_MANIFEST_DIGEST = snapshotAt(4).manifestDigest +export const LOCAL_SCHEMA_V5_SQL = snapshotAt(5).sql +export const LOCAL_SCHEMA_V5_MANIFEST = snapshotAt(5).manifest +export const LOCAL_SCHEMA_V5_MANIFEST_DIGEST = snapshotAt(5).manifestDigest +export const LOCAL_SCHEMA_V6_SQL = snapshotAt(6).sql +export const LOCAL_SCHEMA_V6_MANIFEST = snapshotAt(6).manifest +export const LOCAL_SCHEMA_V6_MANIFEST_DIGEST = snapshotAt(6).manifestDigest +export const LOCAL_SCHEMA_V7_SQL = snapshotAt(7).sql +export const LOCAL_SCHEMA_V7_MANIFEST = snapshotAt(7).manifest +export const LOCAL_SCHEMA_V7_MANIFEST_DIGEST = snapshotAt(7).manifestDigest +export const LOCAL_SCHEMA_V8_SQL = snapshotAt(8).sql +export const LOCAL_SCHEMA_V8_MANIFEST = snapshotAt(8).manifest +export const LOCAL_SCHEMA_V8_MANIFEST_DIGEST = snapshotAt(8).manifestDigest -export const LOCAL_SCHEMA_V8_SQL = schemaV8Sql -export const LOCAL_SCHEMA_V8_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaV8Sql) -export const LOCAL_SCHEMA_V8_MANIFEST_DIGEST = LOCAL_SCHEMA_V8_MANIFEST.digest export interface LocalSchemaIdentity { readonly version: number readonly fingerprint: string @@ -90,81 +118,33 @@ export interface LocalSchemaIdentity { } /** - * The v1 identity is deliberately frozen. Historical migration edges must - * never point at CURRENT_LOCAL_SCHEMA: when v2 ships, v0 -> v1 must still - * construct and verify v1 rather than silently changing its destination. + * Per-version identities, frozen and read straight from the append-only + * history. Historical migration edges must never point at + * CURRENT_LOCAL_SCHEMA: when v9 ships, v0 -> v1 must still construct and verify + * v1 rather than silently changing its destination. */ -export const LOCAL_SCHEMA_V1: LocalSchemaIdentity = Object.freeze({ - version: LOCAL_SCHEMA_HISTORY[1]!.version, - fingerprint: LOCAL_SCHEMA_HISTORY[1]!.fingerprint, - digest: LOCAL_SCHEMA_HISTORY[1]!.digest, - manifestDigest: LOCAL_SCHEMA_HISTORY[1]!.manifestDigest, - chdb: CHDB_VERSION, - projectRevision: LOCAL_SCHEMA_HISTORY[1]!.projectRevision, -}) - -export const LOCAL_SCHEMA_V2: LocalSchemaIdentity = Object.freeze({ - version: LOCAL_SCHEMA_HISTORY[2]!.version, - fingerprint: LOCAL_SCHEMA_HISTORY[2]!.fingerprint, - digest: LOCAL_SCHEMA_HISTORY[2]!.digest, - manifestDigest: LOCAL_SCHEMA_HISTORY[2]!.manifestDigest, - chdb: CHDB_VERSION, - projectRevision: LOCAL_SCHEMA_HISTORY[2]!.projectRevision, -}) - -export const LOCAL_SCHEMA_V3: LocalSchemaIdentity = Object.freeze({ - version: LOCAL_SCHEMA_HISTORY[3]!.version, - fingerprint: LOCAL_SCHEMA_HISTORY[3]!.fingerprint, - digest: LOCAL_SCHEMA_HISTORY[3]!.digest, - manifestDigest: LOCAL_SCHEMA_HISTORY[3]!.manifestDigest, - chdb: CHDB_VERSION, - projectRevision: LOCAL_SCHEMA_HISTORY[3]!.projectRevision, -}) - -export const LOCAL_SCHEMA_V4: LocalSchemaIdentity = Object.freeze({ - version: LOCAL_SCHEMA_HISTORY[4]!.version, - fingerprint: LOCAL_SCHEMA_HISTORY[4]!.fingerprint, - digest: LOCAL_SCHEMA_HISTORY[4]!.digest, - manifestDigest: LOCAL_SCHEMA_HISTORY[4]!.manifestDigest, - chdb: CHDB_VERSION, - projectRevision: LOCAL_SCHEMA_HISTORY[4]!.projectRevision, -}) - -export const LOCAL_SCHEMA_V5: LocalSchemaIdentity = Object.freeze({ - version: LOCAL_SCHEMA_HISTORY[5]!.version, - fingerprint: LOCAL_SCHEMA_HISTORY[5]!.fingerprint, - digest: LOCAL_SCHEMA_HISTORY[5]!.digest, - manifestDigest: LOCAL_SCHEMA_HISTORY[5]!.manifestDigest, - chdb: CHDB_VERSION, - projectRevision: LOCAL_SCHEMA_HISTORY[5]!.projectRevision, -}) - -export const LOCAL_SCHEMA_V6: LocalSchemaIdentity = Object.freeze({ - version: LOCAL_SCHEMA_HISTORY[6]!.version, - fingerprint: LOCAL_SCHEMA_HISTORY[6]!.fingerprint, - digest: LOCAL_SCHEMA_HISTORY[6]!.digest, - manifestDigest: LOCAL_SCHEMA_HISTORY[6]!.manifestDigest, - chdb: CHDB_VERSION, - projectRevision: LOCAL_SCHEMA_HISTORY[6]!.projectRevision, -}) - -export const LOCAL_SCHEMA_V7: LocalSchemaIdentity = Object.freeze({ - version: LOCAL_SCHEMA_HISTORY[7]!.version, - fingerprint: LOCAL_SCHEMA_HISTORY[7]!.fingerprint, - digest: LOCAL_SCHEMA_HISTORY[7]!.digest, - manifestDigest: LOCAL_SCHEMA_HISTORY[7]!.manifestDigest, - chdb: CHDB_VERSION, - projectRevision: LOCAL_SCHEMA_HISTORY[7]!.projectRevision, -}) +const identityAt = (version: number): LocalSchemaIdentity => { + const entry = LOCAL_SCHEMA_HISTORY[version] + if (!entry || entry.version !== version) + throw new Error(`local schema history has no entry for version ${version}`) + return Object.freeze({ + version: entry.version, + fingerprint: entry.fingerprint, + digest: entry.digest, + manifestDigest: entry.manifestDigest, + chdb: CHDB_VERSION, + projectRevision: entry.projectRevision, + }) +} -export const LOCAL_SCHEMA_V8: LocalSchemaIdentity = Object.freeze({ - version: LOCAL_SCHEMA_HISTORY[8]!.version, - fingerprint: LOCAL_SCHEMA_HISTORY[8]!.fingerprint, - digest: LOCAL_SCHEMA_HISTORY[8]!.digest, - manifestDigest: LOCAL_SCHEMA_HISTORY[8]!.manifestDigest, - chdb: CHDB_VERSION, - projectRevision: LOCAL_SCHEMA_HISTORY[8]!.projectRevision, -}) +export const LOCAL_SCHEMA_V1 = identityAt(1) +export const LOCAL_SCHEMA_V2 = identityAt(2) +export const LOCAL_SCHEMA_V3 = identityAt(3) +export const LOCAL_SCHEMA_V4 = identityAt(4) +export const LOCAL_SCHEMA_V5 = identityAt(5) +export const LOCAL_SCHEMA_V6 = identityAt(6) +export const LOCAL_SCHEMA_V7 = identityAt(7) +export const LOCAL_SCHEMA_V8 = identityAt(8) export const CURRENT_LOCAL_SCHEMA: LocalSchemaIdentity = Object.freeze({ version: LOCAL_SCHEMA_VERSION, diff --git a/apps/cli/src/server/schema-physical.ts b/apps/cli/src/server/schema-physical.ts index 688aadd6f..c9ac11880 100644 --- a/apps/cli/src/server/schema-physical.ts +++ b/apps/cli/src/server/schema-physical.ts @@ -1,5 +1,7 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. +import { Schema } from "effect" import { Chdb, RAW_TELEMETRY_TTL_COLUMNS } from "./chdb" +import { decodeJsonEachRow } from "./chdb-rows" import { LOCAL_SCHEMA_MANIFEST } from "./schema-identity" import { comparePhysicalSchema, @@ -9,45 +11,52 @@ import { withRawTelemetryRetentionFloor, } from "./schema-manifest" -const parseJsonEachRow = (value: string): A[] => - value - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length > 0) - .map((line) => JSON.parse(line) as A) +/** The `system.*` shapes this inspector reads. Each is fixed by the SELECT + * directly above its use, so a row that does not match means the query and the + * decoder have drifted apart. */ +const TableRow = Schema.Struct({ + name: Schema.String, + engine: Schema.String, + partition_key: Schema.String, + sorting_key: Schema.String, + create_table_query: Schema.String, +}) -interface ColumnRow { - table: string - name: string - type: string - position: number - default_kind: string - default_expression: string - compression_codec: string -} +const ColumnRowSchema = Schema.Struct({ + table: Schema.String, + name: Schema.String, + type: Schema.String, + position: Schema.Number, + default_kind: Schema.String, + default_expression: Schema.String, + compression_codec: Schema.String, +}) + +const IndexRow = Schema.Struct({ table: Schema.String, name: Schema.String }) + +const FormattedRow = Schema.Struct({ formatted: Schema.String }) + +const decodeTableRows = decodeJsonEachRow(TableRow) +const decodeColumnRows = decodeJsonEachRow(ColumnRowSchema) +const decodeIndexRows = decodeJsonEachRow(IndexRow) +const decodeFormattedRows = decodeJsonEachRow(FormattedRow) /** Inspect the physical definitions chDB reports, including the objects that * are easy to miss when relying on a bundled DDL fingerprint alone. */ export const inspectPhysicalSchema = (db: Chdb): PhysicalSchema => { - const tables = parseJsonEachRow<{ - name: string - engine: string - partition_key: string - sorting_key: string - create_table_query: string - }>( + const tables = decodeTableRows( db.query( "SELECT name, engine, partition_key, sorting_key, create_table_query FROM system.tables WHERE database = 'default'", ), ) - const columns = parseJsonEachRow( + const columns = decodeColumnRows( db.query( "SELECT table, name, type, position, default_kind, default_expression, compression_codec FROM system.columns WHERE database = 'default' ORDER BY table, position", ), ) - let indexes: Array<{ table: string; name: string }> = [] + let indexes: ReadonlyArray<{ readonly table: string; readonly name: string }> = [] try { - indexes = parseJsonEachRow<{ table: string; name: string }>( + indexes = decodeIndexRows( db.query("SELECT table, name FROM system.data_skipping_indices WHERE database = 'default'"), ) } catch { @@ -119,7 +128,7 @@ const formatViewBody = (db: Chdb): ((sql: string) => string | undefined) => { if (cached !== undefined || cache.has(sql)) return cached let formatted: string | undefined try { - const rows = parseJsonEachRow<{ formatted: string }>( + const rows = decodeFormattedRows( db.query(`SELECT formatQuery($maple_fmt$${sql}$maple_fmt$) AS formatted FORMAT JSONEachRow`), ) // The local store is always the `default` database, and ClickHouse diff --git a/apps/cli/src/server/store-version.ts b/apps/cli/src/server/store-version.ts index 507debec0..95485e525 100644 --- a/apps/cli/src/server/store-version.ts +++ b/apps/cli/src/server/store-version.ts @@ -9,7 +9,9 @@ import { createHash, randomUUID } from "node:crypto" import { existsSync, readFileSync, unlinkSync, writeFileSync } from "node:fs" import { dirname, join } from "node:path" +import { Effect, Schema, SchemaGetter } from "effect" import { CHDB_VERSION } from "../version" +import { Digest64, Fingerprint16, IsoOrUnknown } from "./identity-schema" import { durableRemove, durableWrite } from "./durable-files" export const STORE_MARKER_FORMAT_VERSION = 2 as const @@ -112,83 +114,94 @@ export const markStoreClosedDurable = async (dataDir: string): Promise => export const isStoreDirty = (dataDir: string): boolean => storeHasData(dataDir) && existsSync(storeOpenMarkerPath(dataDir)) -const isRecord = (value: unknown): value is Record => - typeof value === "object" && value !== null && !Array.isArray(value) - -const isIsoOrUnknown = (value: unknown): value is string => - typeof value === "string" && (value === "unknown" || Number.isFinite(Date.parse(value))) - -const isHexDigest = (value: unknown): value is string => - typeof value === "string" && /^[0-9a-f]{64}$/i.test(value) +/** + * Provenance fields are read leniently. + * + * `maple`, `createdAt`, and `createdByMaple` describe who made the store, not + * what it contains. A store whose provenance is missing or garbled is still a + * perfectly openable store, so a bad value degrades to "unknown" instead of + * making the marker malformed and refusing to start. Everything below this + * comment — the identity fields the loader actually acts on — is strict. + */ +const LenientProvenance = Schema.Unknown.pipe( + Schema.decodeTo(Schema.String, { + decode: SchemaGetter.transform((value) => (typeof value === "string" ? value : "unknown")), + encode: SchemaGetter.passthrough(), + }), + Schema.withDecodingDefaultKey(Effect.succeed(undefined)), +) + +const StoreMigrationStampSchema = Schema.Struct({ + id: Schema.String, + completedAt: IsoOrUnknown, + fromVersion: Schema.Int, + toVersion: Schema.Int, +}) + +const StoreMarkerV2Schema = Schema.Struct({ + formatVersion: Schema.Literal(STORE_MARKER_FORMAT_VERSION), + storeId: Schema.String.check(Schema.isMinLength(1)), + chdb: Schema.String.check(Schema.isMinLength(1)), + maple: LenientProvenance, + createdAt: LenientProvenance.check( + Schema.makeFilter((value: string) => + value === "unknown" || Number.isFinite(Date.parse(value)) + ? undefined + : "marker createdAt is invalid", + ), + ), + createdByMaple: LenientProvenance, + schemaVersion: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), + schemaDigest: Digest64, + schema: Fingerprint16, + activation: Schema.Literals(["active", "staging"]), + lastMigration: Schema.optionalKey(StoreMigrationStampSchema), +}) + +/** + * The original marker had no `formatVersion` at all, so v1 is recognised by its + * absence. A malformed object must not silently become a legacy store, which is + * why `chdb` stays required here: it is the only field the original format + * guaranteed. + */ +const StoreMarkerV1Schema = Schema.Struct({ + formatVersion: Schema.Literal(1).pipe(Schema.withDecodingDefaultKey(Effect.succeed(1 as const))), + chdb: Schema.String.check(Schema.isMinLength(1)), + maple: LenientProvenance, + createdAt: LenientProvenance.check( + Schema.makeFilter((value: string) => + value === "unknown" || Number.isFinite(Date.parse(value)) + ? undefined + : "marker createdAt is invalid", + ), + ), + schema: Schema.Unknown.pipe( + Schema.decodeTo(Schema.String, { + decode: SchemaGetter.transform((value) => (typeof value === "string" ? value : "")), + encode: SchemaGetter.passthrough(), + }), + Schema.withDecodingDefaultKey(Effect.succeed(undefined)), + ), +}) + +const decodeMarkerV1 = Schema.decodeUnknownSync(StoreMarkerV1Schema) +const decodeMarkerV2 = Schema.decodeUnknownSync(StoreMarkerV2Schema) const parseMarker = (value: unknown): StoreMarker => { - if (!isRecord(value) || typeof value.chdb !== "string" || value.chdb.length === 0) { + if (typeof value !== "object" || value === null || Array.isArray(value)) { throw new Error("marker must contain a non-empty chdb string") } - const maple = typeof value.maple === "string" ? value.maple : "unknown" - const createdAt = typeof value.createdAt === "string" ? value.createdAt : "unknown" - const schema = typeof value.schema === "string" ? value.schema : "" - if (!isIsoOrUnknown(createdAt)) throw new Error("marker createdAt is invalid") - if ( - value.formatVersion !== undefined && - value.formatVersion !== 1 && - value.formatVersion !== STORE_MARKER_FORMAT_VERSION - ) { - throw new Error(`unsupported marker format ${String(value.formatVersion)}`) + const formatVersion = (value as { readonly formatVersion?: unknown }).formatVersion + if (formatVersion !== undefined && formatVersion !== 1 && formatVersion !== STORE_MARKER_FORMAT_VERSION) { + throw new Error(`unsupported marker format ${String(formatVersion)}`) } - - if (value.formatVersion === STORE_MARKER_FORMAT_VERSION) { - if (typeof value.storeId !== "string" || value.storeId.length === 0) { - throw new Error("v2 marker storeId is missing") - } - if (!Number.isInteger(value.schemaVersion) || (value.schemaVersion as number) < 0) { - throw new Error("v2 marker schemaVersion is invalid") - } - if (!isHexDigest(value.schemaDigest)) throw new Error("v2 marker schemaDigest is invalid") - if (!/^[0-9a-f]{16}$/i.test(schema)) throw new Error("v2 marker schema fingerprint is invalid") - if (value.activation !== "active" && value.activation !== "staging") { - throw new Error("v2 marker activation is invalid") - } - const lastMigration = value.lastMigration - if (lastMigration !== undefined) { - if ( - !isRecord(lastMigration) || - typeof lastMigration.id !== "string" || - !Number.isInteger(lastMigration.fromVersion) || - !Number.isInteger(lastMigration.toVersion) || - !isIsoOrUnknown(lastMigration.completedAt) - ) { - throw new Error("v2 marker lastMigration is invalid") - } - } - return { - formatVersion: STORE_MARKER_FORMAT_VERSION, - storeId: value.storeId, - chdb: value.chdb, - maple, - createdAt, - createdByMaple: typeof value.createdByMaple === "string" ? value.createdByMaple : maple, - schemaVersion: value.schemaVersion as number, - schemaDigest: value.schemaDigest, - schema, - activation: value.activation, - ...(!(lastMigration === undefined) - ? { - lastMigration: { - id: lastMigration.id as string, - completedAt: lastMigration.completedAt as string, - fromVersion: lastMigration.fromVersion as number, - toVersion: lastMigration.toVersion as number, - }, - } - : undefined), - } + if (formatVersion === STORE_MARKER_FORMAT_VERSION) { + const marker = decodeMarkerV2(value) + // The one default a struct cannot express: an unrecorded creator is the + // running version, not "unknown". + return marker.createdByMaple === "unknown" ? { ...marker, createdByMaple: marker.maple } : marker } - - // The original marker had no formatVersion. Treat it as v1. A malformed - // object must not silently become a legacy store, so the required chdb field - // above is the only compatibility relaxation. - return { formatVersion: 1, chdb: value.chdb, maple, createdAt, schema } + return decodeMarkerV1(value) } /** Read the marker with an explicit missing/malformed distinction. */ @@ -226,16 +239,13 @@ export const makeStoreMarker = ( schema: string, options: StoreMarkerWriteOptions = {}, ): StoreMarkerV2 => { - if (!/^[0-9a-f]{16}$/i.test(schema)) { - throw new Error("a 16-character schema fingerprint is required for a v2 marker") - } - if (!options.schemaDigest || !/^[0-9a-f]{64}$/i.test(options.schemaDigest)) { - throw new Error("a full 64-character schemaDigest is required for a v2 marker") - } - if (options.schemaVersion === undefined || !Number.isInteger(options.schemaVersion)) { + if (options.schemaVersion === undefined) { throw new Error("a schemaVersion is required for a v2 marker") } - return { + // Constructing through the same schema the reader decodes with is the point: + // the fingerprint and digest rules are stated once, and a marker this build + // writes is a marker this build can read back. + return decodeMarkerV2({ formatVersion: STORE_MARKER_FORMAT_VERSION, storeId: options.storeId ?? randomUUID(), chdb: CHDB_VERSION, @@ -247,7 +257,7 @@ export const makeStoreMarker = ( schema, activation: options.activation ?? "active", ...(!(options.lastMigration === undefined) ? { lastMigration: options.lastMigration } : undefined), - } + }) } /** Serialize a current marker for a known identity. */ diff --git a/apps/cli/test/journal-schema.test.ts b/apps/cli/test/journal-schema.test.ts new file mode 100644 index 000000000..f1bc07bdf --- /dev/null +++ b/apps/cli/test/journal-schema.test.ts @@ -0,0 +1,143 @@ +// Golden decode tests for the coordinator journal envelope. +// +// The fixture below is a literal copy of what a shipped build writes to +// `maple-store-migration.json`. Its purpose is to fail loudly if the schema +// ever stops accepting a journal an installed store already holds — the +// declaration is only safe because this pins the accepted form. +import { Schema } from "effect" +import { describe, expect, it } from "vitest" +import { + decodeMigrationJournal, + MigrationJournalSchema, +} from "../src/server/local-store-migrations/journal-schema" + +const golden = { + formatVersion: 2, + migrationId: "local-0007-to-0008-apple-crash-frames-20260819T101500000Z", + phase: "copying", + chain: [ + { + id: "local-0007-to-0008-apple-crash-frames", + moduleVersion: 1, + from: { + version: 7, + fingerprint: "0123456789abcdef", + digest: "a".repeat(64), + manifestDigest: "b".repeat(64), + chdb: "3.6.0", + projectRevision: "c".repeat(64), + }, + to: { + version: 8, + fingerprint: "51081e951066442a", + digest: "d".repeat(64), + manifestDigest: "e".repeat(64), + chdb: "3.6.0", + projectRevision: "f".repeat(64), + }, + status: "running", + state: { module: "local-0007-to-0008-apple-crash-frames", version: 1, rawRows: {} }, + progress: { installed: true }, + }, + ], + currentStepIndex: 0, + sourceDataDir: "/var/lib/maple/data", + sourceStoreId: "8b7d9f1e-0000-4000-8000-000000000001", + sourceChdb: "3.6.0", + sourceFingerprint: "0123456789abcdef", + sourceDigest: "a".repeat(64), + sourceVersion: 7, + targetDataDir: "/var/lib/maple/.maple-migrations/m/target/data", + targetStoreId: "8b7d9f1e-0000-4000-8000-000000000002", + targetChdb: "3.6.0", + targetFingerprint: "51081e951066442a", + targetDigest: "d".repeat(64), + targetVersion: 8, + cutoffAt: "2026-08-19T10:15:00.000Z", + createdAt: "2026-08-19T10:15:00.000Z", +} + +describe("migration journal envelope", () => { + it("accepts a journal written by a shipped build, unchanged", () => { + expect(decodeMigrationJournal(structuredClone(golden))).toEqual(golden) + }) + + it("normalizes data directories so an unnormalized journal still matches its store", () => { + const decoded = decodeMigrationJournal({ + ...structuredClone(golden), + sourceDataDir: "/var/lib/maple/./data", + targetDataDir: "/var/lib/maple/.maple-migrations/m/target/../target/data", + }) + expect(decoded.sourceDataDir).toBe("/var/lib/maple/data") + expect(decoded.targetDataDir).toBe("/var/lib/maple/.maple-migrations/m/target/data") + }) + + it("defaults a missing source digest, because the v0 legacy identity has none", () => { + const { sourceDigest: _dropped, ...withoutSourceDigest } = structuredClone(golden) + expect(decodeMigrationJournal(withoutSourceDigest).sourceDigest).toBe("") + }) + + it("keeps absent optional identity fields absent rather than present-undefined", () => { + const journal = structuredClone(golden) + const { manifestDigest: _m, projectRevision: _p, ...leanFrom } = journal.chain[0]!.from + const decoded = decodeMigrationJournal({ + ...journal, + chain: [{ ...journal.chain[0]!, from: leanFrom }], + }) + expect("manifestDigest" in decoded.chain[0]!.from).toBe(false) + expect(Object.keys(decoded.chain[0]!.from)).not.toContain("projectRevision") + }) + + it("rejects a journal written by a build this one does not know", () => { + expect(() => decodeMigrationJournal({ ...structuredClone(golden), somethingElse: 1 })).toThrow() + expect(() => decodeMigrationJournal({ ...structuredClone(golden), formatVersion: 3 })).toThrow() + }) + + it("rejects the shapes the hand-rolled parser rejected", () => { + const bad: ReadonlyArray> = [ + { migrationId: "" }, + { migrationId: "../escape" }, + { phase: "somewhere-else" }, + { currentStepIndex: -1 }, + { currentStepIndex: 1.5 }, + { chain: [] }, + { sourceVersion: "7" }, + { targetDigest: "" }, + { sourceStoreId: "" }, + { failure: 42 }, + ] + for (const patch of bad) { + expect(() => decodeMigrationJournal({ ...structuredClone(golden), ...patch })).toThrow() + } + for (const value of [null, [], "x", 1, undefined]) { + expect(() => decodeMigrationJournal(value)).toThrow() + } + }) + + it("rejects a malformed step without interpreting module state", () => { + const step = golden.chain[0]! + for (const patch of [ + { status: "halfway" }, + { moduleVersion: 0 }, + { id: "" }, + { from: { ...step.from, version: -1 } }, + { to: { ...step.to, chdb: "" } }, + ]) { + expect(() => + decodeMigrationJournal({ ...structuredClone(golden), chain: [{ ...step, ...patch }] }), + ).toThrow() + } + // Opaque module state is the module's business, not the coordinator's. + expect(() => + decodeMigrationJournal({ + ...structuredClone(golden), + chain: [{ ...step, state: { anything: [1, 2, 3] } }], + }), + ).not.toThrow() + }) + + it("round-trips through the schema without changing the persisted form", () => { + const journal = decodeMigrationJournal(structuredClone(golden)) + expect(Schema.encodeSync(MigrationJournalSchema)(journal)).toEqual(golden) + }) +}) diff --git a/apps/cli/test/store-marker-schema.test.ts b/apps/cli/test/store-marker-schema.test.ts new file mode 100644 index 000000000..1acadc824 --- /dev/null +++ b/apps/cli/test/store-marker-schema.test.ts @@ -0,0 +1,121 @@ +// Golden decode tests for the on-disk store marker. +// +// Both fixtures are literal copies of markers shipped builds have written. A +// marker that stops decoding is a store that stops opening, so the accepted +// form is pinned here rather than left implicit in the schema. +import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, expect, it } from "vitest" +import { makeStoreMarker, readMarkerState, storeMarkerPath } from "../src/server/store-version" +import { CHDB_VERSION } from "../src/version" + +const goldenV1 = { + chdb: "1.4.0", + maple: "0.1.7", + createdAt: "2026-01-14T09:30:00.000Z", + schema: "428701854f9fd30e", +} + +const goldenV2 = { + formatVersion: 2, + storeId: "8b7d9f1e-0000-4000-8000-000000000001", + chdb: "3.6.0", + maple: "0.4.2", + createdAt: "2026-08-19T10:15:00.000Z", + createdByMaple: "0.3.9", + schemaVersion: 8, + schemaDigest: "a".repeat(64), + schema: "51081e951066442a", + activation: "active", + lastMigration: { + id: "local-0007-to-0008-apple-crash-frames", + completedAt: "2026-08-19T10:20:00.000Z", + fromVersion: 7, + toVersion: 8, + }, +} + +const readAs = (value: unknown) => { + const root = mkdtempSync(join(tmpdir(), "maple-marker-")) + const dataDir = join(root, "data") + mkdirSync(dataDir, { recursive: true }) + writeFileSync(storeMarkerPath(dataDir), JSON.stringify(value)) + return readMarkerState(dataDir) +} + +describe("store marker", () => { + it("accepts a v1 marker written before the format was versioned", () => { + const state = readAs(goldenV1) + expect(state).toEqual({ kind: "valid", marker: { formatVersion: 1, ...goldenV1 } }) + }) + + it("accepts a v2 marker written by a shipped build, unchanged", () => { + const state = readAs(goldenV2) + expect(state.kind === "valid" && state.marker).toEqual(goldenV2) + }) + + it("degrades unusable provenance rather than refusing to open the store", () => { + // Who made the store is not something the loader acts on. Identity is. + const state = readAs({ ...goldenV2, maple: 42, createdAt: null }) + expect(state.kind === "valid" && state.marker.maple).toBe("unknown") + expect(state.kind === "valid" && state.marker.createdAt).toBe("unknown") + }) + + it("falls back to the recorded maple version when no creator was stamped", () => { + const { createdByMaple: _dropped, ...withoutCreator } = goldenV2 + const state = readAs(withoutCreator) + expect(state.kind === "valid" && state.marker.createdByMaple).toBe(goldenV2.maple) + }) + + it("reports a malformed identity instead of guessing at it", () => { + for (const patch of [ + { schema: "not-hex" }, + { schema: "51081e951066442" }, + { schemaDigest: "a".repeat(63) }, + { schemaVersion: -1 }, + { schemaVersion: 1.5 }, + { storeId: "" }, + { activation: "somewhere-else" }, + { chdb: "" }, + { lastMigration: { id: "x", completedAt: "nope", fromVersion: 7, toVersion: 8 } }, + ]) { + expect(readAs({ ...goldenV2, ...patch }).kind).toBe("malformed") + } + expect(readAs({ ...goldenV2, formatVersion: 3 }).kind).toBe("malformed") + expect(readAs({ maple: "0.1.0" }).kind).toBe("malformed") + expect(readAs([]).kind).toBe("malformed") + }) + + it("constructs a marker through the same rules it reads one with", () => { + const marker = makeStoreMarker("0.4.2", "2026-08-19T10:15:00.000Z", "51081e951066442a", { + storeId: goldenV2.storeId, + schemaVersion: 8, + schemaDigest: "a".repeat(64), + }) + expect(marker).toEqual({ + formatVersion: 2, + storeId: goldenV2.storeId, + chdb: CHDB_VERSION, + maple: "0.4.2", + createdAt: "2026-08-19T10:15:00.000Z", + createdByMaple: "0.4.2", + schemaVersion: 8, + schemaDigest: "a".repeat(64), + schema: "51081e951066442a", + activation: "active", + }) + expect(readAs(marker).kind).toBe("valid") + }) + + it("refuses to construct a marker without a full identity", () => { + const now = "2026-08-19T10:15:00.000Z" + expect(() => + makeStoreMarker("0.4.2", now, "not-hex", { schemaVersion: 8, schemaDigest: "a".repeat(64) }), + ).toThrow() + expect(() => makeStoreMarker("0.4.2", now, "51081e951066442a", { schemaVersion: 8 })).toThrow() + expect(() => + makeStoreMarker("0.4.2", now, "51081e951066442a", { schemaDigest: "a".repeat(64) }), + ).toThrow(/schemaVersion/) + }) +}) diff --git a/scripts/check-local-schema-manifest.ts b/scripts/check-local-schema-manifest.ts index e1b06e11a..66874a1de 100644 --- a/scripts/check-local-schema-manifest.ts +++ b/scripts/check-local-schema-manifest.ts @@ -4,27 +4,7 @@ import { LOCAL_SCHEMA_HISTORY, LOCAL_SCHEMA_MANIFEST, LOCAL_SCHEMA_MANIFEST_DIGEST, - LOCAL_SCHEMA_V1, - LOCAL_SCHEMA_V1_MANIFEST_DIGEST, - LOCAL_SCHEMA_V1_SQL, - LOCAL_SCHEMA_V2, - LOCAL_SCHEMA_V2_MANIFEST_DIGEST, - LOCAL_SCHEMA_V2_SQL, - LOCAL_SCHEMA_V3, - LOCAL_SCHEMA_V3_MANIFEST_DIGEST, - LOCAL_SCHEMA_V3_SQL, - LOCAL_SCHEMA_V4, - LOCAL_SCHEMA_V4_MANIFEST_DIGEST, - LOCAL_SCHEMA_V4_SQL, - LOCAL_SCHEMA_V5, - LOCAL_SCHEMA_V5_MANIFEST_DIGEST, - LOCAL_SCHEMA_V5_SQL, - LOCAL_SCHEMA_V6, - LOCAL_SCHEMA_V6_MANIFEST_DIGEST, - LOCAL_SCHEMA_V6_SQL, - LOCAL_SCHEMA_V7, - LOCAL_SCHEMA_V7_MANIFEST_DIGEST, - LOCAL_SCHEMA_V7_SQL, + LOCAL_SCHEMA_SNAPSHOTS, LOCAL_SCHEMA_VERSION, } from "../apps/cli/src/server/schema-identity" import { resolveMigrationChain } from "../apps/cli/src/server/local-store-migrations" @@ -65,88 +45,31 @@ if (!sameIdentity(latest, CURRENT_LOCAL_SCHEMA)) { ) } -const v1 = LOCAL_SCHEMA_HISTORY.find((entry) => entry.version === LOCAL_SCHEMA_V1.version) -if ( - !v1 || - LOCAL_SCHEMA_V1_MANIFEST_DIGEST !== v1.manifestDigest || - schemaFingerprint(LOCAL_SCHEMA_V1_SQL) !== v1.fingerprint || - schemaDigest(LOCAL_SCHEMA_V1_SQL) !== v1.digest || - LOCAL_SCHEMA_V1.fingerprint !== v1.fingerprint || - LOCAL_SCHEMA_V1.digest !== v1.digest -) { - fail("the immutable local schema v1 snapshot no longer matches its historical identity") -} - -const v2 = LOCAL_SCHEMA_HISTORY.find((entry) => entry.version === LOCAL_SCHEMA_V2.version) -if ( - !v2 || - LOCAL_SCHEMA_V2_MANIFEST_DIGEST !== v2.manifestDigest || - schemaFingerprint(LOCAL_SCHEMA_V2_SQL) !== v2.fingerprint || - schemaDigest(LOCAL_SCHEMA_V2_SQL) !== v2.digest || - LOCAL_SCHEMA_V2.fingerprint !== v2.fingerprint || - LOCAL_SCHEMA_V2.digest !== v2.digest -) { - fail("the immutable local schema v2 snapshot no longer matches its historical identity") -} - -const v3 = LOCAL_SCHEMA_HISTORY.find((entry) => entry.version === LOCAL_SCHEMA_V3.version) -if ( - !v3 || - LOCAL_SCHEMA_V3_MANIFEST_DIGEST !== v3.manifestDigest || - schemaFingerprint(LOCAL_SCHEMA_V3_SQL) !== v3.fingerprint || - schemaDigest(LOCAL_SCHEMA_V3_SQL) !== v3.digest || - LOCAL_SCHEMA_V3.fingerprint !== v3.fingerprint || - LOCAL_SCHEMA_V3.digest !== v3.digest -) { - fail("the immutable local schema v3 snapshot no longer matches its historical identity") -} - -const v4 = LOCAL_SCHEMA_HISTORY.find((entry) => entry.version === LOCAL_SCHEMA_V4.version) -if ( - !v4 || - LOCAL_SCHEMA_V4_MANIFEST_DIGEST !== v4.manifestDigest || - schemaFingerprint(LOCAL_SCHEMA_V4_SQL) !== v4.fingerprint || - schemaDigest(LOCAL_SCHEMA_V4_SQL) !== v4.digest || - LOCAL_SCHEMA_V4.fingerprint !== v4.fingerprint || - LOCAL_SCHEMA_V4.digest !== v4.digest -) { - fail("the immutable local schema v4 snapshot no longer matches its historical identity") -} - -const v5 = LOCAL_SCHEMA_HISTORY.find((entry) => entry.version === LOCAL_SCHEMA_V5.version) -if ( - !v5 || - LOCAL_SCHEMA_V5_MANIFEST_DIGEST !== v5.manifestDigest || - schemaFingerprint(LOCAL_SCHEMA_V5_SQL) !== v5.fingerprint || - schemaDigest(LOCAL_SCHEMA_V5_SQL) !== v5.digest || - LOCAL_SCHEMA_V5.fingerprint !== v5.fingerprint || - LOCAL_SCHEMA_V5.digest !== v5.digest -) { - fail("the immutable local schema v5 snapshot no longer matches its historical identity") -} - -const v6 = LOCAL_SCHEMA_HISTORY.find((entry) => entry.version === LOCAL_SCHEMA_V6.version) -if ( - !v6 || - LOCAL_SCHEMA_V6_MANIFEST_DIGEST !== v6.manifestDigest || - schemaFingerprint(LOCAL_SCHEMA_V6_SQL) !== v6.fingerprint || - schemaDigest(LOCAL_SCHEMA_V6_SQL) !== v6.digest || - LOCAL_SCHEMA_V6.fingerprint !== v6.fingerprint || - LOCAL_SCHEMA_V6.digest !== v6.digest -) { - fail("the immutable local schema v6 snapshot no longer matches its historical identity") +// A loop is only a gate while it has something to iterate. The history carries +// one entry per schema version plus v0, which has no DDL of its own. +const snapshotCount = LOCAL_SCHEMA_SNAPSHOTS.filter(Boolean).length +if (snapshotCount !== LOCAL_SCHEMA_HISTORY.length - 1) { + fail( + `local schema snapshots (${snapshotCount}) do not cover every versioned history entry (${LOCAL_SCHEMA_HISTORY.length - 1})`, + ) } -const v7 = LOCAL_SCHEMA_HISTORY.find((entry) => entry.version === LOCAL_SCHEMA_V7.version) -if ( - !v7 || - LOCAL_SCHEMA_V7_MANIFEST_DIGEST !== v7.manifestDigest || - schemaFingerprint(LOCAL_SCHEMA_V7_SQL) !== v7.fingerprint || - schemaDigest(LOCAL_SCHEMA_V7_SQL) !== v7.digest || - LOCAL_SCHEMA_V7.fingerprint !== v7.fingerprint || - LOCAL_SCHEMA_V7.digest !== v7.digest -) { - fail("the immutable local schema v7 snapshot no longer matches its historical identity") +// Every frozen snapshot must still hash to the identity the history recorded +// for it. A snapshot that drifts silently retargets a historical migration +// edge, which is the failure this gate exists to make impossible. +for (const snapshot of LOCAL_SCHEMA_SNAPSHOTS) { + if (!snapshot) continue + const entry = LOCAL_SCHEMA_HISTORY.find((candidate) => candidate.version === snapshot.version) + if ( + !entry || + snapshot.manifestDigest !== entry.manifestDigest || + schemaFingerprint(snapshot.sql) !== entry.fingerprint || + schemaDigest(snapshot.sql) !== entry.digest + ) { + fail( + `the immutable local schema v${snapshot.version} snapshot no longer matches its historical identity`, + ) + } } const names = LOCAL_SCHEMA_MANIFEST.objects.map((object) => object.name)