diff --git a/apps/cli/src/server/archives/journal.ts b/apps/cli/src/server/archives/journal.ts index 7c481b4f2..45c0fe83d 100644 --- a/apps/cli/src/server/archives/journal.ts +++ b/apps/cli/src/server/archives/journal.ts @@ -12,6 +12,8 @@ import { validateArchiveId, validateRangeDate, } from "./paths" +import { Schema } from "effect" +import { NonEmptyString, NonNegativeSafeInt, Sha256Lower } from "./schemas" import { parseArchiveActivePointer } from "./manifest" import { archiveSignal } from "./signals" @@ -177,6 +179,19 @@ const intentPath = (archiveDir: string, operationId: string): string => const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null +const decodeCountValue = Schema.decodeUnknownSync(NonNegativeSafeInt) + +/** Same rule as the GC targets' byte counts, with the field named in the error. */ +const decodeCount = (value: unknown, field: string): number => { + try { + return decodeCountValue(value) + } catch { + throw new Error(`archive operation gc intent has invalid ${field}: ${String(value)}`) + } +} + +const decodeSha256 = Schema.decodeUnknownSync(Sha256Lower) + const requiredString = (value: unknown, field: string): string => { if (typeof value !== "string" || value.length === 0) throw new Error(`journal field ${field} missing or not a string`) @@ -289,9 +304,10 @@ const parseCreateIntent = ( throw new Error(`archive operation pin purpose does not match generation: ${pinPurpose}`) } const manifestSha256Raw = raw.manifestSha256 - const manifestSha256 = - manifestSha256Raw === null ? null : requiredString(manifestSha256Raw, "manifestSha256").toLowerCase() - if (manifestSha256 !== null && !/^[0-9a-f]{64}$/.test(manifestSha256)) { + let manifestSha256: string | null + try { + manifestSha256 = manifestSha256Raw === null ? null : decodeSha256(manifestSha256Raw) + } catch { throw new Error("invalid archive operation manifestSha256") } if (phaseRequiresManifest(phase) !== (manifestSha256 !== null)) { @@ -337,20 +353,8 @@ const parseGcIntent = ( createdAt: string, updatedAt: string, ): GcOperationIntent => { - const keepRaw = raw.keep - if (typeof keepRaw !== "number" || !Number.isSafeInteger(keepRaw) || keepRaw < 0) { - throw new Error(`archive operation gc intent has invalid keep: ${String(keepRaw)}`) - } - const completedTargetsRaw = raw.completedTargets - if ( - typeof completedTargetsRaw !== "number" || - !Number.isSafeInteger(completedTargetsRaw) || - completedTargetsRaw < 0 - ) { - throw new Error( - `archive operation gc intent has invalid completedTargets: ${String(completedTargetsRaw)}`, - ) - } + const keepRaw = decodeCount(raw.keep, "keep") + const completedTargetsRaw = decodeCount(raw.completedTargets, "completedTargets") const targetsRaw = raw.targets if (!Array.isArray(targetsRaw)) { throw new Error("archive operation gc intent targets is not an array") @@ -402,50 +406,54 @@ const parseGcIntent = ( } } +/** + * One GC target exactly as the journal recorded it. + * + * Only the recorded fields are decoded here. The identities are re-validated + * through the same + * canonicalizing validators the writer used, because a hand-edited journal must + * not be able to point collection at an arbitrary path. + */ +const RecordedGcTarget = Schema.Struct({ + signal: NonEmptyString, + rangeStart: NonEmptyString, + generationId: NonEmptyString, + createdAt: NonEmptyString, + manifestSha256: Sha256Lower, + bytes: NonNegativeSafeInt, + recordedActiveGenerationId: NonEmptyString, + shards: Schema.Array( + Schema.Struct({ + name: NonEmptyString, + bytes: NonNegativeSafeInt, + sha256: Sha256Lower, + }), + ), +}) + +const decodeRecordedGcTarget = Schema.decodeUnknownSync(RecordedGcTarget) + const parseGcTarget = (raw: unknown, index: number): GcTarget => { - if (!isRecord(raw)) throw new Error(`archive gc target ${index} is not a record`) - const signal = archiveSignal(requiredString(raw.signal, "signal")).name - const rangeStart = validateRangeDate(requiredString(raw.rangeStart, "rangeStart")) - const generationId = validateArchiveId(requiredString(raw.generationId, "generationId"), "generation") - const createdAt = requiredString(raw.createdAt, "createdAt") - const manifestSha256 = requiredString(raw.manifestSha256, "manifestSha256").toLowerCase() - if (!/^[0-9a-f]{64}$/.test(manifestSha256)) { - throw new Error(`archive gc target ${index} has invalid manifestSha256`) - } - const bytesRaw = raw.bytes - if (typeof bytesRaw !== "number" || !Number.isSafeInteger(bytesRaw) || bytesRaw < 0) { - throw new Error(`archive gc target ${index} has invalid bytes: ${String(bytesRaw)}`) + let recorded: typeof RecordedGcTarget.Type + try { + recorded = decodeRecordedGcTarget(raw) + } catch (error) { + throw new Error( + `archive gc target ${index} is malformed: ${error instanceof Error ? error.message : String(error)}`, + ) } - const recordedActiveGenerationId = validateArchiveId( - requiredString(raw.recordedActiveGenerationId, "recordedActiveGenerationId"), - "active generation", - ) - const shardsRaw = raw.shards - if (!Array.isArray(shardsRaw)) { - throw new Error(`archive gc target ${index} shards is not an array`) - } - const shards = shardsRaw.map((s, j) => { - if (!isRecord(s)) throw new Error(`archive gc target ${index} shard ${j} is not a record`) - const name = requiredString(s.name, "name") - const bytes = s.bytes - if (typeof bytes !== "number" || !Number.isSafeInteger(bytes) || bytes < 0) { - throw new Error(`archive gc target ${index} shard ${j} invalid bytes`) - } - const sha256 = requiredString(s.sha256, "sha256").toLowerCase() - if (!/^[0-9a-f]{64}$/.test(sha256)) { - throw new Error(`archive gc target ${index} shard ${j} invalid sha256`) - } - return { name, bytes, sha256 } - }) return { - signal, - rangeStart, - generationId, - createdAt, - manifestSha256, - bytes: bytesRaw, - shards, - recordedActiveGenerationId, + signal: archiveSignal(recorded.signal).name, + rangeStart: validateRangeDate(recorded.rangeStart), + generationId: validateArchiveId(recorded.generationId, "generation"), + createdAt: recorded.createdAt, + manifestSha256: recorded.manifestSha256, + bytes: recorded.bytes, + shards: recorded.shards, + recordedActiveGenerationId: validateArchiveId( + recorded.recordedActiveGenerationId, + "active generation", + ), } } diff --git a/apps/cli/src/server/archives/manifest.ts b/apps/cli/src/server/archives/manifest.ts index cb61b70c1..dd684db0a 100644 --- a/apps/cli/src/server/archives/manifest.ts +++ b/apps/cli/src/server/archives/manifest.ts @@ -1,11 +1,13 @@ import { readFileSync } from "node:fs" import { join } from "node:path" +import { Schema } from "effect" import { type ArchiveTuningRecord, type TuningConfigIdentity, LEGACY_TUNING_CONFIG_FORMAT_VERSION, TUNING_CONFIG_FORMAT_VERSION, } from "./config" +import { NonNegativeSafeInt, Sha256Lower } from "./schemas" import { KNOWN_COMPLEX_DIGEST_ALGORITHMS } from "./export" import { assertNoSymlinkSync, @@ -150,12 +152,14 @@ const requiredString = (record: Record, key: string): string => return value } +const decodeCount = Schema.decodeUnknownSync(NonNegativeSafeInt) + const requiredCount = (record: Record, key: string): number => { - const value = record[key] - if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) { + try { + return decodeCount(record[key]) + } catch { throw new Error(`invalid archive manifest field: ${key} (must be a safe non-negative integer)`) } - return value } const requiredPositiveInteger = (record: Record, key: string): number => { @@ -166,7 +170,9 @@ const requiredPositiveInteger = (record: Record, key: string): return value } -const SHA256_HEX = /^[0-9a-f]{64}$/ +/** The same 64-hex rule the journal enforces, stated once in ./schemas. */ +const isSha256Hex = (value: string): boolean => + Schema.decodeUnknownResult(Sha256Lower)(value)._tag === "Success" /** A safe logical config name (no path separators, no traversal). */ const SAFE_CONFIG_NAME = /^[A-Za-z0-9._-]+$/ @@ -208,7 +214,7 @@ const parseTuningConfig = (value: unknown): TuningConfigIdentity | null => { throw new Error(`invalid archive manifest tuningConfig.configName (unsafe name): ${configName}`) } const sha256 = requiredString(value, "sha256") - if (!SHA256_HEX.test(sha256)) { + if (!isSha256Hex(sha256)) { throw new Error(`invalid archive manifest tuningConfig.sha256 (must be 64 hex chars): ${sha256}`) } return { formatVersion, configName, sha256 } @@ -297,7 +303,7 @@ const parseShardRecord = ( return c }) const sha256 = requiredString(value, "sha256") - if (!SHA256_HEX.test(sha256)) + if (!isSha256Hex(sha256)) throw new Error(`invalid archive shard sha256 (must be 64 hex chars): ${sha256}`) const rowCount = requiredCount(value, "rowCount") const minNano = requiredNanoDecimal(value, "minEventTimeUnixNano") diff --git a/apps/cli/src/server/archives/schemas.ts b/apps/cli/src/server/archives/schemas.ts new file mode 100644 index 000000000..77c5a554f --- /dev/null +++ b/apps/cli/src/server/archives/schemas.ts @@ -0,0 +1,36 @@ +// Shape primitives shared by the archive journal and manifest readers. +// +// These are the rules that were genuinely copy-pasted: a non-negative safe +// integer was re-checked at four sites in the GC intent alone, and the sha256 +// pattern at six across the two files. The rest of the archive parsers are +// domain invariants — path containment, phase-vs-kind compatibility, cursor +// consistency — each with its own tested message, and those stay where they +// are rather than being flattened into a struct declaration. +import { Schema, SchemaGetter } from "effect" + +/** Byte counts and cursors: never negative, never beyond exact integer range. */ +export const NonNegativeSafeInt = Schema.Number.check( + Schema.makeFilter((value: number) => + Number.isSafeInteger(value) && value >= 0 ? undefined : "must be a safe non-negative integer", + ), +) + +/** + * A sha256 digest, normalized to lower case before it is compared. + * + * Case normalization is part of the rule, not a courtesy: these digests gate + * whether a generation may be deleted, and an upper-case digest that failed to + * compare equal would look like corruption of data that is in fact intact. + */ +export const Sha256Lower = Schema.String.pipe( + Schema.decodeTo(Schema.String, { + decode: SchemaGetter.transform((value: string) => value.toLowerCase()), + encode: SchemaGetter.passthrough(), + }), +).check( + Schema.makeFilter((value: string) => + /^[0-9a-f]{64}$/.test(value) ? undefined : "must be 64 hex characters", + ), +) + +export const NonEmptyString = Schema.String.check(Schema.isMinLength(1)) diff --git a/apps/cli/src/server/chdb-rows.ts b/apps/cli/src/server/chdb-rows.ts new file mode 100644 index 000000000..d7580e4f3 --- /dev/null +++ b/apps/cli/src/server/chdb-rows.ts @@ -0,0 +1,61 @@ +// Decoding rows out of chDB's JSONEachRow output. +// +// Nine copies of a `split("\n") … JSON.parse(line) as A` helper used to live +// across the migration coordinator, the physical-schema inspector, and every +// versioned edge — each one ending in an unchecked cast at the point where an +// external process hands us bytes. The shape is declared once here and the +// rows are decoded, not asserted. +import { Schema } from "effect" + +/** + * Decode chDB JSONEachRow output. + * + * A row that does not match the schema is a query returning something other + * than what the caller asked for, which is a bug in the SQL rather than data to + * be tolerated — so this throws instead of skipping the row. + */ +export const decodeJsonEachRow = >( + rowSchema: S, +): ((value: string) => Array) => { + const decodeRow = Schema.decodeUnknownSync(rowSchema) + return (value) => + value + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => decodeRow(JSON.parse(line))) +} + +/** + * `sum(rows)` per table, as returned by the raw-telemetry inventory query. + * + * The count is `toString()`-wrapped in SQL because it is a ClickHouse UInt64: + * above 2^53 a JS number silently loses the low bits, and these values are + * compared for exact equality when a migration verifies that no row was lost. + */ +export const TableRowCountSchema = Schema.Struct({ + table: Schema.String, + rowCount: Schema.String.check(Schema.isPattern(/^\d+$/)), +}) + +export const decodeTableRowCounts = decodeJsonEachRow(TableRowCountSchema) + +/** A single `toString(count())`-style scalar, for the same UInt64 reason. */ +export const RowCountSchema = Schema.Struct({ + rowCount: Schema.String.check(Schema.isPattern(/^\d+$/)), +}) + +export const decodeRowCounts = decodeJsonEachRow(RowCountSchema) + +/** + * Rows whose columns are not known ahead of time. + * + * The v0 -> v1 raw replay copies whatever columns the source table happens to + * have, so there is no field list to declare. Requiring each line to be a JSON + * object is the only claim that can honestly be made about it, and it is still + * a claim worth making: a bare scalar or array here means the query returned + * something other than rows. + */ +const OpaqueRowSchema = Schema.Record(Schema.String, Schema.Unknown) + +export const decodeJsonObjectRows = decodeJsonEachRow(OpaqueRowSchema) diff --git a/apps/cli/src/server/chdb.ts b/apps/cli/src/server/chdb.ts index 59c7f43fe..92089e3a8 100644 --- a/apps/cli/src/server/chdb.ts +++ b/apps/cli/src/server/chdb.ts @@ -104,32 +104,34 @@ export const RAW_TELEMETRY_TTL_COLUMNS = [ export const MINIMUM_RAW_TELEMETRY_RETENTION_DAYS = 90 export const MAXIMUM_RAW_TELEMETRY_RETENTION_DAYS = 3_650 -interface RawTelemetryRetentionConfig { - readonly formatVersion: 1 - readonly minimumDays: number -} +/** + * The retention floor an operator has pinned for this store. + * + * Unknown fields are rejected rather than ignored: a config carrying a field + * this build does not understand was written by a different build, and reading + * only the half we recognise would silently apply a policy nobody chose. + */ +const RawTelemetryRetentionConfigSchema = Schema.Struct({ + formatVersion: Schema.Literal(1), + minimumDays: Schema.Int.check( + Schema.makeFilter((days: number) => + days >= MINIMUM_RAW_TELEMETRY_RETENTION_DAYS && days <= MAXIMUM_RAW_TELEMETRY_RETENTION_DAYS + ? undefined + : `raw telemetry retention minimum must be an integer from ${MINIMUM_RAW_TELEMETRY_RETENTION_DAYS} through ${MAXIMUM_RAW_TELEMETRY_RETENTION_DAYS} days`, + ), + ), +}) + +type RawTelemetryRetentionConfig = typeof RawTelemetryRetentionConfigSchema.Type export const rawTelemetryRetentionConfigPath = (dataDir: string): string => `${resolve(dataDir)}.raw-telemetry-retention.json` -const parseRawTelemetryRetentionDays = (value: unknown): number => { - if (typeof value !== "object" || value === null || Array.isArray(value)) - throw new Error("raw telemetry retention config must be a record") - const record = value as Record - if (Object.keys(record).sort().join(",") !== "formatVersion,minimumDays" || record.formatVersion !== 1) - throw new Error("unsupported or malformed raw telemetry retention config") - const days = record.minimumDays - if ( - typeof days !== "number" || - !Number.isSafeInteger(days) || - days < MINIMUM_RAW_TELEMETRY_RETENTION_DAYS || - days > MAXIMUM_RAW_TELEMETRY_RETENTION_DAYS - ) - throw new Error( - `raw telemetry retention minimum must be an integer from ${MINIMUM_RAW_TELEMETRY_RETENTION_DAYS} through ${MAXIMUM_RAW_TELEMETRY_RETENTION_DAYS} days`, - ) - return days -} +const decodeRetentionConfig = Schema.decodeUnknownSync(RawTelemetryRetentionConfigSchema, { + onExcessProperty: "error", +}) + +const parseRawTelemetryRetentionDays = (value: unknown): number => decodeRetentionConfig(value).minimumDays export const readRawTelemetryRetentionDays = (dataDir: string): number | undefined => { const path = rawTelemetryRetentionConfigPath(dataDir) diff --git a/apps/cli/src/server/identity-schema.ts b/apps/cli/src/server/identity-schema.ts new file mode 100644 index 000000000..32c892361 --- /dev/null +++ b/apps/cli/src/server/identity-schema.ts @@ -0,0 +1,29 @@ +// Shared identity primitives for the local store. +// +// The 16-character bundle fingerprint and the 64-character sha256 digest used +// to be re-checked with an inline regex at every site that read or wrote one: +// twice in the marker reader, twice more in the marker constructor, and once +// again inside the v0 -> v1 migration. Four copies of the same rule is four +// chances to relax one of them. +import { Schema } from "effect" + +/** Legacy 16-character bundled-schema fingerprint. */ +export const Fingerprint16 = Schema.String.check(Schema.isPattern(/^[0-9a-f]{16}$/i)) + +/** Full sha256 hex digest. */ +export const Digest64 = Schema.String.check(Schema.isPattern(/^[0-9a-f]{64}$/i)) + +/** + * An ISO timestamp, or the literal `"unknown"`. + * + * Provenance timestamps on a store predate the versioned marker, so a store + * created by an old build genuinely has no creation time. That is recorded + * rather than guessed. + */ +export const IsoOrUnknown = Schema.String.check( + Schema.makeFilter((value: string) => + value === "unknown" || Number.isFinite(Date.parse(value)) + ? undefined + : 'must be an ISO timestamp or "unknown"', + ), +) diff --git a/apps/cli/src/server/local-schema-history.ts b/apps/cli/src/server/local-schema-history.ts index 337e192f8..01f47a706 100644 --- a/apps/cli/src/server/local-schema-history.ts +++ b/apps/cli/src/server/local-schema-history.ts @@ -72,4 +72,11 @@ export const LOCAL_SCHEMA_HISTORY: ReadonlyArray = Obje manifestDigest: "2814f51596f9eabcdb32fb992249b9c8378d15f0c2a60dcbc2ccb49f3ad9dae6", projectRevision: "73f8f3249a508cd05598289b67b3773a049e38db0302efa6aa9e45e3501d2182", }), + Object.freeze({ + version: 8, + fingerprint: "51081e951066442a", + digest: "51081e951066442a8e5b53df2c4bdda933edd20fc89132a54ed9b4dbb7e55a05", + manifestDigest: "60908c2e8307e24885227d4553916eef64df7f9b23abec23b5697cfea0d84d94", + projectRevision: "bb7da950a3a65af75fcf627bf4ed0436308c98fee86906048b05b6d40d9f7534", + }), ] as const) diff --git a/apps/cli/src/server/local-schema-version.ts b/apps/cli/src/server/local-schema-version.ts index e3571c962..8bbe13f03 100644 --- a/apps/cli/src/server/local-schema-version.ts +++ b/apps/cli/src/server/local-schema-version.ts @@ -1,4 +1,4 @@ // Increment this value for every structural change to the generated local // schema. The compatibility manifest and migration registry must be updated in // the same change before a new value can ship. -export const LOCAL_SCHEMA_VERSION = 7 as const +export const LOCAL_SCHEMA_VERSION = 8 as const diff --git a/apps/cli/src/server/local-store-migration-module.ts b/apps/cli/src/server/local-store-migration-module.ts index f0d2f2193..1e47a0394 100644 --- a/apps/cli/src/server/local-store-migration-module.ts +++ b/apps/cli/src/server/local-store-migration-module.ts @@ -1,20 +1,17 @@ // BOUNDARY: This module owns unparsed external values and narrows them before domain use. import type { Chdb } from "./chdb" import type { LocalSchemaIdentity } from "./schema-identity" +import type { + MigrationPhase, + MigrationStepJournalSchema, + MigrationStepStatus, +} from "./local-store-migrations/journal-schema" /** Coordinator-owned transaction phases. Modules may report progress, but - * only the coordinator advances this top-level state machine. */ -export type MigrationPhase = - | "planned" - | "preflight-complete" - | "target-created" - | "copying" - | "copy-verified" - | "promotion-started" - | "promoted" - | "failed" - -export type MigrationStepStatus = "pending" | "running" | "verified" | "completed" + * only the coordinator advances this top-level state machine. Both unions are + * derived from the journal schema so the persisted form and the in-memory type + * cannot drift. */ +export type { MigrationPhase, MigrationStepStatus } export type StateDisposition = | "preserve-exact" @@ -48,15 +45,7 @@ export interface StateDispositionEntry { /** The persisted binding for one edge in a migration chain. The identities * are copied into the journal so a later build cannot reinterpret an old * step through its current-schema constant. */ -export interface MigrationStepJournal { - readonly id: string - readonly moduleVersion: number - readonly from: LocalSchemaIdentity - readonly to: LocalSchemaIdentity - readonly status: MigrationStepStatus - readonly state?: unknown - readonly progress?: unknown -} +export type MigrationStepJournal = typeof MigrationStepJournalSchema.Type export interface MigrationDbOptions { readonly schemaSql?: string diff --git a/apps/cli/src/server/local-store-migrations.ts b/apps/cli/src/server/local-store-migrations.ts index 1b9e1ad05..f41dc937c 100644 --- a/apps/cli/src/server/local-store-migrations.ts +++ b/apps/cli/src/server/local-store-migrations.ts @@ -9,7 +9,9 @@ import { randomUUID } from "node:crypto" import { existsSync, readFileSync, statfsSync } from "node:fs" import { lstat, readdir, readFile, stat } from "node:fs/promises" import { dirname, join, relative, resolve } from "node:path" +import { Schema } from "effect" import { Chdb } from "./chdb" +import { decodeJsonEachRow } from "./chdb-rows" import { CURRENT_LOCAL_SCHEMA, LEGACY_LOCAL_SCHEMA, @@ -33,6 +35,7 @@ import { } from "./store-version" import { durableJson, durableRename, ensurePrivateDirectory } from "./durable-files" import { MAPLE_VERSION } from "../version" +import { decodeMigrationJournal, type MigrationJournalSchema } from "./local-store-migrations/journal-schema" import { legacyToCurrentModule } from "./local-store-migrations/legacy-to-current" import { v1ToV2ErrorRollupModule } from "./local-store-migrations/v1-to-v2-error-rollup" import { v2ToV3ServiceMapIngestBridgeModule } from "./local-store-migrations/v2-to-v3-service-map-ingest-bridge" @@ -40,6 +43,7 @@ import { v3ToV4WebEventsModule } from "./local-store-migrations/v3-to-v4-web-eve import { v4ToV5ServiceOverviewMinutelyModule } from "./local-store-migrations/v4-to-v5-service-overview-minutely" import { v5ToV6ErrorEventsFingerprintHygieneModule } from "./local-store-migrations/v5-to-v6-error-events-fingerprint-hygiene" import { v6ToV7ErrorServiceVersionModule } from "./local-store-migrations/v6-to-v7-error-service-version" +import { v7ToV8AppleCrashFramesModule } from "./local-store-migrations/v7-to-v8-apple-crash-frames" import type { AnyLocalStoreMigrationModule, LocalStoreMigration, @@ -47,7 +51,6 @@ import type { MigrationModuleContext, MigrationPhase, MigrationStepJournal, - MigrationStepStatus, } from "./local-store-migration-module" export { @@ -83,28 +86,7 @@ export interface MigrationPlan { readonly checkpointDisposition: string } -export interface MigrationJournal { - readonly formatVersion: 2 - readonly migrationId: string - readonly phase: MigrationPhase - readonly chain: ReadonlyArray - readonly currentStepIndex: number - readonly sourceDataDir: string - readonly sourceStoreId: string - readonly sourceChdb: string - readonly sourceFingerprint: string - readonly sourceDigest: string - readonly sourceVersion: number - readonly targetDataDir: string - readonly targetStoreId: string - readonly targetChdb: string - readonly targetFingerprint: string - readonly targetDigest: string - readonly targetVersion: number - readonly cutoffAt: string - readonly createdAt: string - readonly failure?: string -} +export type MigrationJournal = typeof MigrationJournalSchema.Type export interface MigrationResult { readonly migrationId: string @@ -126,6 +108,7 @@ export const localStoreMigrations: ReadonlyArray = v4ToV5ServiceOverviewMinutelyModule, v5ToV6ErrorEventsFingerprintHygieneModule, v6ToV7ErrorServiceVersionModule, + v7ToV8AppleCrashFramesModule, ] export const validateMigrationRegistry = ( @@ -321,8 +304,6 @@ export const migrationRootPath = (dataDir: string, migrationId: string): string export const migrationHistoryPath = (dataDir: string, migrationId: string): string => join(migrationRootPath(dataDir, migrationId), "journal.json") -const migrationIdPattern = /^[A-Za-z0-9._-]+$/ - const safeMigrationPath = (path: string, root: string, label: string): string => { const absolute = resolve(path) const relativePath = relative(resolve(root), absolute) @@ -360,73 +341,6 @@ const assertJournalPaths = (dataDir: string, journal: MigrationJournal): void => throw new Error("local migration journal final target does not match its chain") } -const parsePhase = (value: unknown): MigrationPhase => { - if ( - value !== "planned" && - value !== "preflight-complete" && - value !== "target-created" && - value !== "copying" && - value !== "copy-verified" && - value !== "promotion-started" && - value !== "promoted" && - value !== "failed" - ) - throw new Error(`invalid local migration phase: ${String(value)}`) - return value -} - -const parseIdentity = (value: unknown, label: string): LocalSchemaIdentity => { - if (typeof value !== "object" || value === null || Array.isArray(value)) - throw new Error(`migration journal ${label} identity is invalid`) - const identity = value as Record - if ( - !Number.isInteger(identity.version) || - (identity.version as number) < 0 || - typeof identity.fingerprint !== "string" || - identity.fingerprint.length === 0 || - typeof identity.digest !== "string" || - typeof identity.chdb !== "string" || - identity.chdb.length === 0 - ) - throw new Error(`migration journal ${label} identity is invalid`) - return { - version: identity.version as number, - fingerprint: identity.fingerprint, - digest: identity.digest, - chdb: identity.chdb, - ...(typeof identity.manifestDigest === "string" - ? { manifestDigest: identity.manifestDigest } - : undefined), - ...(typeof identity.projectRevision === "string" - ? { projectRevision: identity.projectRevision } - : undefined), - } -} - -const parseStep = (value: unknown, index: number): MigrationStepJournal => { - if (typeof value !== "object" || value === null || Array.isArray(value)) - throw new Error(`migration journal step ${index} is invalid`) - const step = value as Record - const status = step.status - if ( - typeof step.id !== "string" || - step.id.length === 0 || - !Number.isInteger(step.moduleVersion) || - (step.moduleVersion as number) < 1 || - (status !== "pending" && status !== "running" && status !== "verified" && status !== "completed") - ) - throw new Error(`migration journal step ${index} is invalid`) - return { - id: step.id, - moduleVersion: step.moduleVersion as number, - from: parseIdentity(step.from, `step ${index} from`), - to: parseIdentity(step.to, `step ${index} to`), - status: status as MigrationStepStatus, - ...(!(step.state === undefined) ? { state: step.state } : undefined), - ...(!(step.progress === undefined) ? { progress: step.progress } : undefined), - } -} - const sameJournalIdentity = (a: LocalSchemaIdentity, b: LocalSchemaIdentity): boolean => a.version === b.version && a.fingerprint === b.fingerprint && a.digest === b.digest && a.chdb === b.chdb @@ -501,59 +415,9 @@ const assertJournalChainInvariants = (journal: MigrationJournal): void => { } const parseJournal = (value: unknown): MigrationJournal => { - if (typeof value !== "object" || value === null || Array.isArray(value)) - throw new Error("migration journal is not an object") - const record = value as Record - const requiredStrings = [ - "migrationId", - "sourceDataDir", - "sourceStoreId", - "sourceChdb", - "sourceFingerprint", - "targetDataDir", - "targetStoreId", - "targetChdb", - "targetFingerprint", - "targetDigest", - "cutoffAt", - "createdAt", - ] as const - for (const key of requiredStrings) - if (typeof record[key] !== "string" || record[key] === "") - throw new Error(`migration journal ${key} is invalid`) - for (const key of ["sourceVersion", "targetVersion", "currentStepIndex"] as const) - if (!Number.isInteger(record[key])) throw new Error(`migration journal ${key} is invalid`) - if (record.formatVersion !== 2) - throw new Error(`unsupported migration journal format ${String(record.formatVersion)}`) - if (!migrationIdPattern.test(record.migrationId as string)) - throw new Error("migration journal id is unsafe") - if (!Array.isArray(record.chain) || record.chain.length === 0) - throw new Error("migration journal chain is invalid") - const chain = record.chain.map(parseStep) - if ((record.currentStepIndex as number) < 0 || (record.currentStepIndex as number) > chain.length) + const journal = decodeMigrationJournal(value) + if (journal.currentStepIndex > journal.chain.length) throw new Error("migration journal currentStepIndex is invalid") - const journal: MigrationJournal = { - formatVersion: 2, - migrationId: record.migrationId as string, - phase: parsePhase(record.phase), - chain, - currentStepIndex: record.currentStepIndex as number, - sourceDataDir: resolve(record.sourceDataDir as string), - sourceStoreId: record.sourceStoreId as string, - sourceChdb: record.sourceChdb as string, - sourceFingerprint: record.sourceFingerprint as string, - sourceDigest: typeof record.sourceDigest === "string" ? record.sourceDigest : "", - sourceVersion: record.sourceVersion as number, - targetDataDir: resolve(record.targetDataDir as string), - targetStoreId: record.targetStoreId as string, - targetChdb: record.targetChdb as string, - targetFingerprint: record.targetFingerprint as string, - targetDigest: record.targetDigest as string, - targetVersion: record.targetVersion as number, - cutoffAt: record.cutoffAt as string, - createdAt: record.createdAt as string, - ...(!(record.failure === undefined) ? { failure: String(record.failure) } : undefined), - } assertJournalChainInvariants(journal) for (const [index, step] of journal.chain.entries()) { if (step.status === "verified" || step.status === "completed") { @@ -660,12 +524,11 @@ const assertNoLiveServer = (dataDir: string): void => { } } -const parseJsonEachRow = (value: string): A[] => - value - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length > 0) - .map((line) => JSON.parse(line) as A) +/** Total bytes on disk. A `string | number` because the setting that unquotes + * 64-bit integers is not in force for every libchdb build this CLI supports. */ +const DiskUsageRow = Schema.Struct({ bytes: Schema.Union([Schema.String, Schema.Number]) }) + +const decodeDiskUsageRows = decodeJsonEachRow(DiskUsageRow) const MIN_MIGRATION_FREE_BYTES = 128 * 1024 * 1024 @@ -752,7 +615,7 @@ const ensureMigrationCapacity = async (dataDir: string, session: MigrationDbSess const rows = await session.use( dataDir, (db) => - parseJsonEachRow<{ bytes: string | number }>( + decodeDiskUsageRows( db.query( "SELECT coalesce(sum(bytes_on_disk), 0) AS bytes FROM system.parts WHERE database = 'default' AND active = 1", ), diff --git a/apps/cli/src/server/local-store-migrations/journal-codecs.ts b/apps/cli/src/server/local-store-migrations/journal-codecs.ts new file mode 100644 index 000000000..b2615af33 --- /dev/null +++ b/apps/cli/src/server/local-store-migrations/journal-codecs.ts @@ -0,0 +1,122 @@ +// Shared journal codecs for the local-store migration edges. +// +// Every versioned edge from v1 onward round-trips the same journal state: which +// module wrote it, the raw-telemetry row counts it must preserve, and the +// retention floor in force at the time. Each edge used to carry its own copy of +// `isRecord` + a `decodeCounts` loop + five `throw new Error` branches — six +// identical copies, drifting only in the strings. +// +// The decoders here are synchronous and throwing because +// `LocalStoreMigrationModule.decodeState` is: the runner that drives these +// modules is plain TypeScript. The schemas are the declarative part, and +// `Schema.decodeUnknownEffect` is a one-line swap if that runner ever moves +// into Effect. +import { Schema } from "effect" +import { RAW_TELEMETRY_TTL_COLUMNS, type Chdb } from "../chdb" +import { decodeTableRowCounts } from "../chdb-rows" +import { withRawTelemetryRetentionFloor, type LocalSchemaManifest } from "../schema-manifest" + +const RAW_TABLES_INTERNAL = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) + +/** + * Unsigned decimal string. + * + * Row counts and ClickHouse UInt64 cursors are carried as text because they can + * exceed `Number.MAX_SAFE_INTEGER`, and the journal has to round-trip them + * exactly. The pattern is not cosmetic: these values are interpolated into SQL + * comparisons, so anything that could change their meaning has to fail here. + */ +export const UnsignedDecimal = Schema.String.check(Schema.isPattern(/^\d+$/)) + +/** + * Exactly the raw tables, each required. + * + * Built from `RAW_TELEMETRY_TTL_COLUMNS` rather than written out, so a table + * added there is covered without touching any edge. With + * `onExcessProperty: "error"` at the decode site this rejects a missing table + * and an unknown one alike, which is what the hand-rolled loop did in two + * separate passes. + */ +export const RawRowsSchema = Schema.Struct( + Object.fromEntries(RAW_TABLES_INTERNAL.map((table) => [table, UnsignedDecimal])), +) + +/** + * Rejecting unknown fields is not tidiness. A journal carrying a field this + * build does not know about was written by a different build, and silently + * dropping it would resume someone else's migration under our assumptions. + */ +const strict = { onExcessProperty: "error" } as const + +/** + * The journal state shared by every versioned edge. + * + * `retentionDays` is `optionalKey`, not `optional`: the journal is JSON, where + * an absent retention floor is an absent key rather than a present `undefined`. + */ +export const makeRawRowsState = (moduleId: Id) => { + const schema = Schema.Struct({ + module: Schema.Literal(moduleId), + version: Schema.Literal(1), + rawRows: RawRowsSchema, + retentionDays: Schema.optionalKey(Schema.Int), + }) + return { schema, decode: Schema.decodeUnknownSync(schema, strict) } +} + +/** Progress for an edge whose apply step is a single idempotent install. */ +export const InstalledProgressSchema = Schema.Struct({ installed: Schema.Literal(true) }) + +export type InstalledProgress = typeof InstalledProgressSchema.Type + +const decodeInstalled = Schema.decodeUnknownSync(InstalledProgressSchema, strict) + +/** + * Absent progress means the step has not run; it is not the same as invalid + * progress, which means the journal disagrees with this build. + */ +export const decodeInstalledProgress = (value: unknown): InstalledProgress | undefined => + value === undefined ? undefined : decodeInstalled(value) + +/** + * `decodeUnknownSync` with the strict excess-property policy, for an edge whose + * progress is not the plain `installed` flag. + */ +export const strictDecoder = >(schema: S) => + Schema.decodeUnknownSync(schema, strict) + +/** Exactly the raw telemetry tables a migration must preserve, in a stable order. */ +export const RAW_TABLES: ReadonlyArray = RAW_TABLES_INTERNAL + +/** + * Row counts per raw table, straight from `system.parts`. + * + * Every versioned edge carried a byte-identical copy of this. It is the input + * to the only guarantee those edges make — that a structural DDL change moves + * no telemetry — so it belongs in one place where that query can be reasoned + * about once. + */ +export const rawRowCounts = (db: Chdb): Readonly> => { + const quotedTables = RAW_TABLES_INTERNAL.map((table) => `'${table}'`).join(", ") + const rows = decodeTableRowCounts( + db.query( + `SELECT table, toString(sum(rows)) AS rowCount FROM system.parts WHERE database = 'default' AND active = 1 AND table IN (${quotedTables}) GROUP BY table`, + ), + ) + const byTable = new Map(rows.map((row) => [row.table, row.rowCount])) + return Object.fromEntries(RAW_TABLES_INTERNAL.map((table) => [table, byTable.get(table) ?? "0"])) +} + +/** + * The manifest an edge should expect to find, given the retention floor an + * operator pinned for this store. A pinned floor rewrites the raw tables' TTL + * intervals, so comparing against the bundled manifest verbatim would report a + * drift the operator asked for. + */ +export const expectedManifest = ( + manifest: LocalSchemaManifest, + retentionDays: number | undefined, +): LocalSchemaManifest => + retentionDays === undefined + ? manifest + : withRawTelemetryRetentionFloor(manifest, RAW_TABLES_INTERNAL, retentionDays) diff --git a/apps/cli/src/server/local-store-migrations/journal-schema.ts b/apps/cli/src/server/local-store-migrations/journal-schema.ts new file mode 100644 index 000000000..9b613547e --- /dev/null +++ b/apps/cli/src/server/local-store-migrations/journal-schema.ts @@ -0,0 +1,128 @@ +// Declarative shape of the coordinator-owned migration journal. +// +// The journal is untrusted JSON: it was written by some build of this CLI, not +// necessarily this one. Everything below used to be a hand-rolled parser in +// `local-store-migrations.ts` — `typeof` chains, `Number.isInteger` guards, and +// an `as string` on every field read. The shape is now declared once and the +// casts are gone; what remains imperative in that file are the cross-field +// invariants (phase x currentStepIndex x per-step status), which are a state +// machine rather than a struct. +// +// Decoding is synchronous and throwing to match the coordinator, which is plain +// TypeScript throw/catch code. `Schema.decodeUnknownEffect` is a one-line swap +// if that ever moves into Effect. +import { Effect, Schema, SchemaGetter } from "effect" +import { resolve } from "node:path" + +/** + * Rejecting unknown fields is not tidiness. A journal carrying a field this + * build does not know about was written by a different build, and silently + * dropping it would resume someone else's migration under our assumptions. + */ +const strict = { onExcessProperty: "error" } as const + +const NonEmptyString = Schema.String.check(Schema.isMinLength(1)) + +/** A migration id is interpolated into filesystem paths; keep it path-safe. */ +const migrationIdPattern = /^[A-Za-z0-9._-]+$/ + +/** + * Paths are normalized on the way in so a journal written with a relative or + * unnormalized path compares equal to the configured data directory. `resolve` + * is idempotent on an absolute path, so the encoding side is a passthrough. + */ +const ResolvedPath = NonEmptyString.pipe( + Schema.decodeTo(Schema.String, { + decode: SchemaGetter.transform(resolve), + encode: SchemaGetter.passthrough(), + }), +) + +/** Coordinator-owned transaction phases. */ +export const MigrationPhaseSchema = Schema.Literals([ + "planned", + "preflight-complete", + "target-created", + "copying", + "copy-verified", + "promotion-started", + "promoted", + "failed", +]) + +export type MigrationPhase = typeof MigrationPhaseSchema.Type + +export const MigrationStepStatusSchema = Schema.Literals(["pending", "running", "verified", "completed"]) + +export type MigrationStepStatus = typeof MigrationStepStatusSchema.Type + +/** + * A schema identity as persisted in a journal. + * + * `manifestDigest` and `projectRevision` are `optionalKey`, not `optional`: the + * journal is JSON, where an absent field is an absent key rather than a present + * `undefined`. That is also what lets the coordinator write them back without a + * conditional spread. + */ +export const LocalSchemaIdentitySchema = Schema.Struct({ + version: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), + fingerprint: NonEmptyString, + digest: Schema.String, + manifestDigest: Schema.optionalKey(Schema.String), + chdb: NonEmptyString, + projectRevision: Schema.optionalKey(Schema.String), +}) + +/** + * One edge in a migration chain. + * + * `state` and `progress` stay `Unknown` on purpose. The coordinator does not + * interpret them; each module decodes its own through `decodeState` / + * `decodeProgress`. Keeping that split is what lets a staged target be + * abandoned when a module's persisted state is corrupt or its code is no + * longer bindable. + */ +export const MigrationStepJournalSchema = Schema.Struct({ + id: NonEmptyString, + moduleVersion: Schema.Int.check(Schema.isGreaterThanOrEqualTo(1)), + from: LocalSchemaIdentitySchema, + to: LocalSchemaIdentitySchema, + status: MigrationStepStatusSchema, + state: Schema.optionalKey(Schema.Unknown), + progress: Schema.optionalKey(Schema.Unknown), +}) + +export const MigrationJournalSchema = Schema.Struct({ + formatVersion: Schema.Literal(2), + migrationId: Schema.String.check(Schema.isPattern(migrationIdPattern)), + phase: MigrationPhaseSchema, + chain: Schema.Array(MigrationStepJournalSchema).check(Schema.isMinLength(1)), + currentStepIndex: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), + sourceDataDir: ResolvedPath, + sourceStoreId: NonEmptyString, + sourceChdb: NonEmptyString, + sourceFingerprint: NonEmptyString, + // Asymmetric with `targetDigest` on purpose: the v0 legacy identity has no + // digest at all (see LOCAL_SCHEMA_HISTORY), so a journal migrating away from + // it carries an empty source digest. A target is always a known schema. + sourceDigest: Schema.String.pipe(Schema.withDecodingDefaultKey(Effect.succeed(""))), + sourceVersion: Schema.Int, + targetDataDir: ResolvedPath, + targetStoreId: NonEmptyString, + targetChdb: NonEmptyString, + targetFingerprint: NonEmptyString, + targetDigest: NonEmptyString, + targetVersion: Schema.Int, + cutoffAt: NonEmptyString, + createdAt: NonEmptyString, + failure: Schema.optionalKey(Schema.String), +}) + +const decodeJournal = Schema.decodeUnknownSync(MigrationJournalSchema, strict) + +/** + * Decode the journal envelope. Cross-field chain invariants are asserted by the + * coordinator after this returns; this only establishes the shape. + */ +export const decodeMigrationJournal = (value: unknown): typeof MigrationJournalSchema.Type => + decodeJournal(value) diff --git a/apps/cli/src/server/local-store-migrations/legacy-to-current.ts b/apps/cli/src/server/local-store-migrations/legacy-to-current.ts index 02a20b732..fddec8fd4 100644 --- a/apps/cli/src/server/local-store-migrations/legacy-to-current.ts +++ b/apps/cli/src/server/local-store-migrations/legacy-to-current.ts @@ -1,5 +1,7 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. import { createHash } from "node:crypto" +import { Schema } from "effect" +import { decodeJsonEachRow, decodeJsonObjectRows } from "../chdb-rows" import { LOCAL_SCHEMA_V1_MANIFEST, LOCAL_SCHEMA_V1_SQL, @@ -7,6 +9,7 @@ import { LOCAL_SCHEMA_V1, } from "../schema-identity" import { assertPhysicalSchema } from "../schema-physical" +import { strictDecoder, UnsignedDecimal } from "./journal-codecs" import type { LocalSchemaColumn } from "../schema-manifest" import type { LocalStoreMigrationModule, @@ -135,175 +138,142 @@ const emptyCopyProgress = (): CopyProgress => ({ const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null && !Array.isArray(value) -const record = (value: unknown, label: string): Record => { - if (!isRecord(value)) throw new Error(`${label} must be an object`) - return value -} - -const nonEmptyString = (value: unknown, label: string): string => { - if (typeof value !== "string" || value.length === 0) - throw new Error(`${label} must be a non-empty string`) - return value -} - -const nullableString = (value: unknown, label: string): string | null => { - if (value !== null && typeof value !== "string") throw new Error(`${label} must be a string or null`) - return value -} - -/** ClickHouse emits UInt64 cursor values as decimal strings in JSONEachRow. - * Keep the journal representation textual, but reject anything that could - * change the meaning of the numeric SQL comparisons when interpolated. */ -const uint64String = (value: unknown, label: string): string | null => { - if (value === null) return null - if (typeof value !== "string" || !/^\d+$/.test(value)) - throw new Error(`${label} must be an unsigned decimal string or null`) - return value -} +const NonEmptyString = Schema.String.check(Schema.isMinLength(1)) +const NullableString = Schema.NullOr(Schema.String) +const NonNegativeInt = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)) + +/** + * A ClickHouse UInt64 cursor, or `null` for "no cursor yet". + * + * Textual because a UInt64 does not survive a JS number, and pattern-checked + * because these values are interpolated into the numeric SQL comparisons that + * drive the resumable copy — anything that could change their meaning has to + * fail here rather than there. + */ +const Uint64Cursor = Schema.NullOr(UnsignedDecimal) + +/** Only tables this build knows how to replay. */ +const RawTable = Schema.Literals([...RAW_TABLE_NAMES]) + +const Sha256Hex = Schema.String.check(Schema.isPattern(/^[0-9a-f]{64}$/i)) + +const TableInventorySchema = Schema.Struct({ + table: NonEmptyString, + rowCount: NonEmptyString, + retentionStartAt: NonEmptyString, + minTime: NullableString, + maxTime: NullableString, + hashSum: NonEmptyString, + hashXor: NonEmptyString, +}) -const nonNegativeInteger = (value: unknown, label: string): number => { - if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) - throw new Error(`${label} must be a non-negative integer`) - return value -} +const CopyProgressSchema = Schema.Struct({ + rows: NonNegativeInt, + bytes: NonNegativeInt, + lastTimestamp: NullableString, + lastHash: Uint64Cursor, + lastTieBreak: Uint64Cursor, + /** Cumulative ordinal consumed within the final composite-key group. */ + duplicateCount: NonNegativeInt, + duplicateGroupExhausted: Schema.Boolean, +}) -const exactKeys = (value: Record, allowed: ReadonlyArray, label: string): void => { - const allowedKeys = new Set(allowed) - for (const key of Object.keys(value)) { - if (!allowedKeys.has(key)) throw new Error(`${label} contains unknown field ${key}`) - } -} +const PendingBatchSchema = Schema.Struct({ + table: RawTable, + // Positive, not merely non-negative: an empty batch would commit nothing + // while advancing the cursor past the rows it claims to cover. + rowCount: Schema.Int.check(Schema.isGreaterThanOrEqualTo(1)), + byteLength: NonNegativeInt, + firstTimestamp: NullableString, + firstHash: Uint64Cursor, + firstTieBreak: Uint64Cursor, + lastTimestamp: NullableString, + lastHash: Uint64Cursor, + lastTieBreak: Uint64Cursor, + lastKeyCount: NonNegativeInt, + lastKeyExhausted: Schema.Boolean, + signature: Sha256Hex, +}) -const decodeTableInventory = (value: unknown, label: string): TableInventory => { - const inventory = record(value, label) - exactKeys( - inventory, - ["table", "rowCount", "retentionStartAt", "minTime", "maxTime", "hashSum", "hashXor"], - label, - ) - return { - table: nonEmptyString(inventory.table, `${label}.table`), - rowCount: nonEmptyString(inventory.rowCount, `${label}.rowCount`), - retentionStartAt: nonEmptyString(inventory.retentionStartAt, `${label}.retentionStartAt`), - minTime: nullableString(inventory.minTime, `${label}.minTime`), - maxTime: nullableString(inventory.maxTime, `${label}.maxTime`), - hashSum: nonEmptyString(inventory.hashSum, `${label}.hashSum`), - hashXor: nonEmptyString(inventory.hashXor, `${label}.hashXor`), - } -} +/** + * Keyed by `Schema.String`, not by `RawTable`, and the keys are checked below. + * + * A record keyed on the literal union requires *every* table to be present, but + * both maps legitimately hold a subset: `copied` starts empty and fills one + * table at a time as the replay progresses. Keying it strictly would reject the + * journal of every partially-progressed migration and strand it. + */ +const RawReplayProgressSchema = Schema.Struct({ + sourceInventory: Schema.Record(Schema.String, TableInventorySchema), + copied: Schema.Record(Schema.String, CopyProgressSchema), + pendingBatch: Schema.optionalKey(PendingBatchSchema), +}) -const decodeCopyProgress = (value: unknown, label: string): CopyProgress => { - const progress = record(value, label) - exactKeys( - progress, - [ - "rows", - "bytes", - "lastTimestamp", - "lastHash", - "lastTieBreak", - "duplicateCount", - "duplicateGroupExhausted", - ], - label, - ) - if (typeof progress.duplicateGroupExhausted !== "boolean") - throw new Error(`${label}.duplicateGroupExhausted must be a boolean`) - return { - rows: nonNegativeInteger(progress.rows, `${label}.rows`), - bytes: nonNegativeInteger(progress.bytes, `${label}.bytes`), - lastTimestamp: nullableString(progress.lastTimestamp, `${label}.lastTimestamp`), - lastHash: uint64String(progress.lastHash, `${label}.lastHash`), - lastTieBreak: uint64String(progress.lastTieBreak, `${label}.lastTieBreak`), - duplicateCount: nonNegativeInteger(progress.duplicateCount, `${label}.duplicateCount`), - duplicateGroupExhausted: progress.duplicateGroupExhausted, - } -} +const LegacyStateSchema = Schema.Struct({ + module: Schema.Literal("local-0000-to-0001-raw-replay"), + version: Schema.Literal(1), +}) -const decodePendingBatch = (value: unknown, label: string): PendingBatch => { - const pending = record(value, label) - exactKeys( - pending, - [ - "table", - "rowCount", - "byteLength", - "firstTimestamp", - "firstHash", - "firstTieBreak", - "lastTimestamp", - "lastHash", - "lastTieBreak", - "lastKeyCount", - "lastKeyExhausted", - "signature", - ], - label, - ) - const table = nonEmptyString(pending.table, `${label}.table`) - if (!RAW_TABLE_NAMES.has(table)) throw new Error(`${label}.table is not a registered raw table`) - if (typeof pending.lastKeyExhausted !== "boolean") - throw new Error(`${label}.lastKeyExhausted must be a boolean`) - if (typeof pending.signature !== "string" || !/^[0-9a-f]{64}$/i.test(pending.signature)) - throw new Error(`${label}.signature must be a SHA-256 hex digest`) - const rowCount = nonNegativeInteger(pending.rowCount, `${label}.rowCount`) - if (rowCount === 0) throw new Error(`${label}.rowCount must be positive`) - return { - table, - rowCount, - byteLength: nonNegativeInteger(pending.byteLength, `${label}.byteLength`), - firstTimestamp: nullableString(pending.firstTimestamp, `${label}.firstTimestamp`), - firstHash: uint64String(pending.firstHash, `${label}.firstHash`), - firstTieBreak: uint64String(pending.firstTieBreak, `${label}.firstTieBreak`), - lastTimestamp: nullableString(pending.lastTimestamp, `${label}.lastTimestamp`), - lastHash: uint64String(pending.lastHash, `${label}.lastHash`), - lastTieBreak: uint64String(pending.lastTieBreak, `${label}.lastTieBreak`), - lastKeyCount: nonNegativeInteger(pending.lastKeyCount, `${label}.lastKeyCount`), - lastKeyExhausted: pending.lastKeyExhausted, - signature: pending.signature, - } -} +const decodeProgressStrict = strictDecoder(RawReplayProgressSchema) const decodeRawReplayProgress = (value: unknown): RawReplayProgress => { - const progress = record(value, "legacy raw replay progress") - exactKeys(progress, ["sourceInventory", "copied", "pendingBatch"], "legacy raw replay progress") - const sourceInventoryRecord = record(progress.sourceInventory, "legacy raw replay sourceInventory") - const copiedRecord = record(progress.copied, "legacy raw replay copied") - const sourceInventory: Record = {} - for (const [table, inventory] of Object.entries(sourceInventoryRecord)) { + const progress = decodeProgressStrict(value) + // Two invariants the schema is not the right place to state, both about the + // map *keys* rather than the values. + for (const [table, inventory] of Object.entries(progress.sourceInventory)) { if (!RAW_TABLE_NAMES.has(table)) throw new Error(`legacy raw replay sourceInventory has unknown table ${table}`) - const decoded = decodeTableInventory(inventory, `legacy raw replay sourceInventory.${table}`) - if (decoded.table !== table) + // An inventory is keyed by table and also carries its own `table` field. + // If those disagree the journal describes a copy of one table under + // another's cursor. + if (inventory.table !== table) throw new Error(`legacy raw replay sourceInventory.${table}.table does not match its key`) - sourceInventory[table] = decoded } - const copied: Record = {} - for (const [table, progressValue] of Object.entries(copiedRecord)) { + for (const table of Object.keys(progress.copied)) { if (!RAW_TABLE_NAMES.has(table)) throw new Error(`legacy raw replay copied has unknown table ${table}`) - copied[table] = decodeCopyProgress(progressValue, `legacy raw replay copied.${table}`) - } - return { - sourceInventory, - copied, - ...(!(progress.pendingBatch === undefined) - ? { - pendingBatch: decodePendingBatch(progress.pendingBatch, "legacy raw replay pendingBatch"), - } - : undefined), } + return progress } const asProgress = (value: RawReplayProgress | undefined): RawReplayProgress => value ?? { sourceInventory: {}, copied: {} } -const parseJsonEachRow = (value: string): A[] => - value - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length > 0) - .map((line) => JSON.parse(line) as A) +/** + * Row shapes read back out of chDB. + * + * `rowCount` and the hash aggregates are `string | number` on purpose: a UInt64 + * arrives quoted or unquoted depending on the libchdb build, and `numberString` + * normalizes it. What must not happen is a `Number` decode above 2^53, which is + * why nothing here is `Schema.Number` alone. + */ +const Uint64Wire = Schema.Union([Schema.String, Schema.Number]) + +const ColumnRowSchema = Schema.Struct({ + name: Schema.String, + type: Schema.String, + position: Schema.Number, + default_kind: Schema.String, + default_expression: Schema.String, + compression_codec: Schema.String, +}) + +const NameRow = Schema.Struct({ name: Schema.String }) + +const InventoryRow = Schema.Struct({ + rowCount: Uint64Wire, + minTime: Schema.NullOr(Schema.String), + maxTime: Schema.NullOr(Schema.String), + hashSum: Uint64Wire, + hashXor: Uint64Wire, +}) + +const RowCountRow = Schema.Struct({ rowCount: Uint64Wire }) + +const decodeColumnRows = decodeJsonEachRow(ColumnRowSchema) +const decodeNameRows = decodeJsonEachRow(NameRow) +const decodeInventoryRows = decodeJsonEachRow(InventoryRow) +const decodeRowCountRows = decodeJsonEachRow(RowCountRow) const identifier = (value: string): string => { if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(value)) throw new Error(`unsafe ClickHouse identifier: ${value}`) @@ -346,22 +316,13 @@ const queryTarget = (context: MigrationModuleContext, sql: string): Promise => context.openTarget((db) => db.exec(sql), { schemaSql: LOCAL_SCHEMA_V1_SQL, bootstrapSchema: false }) -interface ColumnRow { - name: string - type: string - position: number - default_kind: string - default_expression: string - compression_codec: string -} - const tableColumns = async ( context: MigrationModuleContext, side: "source" | "target", table: string, ): Promise> => { const query = `SELECT name, type, position, default_kind, default_expression, compression_codec FROM system.columns WHERE database = 'default' AND table = ${sqlString(table)} ORDER BY position` - const rows = parseJsonEachRow( + const rows = decodeColumnRows( await (side === "source" ? querySource(context, query) : queryTarget(context, query)), ) return rows.map((row) => ({ @@ -378,7 +339,7 @@ const tableColumns = async ( } const tableExists = async (context: MigrationModuleContext, side: "source" | "target", table: string) => { - const rows = parseJsonEachRow<{ name: string }>( + const rows = decodeNameRows( await (side === "source" ? querySource( context, @@ -394,7 +355,7 @@ const tableExists = async (context: MigrationModuleContext, side: "source" | "ta const assertKnownSourceObjects = async (context: MigrationModuleContext): Promise => { const known = new Set(LOCAL_SCHEMA_V1_MANIFEST.objects.map((object) => object.name)) - const rows = parseJsonEachRow<{ name: string }>( + const rows = decodeNameRows( await querySource(context, "SELECT name FROM system.tables WHERE database = 'default' ORDER BY name"), ) for (const row of rows) { @@ -449,13 +410,9 @@ const inventory = async ( // UInt64 aggregates leave SQL as strings: chDB emits 64-bit integers as // JSON numbers, and JS Number rounding above 2^53 corrupts them. const sql = `SELECT count() AS rowCount, min(${identifier(table.timeColumn)}) AS minTime, max(${identifier(table.timeColumn)}) AS maxTime, toString(sum(${hash})) AS hashSum, toString(groupBitXor(${hash})) AS hashXor FROM ${identifier(table.name)} WHERE ${identifier(table.timeColumn)} >= ${timestampLiteral(lowerBound)} AND ${identifier(table.timeColumn)} <= ${timestampLiteral(cutoffAt)}` - const rows = parseJsonEachRow<{ - rowCount: string | number - minTime: string | null - maxTime: string | null - hashSum: string | number - hashXor: string | number - }>(side === "source" ? await querySource(context, sql) : await queryTarget(context, sql)) + const rows = decodeInventoryRows( + side === "source" ? await querySource(context, sql) : await queryTarget(context, sql), + ) const row = rows[0] if (!row) throw new Error(`inventory query returned no row for ${table.name}`) return { @@ -484,7 +441,7 @@ const tableTotalRowCount = async ( table: string, ): Promise => { const sql = `SELECT count() AS rowCount FROM ${identifier(table)}` - const rows = parseJsonEachRow<{ rowCount: string | number }>( + const rows = decodeRowCountRows( side === "source" ? await querySource(context, sql) : await queryTarget(context, sql), ) if (!rows[0]) throw new Error(`row-count query returned no row for ${table}`) @@ -567,7 +524,7 @@ const duplicateGroupCount = async ( const hash = `cityHash64(toString(tuple(${names})))` const tie = `sipHash64(toString(tuple(${names})))` const sql = `SELECT count() AS rowCount FROM ${identifier(table.name)} WHERE ${nsExpression(identifier(table.timeColumn))} = ${uint64Literal(progress.lastTimestamp, "lastTimestamp")} AND ${hash} = ${uint64Literal(progress.lastHash, "lastHash")} AND ${tie} = ${uint64Literal(progress.lastTieBreak, "lastTieBreak")}` - const rows = parseJsonEachRow<{ rowCount: string | number }>(await querySource(context, sql)) + const rows = decodeRowCountRows(await querySource(context, sql)) return rows[0] === undefined ? 0 : Number(rows[0].rowCount) } @@ -608,7 +565,7 @@ const copyTable = async ( context, `SELECT ${columnList}, toString(${timeNs}) AS __maple_timestamp, toString(${hashExpression}) AS __maple_hash, toString(${tieBreakExpression}) AS __maple_tie_break FROM ${identifier(table.name)} WHERE ${identifier(table.timeColumn)} >= ${timestampLiteral(retentionStartAt(context.cutoffAt, table.retentionDays))} AND ${identifier(table.timeColumn)} <= ${timestampLiteral(context.cutoffAt)} ${cursor} ORDER BY ${timeNs}, ${hashExpression}, ${tieBreakExpression} LIMIT ${table.batchRows}${offset}`, ) - const rawRows = parseJsonEachRow>(output) + const rawRows = decodeJsonObjectRows(output) let rows = rawRows if (rows.length === 0) break const candidates = rows.map((row) => { @@ -702,7 +659,7 @@ const recoverPendingBatch = async ( ? "" : `AND (${timeNs} > ${uint64Literal(previous.lastTimestamp, "lastTimestamp")} OR (${timeNs} = ${uint64Literal(previous.lastTimestamp, "lastTimestamp")} AND (${hashExpression} > ${uint64Literal(previous.lastHash, "lastHash")} OR (${hashExpression} = ${uint64Literal(previous.lastHash, "lastHash")} AND ${tieBreakExpression} ${continuation.comparison} ${uint64Literal(previous.lastTieBreak, "lastTieBreak")}))))` const offset = continuation.offset === 0 ? "" : ` OFFSET ${continuation.offset}` - const inserted = parseJsonEachRow>( + const inserted = decodeJsonObjectRows( await queryTarget( context, `SELECT ${columnList}, toString(${timeNs}) AS __maple_timestamp, toString(${hashExpression}) AS __maple_hash, toString(${tieBreakExpression}) AS __maple_tie_break FROM ${identifier(table.name)} WHERE ${identifier(table.timeColumn)} >= ${timestampLiteral(retentionStartAt(context.cutoffAt, table.retentionDays))} AND ${identifier(table.timeColumn)} <= ${timestampLiteral(context.cutoffAt)} ${cursor} ORDER BY ${timeNs}, ${hashExpression}, ${tieBreakExpression} LIMIT ${pending.rowCount}${offset}`, @@ -882,13 +839,7 @@ const legacyPreflight = async (context: MigrationModuleContext): Promise { - const state = record(value, "legacy raw replay state") - exactKeys(state, ["module", "version"], "legacy raw replay state") - if (state.module !== "local-0000-to-0001-raw-replay" || state.version !== 1) - throw new Error("legacy raw replay state has an unsupported module or version") - return { module: "local-0000-to-0001-raw-replay", version: 1 } -} +const decodeLegacyState = strictDecoder(LegacyStateSchema) const decodeLegacyProgress = (value: unknown): RawReplayProgress | undefined => value === undefined ? undefined : decodeRawReplayProgress(value) diff --git a/apps/cli/src/server/local-store-migrations/v1-to-v2-error-rollup.ts b/apps/cli/src/server/local-store-migrations/v1-to-v2-error-rollup.ts index 9cccc374e..9f22845dc 100644 --- a/apps/cli/src/server/local-store-migrations/v1-to-v2-error-rollup.ts +++ b/apps/cli/src/server/local-store-migrations/v1-to-v2-error-rollup.ts @@ -1,19 +1,23 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. import { cp, mkdir, rm } from "node:fs/promises" import { dirname, resolve } from "node:path" +import { Schema } from "effect" +import { decodeRowCounts } from "../chdb-rows" import { - applyRawTelemetryRetentionFloor, - RAW_TELEMETRY_TTL_COLUMNS, - readRawTelemetryRetentionDays, - type Chdb, -} from "../chdb" + UnsignedDecimal, + makeRawRowsState, + strictDecoder, + RAW_TABLES, + rawRowCounts, + expectedManifest, +} from "./journal-codecs" +import { applyRawTelemetryRetentionFloor, readRawTelemetryRetentionDays } from "../chdb" import type { LocalStoreMigrationModule, MigrationModuleContext, MigrationOperation, StateDispositionEntry, } from "../local-store-migration-module" -import { withRawTelemetryRetentionFloor } from "../schema-manifest" import { LOCAL_SCHEMA_V1, LOCAL_SCHEMA_V1_MANIFEST, @@ -24,90 +28,26 @@ import { } from "../schema-identity" import { assertPhysicalSchema } from "../schema-physical" -const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) +/** Stamped into the journal and matched on the way back out. */ +const MODULE_ID = "local-0001-to-0002-error-rollup" as const -interface V1ToV2State { - readonly module: "local-0001-to-0002-error-rollup" - readonly version: 1 - readonly rawRows: Readonly> - readonly retentionDays?: number -} - -interface V1ToV2Progress { - readonly backfilledErrorEvents: string -} - -const isRecord = (value: unknown): value is Record => - typeof value === "object" && value !== null && !Array.isArray(value) - -const decodeCounts = (value: unknown): Readonly> => { - if (!isRecord(value)) throw new Error("v1 -> v2 rawRows must be an object") - const counts: Record = {} - for (const table of RAW_TABLES) { - const count = value[table] - if (typeof count !== "string" || !/^\d+$/.test(count)) - throw new Error(`v1 -> v2 rawRows.${table} must be an unsigned decimal string`) - counts[table] = count - } - if (Object.keys(value).some((table) => !RAW_TABLES.includes(table as (typeof RAW_TABLES)[number]))) - throw new Error("v1 -> v2 rawRows contains an unknown table") - return counts -} - -const decodeState = (value: unknown): V1ToV2State => { - if (!isRecord(value)) throw new Error("v1 -> v2 state must be an object") - const allowed = new Set(["module", "version", "rawRows", "retentionDays"]) - if (Object.keys(value).some((key) => !allowed.has(key))) - throw new Error("v1 -> v2 state contains an unknown field") - if (value.module !== "local-0001-to-0002-error-rollup" || value.version !== 1) - throw new Error("v1 -> v2 state has an unsupported module or version") - if ( - value.retentionDays !== undefined && - (typeof value.retentionDays !== "number" || !Number.isSafeInteger(value.retentionDays)) - ) - throw new Error("v1 -> v2 retentionDays must be an integer") - return { - module: "local-0001-to-0002-error-rollup", - version: 1, - rawRows: decodeCounts(value.rawRows), - ...(!(value.retentionDays === undefined) ? { retentionDays: value.retentionDays } : undefined), - } -} +const V1ToV2StateCodec = makeRawRowsState(MODULE_ID) -const decodeProgress = (value: unknown): V1ToV2Progress | undefined => { - if (value === undefined) return undefined - if ( - !isRecord(value) || - Object.keys(value).some((key) => key !== "backfilledErrorEvents") || - typeof value.backfilledErrorEvents !== "string" || - !/^\d+$/.test(value.backfilledErrorEvents) - ) - throw new Error("v1 -> v2 progress is invalid") - return { backfilledErrorEvents: value.backfilledErrorEvents } -} +/** + * Unlike its siblings this edge backfills, so its progress is a resumable + * cursor rather than an installed flag: how many `error_events` rows the + * backfill has written so far, as an unsigned decimal because the count is a + * ClickHouse UInt64. + */ +const V1ToV2ProgressSchema = Schema.Struct({ backfilledErrorEvents: UnsignedDecimal }) -const parseJsonEachRow = (value: string): A[] => - value - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length > 0) - .map((line) => JSON.parse(line) as A) +type V1ToV2State = typeof V1ToV2StateCodec.schema.Type +type V1ToV2Progress = typeof V1ToV2ProgressSchema.Type -const rawRowCounts = (db: Chdb): Readonly> => { - const quotedTables = RAW_TABLES.map((table) => `'${table}'`).join(", ") - const rows = parseJsonEachRow<{ table: string; rowCount: string }>( - db.query( - `SELECT table, toString(sum(rows)) AS rowCount FROM system.parts WHERE database = 'default' AND active = 1 AND table IN (${quotedTables}) GROUP BY table`, - ), - ) - const byTable = new Map(rows.map((row) => [row.table, row.rowCount])) - return Object.fromEntries(RAW_TABLES.map((table) => [table, byTable.get(table) ?? "0"])) -} - -const expectedManifest = (manifest: typeof LOCAL_SCHEMA_V1_MANIFEST, retentionDays: number | undefined) => - retentionDays === undefined - ? manifest - : withRawTelemetryRetentionFloor(manifest, RAW_TABLES, retentionDays) +const decodeState = V1ToV2StateCodec.decode +const decodeV1ToV2Progress = strictDecoder(V1ToV2ProgressSchema) +const decodeProgress = (value: unknown): V1ToV2Progress | undefined => + value === undefined ? undefined : decodeV1ToV2Progress(value) const preflight = async (context: MigrationModuleContext): Promise => { await context.ensureCapacity() @@ -119,12 +59,12 @@ const preflight = async (context: MigrationModuleContext): Promise }, { schemaSql: LOCAL_SCHEMA_V1_SQL, bootstrapSchema: false }, ) - return { - module: "local-0001-to-0002-error-rollup", - version: 1, - rawRows, - ...(!(retentionDays === undefined) ? { retentionDays } : undefined), - } + // Two literals rather than a conditional spread: `retentionDays` is an + // `optionalKey`, so an absent floor has to be an absent key, not a present + // `undefined`. + return retentionDays === undefined + ? { module: MODULE_ID, version: 1, rawRows } + : { module: MODULE_ID, version: 1, rawRows, retentionDays } } const prepareTarget = async (context: MigrationModuleContext, state: V1ToV2State): Promise => { @@ -172,7 +112,7 @@ const apply = async ( (db) => { if (state.retentionDays !== undefined) applyRawTelemetryRetentionFloor(db, state.retentionDays) db.exec(backfillSql) - const [row] = parseJsonEachRow<{ rowCount: string }>( + const [row] = decodeRowCounts( db.query( "SELECT toString(sum(OccurrenceCount)) AS rowCount FROM error_fingerprints_minutely", ), @@ -196,9 +136,7 @@ const verify = async ( if (targetRows[table] !== state.rawRows[table]) throw new Error(`v1 -> v2 raw telemetry verification failed for ${table}`) } - const [row] = parseJsonEachRow<{ rowCount: string }>( - db.query("SELECT toString(count()) AS rowCount FROM error_events"), - ) + const [row] = decodeRowCounts(db.query("SELECT toString(count()) AS rowCount FROM error_events")) if ((row?.rowCount ?? "0") !== progress.backfilledErrorEvents) throw new Error("v1 -> v2 error rollup backfill verification failed") }, @@ -246,7 +184,7 @@ const dispositions: ReadonlyArray = [ ] export const v1ToV2ErrorRollupModule: LocalStoreMigrationModule = { - id: "local-0001-to-0002-error-rollup", + id: MODULE_ID, moduleVersion: 1, description: "Add the durable minutely error-fingerprint rollup to a v1 local store", from: LOCAL_SCHEMA_V1, diff --git a/apps/cli/src/server/local-store-migrations/v2-to-v3-service-map-ingest-bridge.ts b/apps/cli/src/server/local-store-migrations/v2-to-v3-service-map-ingest-bridge.ts index 719cc767d..10e4e60ac 100644 --- a/apps/cli/src/server/local-store-migrations/v2-to-v3-service-map-ingest-bridge.ts +++ b/apps/cli/src/server/local-store-migrations/v2-to-v3-service-map-ingest-bridge.ts @@ -1,14 +1,21 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. import { cp, mkdir, rm } from "node:fs/promises" import { dirname, resolve } from "node:path" -import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" +import { + decodeInstalledProgress, + makeRawRowsState, + type InstalledProgress, + RAW_TABLES, + rawRowCounts, + expectedManifest, +} from "./journal-codecs" +import { readRawTelemetryRetentionDays } from "../chdb" import type { LocalStoreMigrationModule, MigrationModuleContext, MigrationOperation, StateDispositionEntry, } from "../local-store-migration-module" -import { withRawTelemetryRetentionFloor } from "../schema-manifest" import { LOCAL_SCHEMA_V2, LOCAL_SCHEMA_V2_MANIFEST, @@ -19,85 +26,16 @@ import { } from "../schema-identity" import { assertPhysicalSchema } from "../schema-physical" -const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) - -interface V2ToV3State { - readonly module: "local-0002-to-0003-service-map-ingest-bridge" - readonly version: 1 - readonly rawRows: Readonly> - readonly retentionDays?: number -} - -interface V2ToV3Progress { - readonly installed: true -} - -const isRecord = (value: unknown): value is Record => - typeof value === "object" && value !== null && !Array.isArray(value) - -const decodeCounts = (value: unknown): Readonly> => { - if (!isRecord(value)) throw new Error("v2 -> v3 rawRows must be an object") - const counts: Record = {} - for (const table of RAW_TABLES) { - const count = value[table] - if (typeof count !== "string" || !/^\d+$/.test(count)) - throw new Error(`v2 -> v3 rawRows.${table} must be an unsigned decimal string`) - counts[table] = count - } - if (Object.keys(value).some((table) => !RAW_TABLES.includes(table as (typeof RAW_TABLES)[number]))) - throw new Error("v2 -> v3 rawRows contains an unknown table") - return counts -} - -const decodeState = (value: unknown): V2ToV3State => { - if (!isRecord(value)) throw new Error("v2 -> v3 state must be an object") - const allowed = new Set(["module", "version", "rawRows", "retentionDays"]) - if (Object.keys(value).some((key) => !allowed.has(key))) - throw new Error("v2 -> v3 state contains an unknown field") - if (value.module !== "local-0002-to-0003-service-map-ingest-bridge" || value.version !== 1) - throw new Error("v2 -> v3 state has an unsupported module or version") - if ( - value.retentionDays !== undefined && - (typeof value.retentionDays !== "number" || !Number.isSafeInteger(value.retentionDays)) - ) - throw new Error("v2 -> v3 retentionDays must be an integer") - return { - module: "local-0002-to-0003-service-map-ingest-bridge", - version: 1, - rawRows: decodeCounts(value.rawRows), - ...(!(value.retentionDays === undefined) ? { retentionDays: value.retentionDays } : undefined), - } -} +/** Stamped into the journal and matched on the way back out. */ +const MODULE_ID = "local-0002-to-0003-service-map-ingest-bridge" as const -const decodeProgress = (value: unknown): V2ToV3Progress | undefined => { - if (value === undefined) return undefined - if (!isRecord(value) || Object.keys(value).some((key) => key !== "installed") || value.installed !== true) - throw new Error("v2 -> v3 progress is invalid") - return { installed: true } -} +const V2ToV3StateCodec = makeRawRowsState(MODULE_ID) -const parseJsonEachRow = (value: string): A[] => - value - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length > 0) - .map((line) => JSON.parse(line) as A) +type V2ToV3State = typeof V2ToV3StateCodec.schema.Type +type V2ToV3Progress = InstalledProgress -const rawRowCounts = (db: Chdb): Readonly> => { - const quotedTables = RAW_TABLES.map((table) => `'${table}'`).join(", ") - const rows = parseJsonEachRow<{ table: string; rowCount: string }>( - db.query( - `SELECT table, toString(sum(rows)) AS rowCount FROM system.parts WHERE database = 'default' AND active = 1 AND table IN (${quotedTables}) GROUP BY table`, - ), - ) - const byTable = new Map(rows.map((row) => [row.table, row.rowCount])) - return Object.fromEntries(RAW_TABLES.map((table) => [table, byTable.get(table) ?? "0"])) -} - -const expectedManifest = (manifest: typeof LOCAL_SCHEMA_V2_MANIFEST, retentionDays: number | undefined) => - retentionDays === undefined - ? manifest - : withRawTelemetryRetentionFloor(manifest, RAW_TABLES, retentionDays) +const decodeState = V2ToV3StateCodec.decode +const decodeProgress = decodeInstalledProgress const preflight = async (context: MigrationModuleContext): Promise => { await context.ensureCapacity() @@ -109,12 +47,12 @@ const preflight = async (context: MigrationModuleContext): Promise }, { schemaSql: LOCAL_SCHEMA_V2_SQL, bootstrapSchema: false }, ) - return { - module: "local-0002-to-0003-service-map-ingest-bridge", - version: 1, - rawRows, - ...(!(retentionDays === undefined) ? { retentionDays } : undefined), - } + // Two literals rather than a conditional spread: `retentionDays` is an + // `optionalKey`, so an absent floor has to be an absent key, not a present + // `undefined`. + return retentionDays === undefined + ? { module: MODULE_ID, version: 1, rawRows } + : { module: MODULE_ID, version: 1, rawRows, retentionDays } } const prepareTarget = async (context: MigrationModuleContext, state: V2ToV3State): Promise => { @@ -207,7 +145,7 @@ const dispositions: ReadonlyArray = [ ] export const v2ToV3ServiceMapIngestBridgeModule: LocalStoreMigrationModule = { - id: "local-0002-to-0003-service-map-ingest-bridge", + id: MODULE_ID, moduleVersion: 1, description: "Restore the deployment-safe error view trigger and add the service-map ingress bridge to v2", diff --git a/apps/cli/src/server/local-store-migrations/v3-to-v4-web-events.ts b/apps/cli/src/server/local-store-migrations/v3-to-v4-web-events.ts index 863361f02..3a9af3138 100644 --- a/apps/cli/src/server/local-store-migrations/v3-to-v4-web-events.ts +++ b/apps/cli/src/server/local-store-migrations/v3-to-v4-web-events.ts @@ -1,14 +1,21 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. import { cp, mkdir, rm } from "node:fs/promises" import { dirname, resolve } from "node:path" -import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" +import { + decodeInstalledProgress, + makeRawRowsState, + type InstalledProgress, + RAW_TABLES, + rawRowCounts, + expectedManifest, +} from "./journal-codecs" +import { readRawTelemetryRetentionDays } from "../chdb" import type { LocalStoreMigrationModule, MigrationModuleContext, MigrationOperation, StateDispositionEntry, } from "../local-store-migration-module" -import { withRawTelemetryRetentionFloor } from "../schema-manifest" import { LOCAL_SCHEMA_V3, LOCAL_SCHEMA_V3_MANIFEST, @@ -19,85 +26,16 @@ import { } from "../schema-identity" import { assertPhysicalSchema } from "../schema-physical" -const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) - -interface V3ToV4State { - readonly module: "local-0003-to-0004-web-events" - readonly version: 1 - readonly rawRows: Readonly> - readonly retentionDays?: number -} - -interface V3ToV4Progress { - readonly installed: true -} - -const isRecord = (value: unknown): value is Record => - typeof value === "object" && value !== null && !Array.isArray(value) - -const decodeCounts = (value: unknown): Readonly> => { - if (!isRecord(value)) throw new Error("v3 -> v4 rawRows must be an object") - const counts: Record = {} - for (const table of RAW_TABLES) { - const count = value[table] - if (typeof count !== "string" || !/^\d+$/.test(count)) - throw new Error(`v3 -> v4 rawRows.${table} must be an unsigned decimal string`) - counts[table] = count - } - if (Object.keys(value).some((table) => !RAW_TABLES.includes(table as (typeof RAW_TABLES)[number]))) - throw new Error("v3 -> v4 rawRows contains an unknown table") - return counts -} - -const decodeState = (value: unknown): V3ToV4State => { - if (!isRecord(value)) throw new Error("v3 -> v4 state must be an object") - const allowed = new Set(["module", "version", "rawRows", "retentionDays"]) - if (Object.keys(value).some((key) => !allowed.has(key))) - throw new Error("v3 -> v4 state contains an unknown field") - if (value.module !== "local-0003-to-0004-web-events" || value.version !== 1) - throw new Error("v3 -> v4 state has an unsupported module or version") - if ( - value.retentionDays !== undefined && - (typeof value.retentionDays !== "number" || !Number.isSafeInteger(value.retentionDays)) - ) - throw new Error("v3 -> v4 retentionDays must be an integer") - return { - module: "local-0003-to-0004-web-events", - version: 1, - rawRows: decodeCounts(value.rawRows), - ...(!(value.retentionDays === undefined) ? { retentionDays: value.retentionDays } : undefined), - } -} +/** Stamped into the journal and matched on the way back out. */ +const MODULE_ID = "local-0003-to-0004-web-events" as const -const decodeProgress = (value: unknown): V3ToV4Progress | undefined => { - if (value === undefined) return undefined - if (!isRecord(value) || Object.keys(value).some((key) => key !== "installed") || value.installed !== true) - throw new Error("v3 -> v4 progress is invalid") - return { installed: true } -} +const V3ToV4StateCodec = makeRawRowsState(MODULE_ID) -const parseJsonEachRow = (value: string): A[] => - value - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length > 0) - .map((line) => JSON.parse(line) as A) +type V3ToV4State = typeof V3ToV4StateCodec.schema.Type +type V3ToV4Progress = InstalledProgress -const rawRowCounts = (db: Chdb): Readonly> => { - const quotedTables = RAW_TABLES.map((table) => `'${table}'`).join(", ") - const rows = parseJsonEachRow<{ table: string; rowCount: string }>( - db.query( - `SELECT table, toString(sum(rows)) AS rowCount FROM system.parts WHERE database = 'default' AND active = 1 AND table IN (${quotedTables}) GROUP BY table`, - ), - ) - const byTable = new Map(rows.map((row) => [row.table, row.rowCount])) - return Object.fromEntries(RAW_TABLES.map((table) => [table, byTable.get(table) ?? "0"])) -} - -const expectedManifest = (manifest: typeof LOCAL_SCHEMA_V3_MANIFEST, retentionDays: number | undefined) => - retentionDays === undefined - ? manifest - : withRawTelemetryRetentionFloor(manifest, RAW_TABLES, retentionDays) +const decodeState = V3ToV4StateCodec.decode +const decodeProgress = decodeInstalledProgress const preflight = async (context: MigrationModuleContext): Promise => { await context.ensureCapacity() @@ -109,12 +47,12 @@ const preflight = async (context: MigrationModuleContext): Promise }, { schemaSql: LOCAL_SCHEMA_V3_SQL, bootstrapSchema: false }, ) - return { - module: "local-0003-to-0004-web-events", - version: 1, - rawRows, - ...(!(retentionDays === undefined) ? { retentionDays } : undefined), - } + // Two literals rather than a conditional spread: `retentionDays` is an + // `optionalKey`, so an absent floor has to be an absent key, not a present + // `undefined`. + return retentionDays === undefined + ? { module: MODULE_ID, version: 1, rawRows } + : { module: MODULE_ID, version: 1, rawRows, retentionDays } } const prepareTarget = async (context: MigrationModuleContext, state: V3ToV4State): Promise => { @@ -220,7 +158,7 @@ const dispositions: ReadonlyArray = [ ] export const v3ToV4WebEventsModule: LocalStoreMigrationModule = { - id: "local-0003-to-0004-web-events", + id: MODULE_ID, moduleVersion: 1, description: "Add the web_events analytics fact table and its materialized view to v3", from: LOCAL_SCHEMA_V3, diff --git a/apps/cli/src/server/local-store-migrations/v4-to-v5-service-overview-minutely.ts b/apps/cli/src/server/local-store-migrations/v4-to-v5-service-overview-minutely.ts index da8a4c560..e0429be07 100644 --- a/apps/cli/src/server/local-store-migrations/v4-to-v5-service-overview-minutely.ts +++ b/apps/cli/src/server/local-store-migrations/v4-to-v5-service-overview-minutely.ts @@ -1,14 +1,21 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. import { cp, mkdir, rm } from "node:fs/promises" import { dirname, resolve } from "node:path" -import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" +import { + decodeInstalledProgress, + makeRawRowsState, + type InstalledProgress, + RAW_TABLES, + rawRowCounts, + expectedManifest, +} from "./journal-codecs" +import { readRawTelemetryRetentionDays } from "../chdb" import type { LocalStoreMigrationModule, MigrationModuleContext, MigrationOperation, StateDispositionEntry, } from "../local-store-migration-module" -import { withRawTelemetryRetentionFloor } from "../schema-manifest" import { LOCAL_SCHEMA_V4, LOCAL_SCHEMA_V4_MANIFEST, @@ -19,85 +26,16 @@ import { } from "../schema-identity" import { assertPhysicalSchema } from "../schema-physical" -const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) - -interface V4ToV5State { - readonly module: "local-0004-to-0005-service-overview-minutely" - readonly version: 1 - readonly rawRows: Readonly> - readonly retentionDays?: number -} - -interface V4ToV5Progress { - readonly installed: true -} - -const isRecord = (value: unknown): value is Record => - typeof value === "object" && value !== null && !Array.isArray(value) - -const decodeCounts = (value: unknown): Readonly> => { - if (!isRecord(value)) throw new Error("v4 -> v5 rawRows must be an object") - const counts: Record = {} - for (const table of RAW_TABLES) { - const count = value[table] - if (typeof count !== "string" || !/^\d+$/.test(count)) - throw new Error(`v4 -> v5 rawRows.${table} must be an unsigned decimal string`) - counts[table] = count - } - if (Object.keys(value).some((table) => !RAW_TABLES.includes(table as (typeof RAW_TABLES)[number]))) - throw new Error("v4 -> v5 rawRows contains an unknown table") - return counts -} - -const decodeState = (value: unknown): V4ToV5State => { - if (!isRecord(value)) throw new Error("v4 -> v5 state must be an object") - const allowed = new Set(["module", "version", "rawRows", "retentionDays"]) - if (Object.keys(value).some((key) => !allowed.has(key))) - throw new Error("v4 -> v5 state contains an unknown field") - if (value.module !== "local-0004-to-0005-service-overview-minutely" || value.version !== 1) - throw new Error("v4 -> v5 state has an unsupported module or version") - if ( - value.retentionDays !== undefined && - (typeof value.retentionDays !== "number" || !Number.isSafeInteger(value.retentionDays)) - ) - throw new Error("v4 -> v5 retentionDays must be an integer") - return { - module: "local-0004-to-0005-service-overview-minutely", - version: 1, - rawRows: decodeCounts(value.rawRows), - ...(!(value.retentionDays === undefined) ? { retentionDays: value.retentionDays } : undefined), - } -} +/** Stamped into the journal and matched on the way back out. */ +const MODULE_ID = "local-0004-to-0005-service-overview-minutely" as const -const decodeProgress = (value: unknown): V4ToV5Progress | undefined => { - if (value === undefined) return undefined - if (!isRecord(value) || Object.keys(value).some((key) => key !== "installed") || value.installed !== true) - throw new Error("v4 -> v5 progress is invalid") - return { installed: true } -} +const V4ToV5StateCodec = makeRawRowsState(MODULE_ID) -const parseJsonEachRow = (value: string): A[] => - value - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length > 0) - .map((line) => JSON.parse(line) as A) +type V4ToV5State = typeof V4ToV5StateCodec.schema.Type +type V4ToV5Progress = InstalledProgress -const rawRowCounts = (db: Chdb): Readonly> => { - const quotedTables = RAW_TABLES.map((table) => `'${table}'`).join(", ") - const rows = parseJsonEachRow<{ table: string; rowCount: string }>( - db.query( - `SELECT table, toString(sum(rows)) AS rowCount FROM system.parts WHERE database = 'default' AND active = 1 AND table IN (${quotedTables}) GROUP BY table`, - ), - ) - const byTable = new Map(rows.map((row) => [row.table, row.rowCount])) - return Object.fromEntries(RAW_TABLES.map((table) => [table, byTable.get(table) ?? "0"])) -} - -const expectedManifest = (manifest: typeof LOCAL_SCHEMA_V4_MANIFEST, retentionDays: number | undefined) => - retentionDays === undefined - ? manifest - : withRawTelemetryRetentionFloor(manifest, RAW_TABLES, retentionDays) +const decodeState = V4ToV5StateCodec.decode +const decodeProgress = decodeInstalledProgress const preflight = async (context: MigrationModuleContext): Promise => { await context.ensureCapacity() @@ -109,12 +47,12 @@ const preflight = async (context: MigrationModuleContext): Promise }, { schemaSql: LOCAL_SCHEMA_V4_SQL, bootstrapSchema: false }, ) - return { - module: "local-0004-to-0005-service-overview-minutely", - version: 1, - rawRows, - ...(!(retentionDays === undefined) ? { retentionDays } : undefined), - } + // Two literals rather than a conditional spread: `retentionDays` is an + // `optionalKey`, so an absent floor has to be an absent key, not a present + // `undefined`. + return retentionDays === undefined + ? { module: MODULE_ID, version: 1, rawRows } + : { module: MODULE_ID, version: 1, rawRows, retentionDays } } const prepareTarget = async (context: MigrationModuleContext, state: V4ToV5State): Promise => { @@ -219,7 +157,7 @@ const dispositions: ReadonlyArray = [ ] export const v4ToV5ServiceOverviewMinutelyModule: LocalStoreMigrationModule = { - id: "local-0004-to-0005-service-overview-minutely", + id: MODULE_ID, moduleVersion: 1, description: "Add the service_overview_minutely rollup and its materialized view to v4", from: LOCAL_SCHEMA_V4, diff --git a/apps/cli/src/server/local-store-migrations/v5-to-v6-error-events-fingerprint-hygiene.ts b/apps/cli/src/server/local-store-migrations/v5-to-v6-error-events-fingerprint-hygiene.ts index 53e5c5e1c..4818e3f45 100644 --- a/apps/cli/src/server/local-store-migrations/v5-to-v6-error-events-fingerprint-hygiene.ts +++ b/apps/cli/src/server/local-store-migrations/v5-to-v6-error-events-fingerprint-hygiene.ts @@ -1,14 +1,21 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. import { cp, mkdir, rm } from "node:fs/promises" import { dirname, resolve } from "node:path" -import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" +import { + decodeInstalledProgress, + makeRawRowsState, + type InstalledProgress, + RAW_TABLES, + rawRowCounts, + expectedManifest, +} from "./journal-codecs" +import { readRawTelemetryRetentionDays } from "../chdb" import type { LocalStoreMigrationModule, MigrationModuleContext, MigrationOperation, StateDispositionEntry, } from "../local-store-migration-module" -import { withRawTelemetryRetentionFloor } from "../schema-manifest" import { LOCAL_SCHEMA_V5, LOCAL_SCHEMA_V5_MANIFEST, @@ -19,85 +26,16 @@ import { } from "../schema-identity" import { assertPhysicalSchema } from "../schema-physical" -const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) - -interface V5ToV6State { - readonly module: "local-0005-to-0006-error-events-fingerprint-hygiene" - readonly version: 1 - readonly rawRows: Readonly> - readonly retentionDays?: number -} - -interface V5ToV6Progress { - readonly installed: true -} - -const isRecord = (value: unknown): value is Record => - typeof value === "object" && value !== null && !Array.isArray(value) - -const decodeCounts = (value: unknown): Readonly> => { - if (!isRecord(value)) throw new Error("v5 -> v6 rawRows must be an object") - const counts: Record = {} - for (const table of RAW_TABLES) { - const count = value[table] - if (typeof count !== "string" || !/^\d+$/.test(count)) - throw new Error(`v5 -> v6 rawRows.${table} must be an unsigned decimal string`) - counts[table] = count - } - if (Object.keys(value).some((table) => !RAW_TABLES.includes(table as (typeof RAW_TABLES)[number]))) - throw new Error("v5 -> v6 rawRows contains an unknown table") - return counts -} - -const decodeState = (value: unknown): V5ToV6State => { - if (!isRecord(value)) throw new Error("v5 -> v6 state must be an object") - const allowed = new Set(["module", "version", "rawRows", "retentionDays"]) - if (Object.keys(value).some((key) => !allowed.has(key))) - throw new Error("v5 -> v6 state contains an unknown field") - if (value.module !== "local-0005-to-0006-error-events-fingerprint-hygiene" || value.version !== 1) - throw new Error("v5 -> v6 state has an unsupported module or version") - if ( - value.retentionDays !== undefined && - (typeof value.retentionDays !== "number" || !Number.isSafeInteger(value.retentionDays)) - ) - throw new Error("v5 -> v6 retentionDays must be an integer") - return { - module: "local-0005-to-0006-error-events-fingerprint-hygiene", - version: 1, - rawRows: decodeCounts(value.rawRows), - ...(!(value.retentionDays === undefined) ? { retentionDays: value.retentionDays } : undefined), - } -} +/** Stamped into the journal and matched on the way back out. */ +const MODULE_ID = "local-0005-to-0006-error-events-fingerprint-hygiene" as const -const decodeProgress = (value: unknown): V5ToV6Progress | undefined => { - if (value === undefined) return undefined - if (!isRecord(value) || Object.keys(value).some((key) => key !== "installed") || value.installed !== true) - throw new Error("v5 -> v6 progress is invalid") - return { installed: true } -} +const V5ToV6StateCodec = makeRawRowsState(MODULE_ID) -const parseJsonEachRow = (value: string): A[] => - value - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length > 0) - .map((line) => JSON.parse(line) as A) +type V5ToV6State = typeof V5ToV6StateCodec.schema.Type +type V5ToV6Progress = InstalledProgress -const rawRowCounts = (db: Chdb): Readonly> => { - const quotedTables = RAW_TABLES.map((table) => `'${table}'`).join(", ") - const rows = parseJsonEachRow<{ table: string; rowCount: string }>( - db.query( - `SELECT table, toString(sum(rows)) AS rowCount FROM system.parts WHERE database = 'default' AND active = 1 AND table IN (${quotedTables}) GROUP BY table`, - ), - ) - const byTable = new Map(rows.map((row) => [row.table, row.rowCount])) - return Object.fromEntries(RAW_TABLES.map((table) => [table, byTable.get(table) ?? "0"])) -} - -const expectedManifest = (manifest: typeof LOCAL_SCHEMA_V5_MANIFEST, retentionDays: number | undefined) => - retentionDays === undefined - ? manifest - : withRawTelemetryRetentionFloor(manifest, RAW_TABLES, retentionDays) +const decodeState = V5ToV6StateCodec.decode +const decodeProgress = decodeInstalledProgress const preflight = async (context: MigrationModuleContext): Promise => { await context.ensureCapacity() @@ -109,12 +47,12 @@ const preflight = async (context: MigrationModuleContext): Promise }, { schemaSql: LOCAL_SCHEMA_V5_SQL, bootstrapSchema: false }, ) - return { - module: "local-0005-to-0006-error-events-fingerprint-hygiene", - version: 1, - rawRows, - ...(!(retentionDays === undefined) ? { retentionDays } : undefined), - } + // Two literals rather than a conditional spread: `retentionDays` is an + // `optionalKey`, so an absent floor has to be an absent key, not a present + // `undefined`. + return retentionDays === undefined + ? { module: MODULE_ID, version: 1, rawRows } + : { module: MODULE_ID, version: 1, rawRows, retentionDays } } const prepareTarget = async (context: MigrationModuleContext, state: V5ToV6State): Promise => { @@ -246,7 +184,7 @@ export const v5ToV6ErrorEventsFingerprintHygieneModule: LocalStoreMigrationModul V5ToV6State, V5ToV6Progress > = { - id: "local-0005-to-0006-error-events-fingerprint-hygiene", + id: MODULE_ID, moduleVersion: 1, description: "Rebuild the error-events views: exclude exception-less 4xx client spans and redact ids from fingerprint frames", diff --git a/apps/cli/src/server/local-store-migrations/v6-to-v7-error-service-version.ts b/apps/cli/src/server/local-store-migrations/v6-to-v7-error-service-version.ts index 658fe29c4..4045f5d6a 100644 --- a/apps/cli/src/server/local-store-migrations/v6-to-v7-error-service-version.ts +++ b/apps/cli/src/server/local-store-migrations/v6-to-v7-error-service-version.ts @@ -1,14 +1,21 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. import { cp, mkdir, rm } from "node:fs/promises" import { dirname, resolve } from "node:path" -import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" +import { + decodeInstalledProgress, + makeRawRowsState, + type InstalledProgress, + RAW_TABLES, + rawRowCounts, + expectedManifest, +} from "./journal-codecs" +import { readRawTelemetryRetentionDays } from "../chdb" import type { LocalStoreMigrationModule, MigrationModuleContext, MigrationOperation, StateDispositionEntry, } from "../local-store-migration-module" -import { withRawTelemetryRetentionFloor } from "../schema-manifest" import { LOCAL_SCHEMA_V6, LOCAL_SCHEMA_V6_MANIFEST, @@ -19,8 +26,6 @@ import { } from "../schema-identity" import { assertPhysicalSchema } from "../schema-physical" -const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) - /** Error-family views replaced by this edge, dropped before the v7 DDL runs. */ const ERROR_VIEWS = ["error_events_mv", "error_events_by_time_mv", "error_fingerprints_minutely_mv"] as const @@ -40,83 +45,16 @@ const SERVICE_VERSION_COLUMNS = [ ], ] as const -interface V6ToV7State { - readonly module: "local-0006-to-0007-error-service-version" - readonly version: 1 - readonly rawRows: Readonly> - readonly retentionDays?: number -} - -interface V6ToV7Progress { - readonly installed: true -} - -const isRecord = (value: unknown): value is Record => - typeof value === "object" && value !== null && !Array.isArray(value) - -const decodeCounts = (value: unknown): Readonly> => { - if (!isRecord(value)) throw new Error("v6 -> v7 rawRows must be an object") - const counts: Record = {} - for (const table of RAW_TABLES) { - const count = value[table] - if (typeof count !== "string" || !/^\d+$/.test(count)) - throw new Error(`v6 -> v7 rawRows.${table} must be an unsigned decimal string`) - counts[table] = count - } - if (Object.keys(value).some((table) => !RAW_TABLES.includes(table as (typeof RAW_TABLES)[number]))) - throw new Error("v6 -> v7 rawRows contains an unknown table") - return counts -} - -const decodeState = (value: unknown): V6ToV7State => { - if (!isRecord(value)) throw new Error("v6 -> v7 state must be an object") - const allowed = new Set(["module", "version", "rawRows", "retentionDays"]) - if (Object.keys(value).some((key) => !allowed.has(key))) - throw new Error("v6 -> v7 state contains an unknown field") - if (value.module !== "local-0006-to-0007-error-service-version" || value.version !== 1) - throw new Error("v6 -> v7 state has an unsupported module or version") - if ( - value.retentionDays !== undefined && - (typeof value.retentionDays !== "number" || !Number.isSafeInteger(value.retentionDays)) - ) - throw new Error("v6 -> v7 retentionDays must be an integer") - return { - module: "local-0006-to-0007-error-service-version", - version: 1, - rawRows: decodeCounts(value.rawRows), - ...(!(value.retentionDays === undefined) ? { retentionDays: value.retentionDays } : undefined), - } -} +/** Stamped into the journal and matched on the way back out. */ +const MODULE_ID = "local-0006-to-0007-error-service-version" as const -const decodeProgress = (value: unknown): V6ToV7Progress | undefined => { - if (value === undefined) return undefined - if (!isRecord(value) || Object.keys(value).some((key) => key !== "installed") || value.installed !== true) - throw new Error("v6 -> v7 progress is invalid") - return { installed: true } -} +const V6ToV7StateCodec = makeRawRowsState(MODULE_ID) -const parseJsonEachRow = (value: string): A[] => - value - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length > 0) - .map((line) => JSON.parse(line) as A) +type V6ToV7State = typeof V6ToV7StateCodec.schema.Type +type V6ToV7Progress = InstalledProgress -const rawRowCounts = (db: Chdb): Readonly> => { - const quotedTables = RAW_TABLES.map((table) => `'${table}'`).join(", ") - const rows = parseJsonEachRow<{ table: string; rowCount: string }>( - db.query( - `SELECT table, toString(sum(rows)) AS rowCount FROM system.parts WHERE database = 'default' AND active = 1 AND table IN (${quotedTables}) GROUP BY table`, - ), - ) - const byTable = new Map(rows.map((row) => [row.table, row.rowCount])) - return Object.fromEntries(RAW_TABLES.map((table) => [table, byTable.get(table) ?? "0"])) -} - -const expectedManifest = (manifest: typeof LOCAL_SCHEMA_V6_MANIFEST, retentionDays: number | undefined) => - retentionDays === undefined - ? manifest - : withRawTelemetryRetentionFloor(manifest, RAW_TABLES, retentionDays) +const decodeState = V6ToV7StateCodec.decode +const decodeProgress = decodeInstalledProgress const preflight = async (context: MigrationModuleContext): Promise => { await context.ensureCapacity() @@ -128,12 +66,12 @@ const preflight = async (context: MigrationModuleContext): Promise }, { schemaSql: LOCAL_SCHEMA_V6_SQL, bootstrapSchema: false }, ) - return { - module: "local-0006-to-0007-error-service-version", - version: 1, - rawRows, - ...(!(retentionDays === undefined) ? { retentionDays } : undefined), - } + // Two literals rather than a conditional spread: `retentionDays` is an + // `optionalKey`, so an absent floor has to be an absent key, not a present + // `undefined`. + return retentionDays === undefined + ? { module: MODULE_ID, version: 1, rawRows } + : { module: MODULE_ID, version: 1, rawRows, retentionDays } } const prepareTarget = async (context: MigrationModuleContext, state: V6ToV7State): Promise => { @@ -280,7 +218,7 @@ const dispositions: ReadonlyArray = [ ] export const v6ToV7ErrorServiceVersionModule: LocalStoreMigrationModule = { - id: "local-0006-to-0007-error-service-version", + id: MODULE_ID, moduleVersion: 1, description: "Add ServiceVersion to the error-events tables and rebuild the error-events views on fingerprint v2", diff --git a/apps/cli/src/server/local-store-migrations/v7-to-v8-apple-crash-frames.ts b/apps/cli/src/server/local-store-migrations/v7-to-v8-apple-crash-frames.ts new file mode 100644 index 000000000..197dffb88 --- /dev/null +++ b/apps/cli/src/server/local-store-migrations/v7-to-v8-apple-crash-frames.ts @@ -0,0 +1,195 @@ +// SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. +import { cp, mkdir, rm } from "node:fs/promises" +import { dirname, resolve } from "node:path" +import { + decodeInstalledProgress, + makeRawRowsState, + type InstalledProgress, + RAW_TABLES, + rawRowCounts, + expectedManifest, +} from "./journal-codecs" +import { readRawTelemetryRetentionDays } from "../chdb" +import type { + LocalStoreMigrationModule, + MigrationModuleContext, + MigrationOperation, + StateDispositionEntry, +} from "../local-store-migration-module" +import { + LOCAL_SCHEMA_V7, + LOCAL_SCHEMA_V7_MANIFEST, + LOCAL_SCHEMA_V7_SQL, + LOCAL_SCHEMA_V8, + LOCAL_SCHEMA_V8_MANIFEST, + LOCAL_SCHEMA_V8_SQL, +} from "../schema-identity" +import { assertPhysicalSchema } from "../schema-physical" + +/** Stamped into the journal and matched on the way back out. */ +const MODULE_ID = "local-0007-to-0008-apple-crash-frames" as const + +const V7ToV8StateCodec = makeRawRowsState(MODULE_ID) + +type V7ToV8State = typeof V7ToV8StateCodec.schema.Type +type V7ToV8Progress = InstalledProgress + +const decodeState = V7ToV8StateCodec.decode +const decodeProgress = decodeInstalledProgress + +const preflight = async (context: MigrationModuleContext): Promise => { + await context.ensureCapacity() + const retentionDays = readRawTelemetryRetentionDays(context.dataDir) + const rawRows = await context.openSource( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V7_MANIFEST, retentionDays)) + return rawRowCounts(db) + }, + { schemaSql: LOCAL_SCHEMA_V7_SQL, bootstrapSchema: false }, + ) + // Two literals rather than a conditional spread: `retentionDays` is an + // `optionalKey`, so an absent floor has to be an absent key, not a present + // `undefined`. + return retentionDays === undefined + ? { module: MODULE_ID, version: 1, rawRows } + : { module: MODULE_ID, version: 1, rawRows, retentionDays } +} + +const prepareTarget = async (context: MigrationModuleContext, state: V7ToV8State): Promise => { + await context.closeStores() + const source = resolve(context.sourceDataDir) + const target = resolve(context.targetDataDir) + if (source !== target) { + await rm(target, { recursive: true, force: true }) + await mkdir(dirname(target), { recursive: true, mode: 0o700 }) + await cp(source, target, { recursive: true, preserveTimestamps: true }) + } + return state +} + +/** + * Like v5 -> v6, this edge replaces the body of two existing views rather than + * adding anything. A materialized view's SELECT is frozen at creation and the + * bundled DDL uses `CREATE ... IF NOT EXISTS`, so both views must be dropped + * before the v8 schema can install its versions. Dropping a view never touches + * rows already in its target table. + * + * The new body adds an Apple alternative to the fingerprint's frame matcher. + * Before it, no frame of an iOS crash matched any alternative, so `_fpFrames` + * was empty and the hash fell through to the message signature — which redacts + * hex and long digit runs, collapsing every crash of an exception type in a + * service into a single issue. + * + * Historical `error_events` / `error_events_by_time` rows are left exactly as + * they are: recomputing FingerprintHash would re-bucket every existing local + * issue. Forward-only, converging as the retention window rolls, and matching + * what a deployed cluster gets from ClickHouse migration 0018. + */ +const apply = async (context: MigrationModuleContext): Promise => { + await context.openTarget( + (db) => { + db.exec("DROP TABLE IF EXISTS error_events_mv") + db.exec("DROP TABLE IF EXISTS error_events_by_time_mv") + }, + { schemaSql: LOCAL_SCHEMA_V7_SQL, bootstrapSchema: false }, + ) + return context.openTarget(() => ({ installed: true }), { + schemaSql: LOCAL_SCHEMA_V8_SQL, + bootstrapSchema: true, + }) +} + +const verify = async ( + context: MigrationModuleContext, + state: V7ToV8State, + _progress: V7ToV8Progress, +): Promise => { + await context.openTarget( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V8_MANIFEST, state.retentionDays)) + const targetRows = rawRowCounts(db) + for (const table of RAW_TABLES) { + if (targetRows[table] !== state.rawRows[table]) + throw new Error(`v7 -> v8 raw telemetry verification failed for ${table}`) + } + }, + { schemaSql: LOCAL_SCHEMA_V8_SQL, bootstrapSchema: false }, + ) +} + +const operations: ReadonlyArray = [ + { + id: "clone-v7-store", + description: "Clone the stopped v7 store into the staged migration target", + requiresQuiescence: true, + phase: "target-created", + }, + { + id: "rebuild-error-events-views", + description: "Drop and recreate the error-events views with Apple crash frames in the fingerprint", + requiresQuiescence: true, + phase: "copying", + }, + { + id: "verify-v8-schema", + description: "Verify the v8 physical schema and retained raw telemetry counts", + requiresQuiescence: true, + phase: "copy-verified", + }, +] + +const dispositions: ReadonlyArray = [ + { + name: "local store", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: "The clean stopped v7 store is cloned byte-for-byte before the views are replaced.", + }, + { + name: "traces", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: + "The source of the replaced views is neither read nor rewritten; only the view definitions change.", + }, + { + // Rows already materialized keep their collapsed iOS fingerprint — + // recomputing hashes would re-bucket every existing issue. Forward-only, + // and bounded by the tables' 90-day TTL. + name: "error_events", + classification: "derived", + disposition: "rebuild-within-retention-horizon", + guarantee: + "Existing rows are preserved untouched; the Apple frame matching applies to events materialized after the migration and converges as the retention window rolls.", + preservationInterval: "error retention horizon", + sourceRetentionDays: 90, + targetRetentionDays: 90, + }, + { + name: "error_events_by_time", + classification: "derived", + disposition: "rebuild-within-retention-horizon", + guarantee: + "Same projection as error_events and treated identically: preserved rows, forward-only correction.", + preservationInterval: "error retention horizon", + sourceRetentionDays: 90, + targetRetentionDays: 90, + }, +] + +export const v7ToV8AppleCrashFramesModule: LocalStoreMigrationModule = { + id: MODULE_ID, + moduleVersion: 1, + description: "Rebuild the error-events views so the fingerprint recognises Apple crash frames", + from: LOCAL_SCHEMA_V7, + to: LOCAL_SCHEMA_V8, + operations, + dispositions, + decodeState, + decodeProgress, + preflight, + prepareTarget, + apply, + verify, + recover: async (_context, state, progress) => ({ state, progress }), +} diff --git a/apps/cli/src/server/schema-identity.ts b/apps/cli/src/server/schema-identity.ts index 0fa21edf9..dcf566f1b 100644 --- a/apps/cli/src/server/schema-identity.ts +++ b/apps/cli/src/server/schema-identity.ts @@ -6,6 +6,7 @@ import schemaV4Sql from "./schema/local-schema-v4.sql" with { type: "text" } import schemaV5Sql from "./schema/local-schema-v5.sql" with { type: "text" } import schemaV6Sql from "./schema/local-schema-v6.sql" with { type: "text" } import schemaV7Sql from "./schema/local-schema-v7.sql" with { type: "text" } +import schemaV8Sql from "./schema/local-schema-v8.sql" with { type: "text" } import { schemaDigest as digestSchema, schemaFingerprint as fingerprintSchema } from "./store-version" import { buildLocalSchemaManifest, type LocalSchemaManifest } from "./schema-manifest" import { LOCAL_SCHEMA_VERSION } from "./local-schema-version" @@ -29,7 +30,7 @@ export const LEGACY_SCHEMA_PROJECT_REVISION = export const LEGACY_SCHEMA_FINGERPRINT = "428701854f9fd30e" export const CURRENT_SCHEMA_PROJECT_REVISION = - "73f8f3249a508cd05598289b67b3773a049e38db0302efa6aa9e45e3501d2182" + "bb7da950a3a65af75fcf627bf4ed0436308c98fee86906048b05b6d40d9f7534" /** Revision recorded by the issue-297 recovery report. The refreshed upstream * generator currently emits CURRENT_SCHEMA_PROJECT_REVISION; the structural * fingerprint is the compatibility identity used by the migration. */ @@ -40,41 +41,73 @@ export const SCHEMA_FINGERPRINT = fingerprintSchema(schemaSql) export const SCHEMA_DIGEST = digestSchema(schemaSql) export const LOCAL_SCHEMA_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaSql) export const LOCAL_SCHEMA_MANIFEST_DIGEST = LOCAL_SCHEMA_MANIFEST.digest -/** Immutable v1 DDL/manifest snapshot used by the v0 -> v1 module even after - * the generated current schema advances. */ -export const LOCAL_SCHEMA_V1_SQL = schemaV1Sql -export const LOCAL_SCHEMA_V1_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaV1Sql) -export const LOCAL_SCHEMA_V1_MANIFEST_DIGEST = LOCAL_SCHEMA_V1_MANIFEST.digest -/** Immutable v2 DDL/manifest snapshot used by the v1 -> v2 module even after - * the generated current schema advances. */ -export const LOCAL_SCHEMA_V2_SQL = schemaV2Sql -export const LOCAL_SCHEMA_V2_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaV2Sql) -export const LOCAL_SCHEMA_V2_MANIFEST_DIGEST = LOCAL_SCHEMA_V2_MANIFEST.digest -/** Immutable v3 DDL/manifest snapshot used by the v2 -> v3 module after the - * generated current schema advances. */ -export const LOCAL_SCHEMA_V3_SQL = schemaV3Sql -export const LOCAL_SCHEMA_V3_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaV3Sql) -export const LOCAL_SCHEMA_V3_MANIFEST_DIGEST = LOCAL_SCHEMA_V3_MANIFEST.digest -/** Immutable v4 DDL/manifest snapshot used by the v3 -> v4 module after the - * generated current schema advances. */ -export const LOCAL_SCHEMA_V4_SQL = schemaV4Sql -export const LOCAL_SCHEMA_V4_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaV4Sql) -export const LOCAL_SCHEMA_V4_MANIFEST_DIGEST = LOCAL_SCHEMA_V4_MANIFEST.digest -/** Immutable v5 DDL/manifest snapshot used by the v4 -> v5 module after the - * generated current schema advances. */ -export const LOCAL_SCHEMA_V5_SQL = schemaV5Sql -export const LOCAL_SCHEMA_V5_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaV5Sql) -export const LOCAL_SCHEMA_V5_MANIFEST_DIGEST = LOCAL_SCHEMA_V5_MANIFEST.digest -/** Immutable v6 DDL/manifest snapshot used by the v5 -> v6 module after the - * generated current schema advances. */ -export const LOCAL_SCHEMA_V6_SQL = schemaV6Sql -export const LOCAL_SCHEMA_V6_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaV6Sql) -export const LOCAL_SCHEMA_V6_MANIFEST_DIGEST = LOCAL_SCHEMA_V6_MANIFEST.digest -/** Immutable v7 DDL/manifest snapshot used by the v6 -> v7 module after the - * generated current schema advances. */ -export const LOCAL_SCHEMA_V7_SQL = schemaV7Sql -export const LOCAL_SCHEMA_V7_MANIFEST: LocalSchemaManifest = buildLocalSchemaManifest(schemaV7Sql) -export const LOCAL_SCHEMA_V7_MANIFEST_DIGEST = LOCAL_SCHEMA_V7_MANIFEST.digest +/** + * Immutable per-version DDL and manifest snapshots. + * + * A historical edge must keep constructing and verifying the schema it was + * written for: when v9 ships, v7 -> v8 must still produce v8 rather than + * silently retargeting whatever the generator currently emits. The SQL is + * imported literally because Bun resolves text imports statically; everything + * derived from it is built once, here. + */ +const SNAPSHOT_SQL: ReadonlyArray = [ + schemaV1Sql, + schemaV2Sql, + schemaV3Sql, + schemaV4Sql, + schemaV5Sql, + schemaV6Sql, + schemaV7Sql, + schemaV8Sql, +] + +export interface LocalSchemaSnapshot { + readonly version: number + readonly sql: string + readonly manifest: LocalSchemaManifest + readonly manifestDigest: string +} + +/** Indexed by schema version; index 0 is the fingerprint-only legacy store, which has no DDL. */ +export const LOCAL_SCHEMA_SNAPSHOTS: ReadonlyArray = Object.freeze([ + undefined, + ...SNAPSHOT_SQL.map((sql, index) => { + const manifest = buildLocalSchemaManifest(sql) + return Object.freeze({ version: index + 1, sql, manifest, manifestDigest: manifest.digest }) + }), +]) + +const snapshotAt = (version: number): LocalSchemaSnapshot => { + const snapshot = LOCAL_SCHEMA_SNAPSHOTS[version] + if (!snapshot) throw new Error(`no bundled DDL snapshot for local schema version ${version}`) + return snapshot +} + +export const LOCAL_SCHEMA_V1_SQL = snapshotAt(1).sql +export const LOCAL_SCHEMA_V1_MANIFEST = snapshotAt(1).manifest +export const LOCAL_SCHEMA_V1_MANIFEST_DIGEST = snapshotAt(1).manifestDigest +export const LOCAL_SCHEMA_V2_SQL = snapshotAt(2).sql +export const LOCAL_SCHEMA_V2_MANIFEST = snapshotAt(2).manifest +export const LOCAL_SCHEMA_V2_MANIFEST_DIGEST = snapshotAt(2).manifestDigest +export const LOCAL_SCHEMA_V3_SQL = snapshotAt(3).sql +export const LOCAL_SCHEMA_V3_MANIFEST = snapshotAt(3).manifest +export const LOCAL_SCHEMA_V3_MANIFEST_DIGEST = snapshotAt(3).manifestDigest +export const LOCAL_SCHEMA_V4_SQL = snapshotAt(4).sql +export const LOCAL_SCHEMA_V4_MANIFEST = snapshotAt(4).manifest +export const LOCAL_SCHEMA_V4_MANIFEST_DIGEST = snapshotAt(4).manifestDigest +export const LOCAL_SCHEMA_V5_SQL = snapshotAt(5).sql +export const LOCAL_SCHEMA_V5_MANIFEST = snapshotAt(5).manifest +export const LOCAL_SCHEMA_V5_MANIFEST_DIGEST = snapshotAt(5).manifestDigest +export const LOCAL_SCHEMA_V6_SQL = snapshotAt(6).sql +export const LOCAL_SCHEMA_V6_MANIFEST = snapshotAt(6).manifest +export const LOCAL_SCHEMA_V6_MANIFEST_DIGEST = snapshotAt(6).manifestDigest +export const LOCAL_SCHEMA_V7_SQL = snapshotAt(7).sql +export const LOCAL_SCHEMA_V7_MANIFEST = snapshotAt(7).manifest +export const LOCAL_SCHEMA_V7_MANIFEST_DIGEST = snapshotAt(7).manifestDigest +export const LOCAL_SCHEMA_V8_SQL = snapshotAt(8).sql +export const LOCAL_SCHEMA_V8_MANIFEST = snapshotAt(8).manifest +export const LOCAL_SCHEMA_V8_MANIFEST_DIGEST = snapshotAt(8).manifestDigest + export interface LocalSchemaIdentity { readonly version: number readonly fingerprint: string @@ -85,72 +118,33 @@ export interface LocalSchemaIdentity { } /** - * The v1 identity is deliberately frozen. Historical migration edges must - * never point at CURRENT_LOCAL_SCHEMA: when v2 ships, v0 -> v1 must still - * construct and verify v1 rather than silently changing its destination. + * Per-version identities, frozen and read straight from the append-only + * history. Historical migration edges must never point at + * CURRENT_LOCAL_SCHEMA: when v9 ships, v0 -> v1 must still construct and verify + * v1 rather than silently changing its destination. */ -export const LOCAL_SCHEMA_V1: LocalSchemaIdentity = Object.freeze({ - version: LOCAL_SCHEMA_HISTORY[1]!.version, - fingerprint: LOCAL_SCHEMA_HISTORY[1]!.fingerprint, - digest: LOCAL_SCHEMA_HISTORY[1]!.digest, - manifestDigest: LOCAL_SCHEMA_HISTORY[1]!.manifestDigest, - chdb: CHDB_VERSION, - projectRevision: LOCAL_SCHEMA_HISTORY[1]!.projectRevision, -}) - -export const LOCAL_SCHEMA_V2: LocalSchemaIdentity = Object.freeze({ - version: LOCAL_SCHEMA_HISTORY[2]!.version, - fingerprint: LOCAL_SCHEMA_HISTORY[2]!.fingerprint, - digest: LOCAL_SCHEMA_HISTORY[2]!.digest, - manifestDigest: LOCAL_SCHEMA_HISTORY[2]!.manifestDigest, - chdb: CHDB_VERSION, - projectRevision: LOCAL_SCHEMA_HISTORY[2]!.projectRevision, -}) - -export const LOCAL_SCHEMA_V3: LocalSchemaIdentity = Object.freeze({ - version: LOCAL_SCHEMA_HISTORY[3]!.version, - fingerprint: LOCAL_SCHEMA_HISTORY[3]!.fingerprint, - digest: LOCAL_SCHEMA_HISTORY[3]!.digest, - manifestDigest: LOCAL_SCHEMA_HISTORY[3]!.manifestDigest, - chdb: CHDB_VERSION, - projectRevision: LOCAL_SCHEMA_HISTORY[3]!.projectRevision, -}) - -export const LOCAL_SCHEMA_V4: LocalSchemaIdentity = Object.freeze({ - version: LOCAL_SCHEMA_HISTORY[4]!.version, - fingerprint: LOCAL_SCHEMA_HISTORY[4]!.fingerprint, - digest: LOCAL_SCHEMA_HISTORY[4]!.digest, - manifestDigest: LOCAL_SCHEMA_HISTORY[4]!.manifestDigest, - chdb: CHDB_VERSION, - projectRevision: LOCAL_SCHEMA_HISTORY[4]!.projectRevision, -}) - -export const LOCAL_SCHEMA_V5: LocalSchemaIdentity = Object.freeze({ - version: LOCAL_SCHEMA_HISTORY[5]!.version, - fingerprint: LOCAL_SCHEMA_HISTORY[5]!.fingerprint, - digest: LOCAL_SCHEMA_HISTORY[5]!.digest, - manifestDigest: LOCAL_SCHEMA_HISTORY[5]!.manifestDigest, - chdb: CHDB_VERSION, - projectRevision: LOCAL_SCHEMA_HISTORY[5]!.projectRevision, -}) - -export const LOCAL_SCHEMA_V6: LocalSchemaIdentity = Object.freeze({ - version: LOCAL_SCHEMA_HISTORY[6]!.version, - fingerprint: LOCAL_SCHEMA_HISTORY[6]!.fingerprint, - digest: LOCAL_SCHEMA_HISTORY[6]!.digest, - manifestDigest: LOCAL_SCHEMA_HISTORY[6]!.manifestDigest, - chdb: CHDB_VERSION, - projectRevision: LOCAL_SCHEMA_HISTORY[6]!.projectRevision, -}) +const identityAt = (version: number): LocalSchemaIdentity => { + const entry = LOCAL_SCHEMA_HISTORY[version] + if (!entry || entry.version !== version) + throw new Error(`local schema history has no entry for version ${version}`) + return Object.freeze({ + version: entry.version, + fingerprint: entry.fingerprint, + digest: entry.digest, + manifestDigest: entry.manifestDigest, + chdb: CHDB_VERSION, + projectRevision: entry.projectRevision, + }) +} -export const LOCAL_SCHEMA_V7: LocalSchemaIdentity = Object.freeze({ - version: LOCAL_SCHEMA_HISTORY[7]!.version, - fingerprint: LOCAL_SCHEMA_HISTORY[7]!.fingerprint, - digest: LOCAL_SCHEMA_HISTORY[7]!.digest, - manifestDigest: LOCAL_SCHEMA_HISTORY[7]!.manifestDigest, - chdb: CHDB_VERSION, - projectRevision: LOCAL_SCHEMA_HISTORY[7]!.projectRevision, -}) +export const LOCAL_SCHEMA_V1 = identityAt(1) +export const LOCAL_SCHEMA_V2 = identityAt(2) +export const LOCAL_SCHEMA_V3 = identityAt(3) +export const LOCAL_SCHEMA_V4 = identityAt(4) +export const LOCAL_SCHEMA_V5 = identityAt(5) +export const LOCAL_SCHEMA_V6 = identityAt(6) +export const LOCAL_SCHEMA_V7 = identityAt(7) +export const LOCAL_SCHEMA_V8 = identityAt(8) export const CURRENT_LOCAL_SCHEMA: LocalSchemaIdentity = Object.freeze({ version: LOCAL_SCHEMA_VERSION, diff --git a/apps/cli/src/server/schema-physical.ts b/apps/cli/src/server/schema-physical.ts index 688aadd6f..c9ac11880 100644 --- a/apps/cli/src/server/schema-physical.ts +++ b/apps/cli/src/server/schema-physical.ts @@ -1,5 +1,7 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. +import { Schema } from "effect" import { Chdb, RAW_TELEMETRY_TTL_COLUMNS } from "./chdb" +import { decodeJsonEachRow } from "./chdb-rows" import { LOCAL_SCHEMA_MANIFEST } from "./schema-identity" import { comparePhysicalSchema, @@ -9,45 +11,52 @@ import { withRawTelemetryRetentionFloor, } from "./schema-manifest" -const parseJsonEachRow = (value: string): A[] => - value - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length > 0) - .map((line) => JSON.parse(line) as A) +/** The `system.*` shapes this inspector reads. Each is fixed by the SELECT + * directly above its use, so a row that does not match means the query and the + * decoder have drifted apart. */ +const TableRow = Schema.Struct({ + name: Schema.String, + engine: Schema.String, + partition_key: Schema.String, + sorting_key: Schema.String, + create_table_query: Schema.String, +}) -interface ColumnRow { - table: string - name: string - type: string - position: number - default_kind: string - default_expression: string - compression_codec: string -} +const ColumnRowSchema = Schema.Struct({ + table: Schema.String, + name: Schema.String, + type: Schema.String, + position: Schema.Number, + default_kind: Schema.String, + default_expression: Schema.String, + compression_codec: Schema.String, +}) + +const IndexRow = Schema.Struct({ table: Schema.String, name: Schema.String }) + +const FormattedRow = Schema.Struct({ formatted: Schema.String }) + +const decodeTableRows = decodeJsonEachRow(TableRow) +const decodeColumnRows = decodeJsonEachRow(ColumnRowSchema) +const decodeIndexRows = decodeJsonEachRow(IndexRow) +const decodeFormattedRows = decodeJsonEachRow(FormattedRow) /** Inspect the physical definitions chDB reports, including the objects that * are easy to miss when relying on a bundled DDL fingerprint alone. */ export const inspectPhysicalSchema = (db: Chdb): PhysicalSchema => { - const tables = parseJsonEachRow<{ - name: string - engine: string - partition_key: string - sorting_key: string - create_table_query: string - }>( + const tables = decodeTableRows( db.query( "SELECT name, engine, partition_key, sorting_key, create_table_query FROM system.tables WHERE database = 'default'", ), ) - const columns = parseJsonEachRow( + const columns = decodeColumnRows( db.query( "SELECT table, name, type, position, default_kind, default_expression, compression_codec FROM system.columns WHERE database = 'default' ORDER BY table, position", ), ) - let indexes: Array<{ table: string; name: string }> = [] + let indexes: ReadonlyArray<{ readonly table: string; readonly name: string }> = [] try { - indexes = parseJsonEachRow<{ table: string; name: string }>( + indexes = decodeIndexRows( db.query("SELECT table, name FROM system.data_skipping_indices WHERE database = 'default'"), ) } catch { @@ -119,7 +128,7 @@ const formatViewBody = (db: Chdb): ((sql: string) => string | undefined) => { if (cached !== undefined || cache.has(sql)) return cached let formatted: string | undefined try { - const rows = parseJsonEachRow<{ formatted: string }>( + const rows = decodeFormattedRows( db.query(`SELECT formatQuery($maple_fmt$${sql}$maple_fmt$) AS formatted FORMAT JSONEachRow`), ) // The local store is always the `default` database, and ClickHouse diff --git a/apps/cli/src/server/schema/local-inserts.json b/apps/cli/src/server/schema/local-inserts.json index 715dd7a8c..a4ab5e71b 100644 --- a/apps/cli/src/server/schema/local-inserts.json +++ b/apps/cli/src/server/schema/local-inserts.json @@ -1,5 +1,5 @@ { - "projectRevision": "73f8f3249a508cd05598289b67b3773a049e38db0302efa6aa9e45e3501d2182", + "projectRevision": "bb7da950a3a65af75fcf627bf4ed0436308c98fee86906048b05b6d40d9f7534", "orgPlaceholder": "__ORG__", "datasources": { "traces": { diff --git a/apps/cli/src/server/schema/local-schema-v8.sql b/apps/cli/src/server/schema/local-schema-v8.sql new file mode 100644 index 000000000..60ba1b4b1 --- /dev/null +++ b/apps/cli/src/server/schema/local-schema-v8.sql @@ -0,0 +1,1842 @@ +-- This file is generated by scripts/generate-clickhouse-schema-sql.ts +-- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. +-- projectRevision: bb7da950a3a65af75fcf627bf4ed0436308c98fee86906048b05b6d40d9f7534 +-- localSchemaVersion: 8 + +CREATE TABLE IF NOT EXISTS alert_checks ( + OrgId LowCardinality(String), + RuleId String, + GroupKey String, + Timestamp DateTime64(3), + Status LowCardinality(String), + SignalType LowCardinality(String), + Comparator LowCardinality(String), + Threshold Float64, + ObservedValue Nullable(Float64), + SampleCount UInt32, + WindowMinutes UInt16, + WindowStart DateTime64(3), + WindowEnd DateTime64(3), + ConsecutiveBreaches UInt16, + ConsecutiveHealthy UInt16, + IncidentId Nullable(String), + IncidentTransition LowCardinality(String), + EvaluationDurationMs UInt32, + ErrorMessage Nullable(String), + ErrorCategory LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, RuleId, GroupKey, Timestamp) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS attribute_keys_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, Hour, AttributeKey) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS attribute_values_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeValue String, + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, AttributeKey, Hour, AttributeValue) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String, + ServiceVersion LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, FingerprintHash, Timestamp) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events_by_time ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String, + ServiceVersion LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, FingerprintHash) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_fingerprints_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + FingerprintHash UInt64, + ServiceName SimpleAggregateFunction(anyLast, String), + ExceptionType SimpleAggregateFunction(anyLast, String), + ExceptionMessage SimpleAggregateFunction(anyLast, String), + ErrorLabel SimpleAggregateFunction(anyLast, String), + TopFrame SimpleAggregateFunction(anyLast, String), + OccurrenceCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime), + ServiceVersions SimpleAggregateFunction(groupUniqArrayArray, Array(String)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Minute) +ORDER BY (OrgId, Minute, FingerprintHash) +TTL Minute + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + StatusMessage String, + Duration UInt64, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, Timestamp) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS logs ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TimestampTime DateTime, + TraceId String, + SpanId String, + TraceFlags UInt8, + SeverityText LowCardinality(String), + SeverityNumber UInt8, + ServiceName LowCardinality(String), + Body String, + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + LogAttributes Map(LowCardinality(String), String), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + LogAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(LogAttributes), mapValues(LogAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 +) +ENGINE = MergeTree +PARTITION BY toDate(TimestampTime) +ORDER BY (OrgId, toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) +TTL toDate(TimestampTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS logs_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SeverityText LowCardinality(String), + DeploymentEnv LowCardinality(String), + Count SimpleAggregateFunction(sum, UInt64), + SizeBytes SimpleAggregateFunction(sum, UInt64), + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS metric_catalog ( + OrgId LowCardinality(String), + Hour DateTime, + MetricType LowCardinality(String), + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription SimpleAggregateFunction(anyLast, String), + MetricUnit SimpleAggregateFunction(anyLast, String), + IsMonotonic SimpleAggregateFunction(anyLast, UInt8), + DataPointCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, MetricType, ServiceName, MetricName, Hour) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_exponential_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + Scale Int32, + ZeroCount UInt64, + PositiveOffset Int32, + PositiveBucketCounts Array(UInt64), + NegativeOffset Int32, + NegativeBucketCounts Array(UInt64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_gauge ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + BucketCounts Array(UInt64), + ExplicitBounds Array(Float64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_sum ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + AggregationTemporality Int32, + IsMonotonic Bool +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_address_resolutions_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + ParentServerAddress String, + ResolvedTargetService LowCardinality(String), + DeploymentEnv LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_external_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + TargetType LowCardinality(String), + TargetSystem LowCardinality(String), + TargetName String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, TargetType, TargetSystem, TargetName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_children ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, ParentSpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_query_shapes_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + QueryKey String, + QueryLabel SimpleAggregateFunction(any, String), + SampleStatement SimpleAggregateFunction(any, String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedCount SimpleAggregateFunction(sum, Float64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSumMs SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace, QueryKey) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, TargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly_ingest ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount UInt64, + ErrorCount UInt64, + DurationSumMs Float64, + MaxDurationMs Float64, + SampledSpanCount UInt64, + UnsampledSpanCount UInt64, + SampleRateSum Float64 +) +ENGINE = Null; + +CREATE TABLE IF NOT EXISTS service_map_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Hour, SpanName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Minute, SpanName) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, Hour, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, Minute, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + ServiceName LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String), + CommitSha LowCardinality(String), + SampleRate Float64 DEFAULT 1, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_platforms_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + K8sCluster SimpleAggregateFunction(max, String), + K8sPodName SimpleAggregateFunction(max, String), + K8sDeploymentName SimpleAggregateFunction(max, String), + K8sStatefulSetName SimpleAggregateFunction(max, String), + K8sDaemonSetName SimpleAggregateFunction(max, String), + K8sNamespaceName SimpleAggregateFunction(max, String), + CloudPlatform SimpleAggregateFunction(max, String), + CloudProvider SimpleAggregateFunction(max, String), + FaasName SimpleAggregateFunction(max, String), + MapleSdkType SimpleAggregateFunction(max, String), + ProcessRuntimeName SimpleAggregateFunction(max, String), + SpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_usage ( + OrgId LowCardinality(String), + ServiceName LowCardinality(String), + Hour DateTime, + LogCount UInt64, + LogSizeBytes UInt64, + TraceCount UInt64, + TraceSizeBytes UInt64, + SumMetricCount UInt64, + SumMetricSizeBytes UInt64, + GaugeMetricCount UInt64, + GaugeMetricSizeBytes UInt64, + HistogramMetricCount UInt64, + HistogramMetricSizeBytes UInt64, + ExpHistogramMetricCount UInt64, + ExpHistogramMetricSizeBytes UInt64 +) +ENGINE = SummingMergeTree +ORDER BY (OrgId, ServiceName, Hour) +TTL Hour + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS session_events ( + OrgId LowCardinality(String), + SessionId String, + Timestamp DateTime64(9), + Seq UInt32 DEFAULT 0, + Type LowCardinality(String), + Url String DEFAULT '', + TraceId String DEFAULT '', + Level LowCardinality(String) DEFAULT '', + Message String DEFAULT '', + TargetSelector String DEFAULT '', + TargetText String DEFAULT '', + NetMethod LowCardinality(String) DEFAULT '', + NetUrl String DEFAULT '', + NetStatus UInt16 DEFAULT 0, + NetDurationMs UInt32 DEFAULT 0, + ErrorStack String DEFAULT '', + Attributes Map(String, String), + INDEX idx_type Type TYPE set(16) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, Timestamp, Seq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replay_events ( + OrgId LowCardinality(String), + SessionId String, + ChunkSeq UInt32, + Timestamp DateTime64(9), + DurationMs UInt32 DEFAULT 0, + EventCount UInt32 DEFAULT 0, + ByteSize UInt32 DEFAULT 0, + Events String, + IsCheckpoint UInt8 DEFAULT 0 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, ChunkSeq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replays ( + OrgId LowCardinality(String), + SessionId String, + StartTime DateTime64(9), + EndTime Nullable(DateTime64(9)), + DurationMs Nullable(UInt32), + Status LowCardinality(String), + UserId String, + UrlInitial String, + UserAgent String, + BrowserName LowCardinality(String), + OsName LowCardinality(String), + DeviceType LowCardinality(String), + Country LowCardinality(String) DEFAULT '', + ServiceName LowCardinality(String), + PageViews UInt32 DEFAULT 0, + ClickCount UInt32 DEFAULT 0, + ErrorCount UInt32 DEFAULT 0, + TraceIds Array(String) DEFAULT [], + ResourceAttributes Map(LowCardinality(String), String), + Version UInt32, + VisitorId String DEFAULT '', + VisitorIsNew UInt8 DEFAULT 0, + UserEmail String DEFAULT '', + UserName String DEFAULT '', + GroupId String DEFAULT '', + GroupName String DEFAULT '', + UserTraits Map(String, String) DEFAULT map(), + Referrer String DEFAULT '', + ReferrerHost LowCardinality(String) DEFAULT '', + UtmSource LowCardinality(String) DEFAULT '', + UtmMedium LowCardinality(String) DEFAULT '', + UtmCampaign LowCardinality(String) DEFAULT '', + UtmTerm String DEFAULT '', + UtmContent String DEFAULT '', + Host LowCardinality(String) DEFAULT '', + EntryPath String DEFAULT '', + ExitPath String DEFAULT '', + Language LowCardinality(String) DEFAULT '', + LastActivityAt Nullable(DateTime64(9)) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(StartTime) +ORDER BY (OrgId, SessionId) +TTL toDate(StartTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS span_metrics_calls_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + SpanKind LowCardinality(String), + AttrFingerprint UInt64, + ResourceFingerprint UInt64, + StartTimeUnix DateTime64(9), + LastValue AggregateFunction(argMax, Float64, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS trace_detail_spans ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + ResourceAttributes Map(LowCardinality(String), String), + EventsTimestamp Array(DateTime64(9)), + EventsName Array(LowCardinality(String)), + EventsAttributes Array(Map(LowCardinality(String), String)) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS trace_list_mv ( + OrgId LowCardinality(String), + TraceId String, + Timestamp DateTime, + ServiceName LowCardinality(String), + SpanName String, + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + HttpMethod LowCardinality(String), + HttpRoute String, + HttpStatusCode LowCardinality(String), + DeploymentEnv LowCardinality(String), + HasError UInt8, + TraceState String, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + TraceState String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + EventsTimestamp Array(DateTime64(9)), + EventsName Array(LowCardinality(String)), + EventsAttributes Array(Map(LowCardinality(String), String)), + LinksTraceId Array(String), + LinksSpanId Array(String), + LinksTraceState Array(String), + LinksAttributes Array(Map(LowCardinality(String), String)), + SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0), + IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp)) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + StatusCode LowCardinality(String), + IsEntryPoint UInt8, + DeploymentEnv LowCardinality(String), + WeightedCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSum SimpleAggregateFunction(sum, Float64), + WeightedErrorCount SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32), + DurationMin SimpleAggregateFunction(min, UInt64), + DurationMax SimpleAggregateFunction(max, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS web_events ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + SessionId String, + Seq UInt32, + Kind LowCardinality(String), + EventName String, + Host LowCardinality(String), + PagePath String, + Url String, + Attributes Map(String, String), + INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, SessionId, Seq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arraySlice( + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection for the message signature below. + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + 1, 120 + ) + ) AS _msgSig, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel, + ResourceAttributes['service.version'] AS ServiceVersion + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all three hold: 4xx, no exception event, and no + -- exception type. 5xx and anything carrying an exception still count, + -- and SpanKind is deliberately not consulted — these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND _exType = '' + ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arraySlice( + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection for the message signature below. + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + 1, 120 + ) + ) AS _msgSig, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel, + ResourceAttributes['service.version'] AS ServiceVersion + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all three hold: 4xx, no exception event, and no + -- exception type. 5xx and anything carrying an exception still count, + -- and SpanKind is deliberately not consulted — these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND _exType = '' + ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS +SELECT + OrgId, + toStartOfMinute(Timestamp) AS Minute, + FingerprintHash, + anyLast(ServiceName) AS ServiceName, + anyLast(ExceptionType) AS ExceptionType, + anyLast(ExceptionMessage) AS ExceptionMessage, + anyLast(ErrorLabel) AS ErrorLabel, + anyLast(TopFrame) AS TopFrame, + count() AS OccurrenceCount, + min(Timestamp) AS FirstSeen, + max(Timestamp) AS LastSeen, + -- Distinct builds, not a sample: see ServiceVersions on the datasource. + groupUniqArray(ServiceVersion) AS ServiceVersions + FROM error_events + GROUP BY OrgId, Minute, FingerprintHash; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_spans_mv TO error_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + StatusMessage, + Duration, + ResourceAttributes['deployment.environment'] AS DeploymentEnv + FROM traces + WHERE StatusCode = 'Error'; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(LogAttributes)) AS AttributeKey, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + WHERE LogAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + ARRAY JOIN + mapKeys(LogAttributes) AS AttributeKey, + mapValues(LogAttributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS logs_aggregates_hourly_mv TO logs_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(TimestampTime) AS Hour, + ServiceName, + SeverityText, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + count() AS Count, + sum(length(Body) + 200) AS SizeBytes, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM logs + GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + arrayJoin(mapKeys(Attributes)) AS AttributeKey, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + WHERE Attributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + AttributeKey, + AttributeValue, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + ARRAY JOIN + mapKeys(Attributes) AS AttributeKey, + mapValues(Attributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_exp_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'exponential_histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_exponential_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_gauge_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'gauge' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_gauge + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_sum_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'sum' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + anyLast(toUInt8(IsMonotonic)) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_sum + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + multiIf( + SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', 'messaging', + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc', + 'http' + ) AS TargetType, + multiIf( + SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'], + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'], + '' + ) AS TargetSystem, + multiIf( + SpanAttributes['messaging.destination'] != '' OR SpanAttributes['messaging.system'] != '', + if(SpanAttributes['messaging.destination'] != '', SpanAttributes['messaging.destination'], SpanAttributes['messaging.system']), + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', + if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']), + if(SpanAttributes['server.address'] != '', + SpanAttributes['server.address'], + if(SpanAttributes['http.host'] != '', + SpanAttributes['http.host'], + SpanAttributes['url.authority'])) + ) AS TargetName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + sum(SampleRate) AS SampleRateSum + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND SpanAttributes['db.system.name'] = '' + AND ServiceName != '' + AND ( + SpanAttributes['server.address'] != '' + OR SpanAttributes['http.host'] != '' + OR SpanAttributes['url.authority'] != '' + OR SpanAttributes['messaging.destination'] != '' + OR SpanAttributes['messaging.system'] != '' + OR SpanAttributes['rpc.service'] != '' + OR SpanAttributes['rpc.system'] != '' + ) + GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv + HAVING TargetName != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + ResourceAttributes['deployment.environment'] AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') + AND ParentSpanId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + countIf(TraceState LIKE '%th:%') AS SampledSpanCount, + countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount, + sum(SampleRate) AS SampleRateSum + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_query_shapes_hourly_mv TO service_map_db_query_shapes_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + coalesce( + nullIf(SpanAttributes['db.query.fingerprint'], ''), + nullIf(SpanAttributes['db.statement.fingerprint'], ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\'[^\']*\'', '?'), '\\bin\\s*\\([^)]*\\)', 'in (?)'), '[0-9]+(\\.[0-9]+)?', '?'), '\\s+', ' '), '^\\s+|\\s+$', ''))), ''), ''), + toString(cityHash64(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +))) +) AS QueryKey, + any(substring(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +), 1, 220)) AS QueryLabel, + any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(SampleRate) AS EstimatedCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv, QueryKey; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_edges_hourly_ingest_mv TO service_map_edges_hourly AS +SELECT + OrgId, + Hour, + SourceService, + TargetService, + DeploymentEnv, + CallCount, + ErrorCount, + DurationSumMs, + MaxDurationMs, + SampledSpanCount, + UnsampledSpanCount, + SampleRateSum + FROM service_map_edges_hourly_ingest; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_spans_mv TO service_map_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + ResourceAttributes['deployment.environment'] AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_hourly_mv TO service_operations_hourly AS +SELECT + OrgId, + toStartOfHour(Minute) AS Hour, + ServiceName, + DeploymentEnv, + SpanName, + sum(SpanCount) AS SpanCount, + sum(EstimatedSpanCount) AS EstimatedSpanCount, + sum(ErrorCount) AS ErrorCount, + sum(EstimatedErrorCount) AS EstimatedErrorCount, + sum(DurationSum) AS DurationSum, + quantilesTDigestMergeState(0.5, 0.95)(DurationQuantiles) AS DurationQuantiles + FROM service_operations_minutely + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_minutely_mv TO service_operations_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles + FROM traces + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_hourly_mv TO service_overview_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_minutely_mv TO service_overview_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_spans_mv TO service_overview_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + Duration, + StatusCode, + TraceState, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + SampleRate, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster, + max(ResourceAttributes['k8s.pod.name']) AS K8sPodName, + max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName, + max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName, + max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName, + max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName, + max(ResourceAttributes['cloud.platform']) AS CloudPlatform, + max(ResourceAttributes['cloud.provider']) AS CloudProvider, + max(ResourceAttributes['faas.name']) AS FaasName, + max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType, + max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName, + count() AS SpanCount + FROM traces + WHERE ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_logs_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(TimestampTime) AS Hour, + count() AS LogCount, + sum(length(Body) + 200) AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM logs + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_exp_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + count() AS ExpHistogramMetricCount, + count() * 300 AS ExpHistogramMetricSizeBytes + FROM metrics_exponential_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_gauge_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + count() AS GaugeMetricCount, + count() * 150 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_gauge + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + count() AS HistogramMetricCount, + count() * 250 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_sum_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + count() AS SumMetricCount, + count() * 150 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_sum + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_traces_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS TraceCount, + sum(length(SpanName) + 300) AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM traces + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS span_metrics_calls_hourly_mv TO span_metrics_calls_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + ServiceName, + MetricName, + Attributes['span.kind'] AS SpanKind, + cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint, + cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint, + StartTimeUnix, + argMaxState(Value, TimeUnix) AS LastValue + FROM metrics_sum + WHERE MetricName IN ('span.metrics.calls', 'calls') AND IsMonotonic + GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_detail_spans_mv TO trace_detail_spans AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanId, + ParentSpanId, + SpanName, + SpanKind, + ServiceName, + Duration, + StatusCode, + StatusMessage, + SpanAttributes, + ResourceAttributes, + EventsTimestamp, + EventsName, + EventsAttributes + FROM traces; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_list_mv_mv TO trace_list_mv AS +SELECT + OrgId, + TraceId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + if( + (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS')) + AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''), + concat( + if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), + ' ', + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path']) + ), + SpanName + ) AS SpanName, + SpanKind, + Duration, + StatusCode, + if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod, + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute, + if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + toUInt8( + StatusCode = 'Error' + OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500) + OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500) + ) AS HasError, + TraceState, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(ResourceAttributes)) AS AttributeKey, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE ResourceAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(ResourceAttributes) AS AttributeKey, + mapValues(ResourceAttributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(SpanAttributes)) AS AttributeKey, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE SpanAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(SpanAttributes) AS AttributeKey, + mapValues(SpanAttributes) AS AttributeValue + WHERE AttributeValue != '' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS traces_aggregates_hourly_mv TO traces_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + SpanName, + SpanKind, + StatusCode, + IsEntryPoint, + ResourceAttributes['deployment.environment'] AS DeploymentEnv, + sum(SampleRate) AS WeightedCount, + sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum, + sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount, + quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles, + min(Duration) AS DurationMin, + max(Duration) AS DurationMax + FROM traces + GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS web_events_mv TO web_events AS +SELECT + OrgId, + Timestamp, + SessionId, + Seq, + Type AS Kind, + if(Type = 'navigation', '$pageview', Message) AS EventName, + domain(Url) AS Host, + path(Url) AS PagePath, + Url, + Attributes + FROM session_events + WHERE Type IN ('navigation', 'custom'); diff --git a/apps/cli/src/server/schema/local-schema.sql b/apps/cli/src/server/schema/local-schema.sql index e0b53e65f..60ba1b4b1 100644 --- a/apps/cli/src/server/schema/local-schema.sql +++ b/apps/cli/src/server/schema/local-schema.sql @@ -1,7 +1,7 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: 73f8f3249a508cd05598289b67b3773a049e38db0302efa6aa9e45e3501d2182 --- localSchemaVersion: 7 +-- projectRevision: bb7da950a3a65af75fcf627bf4ed0436308c98fee86906048b05b6d40d9f7534 +-- localSchemaVersion: 8 CREATE TABLE IF NOT EXISTS alert_checks ( OrgId LowCardinality(String), @@ -875,7 +875,7 @@ WITH -- the SQL that actually runs, silently. arraySlice( arrayFilter( - line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+'), + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+'), splitByChar('\n', _exStack) ), 1, 3 @@ -1015,7 +1015,7 @@ WITH -- the SQL that actually runs, silently. arraySlice( arrayFilter( - line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+'), + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+'), splitByChar('\n', _exStack) ), 1, 3 diff --git a/apps/cli/src/server/store-version.ts b/apps/cli/src/server/store-version.ts index 507debec0..95485e525 100644 --- a/apps/cli/src/server/store-version.ts +++ b/apps/cli/src/server/store-version.ts @@ -9,7 +9,9 @@ import { createHash, randomUUID } from "node:crypto" import { existsSync, readFileSync, unlinkSync, writeFileSync } from "node:fs" import { dirname, join } from "node:path" +import { Effect, Schema, SchemaGetter } from "effect" import { CHDB_VERSION } from "../version" +import { Digest64, Fingerprint16, IsoOrUnknown } from "./identity-schema" import { durableRemove, durableWrite } from "./durable-files" export const STORE_MARKER_FORMAT_VERSION = 2 as const @@ -112,83 +114,94 @@ export const markStoreClosedDurable = async (dataDir: string): Promise => export const isStoreDirty = (dataDir: string): boolean => storeHasData(dataDir) && existsSync(storeOpenMarkerPath(dataDir)) -const isRecord = (value: unknown): value is Record => - typeof value === "object" && value !== null && !Array.isArray(value) - -const isIsoOrUnknown = (value: unknown): value is string => - typeof value === "string" && (value === "unknown" || Number.isFinite(Date.parse(value))) - -const isHexDigest = (value: unknown): value is string => - typeof value === "string" && /^[0-9a-f]{64}$/i.test(value) +/** + * Provenance fields are read leniently. + * + * `maple`, `createdAt`, and `createdByMaple` describe who made the store, not + * what it contains. A store whose provenance is missing or garbled is still a + * perfectly openable store, so a bad value degrades to "unknown" instead of + * making the marker malformed and refusing to start. Everything below this + * comment — the identity fields the loader actually acts on — is strict. + */ +const LenientProvenance = Schema.Unknown.pipe( + Schema.decodeTo(Schema.String, { + decode: SchemaGetter.transform((value) => (typeof value === "string" ? value : "unknown")), + encode: SchemaGetter.passthrough(), + }), + Schema.withDecodingDefaultKey(Effect.succeed(undefined)), +) + +const StoreMigrationStampSchema = Schema.Struct({ + id: Schema.String, + completedAt: IsoOrUnknown, + fromVersion: Schema.Int, + toVersion: Schema.Int, +}) + +const StoreMarkerV2Schema = Schema.Struct({ + formatVersion: Schema.Literal(STORE_MARKER_FORMAT_VERSION), + storeId: Schema.String.check(Schema.isMinLength(1)), + chdb: Schema.String.check(Schema.isMinLength(1)), + maple: LenientProvenance, + createdAt: LenientProvenance.check( + Schema.makeFilter((value: string) => + value === "unknown" || Number.isFinite(Date.parse(value)) + ? undefined + : "marker createdAt is invalid", + ), + ), + createdByMaple: LenientProvenance, + schemaVersion: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), + schemaDigest: Digest64, + schema: Fingerprint16, + activation: Schema.Literals(["active", "staging"]), + lastMigration: Schema.optionalKey(StoreMigrationStampSchema), +}) + +/** + * The original marker had no `formatVersion` at all, so v1 is recognised by its + * absence. A malformed object must not silently become a legacy store, which is + * why `chdb` stays required here: it is the only field the original format + * guaranteed. + */ +const StoreMarkerV1Schema = Schema.Struct({ + formatVersion: Schema.Literal(1).pipe(Schema.withDecodingDefaultKey(Effect.succeed(1 as const))), + chdb: Schema.String.check(Schema.isMinLength(1)), + maple: LenientProvenance, + createdAt: LenientProvenance.check( + Schema.makeFilter((value: string) => + value === "unknown" || Number.isFinite(Date.parse(value)) + ? undefined + : "marker createdAt is invalid", + ), + ), + schema: Schema.Unknown.pipe( + Schema.decodeTo(Schema.String, { + decode: SchemaGetter.transform((value) => (typeof value === "string" ? value : "")), + encode: SchemaGetter.passthrough(), + }), + Schema.withDecodingDefaultKey(Effect.succeed(undefined)), + ), +}) + +const decodeMarkerV1 = Schema.decodeUnknownSync(StoreMarkerV1Schema) +const decodeMarkerV2 = Schema.decodeUnknownSync(StoreMarkerV2Schema) const parseMarker = (value: unknown): StoreMarker => { - if (!isRecord(value) || typeof value.chdb !== "string" || value.chdb.length === 0) { + if (typeof value !== "object" || value === null || Array.isArray(value)) { throw new Error("marker must contain a non-empty chdb string") } - const maple = typeof value.maple === "string" ? value.maple : "unknown" - const createdAt = typeof value.createdAt === "string" ? value.createdAt : "unknown" - const schema = typeof value.schema === "string" ? value.schema : "" - if (!isIsoOrUnknown(createdAt)) throw new Error("marker createdAt is invalid") - if ( - value.formatVersion !== undefined && - value.formatVersion !== 1 && - value.formatVersion !== STORE_MARKER_FORMAT_VERSION - ) { - throw new Error(`unsupported marker format ${String(value.formatVersion)}`) + const formatVersion = (value as { readonly formatVersion?: unknown }).formatVersion + if (formatVersion !== undefined && formatVersion !== 1 && formatVersion !== STORE_MARKER_FORMAT_VERSION) { + throw new Error(`unsupported marker format ${String(formatVersion)}`) } - - if (value.formatVersion === STORE_MARKER_FORMAT_VERSION) { - if (typeof value.storeId !== "string" || value.storeId.length === 0) { - throw new Error("v2 marker storeId is missing") - } - if (!Number.isInteger(value.schemaVersion) || (value.schemaVersion as number) < 0) { - throw new Error("v2 marker schemaVersion is invalid") - } - if (!isHexDigest(value.schemaDigest)) throw new Error("v2 marker schemaDigest is invalid") - if (!/^[0-9a-f]{16}$/i.test(schema)) throw new Error("v2 marker schema fingerprint is invalid") - if (value.activation !== "active" && value.activation !== "staging") { - throw new Error("v2 marker activation is invalid") - } - const lastMigration = value.lastMigration - if (lastMigration !== undefined) { - if ( - !isRecord(lastMigration) || - typeof lastMigration.id !== "string" || - !Number.isInteger(lastMigration.fromVersion) || - !Number.isInteger(lastMigration.toVersion) || - !isIsoOrUnknown(lastMigration.completedAt) - ) { - throw new Error("v2 marker lastMigration is invalid") - } - } - return { - formatVersion: STORE_MARKER_FORMAT_VERSION, - storeId: value.storeId, - chdb: value.chdb, - maple, - createdAt, - createdByMaple: typeof value.createdByMaple === "string" ? value.createdByMaple : maple, - schemaVersion: value.schemaVersion as number, - schemaDigest: value.schemaDigest, - schema, - activation: value.activation, - ...(!(lastMigration === undefined) - ? { - lastMigration: { - id: lastMigration.id as string, - completedAt: lastMigration.completedAt as string, - fromVersion: lastMigration.fromVersion as number, - toVersion: lastMigration.toVersion as number, - }, - } - : undefined), - } + if (formatVersion === STORE_MARKER_FORMAT_VERSION) { + const marker = decodeMarkerV2(value) + // The one default a struct cannot express: an unrecorded creator is the + // running version, not "unknown". + return marker.createdByMaple === "unknown" ? { ...marker, createdByMaple: marker.maple } : marker } - - // The original marker had no formatVersion. Treat it as v1. A malformed - // object must not silently become a legacy store, so the required chdb field - // above is the only compatibility relaxation. - return { formatVersion: 1, chdb: value.chdb, maple, createdAt, schema } + return decodeMarkerV1(value) } /** Read the marker with an explicit missing/malformed distinction. */ @@ -226,16 +239,13 @@ export const makeStoreMarker = ( schema: string, options: StoreMarkerWriteOptions = {}, ): StoreMarkerV2 => { - if (!/^[0-9a-f]{16}$/i.test(schema)) { - throw new Error("a 16-character schema fingerprint is required for a v2 marker") - } - if (!options.schemaDigest || !/^[0-9a-f]{64}$/i.test(options.schemaDigest)) { - throw new Error("a full 64-character schemaDigest is required for a v2 marker") - } - if (options.schemaVersion === undefined || !Number.isInteger(options.schemaVersion)) { + if (options.schemaVersion === undefined) { throw new Error("a schemaVersion is required for a v2 marker") } - return { + // Constructing through the same schema the reader decodes with is the point: + // the fingerprint and digest rules are stated once, and a marker this build + // writes is a marker this build can read back. + return decodeMarkerV2({ formatVersion: STORE_MARKER_FORMAT_VERSION, storeId: options.storeId ?? randomUUID(), chdb: CHDB_VERSION, @@ -247,7 +257,7 @@ export const makeStoreMarker = ( schema, activation: options.activation ?? "active", ...(!(options.lastMigration === undefined) ? { lastMigration: options.lastMigration } : undefined), - } + }) } /** Serialize a current marker for a known identity. */ diff --git a/apps/cli/test/journal-schema.test.ts b/apps/cli/test/journal-schema.test.ts new file mode 100644 index 000000000..f1bc07bdf --- /dev/null +++ b/apps/cli/test/journal-schema.test.ts @@ -0,0 +1,143 @@ +// Golden decode tests for the coordinator journal envelope. +// +// The fixture below is a literal copy of what a shipped build writes to +// `maple-store-migration.json`. Its purpose is to fail loudly if the schema +// ever stops accepting a journal an installed store already holds — the +// declaration is only safe because this pins the accepted form. +import { Schema } from "effect" +import { describe, expect, it } from "vitest" +import { + decodeMigrationJournal, + MigrationJournalSchema, +} from "../src/server/local-store-migrations/journal-schema" + +const golden = { + formatVersion: 2, + migrationId: "local-0007-to-0008-apple-crash-frames-20260819T101500000Z", + phase: "copying", + chain: [ + { + id: "local-0007-to-0008-apple-crash-frames", + moduleVersion: 1, + from: { + version: 7, + fingerprint: "0123456789abcdef", + digest: "a".repeat(64), + manifestDigest: "b".repeat(64), + chdb: "3.6.0", + projectRevision: "c".repeat(64), + }, + to: { + version: 8, + fingerprint: "51081e951066442a", + digest: "d".repeat(64), + manifestDigest: "e".repeat(64), + chdb: "3.6.0", + projectRevision: "f".repeat(64), + }, + status: "running", + state: { module: "local-0007-to-0008-apple-crash-frames", version: 1, rawRows: {} }, + progress: { installed: true }, + }, + ], + currentStepIndex: 0, + sourceDataDir: "/var/lib/maple/data", + sourceStoreId: "8b7d9f1e-0000-4000-8000-000000000001", + sourceChdb: "3.6.0", + sourceFingerprint: "0123456789abcdef", + sourceDigest: "a".repeat(64), + sourceVersion: 7, + targetDataDir: "/var/lib/maple/.maple-migrations/m/target/data", + targetStoreId: "8b7d9f1e-0000-4000-8000-000000000002", + targetChdb: "3.6.0", + targetFingerprint: "51081e951066442a", + targetDigest: "d".repeat(64), + targetVersion: 8, + cutoffAt: "2026-08-19T10:15:00.000Z", + createdAt: "2026-08-19T10:15:00.000Z", +} + +describe("migration journal envelope", () => { + it("accepts a journal written by a shipped build, unchanged", () => { + expect(decodeMigrationJournal(structuredClone(golden))).toEqual(golden) + }) + + it("normalizes data directories so an unnormalized journal still matches its store", () => { + const decoded = decodeMigrationJournal({ + ...structuredClone(golden), + sourceDataDir: "/var/lib/maple/./data", + targetDataDir: "/var/lib/maple/.maple-migrations/m/target/../target/data", + }) + expect(decoded.sourceDataDir).toBe("/var/lib/maple/data") + expect(decoded.targetDataDir).toBe("/var/lib/maple/.maple-migrations/m/target/data") + }) + + it("defaults a missing source digest, because the v0 legacy identity has none", () => { + const { sourceDigest: _dropped, ...withoutSourceDigest } = structuredClone(golden) + expect(decodeMigrationJournal(withoutSourceDigest).sourceDigest).toBe("") + }) + + it("keeps absent optional identity fields absent rather than present-undefined", () => { + const journal = structuredClone(golden) + const { manifestDigest: _m, projectRevision: _p, ...leanFrom } = journal.chain[0]!.from + const decoded = decodeMigrationJournal({ + ...journal, + chain: [{ ...journal.chain[0]!, from: leanFrom }], + }) + expect("manifestDigest" in decoded.chain[0]!.from).toBe(false) + expect(Object.keys(decoded.chain[0]!.from)).not.toContain("projectRevision") + }) + + it("rejects a journal written by a build this one does not know", () => { + expect(() => decodeMigrationJournal({ ...structuredClone(golden), somethingElse: 1 })).toThrow() + expect(() => decodeMigrationJournal({ ...structuredClone(golden), formatVersion: 3 })).toThrow() + }) + + it("rejects the shapes the hand-rolled parser rejected", () => { + const bad: ReadonlyArray> = [ + { migrationId: "" }, + { migrationId: "../escape" }, + { phase: "somewhere-else" }, + { currentStepIndex: -1 }, + { currentStepIndex: 1.5 }, + { chain: [] }, + { sourceVersion: "7" }, + { targetDigest: "" }, + { sourceStoreId: "" }, + { failure: 42 }, + ] + for (const patch of bad) { + expect(() => decodeMigrationJournal({ ...structuredClone(golden), ...patch })).toThrow() + } + for (const value of [null, [], "x", 1, undefined]) { + expect(() => decodeMigrationJournal(value)).toThrow() + } + }) + + it("rejects a malformed step without interpreting module state", () => { + const step = golden.chain[0]! + for (const patch of [ + { status: "halfway" }, + { moduleVersion: 0 }, + { id: "" }, + { from: { ...step.from, version: -1 } }, + { to: { ...step.to, chdb: "" } }, + ]) { + expect(() => + decodeMigrationJournal({ ...structuredClone(golden), chain: [{ ...step, ...patch }] }), + ).toThrow() + } + // Opaque module state is the module's business, not the coordinator's. + expect(() => + decodeMigrationJournal({ + ...structuredClone(golden), + chain: [{ ...step, state: { anything: [1, 2, 3] } }], + }), + ).not.toThrow() + }) + + it("round-trips through the schema without changing the persisted form", () => { + const journal = decodeMigrationJournal(structuredClone(golden)) + expect(Schema.encodeSync(MigrationJournalSchema)(journal)).toEqual(golden) + }) +}) diff --git a/apps/cli/test/local-store-migrations.test.ts b/apps/cli/test/local-store-migrations.test.ts index 6dcedd88c..fba79310a 100644 --- a/apps/cli/test/local-store-migrations.test.ts +++ b/apps/cli/test/local-store-migrations.test.ts @@ -1,3 +1,4 @@ +import { v7ToV8AppleCrashFramesModule } from "../src/server/local-store-migrations/v7-to-v8-apple-crash-frames" import { describe, expect, it } from "vitest" import { CURRENT_LOCAL_SCHEMA, @@ -15,8 +16,10 @@ import { LOCAL_SCHEMA_V4_MANIFEST, LOCAL_SCHEMA_V5, LOCAL_SCHEMA_V5_MANIFEST, + LOCAL_SCHEMA_V7_MANIFEST, LOCAL_SCHEMA_V6, LOCAL_SCHEMA_V7, + LOCAL_SCHEMA_V8, SCHEMA_DIGEST, SCHEMA_FINGERPRINT, } from "../src/server/schema-identity" @@ -58,16 +61,16 @@ import { tmpdir } from "node:os" import { join } from "node:path" describe("current local schema identity", () => { - it("matches the generated v7 revision and keeps the issue-297 identity frozen", () => { - expect(SCHEMA_FINGERPRINT).toBe("bc124f30765c8c56") - expect(SCHEMA_DIGEST).toBe("bc124f30765c8c567daccab1872a0e15afbc8ef2123c264bcb5bcdd8d16b6c3c") + it("matches the generated v8 revision and keeps the issue-297 identity frozen", () => { + expect(SCHEMA_FINGERPRINT).toBe("51081e951066442a") + expect(SCHEMA_DIGEST).toBe("51081e951066442a8e5b53df2c4bdda933edd20fc89132a54ed9b4dbb7e55a05") expect(ISSUE_297_TARGET_SCHEMA_PROJECT_REVISION).toBe( "506bc745f7a7eca202ec905a6403a6815e86413faf0cd3cbbf73881023edce91", ) expect(CURRENT_SCHEMA_PROJECT_REVISION).toMatch(/^[0-9a-f]{64}$/) expect(LOCAL_SCHEMA_MANIFEST.objects.length).toBeGreaterThan(60) - expect(CURRENT_LOCAL_SCHEMA.version).toBe(7) - expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V7) + expect(CURRENT_LOCAL_SCHEMA.version).toBe(8) + expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V8) const logs = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === "logs") expect(logs?.columns.some((column) => column.name.startsWith("idx_"))).toBe(false) expect(logs?.indexes).toContain("idx_lower_body") @@ -149,6 +152,18 @@ describe("current local schema identity", () => { ) expect(v5ErrorEventsView?.definition).not.toContain("_httpStatus") }) + + it("recognises Apple crash frames at v8 but not before", () => { + const applePattern = "^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+" + for (const name of ["error_events_mv", "error_events_by_time_mv"]) { + const view = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === name) + expect(view?.definition).toContain(applePattern) + } + // v7 matched no Apple frame at all, so every iOS crash fell through to the + // message hash and collapsed into one issue per exception type. + const v7View = LOCAL_SCHEMA_V7_MANIFEST.objects.find((object) => object.name === "error_events_mv") + expect(v7View?.definition).not.toContain(applePattern) + }) }) describe("local migration registry", () => { @@ -162,6 +177,7 @@ describe("local migration registry", () => { "local-0004-to-0005-service-overview-minutely", "local-0005-to-0006-error-events-fingerprint-hygiene", "local-0006-to-0007-error-service-version", + "local-0007-to-0008-apple-crash-frames", ]) expect(chain[0]?.from.fingerprint).toBe(LEGACY_SCHEMA_FINGERPRINT) expect(chain[0]?.to).toEqual(LOCAL_SCHEMA_V1) @@ -208,7 +224,7 @@ describe("local migration registry", () => { // One past the current tip — bump alongside LOCAL_SCHEMA_VERSION, or this // stops testing the future-store guard and starts testing the // unknown-fingerprint one. - { ...CURRENT_LOCAL_SCHEMA, version: 8, fingerprint: "future", digest: SCHEMA_DIGEST }, + { ...CURRENT_LOCAL_SCHEMA, version: 9, fingerprint: "future", digest: SCHEMA_DIGEST }, CURRENT_LOCAL_SCHEMA, ), ).toThrow(/newer than this build/) @@ -420,7 +436,9 @@ describe("durable migration recovery", () => { ...base, chain: [{ ...base.chain[0]!, progress: { sourceInventory: [], copied: {} } }], }) - await expect(readMigrationJournal(dataDir)).rejects.toThrow(/sourceInventory must be an object/) + // The message is the schema's, so this asserts the failing field rather + // than the phrasing: an array where the inventory map belongs. + await expect(readMigrationJournal(dataDir)).rejects.toThrow(/sourceInventory/) } finally { await rm(root, { recursive: true, force: true }) } @@ -1000,3 +1018,178 @@ describe("legacy raw replay cursor", () => { expect(() => legacyToCurrentModule.decodeProgress(progress)).toThrow(/lastHash/) }) }) + +describe("v7 -> v8 journal state decoding", () => { + const RAW_TABLES = [ + "logs", + "traces", + "metrics_sum", + "metrics_gauge", + "metrics_histogram", + "metrics_exponential_histogram", + ] + const rawRows = Object.fromEntries(RAW_TABLES.map((table) => [table, "12"])) + const state = { module: "local-0007-to-0008-apple-crash-frames", version: 1, rawRows } + + it("round-trips a valid state, with and without a retention floor", () => { + expect(v7ToV8AppleCrashFramesModule.decodeState(state)).toEqual(state) + expect(v7ToV8AppleCrashFramesModule.decodeState({ ...state, retentionDays: 90 })).toEqual({ + ...state, + retentionDays: 90, + }) + }) + + it("rejects a field this build does not know about", () => { + // A journal carrying an unknown field was written by a different build. + // Dropping it silently would resume someone else's migration under our + // assumptions. + expect(() => v7ToV8AppleCrashFramesModule.decodeState({ ...state, somethingElse: 1 })).toThrow() + }) + + it("rejects another module's state", () => { + expect(() => + v7ToV8AppleCrashFramesModule.decodeState({ + ...state, + module: "local-0006-to-0007-error-service-version", + }), + ).toThrow() + expect(() => v7ToV8AppleCrashFramesModule.decodeState({ ...state, version: 2 })).toThrow() + }) + + it("rejects row counts that are not unsigned decimal strings", () => { + // They are strings precisely because a count can exceed + // Number.MAX_SAFE_INTEGER, so anything lossy has to fail loudly. + for (const bad of [12, "-1", "1.5", "1e3", ""]) { + expect(() => + v7ToV8AppleCrashFramesModule.decodeState({ ...state, rawRows: { ...rawRows, logs: bad } }), + ).toThrow() + } + }) + + it("rejects a missing or unknown raw table", () => { + const { logs: _dropped, ...missing } = rawRows + expect(() => v7ToV8AppleCrashFramesModule.decodeState({ ...state, rawRows: missing })).toThrow() + expect(() => + v7ToV8AppleCrashFramesModule.decodeState({ ...state, rawRows: { ...rawRows, not_a_table: "1" } }), + ).toThrow() + }) + + it("rejects a non-integer retention floor", () => { + expect(() => v7ToV8AppleCrashFramesModule.decodeState({ ...state, retentionDays: 1.5 })).toThrow() + }) + + it("decodes progress, and treats absent progress as absent", () => { + expect(v7ToV8AppleCrashFramesModule.decodeProgress(undefined)).toBeUndefined() + expect(v7ToV8AppleCrashFramesModule.decodeProgress({ installed: true })).toEqual({ installed: true }) + expect(() => v7ToV8AppleCrashFramesModule.decodeProgress({ installed: false })).toThrow() + expect(() => v7ToV8AppleCrashFramesModule.decodeProgress({})).toThrow() + }) +}) + +/** + * Characterization of the legacy raw-replay journal decoder. + * + * It guards a resumable copy out of a pre-v1 store: a journal it wrongly + * accepts resumes someone else's copy under this build's assumptions, and one + * it wrongly rejects strands a user mid-migration. Only one case was pinned + * before this, so these lock the accept/reject boundary in place. + */ +describe("legacy raw replay progress decoding", () => { + const inventory = { + table: "logs", + rowCount: "10", + retentionStartAt: "2026-01-01 00:00:00", + minTime: null, + maxTime: null, + hashSum: "1", + hashXor: "2", + } + const copied = { + rows: 1, + bytes: 2, + lastTimestamp: null, + lastHash: "12", + lastTieBreak: "13", + duplicateCount: 0, + duplicateGroupExhausted: false, + } + const pendingBatch = { + table: "logs", + rowCount: 1, + byteLength: 2, + firstTimestamp: null, + firstHash: "1", + firstTieBreak: "2", + lastTimestamp: null, + lastHash: "3", + lastTieBreak: "4", + lastKeyCount: 1, + lastKeyExhausted: false, + signature: "a".repeat(64), + } + const progress = { sourceInventory: { logs: inventory }, copied: { logs: copied } } + const decode = (value: unknown) => legacyToCurrentModule.decodeProgress(value) + + it("accepts a well-formed progress, with and without a pending batch", () => { + expect(decode(progress)).toEqual(progress) + expect(decode({ ...progress, pendingBatch })).toEqual({ ...progress, pendingBatch }) + // Absent progress is "not started", which is not the same as invalid. + expect(decode(undefined)).toBeUndefined() + }) + + it("rejects a non-object, and unknown top-level fields", () => { + for (const bad of [null, [], "x", 1]) expect(() => decode(bad)).toThrow() + expect(() => decode({ ...progress, somethingElse: 1 })).toThrow() + }) + + it("rejects tables that are not registered raw tables", () => { + expect(() => decode({ ...progress, sourceInventory: { not_a_table: inventory } })).toThrow() + expect(() => decode({ ...progress, copied: { not_a_table: copied } })).toThrow() + expect(() => + decode({ ...progress, pendingBatch: { ...pendingBatch, table: "not_a_table" } }), + ).toThrow() + }) + + it("rejects an inventory whose table disagrees with its key", () => { + expect(() => + decode({ ...progress, sourceInventory: { logs: { ...inventory, table: "traces" } } }), + ).toThrow() + }) + + it("rejects cursors that are not unsigned decimal strings", () => { + // These are interpolated into numeric SQL comparisons, so anything that + // could change their meaning has to fail here rather than there. + for (const bad of ["-1", "1.5", "0x10", "", 12]) { + expect(() => decode({ ...progress, copied: { logs: { ...copied, lastHash: bad } } })).toThrow() + } + // null is a legitimate "no cursor yet". + expect(decode({ ...progress, copied: { logs: { ...copied, lastHash: null } } })).toBeDefined() + }) + + it("rejects counters that are not non-negative safe integers", () => { + for (const bad of [-1, 1.5, Number.MAX_SAFE_INTEGER + 2, "1", null]) { + expect(() => decode({ ...progress, copied: { logs: { ...copied, rows: bad } } })).toThrow() + } + }) + + it("rejects a non-boolean exhaustion flag", () => { + expect(() => + decode({ ...progress, copied: { logs: { ...copied, duplicateGroupExhausted: "no" } } }), + ).toThrow() + }) + + it("rejects a pending batch that is empty or wrongly signed", () => { + // An empty batch would commit nothing while advancing the cursor past it. + expect(() => decode({ ...progress, pendingBatch: { ...pendingBatch, rowCount: 0 } })).toThrow() + for (const bad of ["a".repeat(63), "z".repeat(64), ""]) { + expect(() => decode({ ...progress, pendingBatch: { ...pendingBatch, signature: bad } })).toThrow() + } + }) + + it("rejects missing fields anywhere in the tree", () => { + const { hashSum: _h, ...shortInventory } = inventory + expect(() => decode({ ...progress, sourceInventory: { logs: shortInventory } })).toThrow() + const { rows: _r, ...shortCopied } = copied + expect(() => decode({ ...progress, copied: { logs: shortCopied } })).toThrow() + }) +}) diff --git a/apps/cli/test/native-local-store-migration.sh b/apps/cli/test/native-local-store-migration.sh index 3e79c7a36..0c6444e38 100755 --- a/apps/cli/test/native-local-store-migration.sh +++ b/apps/cli/test/native-local-store-migration.sh @@ -142,7 +142,7 @@ grep -q "local store migrated" "$ROOT/migrate.out" || fail "native migration did # must be bumped in lockstep with LOCAL_SCHEMA_VERSION and the matching # LOCAL_SCHEMA_V.fingerprint in apps/cli/src/server/schema-identity.ts; # leaving it on the previous version is what makes this step fail after a bump. -jq -e '.formatVersion == 2 and .activation == "active" and .schemaVersion == 7 and .schema == "bc124f30765c8c56"' \ +jq -e '.formatVersion == 2 and .activation == "active" and .schemaVersion == 8 and .schema == "51081e951066442a"' \ "$ROOT/maple-store-version.json" >/dev/null || fail "native migration wrote the wrong active identity" step "reopening promoted store in a fresh server" diff --git a/apps/cli/test/store-marker-schema.test.ts b/apps/cli/test/store-marker-schema.test.ts new file mode 100644 index 000000000..1acadc824 --- /dev/null +++ b/apps/cli/test/store-marker-schema.test.ts @@ -0,0 +1,121 @@ +// Golden decode tests for the on-disk store marker. +// +// Both fixtures are literal copies of markers shipped builds have written. A +// marker that stops decoding is a store that stops opening, so the accepted +// form is pinned here rather than left implicit in the schema. +import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, expect, it } from "vitest" +import { makeStoreMarker, readMarkerState, storeMarkerPath } from "../src/server/store-version" +import { CHDB_VERSION } from "../src/version" + +const goldenV1 = { + chdb: "1.4.0", + maple: "0.1.7", + createdAt: "2026-01-14T09:30:00.000Z", + schema: "428701854f9fd30e", +} + +const goldenV2 = { + formatVersion: 2, + storeId: "8b7d9f1e-0000-4000-8000-000000000001", + chdb: "3.6.0", + maple: "0.4.2", + createdAt: "2026-08-19T10:15:00.000Z", + createdByMaple: "0.3.9", + schemaVersion: 8, + schemaDigest: "a".repeat(64), + schema: "51081e951066442a", + activation: "active", + lastMigration: { + id: "local-0007-to-0008-apple-crash-frames", + completedAt: "2026-08-19T10:20:00.000Z", + fromVersion: 7, + toVersion: 8, + }, +} + +const readAs = (value: unknown) => { + const root = mkdtempSync(join(tmpdir(), "maple-marker-")) + const dataDir = join(root, "data") + mkdirSync(dataDir, { recursive: true }) + writeFileSync(storeMarkerPath(dataDir), JSON.stringify(value)) + return readMarkerState(dataDir) +} + +describe("store marker", () => { + it("accepts a v1 marker written before the format was versioned", () => { + const state = readAs(goldenV1) + expect(state).toEqual({ kind: "valid", marker: { formatVersion: 1, ...goldenV1 } }) + }) + + it("accepts a v2 marker written by a shipped build, unchanged", () => { + const state = readAs(goldenV2) + expect(state.kind === "valid" && state.marker).toEqual(goldenV2) + }) + + it("degrades unusable provenance rather than refusing to open the store", () => { + // Who made the store is not something the loader acts on. Identity is. + const state = readAs({ ...goldenV2, maple: 42, createdAt: null }) + expect(state.kind === "valid" && state.marker.maple).toBe("unknown") + expect(state.kind === "valid" && state.marker.createdAt).toBe("unknown") + }) + + it("falls back to the recorded maple version when no creator was stamped", () => { + const { createdByMaple: _dropped, ...withoutCreator } = goldenV2 + const state = readAs(withoutCreator) + expect(state.kind === "valid" && state.marker.createdByMaple).toBe(goldenV2.maple) + }) + + it("reports a malformed identity instead of guessing at it", () => { + for (const patch of [ + { schema: "not-hex" }, + { schema: "51081e951066442" }, + { schemaDigest: "a".repeat(63) }, + { schemaVersion: -1 }, + { schemaVersion: 1.5 }, + { storeId: "" }, + { activation: "somewhere-else" }, + { chdb: "" }, + { lastMigration: { id: "x", completedAt: "nope", fromVersion: 7, toVersion: 8 } }, + ]) { + expect(readAs({ ...goldenV2, ...patch }).kind).toBe("malformed") + } + expect(readAs({ ...goldenV2, formatVersion: 3 }).kind).toBe("malformed") + expect(readAs({ maple: "0.1.0" }).kind).toBe("malformed") + expect(readAs([]).kind).toBe("malformed") + }) + + it("constructs a marker through the same rules it reads one with", () => { + const marker = makeStoreMarker("0.4.2", "2026-08-19T10:15:00.000Z", "51081e951066442a", { + storeId: goldenV2.storeId, + schemaVersion: 8, + schemaDigest: "a".repeat(64), + }) + expect(marker).toEqual({ + formatVersion: 2, + storeId: goldenV2.storeId, + chdb: CHDB_VERSION, + maple: "0.4.2", + createdAt: "2026-08-19T10:15:00.000Z", + createdByMaple: "0.4.2", + schemaVersion: 8, + schemaDigest: "a".repeat(64), + schema: "51081e951066442a", + activation: "active", + }) + expect(readAs(marker).kind).toBe("valid") + }) + + it("refuses to construct a marker without a full identity", () => { + const now = "2026-08-19T10:15:00.000Z" + expect(() => + makeStoreMarker("0.4.2", now, "not-hex", { schemaVersion: 8, schemaDigest: "a".repeat(64) }), + ).toThrow() + expect(() => makeStoreMarker("0.4.2", now, "51081e951066442a", { schemaVersion: 8 })).toThrow() + expect(() => + makeStoreMarker("0.4.2", now, "51081e951066442a", { schemaDigest: "a".repeat(64) }), + ).toThrow(/schemaVersion/) + }) +}) diff --git a/apps/ingest/src/clickhouse_insert_mappings.rs b/apps/ingest/src/clickhouse_insert_mappings.rs index 84ea83c10..e3eae1755 100644 --- a/apps/ingest/src/clickhouse_insert_mappings.rs +++ b/apps/ingest/src/clickhouse_insert_mappings.rs @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-insert-mappings.ts // Do not edit manually. -pub const PROJECT_REVISION: &str = "73f8f3249a508cd05598289b67b3773a049e38db0302efa6aa9e45e3501d2182"; +pub const PROJECT_REVISION: &str = "bb7da950a3a65af75fcf627bf4ed0436308c98fee86906048b05b6d40d9f7534"; // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the // Tinybird-coupled PROJECT_REVISION. Compared against // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse diff --git a/packages/domain/src/clickhouse/migrations/0018_apple_crash_frames.ts b/packages/domain/src/clickhouse/migrations/0018_apple_crash_frames.ts new file mode 100644 index 000000000..6174bcbd4 --- /dev/null +++ b/packages/domain/src/clickhouse/migrations/0018_apple_crash_frames.ts @@ -0,0 +1,53 @@ +/** + * Migration 0018 — recognise Apple crash frames in the error fingerprint. + * + * `FRAME_LINE_PATTERN` matches stack frames by shape, one alternative per + * runtime. It had no alternative for Apple's, so every frame of an iOS crash + * reported by `maple-swift` was skipped, `_fpFrames` came out empty, and the + * hash fell through to `_msgFallback` — which redacts hex and long digit runs, + * collapsing `EXC_BAD_ACCESS at 0x10` and `EXC_BAD_ACCESS at 0xdeadbeef` into + * one signature. The practical result was one issue per exception type per + * service, no matter how many distinct bugs were behind it. + * + * The new alternative, `^[0-9]+ +[^ ]+ +0x[0-9a-fA-F]+`, keys on frame index + + * binary name + hex address. There is no source position to key on: an iOS + * crash arrives unsymbolicated, because the app's symbols live in a dSYM that + * never leaves the build machine. `FRAME_REDACTIONS` then erases the address and + * the (deliberately hex-rendered) offset, so a frame reduces to `index binary +` + * and grouping keys on the sequence of binaries. Coarse, but stable across + * releases — raw offsets shift with any code change and would re-split every iOS + * issue on every build. + * + * `FINGERPRINT_VERSION` is deliberately NOT bumped, even though iOS hashes + * rotate. That constant drives a version-keyed sweep in `ErrorsService` which + * archives every `kind: "error"` issue below the current version, on the premise + * that an older-version row can never receive another occurrence — true only + * when a bump rotates every hash. This change rotates iOS hashes alone, so a + * bump would archive every issue of every other runtime in every org, and + * nothing would un-archive them. See the note on `FINGERPRINT_VERSION`. + * + * NOTHING IS BACKFILLED either, for the reason 0003 and 0016 give: recomputing + * `FingerprintHash` would re-bucket every existing issue. iOS crashes already in + * `error_events` keep their collapsed hashes for the rest of their TTL, and the + * collapsed issues retire through the ordinary resolved window once their hash + * changes and they stop receiving occurrences. + * + * The CREATE statements below are the verbatim DDL as the schema emitter + * produced it at v18. Frozen history: never re-derive them from a later + * snapshot. + * + * `requiredForIngest: false` — nothing writes `error_events` directly, so this + * is a read-path correction, and gating on it would un-ready every + * BYO-ClickHouse org's ingest routing for a change the gateway never sees. + */ +export const migration_0018_apple_crash_frames = { + version: 18, + description: "Recreate the error_events MVs so the fingerprint recognises Apple crash frames", + requiredForIngest: false, + statements: [ + "DROP VIEW IF EXISTS error_events_mv", + "DROP VIEW IF EXISTS error_events_by_time_mv", + "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues \u2014 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist \u2014 works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '\u2514\u2500') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted \u2014 these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", + "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues \u2014 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist \u2014 works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '\u2514\u2500') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted \u2014 these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", + ], +} as const diff --git a/packages/domain/src/clickhouse/migrations/index.test.ts b/packages/domain/src/clickhouse/migrations/index.test.ts index ab14553a7..714722487 100644 --- a/packages/domain/src/clickhouse/migrations/index.test.ts +++ b/packages/domain/src/clickhouse/migrations/index.test.ts @@ -23,6 +23,7 @@ import { } from "./0015_service_overview_minutely" import { migration_0016_error_events_4xx_and_frame_redaction } from "./0016_error_events_4xx_and_frame_redaction" import { migration_0017_error_service_version_columns } from "./0017_error_service_version_columns" +import { migration_0018_apple_crash_frames } from "./0018_apple_crash_frames" import { clickHouseSchemaVersion, latestMigrationVersion, migrations } from "./index" const backfills = migration_0004_service_namespace_projections.statements.filter( @@ -38,12 +39,12 @@ const renderedSql = migration_0004_service_namespace_projections.statements describe("ClickHouse migrations", () => { it("keeps migrations ordered by version", () => { expect(migrations.map((m) => m.version)).toEqual([ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, ]) - expect(migrations.at(-1)).toBe(migration_0017_error_service_version_columns) - expect(latestMigrationVersion).toBe(17) - // 0010, 0014, 0015, 0016 and 0017 are read-path only, so the ingest-gating - // version skips all five and stays at 13 — nothing writes `web_events`, + expect(migrations.at(-1)).toBe(migration_0018_apple_crash_frames) + expect(latestMigrationVersion).toBe(18) + // 0010 and 0014-0018 are read-path only, so the ingest-gating version skips + // all six and stays at 13 — nothing writes `web_events`, // `service_overview_minutely` or `error_events` directly, and bumping it // would un-ready every BYO-CH org's ingest routing for a read-path change. expect(clickHouseSchemaVersion).toBe("13") @@ -52,6 +53,7 @@ describe("ClickHouse migrations", () => { expect(migration_0015_service_overview_minutely.requiredForIngest).toBe(false) expect(migration_0016_error_events_4xx_and_frame_redaction.requiredForIngest).toBe(false) expect(migration_0017_error_service_version_columns.requiredForIngest).toBe(false) + expect(migration_0018_apple_crash_frames.requiredForIngest).toBe(false) }) it("recreates both error-events MVs with the 4xx guard and the widened frame redaction", () => { @@ -498,3 +500,26 @@ describe("ClickHouse migrations", () => { ) }) }) + +describe("migration 0018 — Apple crash frames", () => { + const creates = migration_0018_apple_crash_frames.statements.filter((stmt) => stmt.startsWith("CREATE")) + + it("recreates both error-events MVs with the Apple frame alternative", () => { + expect(creates).toHaveLength(2) + for (const sql of creates) { + // Frame index, binary name, hex address — an iOS crash has no source + // position to key on, because it arrives unsymbolicated. + expect(sql).toContain("^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+") + // The other runtimes' alternatives are untouched; only iOS hashes rotate. + expect(sql).toContain('^[ \\\\t]*at |^[ \\\\t]*File "') + } + }) + + it("does not backfill", () => { + // Recomputing FingerprintHash would re-bucket every existing issue. The + // FINGERPRINT_VERSION bump retires the collapsed iOS issues instead. + expect(migration_0018_apple_crash_frames.statements.some((stmt) => stmt.includes("UPDATE"))).toBe( + false, + ) + }) +}) diff --git a/packages/domain/src/clickhouse/migrations/index.ts b/packages/domain/src/clickhouse/migrations/index.ts index 79dc2395b..82835fa4c 100644 --- a/packages/domain/src/clickhouse/migrations/index.ts +++ b/packages/domain/src/clickhouse/migrations/index.ts @@ -16,6 +16,7 @@ import { migration_0014_web_events } from "./0014_web_events" import { migration_0015_service_overview_minutely } from "./0015_service_overview_minutely" import { migration_0016_error_events_4xx_and_frame_redaction } from "./0016_error_events_4xx_and_frame_redaction" import { migration_0017_error_service_version_columns } from "./0017_error_service_version_columns" +import { migration_0018_apple_crash_frames } from "./0018_apple_crash_frames" /** * A migration statement is either a raw SQL string (structural DDL) or a @@ -64,6 +65,7 @@ export const migrations: ReadonlyArray = [ migration_0015_service_overview_minutely, migration_0016_error_events_4xx_and_frame_redaction, migration_0017_error_service_version_columns, + migration_0018_apple_crash_frames, ] as const /** Highest migration `version` bundled — i.e. the schema level a fully-applied diff --git a/packages/domain/src/generated/clickhouse-schema.ts b/packages/domain/src/generated/clickhouse-schema.ts index 0a70c494a..8600820d3 100644 --- a/packages/domain/src/generated/clickhouse-schema.ts +++ b/packages/domain/src/generated/clickhouse-schema.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-schema.ts // Do not edit manually. -export const projectRevision = "73f8f3249a508cd05598289b67b3773a049e38db0302efa6aa9e45e3501d2182" as const +export const projectRevision = "bb7da950a3a65af75fcf627bf4ed0436308c98fee86906048b05b6d40d9f7534" as const export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS alert_checks (\n OrgId LowCardinality(String),\n RuleId String,\n GroupKey String,\n Timestamp DateTime64(3),\n Status LowCardinality(String),\n SignalType LowCardinality(String),\n Comparator LowCardinality(String),\n Threshold Float64,\n ObservedValue Nullable(Float64),\n SampleCount UInt32,\n WindowMinutes UInt16,\n WindowStart DateTime64(3),\n WindowEnd DateTime64(3),\n ConsecutiveBreaches UInt16,\n ConsecutiveHealthy UInt16,\n IncidentId Nullable(String),\n IncidentTransition LowCardinality(String),\n EvaluationDurationMs UInt32,\n ErrorMessage Nullable(String),\n ErrorCategory LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, RuleId, GroupKey, Timestamp)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", @@ -42,8 +42,8 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS traces (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n ParentSpanId String,\n TraceState String,\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n ServiceName LowCardinality(String),\n ResourceSchemaUrl String,\n ResourceAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n Duration UInt64 DEFAULT 0,\n StatusCode LowCardinality(String),\n StatusMessage String,\n SpanAttributes Map(LowCardinality(String), String),\n EventsTimestamp Array(DateTime64(9)),\n EventsName Array(LowCardinality(String)),\n EventsAttributes Array(Map(LowCardinality(String), String)),\n LinksTraceId Array(String),\n LinksSpanId Array(String),\n LinksTraceState Array(String),\n LinksAttributes Array(Map(LowCardinality(String), String)),\n SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0),\n IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0),\n ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)),\n ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)),\n SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)),\n INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,\n INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp))\nTTL toDate(Timestamp) + INTERVAL 30 DAY", "CREATE TABLE IF NOT EXISTS traces_aggregates_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n SpanName LowCardinality(String),\n SpanKind LowCardinality(String),\n StatusCode LowCardinality(String),\n IsEntryPoint UInt8,\n DeploymentEnv LowCardinality(String),\n WeightedCount SimpleAggregateFunction(sum, Float64),\n WeightedDurationSum SimpleAggregateFunction(sum, Float64),\n WeightedErrorCount SimpleAggregateFunction(sum, Float64),\n DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32),\n DurationMin SimpleAggregateFunction(min, UInt64),\n DurationMax SimpleAggregateFunction(max, UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv)\nTTL toDate(Hour) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS web_events (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n SessionId String,\n Seq UInt32,\n Kind LowCardinality(String),\n EventName String,\n Host LowCardinality(String),\n PagePath String,\n Url String,\n Attributes Map(String, String),\n INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, SessionId, Seq)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", - "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", - "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", + "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", + "CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS\nWITH\n arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei,\n if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType,\n if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg,\n if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack,\n -- Frame lines are matched by SHAPE, not by \"contains :NUMBER\". The old\n -- rule accepted any line with a colon-digit, which let non-frame lines\n -- in: Drizzle's `params: ` line, and the `Type: message`\n -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values\n -- and message text then entered the hash and split one bug into\n -- thousands of issues — 23,035 fingerprints for six real\n -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError\n -- ones.\n --\n -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts,\n -- as is every redaction below. They used to be hand-copied here, which\n -- let the reference implementation the tests exercise drift away from\n -- the SQL that actually runs, silently.\n arraySlice(\n arrayFilter(\n line -> match(line, '^[ \\\\t]*at |^[ \\\\t]*File \"|^[ \\\\t]+from [^ ]+:[0-9]+|^[^ \\\\t@]+@[^ \\\\t]*:[0-9]+|^[ \\\\t]+[^ \\\\t]+\\\\.(go|rs):[0-9]+|^[0-9]+ +\\\\S.* +0x[0-9a-fA-F]+'),\n splitByChar('\\n', _exStack)\n ),\n 1, 3\n ) AS _rawFrames,\n -- Redact every volatile token a frame line can carry: the URL origin\n -- (so preview hosts share one fingerprint), Vite's 8-char bundle\n -- content hash (so a deploy does not re-split every triaged browser and\n -- Worker issue), then line numbers, hex pointers and long id runs. See\n -- FRAME_REDACTIONS for the order and the reasoning.\n arrayMap(\n line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''),\n _rawFrames\n ) AS _topFrames,\n if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame,\n arrayStringConcat(_topFrames, '\\n') AS _fpFrames,\n -- JSON detection for the message signature below.\n isValidJSON(StatusMessage) AS _isJson,\n _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj,\n -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level\n -- keys, redact volatile tokens (long hex / numbers) in each raw value, then\n -- sort by \"key=value\" so key order & whitespace don't matter. No assumption\n -- about which keys exist — works for any producer's JSON shape. (Nested\n -- objects are hashed as their raw substring; only top-level is canonicalized.)\n arrayStringConcat(\n arraySort(\n arrayMap(\n kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')),\n JSONExtractKeysAndValuesRaw(StatusMessage)\n )\n ),\n '|'\n ) AS _jsonSig,\n -- The message signature is folded in ALWAYS, not only when there are no\n -- frames. Bundled runtimes minify every module into one file, so the top\n -- three frames of a Worker error are `toDatabaseError (worker.js)` for\n -- every failing query alike: on frames alone, 25 distinct DatabaseError\n -- bugs (316k occurrences) collapse into a single issue. The signature\n -- restores that discrimination, and it cannot reinflate cardinality the\n -- way a raw prefix would because everything variable is redacted first:\n -- emails, URL origins, home directories, query strings, quoted values,\n -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order,\n -- what is deliberately kept, and the one residual it cannot reach.\n multiIf(\n _isJsonObj, _jsonSig,\n substringUTF8(\n replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )\"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )\"]*', '?#'), '\\'[^\\' ]*/[^\\' ]*\\'|\\'[^\\' ]{25,}\\'', '\\'#\\''), '\"[^\" ]*/[^\" ]*\"|\"[^\" ]{25,}\"', '\"#\"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'),\n 1, 120\n )\n ) AS _msgSig,\n -- Display-only, best-effort human label (decoupled from the fingerprint:\n -- many labels may map to one hash). The broad key list here is a DISPLAY\n -- heuristic only; the fingerprint above makes no key-name assumption.\n multiIf(\n JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'),\n JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'),\n JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'),\n JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'),\n JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'),\n JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'),\n JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'),\n 'JSON error'\n ) AS _jsonLabel,\n multiIf(\n StatusMessage = '', 'Unknown Error',\n position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0,\n if(\n extract(StatusMessage, 'readonly (\\\\w+)') != '',\n concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\\\w+)')),\n 'Schema parse error'\n ),\n _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel,\n left(StatusMessage, multiIf(\n position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1,\n position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1,\n position(StatusMessage, '\\n') > 3, toInt64(position(StatusMessage, '\\n')) - 1,\n least(toInt64(length(StatusMessage)), 150)\n ))\n ) AS _statusLabel,\n if(_exType != '', _exType, _statusLabel) AS _errorLabel,\n -- Both semconv spellings; the current key wins when both are present.\n toUInt16OrZero(\n if(\n SpanAttributes['http.response.status_code'] != '',\n SpanAttributes['http.response.status_code'],\n SpanAttributes['http.status_code']\n )\n ) AS _httpStatus\n SELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv,\n _exType AS ExceptionType,\n _exMsg AS ExceptionMessage,\n _exStack AS ExceptionStacktrace,\n _topFrame AS TopFrame,\n cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash,\n StatusMessage,\n Duration,\n _errorLabel AS ErrorLabel,\n ResourceAttributes['service.version'] AS ServiceVersion\n FROM traces\n WHERE StatusCode = 'Error'\n -- Client-side runtimes (notably the native Cloudflare Workers\n -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot\n -- traffic arrived here as unlabelled \"Unknown Error\" issues. Drop a\n -- span only when all three hold: 4xx, no exception event, and no\n -- exception type. 5xx and anything carrying an exception still count,\n -- and SpanKind is deliberately not consulted — these are Client spans.\n AND NOT (\n _httpStatus >= 400 AND _httpStatus < 500\n AND _ei = 0\n AND _exType = ''\n )", "CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS\nSELECT\n OrgId,\n toStartOfMinute(Timestamp) AS Minute,\n FingerprintHash,\n anyLast(ServiceName) AS ServiceName,\n anyLast(ExceptionType) AS ExceptionType,\n anyLast(ExceptionMessage) AS ExceptionMessage,\n anyLast(ErrorLabel) AS ErrorLabel,\n anyLast(TopFrame) AS TopFrame,\n count() AS OccurrenceCount,\n min(Timestamp) AS FirstSeen,\n max(Timestamp) AS LastSeen,\n -- Distinct builds, not a sample: see ServiceVersions on the datasource.\n groupUniqArray(ServiceVersion) AS ServiceVersions\n FROM error_events\n GROUP BY OrgId, Minute, FingerprintHash", "CREATE MATERIALIZED VIEW IF NOT EXISTS error_spans_mv TO error_spans AS\nSELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n SpanId,\n ParentSpanId,\n ServiceName,\n StatusMessage,\n Duration,\n ResourceAttributes['deployment.environment'] AS DeploymentEnv\n FROM traces\n WHERE StatusCode = 'Error'", "CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n arrayJoin(mapKeys(LogAttributes)) AS AttributeKey,\n 'log' AS AttributeScope,\n count() AS UsageCount\n FROM logs\n WHERE LogAttributes != map()\n GROUP BY OrgId, Hour, AttributeKey, AttributeScope", diff --git a/packages/domain/src/tinybird/fingerprint.test.ts b/packages/domain/src/tinybird/fingerprint.test.ts index 1d36f26e9..d61dced36 100644 --- a/packages/domain/src/tinybird/fingerprint.test.ts +++ b/packages/domain/src/tinybird/fingerprint.test.ts @@ -625,3 +625,107 @@ describe("SQL parity", () => { } }) }) + +describe("Apple crash frames", () => { + // The shape `maple-swift`'s CrashReport.stacktrace renders: frame index, binary + // name, hex address, hex offset. Unsymbolicated — MetricKit has no symbols. + const appleStack = (frames: ReadonlyArray<[string, string, string]>) => + frames + .map(([binary, address, offset], i) => `${i} ${binary} 0x${address} +0x${offset}`) + .join("\n") + + it("matches Apple frames and strips address and offset", () => { + const result = computeFingerprintInputs({ + exceptionType: "EXC_BAD_ACCESS", + exceptionStacktrace: appleStack([ + ["MyApp", "104112600", "1d0f0"], + ["MyApp", "1041125a0", "a112"], + ["UIKitCore", "1a2b3c4d0", "44"], + ["libdyld.dylib", "700000000", "1000"], + ]), + statusMessage: "EXC_BAD_ACCESS (SIGSEGV)", + }) + + expect(result.topFrame).toBe("0 MyApp +") + expect(result.fpFrames.split("\n")).toHaveLength(MAX_FINGERPRINT_FRAMES) + expect(result.fpFrames).toContain("UIKitCore") + expect(result.fpFrames).not.toContain("libdyld") + }) + + it("groups the same crash site across rebuilds", () => { + // An offset moves with any code change above it. If it survived redaction every + // release would re-split every iOS issue. + const before = computeFingerprintInputs({ + exceptionType: "EXC_BAD_ACCESS", + exceptionStacktrace: appleStack([["MyApp", "104112600", "1d0f0"]]), + statusMessage: "", + }) + const after = computeFingerprintInputs({ + exceptionType: "EXC_BAD_ACCESS", + exceptionStacktrace: appleStack([["MyApp", "104999999", "2f4a1"]]), + statusMessage: "", + }) + expect(after.fpFrames).toBe(before.fpFrames) + }) + + it("separates crash sites that differ by binary", () => { + const inApp = computeFingerprintInputs({ + exceptionType: "EXC_BAD_ACCESS", + exceptionStacktrace: appleStack([["MyApp", "104112600", "1d0f0"]]), + statusMessage: "", + }) + const inUIKit = computeFingerprintInputs({ + exceptionType: "EXC_BAD_ACCESS", + exceptionStacktrace: appleStack([["UIKitCore", "1a2b3c4d0", "1d0f0"]]), + statusMessage: "", + }) + expect(inUIKit.fpFrames).not.toBe(inApp.fpFrames) + }) + + it("matches a binary name containing spaces", () => { + // A Mach-O image name is the target's PRODUCT_NAME, and "My App" is an + // ordinary thing to call an app. Requiring one space-free token silently + // excluded those apps from frame matching and left them collapsed on the + // message hash — the exact failure this alternative exists to fix. + const result = computeFingerprintInputs({ + exceptionType: "EXC_BAD_ACCESS", + exceptionStacktrace: [ + "0 My App 0x104112600 +0x1d0f0", + "1 My App 0x1041125a0 +0xa112", + ].join("\n"), + statusMessage: "", + }) + expect(result.fpFrames.split("\n")).toHaveLength(2) + expect(result.topFrame).toContain("My App") + expect(result.msgSignature).toBe("") + }) + + it("does not swallow other runtimes' lines", () => { + // The alternative is anchored on a leading frame index, so a message that merely + // mentions an address must not be read as a frame. + const result = computeFingerprintInputs({ + exceptionType: "Error", + exceptionStacktrace: "Error: mapping failed at 0x1f into region", + statusMessage: "", + }) + expect(result.fpFrames).toBe("") + }) + + it("does not match other runtimes' address-bearing frames", () => { + // The alternative is anchored on a leading frame index with no punctuation + // after it, which is what keeps these out. + for (const line of [ + " 1: 0xb09bc0 node::Abort() [node]", + "1: 0xb09bc0 node::Abort() [node]", + " #00 pc 0000000000045e7c /system/lib/libc.so", + " 0: rust_begin_unwind", + ]) { + const result = computeFingerprintInputs({ + exceptionType: "Error", + exceptionStacktrace: line, + statusMessage: "", + }) + expect(result.fpFrames, line).toBe("") + } + }) +}) diff --git a/packages/domain/src/tinybird/fingerprint.ts b/packages/domain/src/tinybird/fingerprint.ts index 4b03cc16c..f6af87776 100644 --- a/packages/domain/src/tinybird/fingerprint.ts +++ b/packages/domain/src/tinybird/fingerprint.ts @@ -73,9 +73,25 @@ export const MSG_SIGNATURE_CHARS = 120 * Ruby: ` from /app/user.rb:12:in 'find'` * Firefox/Safari: `getUser@https://app/assets/index.js:42:18` * Go/Rust: ` /app/main.go:42 +0x1d` + * Apple: `0 My App 0x104a2c1f0 +0x1d0f0` + * + * The Apple alternative has no source position to key on, because an iOS crash arrives + * unsymbolicated — the app's symbols live in a dSYM that never leaves the build machine. + * It keys on the shape instead: frame index, binary name, hex address. The offset is hex + * (the SDK renders it that way deliberately) so `FRAME_REDACTIONS` erases it along with + * the address, leaving `index binaryName +`. That is coarse — grouping by the sequence of + * binaries rather than of functions — but it is *stable across releases*, which the raw + * offsets are not: any code change shifts every offset below it and would re-split every + * issue on every build. When dSYM symbolication lands, function names drop into the same + * slot. + * + * The binary name is matched as `\\S.*`, not as one space-free token: a Mach-O image name + * is the target's PRODUCT_NAME, and `My App` is an ordinary thing to call an app. Keying + * on a single token silently excluded every such app from frame matching and left it + * collapsed on the message hash — the exact failure this alternative exists to fix. */ export const FRAME_LINE_PATTERN = - '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+' + '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+' /** An ordered `[pattern, replacement]` list, applied outermost-first. */ export type Redactions = ReadonlyArray @@ -312,5 +328,23 @@ export function computeFingerprintInputs(args: { * v1 → v2 (this change): frame lines are matched by shape rather than by * "contains a colon-digit", and the message signature is always folded in and * additionally strips quoted values and query strings. + * + * **Adding the Apple frame alternative deliberately did NOT bump this**, even + * though it rotates hashes for iOS crashes that are still occurring, which is + * what the rule above otherwise calls for. The retirement this version drives is + * version-keyed, not hash-keyed: `ErrorsService` archives every `kind: "error"` + * issue whose `fingerprintVersion` is below this constant, on the premise stated + * in `error_issues.fingerprintVersion` that a row on an older version can never + * receive another occurrence. That premise holds only when a bump rotates + * *every* hash. The Apple change rotates iOS hashes alone, so a bump would + * archive every Node, Python, Go and browser issue in every org — and nothing + * un-archives them, because the tick's upsert conflicts on + * `(orgId, fingerprintHash)` and never clears `archivedAt`. + * + * The cost of not bumping is small and bounded: the collapsed iOS issues stop + * receiving occurrences the moment their hash changes, and retire through the + * ordinary resolved window instead of on sight. Teaching the sweep to retire by + * hash rather than by version is what would let a partial-rotation change bump + * this safely. */ export const FINGERPRINT_VERSION = 2 diff --git a/scripts/check-local-schema-manifest.ts b/scripts/check-local-schema-manifest.ts index e1b06e11a..66874a1de 100644 --- a/scripts/check-local-schema-manifest.ts +++ b/scripts/check-local-schema-manifest.ts @@ -4,27 +4,7 @@ import { LOCAL_SCHEMA_HISTORY, LOCAL_SCHEMA_MANIFEST, LOCAL_SCHEMA_MANIFEST_DIGEST, - LOCAL_SCHEMA_V1, - LOCAL_SCHEMA_V1_MANIFEST_DIGEST, - LOCAL_SCHEMA_V1_SQL, - LOCAL_SCHEMA_V2, - LOCAL_SCHEMA_V2_MANIFEST_DIGEST, - LOCAL_SCHEMA_V2_SQL, - LOCAL_SCHEMA_V3, - LOCAL_SCHEMA_V3_MANIFEST_DIGEST, - LOCAL_SCHEMA_V3_SQL, - LOCAL_SCHEMA_V4, - LOCAL_SCHEMA_V4_MANIFEST_DIGEST, - LOCAL_SCHEMA_V4_SQL, - LOCAL_SCHEMA_V5, - LOCAL_SCHEMA_V5_MANIFEST_DIGEST, - LOCAL_SCHEMA_V5_SQL, - LOCAL_SCHEMA_V6, - LOCAL_SCHEMA_V6_MANIFEST_DIGEST, - LOCAL_SCHEMA_V6_SQL, - LOCAL_SCHEMA_V7, - LOCAL_SCHEMA_V7_MANIFEST_DIGEST, - LOCAL_SCHEMA_V7_SQL, + LOCAL_SCHEMA_SNAPSHOTS, LOCAL_SCHEMA_VERSION, } from "../apps/cli/src/server/schema-identity" import { resolveMigrationChain } from "../apps/cli/src/server/local-store-migrations" @@ -65,88 +45,31 @@ if (!sameIdentity(latest, CURRENT_LOCAL_SCHEMA)) { ) } -const v1 = LOCAL_SCHEMA_HISTORY.find((entry) => entry.version === LOCAL_SCHEMA_V1.version) -if ( - !v1 || - LOCAL_SCHEMA_V1_MANIFEST_DIGEST !== v1.manifestDigest || - schemaFingerprint(LOCAL_SCHEMA_V1_SQL) !== v1.fingerprint || - schemaDigest(LOCAL_SCHEMA_V1_SQL) !== v1.digest || - LOCAL_SCHEMA_V1.fingerprint !== v1.fingerprint || - LOCAL_SCHEMA_V1.digest !== v1.digest -) { - fail("the immutable local schema v1 snapshot no longer matches its historical identity") -} - -const v2 = LOCAL_SCHEMA_HISTORY.find((entry) => entry.version === LOCAL_SCHEMA_V2.version) -if ( - !v2 || - LOCAL_SCHEMA_V2_MANIFEST_DIGEST !== v2.manifestDigest || - schemaFingerprint(LOCAL_SCHEMA_V2_SQL) !== v2.fingerprint || - schemaDigest(LOCAL_SCHEMA_V2_SQL) !== v2.digest || - LOCAL_SCHEMA_V2.fingerprint !== v2.fingerprint || - LOCAL_SCHEMA_V2.digest !== v2.digest -) { - fail("the immutable local schema v2 snapshot no longer matches its historical identity") -} - -const v3 = LOCAL_SCHEMA_HISTORY.find((entry) => entry.version === LOCAL_SCHEMA_V3.version) -if ( - !v3 || - LOCAL_SCHEMA_V3_MANIFEST_DIGEST !== v3.manifestDigest || - schemaFingerprint(LOCAL_SCHEMA_V3_SQL) !== v3.fingerprint || - schemaDigest(LOCAL_SCHEMA_V3_SQL) !== v3.digest || - LOCAL_SCHEMA_V3.fingerprint !== v3.fingerprint || - LOCAL_SCHEMA_V3.digest !== v3.digest -) { - fail("the immutable local schema v3 snapshot no longer matches its historical identity") -} - -const v4 = LOCAL_SCHEMA_HISTORY.find((entry) => entry.version === LOCAL_SCHEMA_V4.version) -if ( - !v4 || - LOCAL_SCHEMA_V4_MANIFEST_DIGEST !== v4.manifestDigest || - schemaFingerprint(LOCAL_SCHEMA_V4_SQL) !== v4.fingerprint || - schemaDigest(LOCAL_SCHEMA_V4_SQL) !== v4.digest || - LOCAL_SCHEMA_V4.fingerprint !== v4.fingerprint || - LOCAL_SCHEMA_V4.digest !== v4.digest -) { - fail("the immutable local schema v4 snapshot no longer matches its historical identity") -} - -const v5 = LOCAL_SCHEMA_HISTORY.find((entry) => entry.version === LOCAL_SCHEMA_V5.version) -if ( - !v5 || - LOCAL_SCHEMA_V5_MANIFEST_DIGEST !== v5.manifestDigest || - schemaFingerprint(LOCAL_SCHEMA_V5_SQL) !== v5.fingerprint || - schemaDigest(LOCAL_SCHEMA_V5_SQL) !== v5.digest || - LOCAL_SCHEMA_V5.fingerprint !== v5.fingerprint || - LOCAL_SCHEMA_V5.digest !== v5.digest -) { - fail("the immutable local schema v5 snapshot no longer matches its historical identity") -} - -const v6 = LOCAL_SCHEMA_HISTORY.find((entry) => entry.version === LOCAL_SCHEMA_V6.version) -if ( - !v6 || - LOCAL_SCHEMA_V6_MANIFEST_DIGEST !== v6.manifestDigest || - schemaFingerprint(LOCAL_SCHEMA_V6_SQL) !== v6.fingerprint || - schemaDigest(LOCAL_SCHEMA_V6_SQL) !== v6.digest || - LOCAL_SCHEMA_V6.fingerprint !== v6.fingerprint || - LOCAL_SCHEMA_V6.digest !== v6.digest -) { - fail("the immutable local schema v6 snapshot no longer matches its historical identity") +// A loop is only a gate while it has something to iterate. The history carries +// one entry per schema version plus v0, which has no DDL of its own. +const snapshotCount = LOCAL_SCHEMA_SNAPSHOTS.filter(Boolean).length +if (snapshotCount !== LOCAL_SCHEMA_HISTORY.length - 1) { + fail( + `local schema snapshots (${snapshotCount}) do not cover every versioned history entry (${LOCAL_SCHEMA_HISTORY.length - 1})`, + ) } -const v7 = LOCAL_SCHEMA_HISTORY.find((entry) => entry.version === LOCAL_SCHEMA_V7.version) -if ( - !v7 || - LOCAL_SCHEMA_V7_MANIFEST_DIGEST !== v7.manifestDigest || - schemaFingerprint(LOCAL_SCHEMA_V7_SQL) !== v7.fingerprint || - schemaDigest(LOCAL_SCHEMA_V7_SQL) !== v7.digest || - LOCAL_SCHEMA_V7.fingerprint !== v7.fingerprint || - LOCAL_SCHEMA_V7.digest !== v7.digest -) { - fail("the immutable local schema v7 snapshot no longer matches its historical identity") +// Every frozen snapshot must still hash to the identity the history recorded +// for it. A snapshot that drifts silently retargets a historical migration +// edge, which is the failure this gate exists to make impossible. +for (const snapshot of LOCAL_SCHEMA_SNAPSHOTS) { + if (!snapshot) continue + const entry = LOCAL_SCHEMA_HISTORY.find((candidate) => candidate.version === snapshot.version) + if ( + !entry || + snapshot.manifestDigest !== entry.manifestDigest || + schemaFingerprint(snapshot.sql) !== entry.fingerprint || + schemaDigest(snapshot.sql) !== entry.digest + ) { + fail( + `the immutable local schema v${snapshot.version} snapshot no longer matches its historical identity`, + ) + } } const names = LOCAL_SCHEMA_MANIFEST.objects.map((object) => object.name)