From fb28e860ff592b0e7b4a2b07355d272e63ccd758 Mon Sep 17 00:00:00 2001 From: Tim Smart Date: Thu, 20 Aug 2026 10:22:22 +0000 Subject: [PATCH 1/2] Add SchemaBinary fingerprint positional mode Co-Authored-By: Claude Opus 5 --- .changeset/schema-binary-fingerprint-mode.md | 11 + .../effect/benchmark/schema/SchemaBinary.md | 89 ++- .../effect/benchmark/schema/SchemaBinary.ts | 64 +- .../src/unstable/encoding/SchemaBinary.ts | 694 +++++++++++++++--- .../unstable/encoding/SchemaBinary.test.ts | 494 +++++++++++++ 5 files changed, 1206 insertions(+), 146 deletions(-) create mode 100644 .changeset/schema-binary-fingerprint-mode.md diff --git a/.changeset/schema-binary-fingerprint-mode.md b/.changeset/schema-binary-fingerprint-mode.md new file mode 100644 index 00000000000..c91bf350693 --- /dev/null +++ b/.changeset/schema-binary-fingerprint-mode.md @@ -0,0 +1,11 @@ +--- +"effect": minor +--- + +Add an opt-in `SchemaBinary` fingerprint / positional wire mode. + +`SchemaBinary.toCodec(schema, { fingerprint: true })` and `SchemaBinary.parser(schema, { fingerprint: true })` select a second wire mode, chosen by envelope flag bit 0. Every frame carries an 8-byte 64-bit FNV-1a hash of the compiled wire layout, and a reader whose layout hashes differently rejects the frame instead of guessing. In exchange, structs are written positionally: no field ids, a presence bitmap for optional fields, no length prefix on fixed-size leaves, and a canonical varint index in place of the union kind byte and 32-bit sentinel tag. + +The hash covers wire-relevant structure only. Checks, annotations, decoded-side transformations, property declaration order, and whether a sub-schema is shared or repeated leave it unchanged; renames, added or removed fields, optionality, leaf types, tuple shape, and union membership change it. + +The default mode is unchanged and remains the default. The two modes are not interchangeable: a frame written in one is rejected by a codec built for the other. On the benchmark payloads, fingerprint mode is 1% to 40% smaller raw depending on the case, with the largest wins on per-frame streams of repeated records and no win on index-signature records, where the fingerprint costs more than the field ids it removes. diff --git a/packages/effect/benchmark/schema/SchemaBinary.md b/packages/effect/benchmark/schema/SchemaBinary.md index 62e166a103c..5a097b2db77 100644 --- a/packages/effect/benchmark/schema/SchemaBinary.md +++ b/packages/effect/benchmark/schema/SchemaBinary.md @@ -6,11 +6,11 @@ Run from the repository root: nix develop -c pnpm --dir packages/effect exec node benchmark/schema/SchemaBinary.ts ``` -The benchmark compares arena-backed `SchemaBinary`, `SchemaBinary` followed by an ownership copy, JSON, and Effect's Msgpack schema codec with the same Schema values. The copy case calls `.slice()` on every arena result, reproducing the allocation that the arena removes. It covers a small scalar-heavy record, a nested order payload, collections, index-signature records below and above the parser's 256-entry cache bound, and 200 repeated records where framing and field-name overhead become visible. +The benchmark compares arena-backed `SchemaBinary`, `SchemaBinary` followed by an ownership copy, `SchemaBinary` in fingerprint mode, JSON, and Effect's Msgpack schema codec with the same Schema values. The copy case calls `.slice()` on every arena result, reproducing the allocation that the arena removes. It covers a small scalar-heavy record, a nested order payload, collections, index-signature records below and above the parser's 256-entry cache bound, and 200 repeated records where framing and field-name overhead become visible. Codec and schema construction happen before timing. Decode uses payloads encoded before timing. Each one-shot encode and decode task gets 100 warmup samples followed by 1,000 measured samples. The output includes UTF-8 payload sizes, average operations per second, median latency, relative margin of error, and sample count. -A second table compares stateful streaming decode with one frame per feed, 32 frames per feed, and each frame split immediately after its first byte. That fragment boundary exercises the incremental frame-header path instead of merely splitting the body. It also includes 200 distinct repeated-record frames, rather than one frame containing a 200-element array. `SchemaBinary.parser(schema).feedSync` processes the binary stream; a reusable msgpackr `Unpackr.unpackMultiple` processes the Msgpack batch, then the same precompiled Schema decoder validates every value. Parser construction, stream encoding, and fragmentation happen before timing. The streaming tasks get 25 warmup samples and 250 measured samples, and report throughput and median latency per decoded value. +A second table compares stateful streaming decode with one frame per feed, 32 frames per feed, and each frame split immediately after its first byte. That fragment boundary exercises the incremental frame-header path instead of merely splitting the body. It also includes 200 distinct repeated-record frames, rather than one frame containing a 200-element array. `SchemaBinary.parser(schema).feedSync` processes the binary stream, once per wire mode; a reusable msgpackr `Unpackr.unpackMultiple` processes the Msgpack batch, then the same precompiled Schema decoder validates every value. Parser construction, stream encoding, and fragmentation happen before timing. The streaming tasks get 25 warmup samples and 250 measured samples, and report throughput and median latency per decoded value. Size output includes raw, gzip level 6, and zstd bytes for each one-shot payload and concatenated stream. Compression is applied to the whole stream so repeated field ids can share the compressor dictionary. @@ -20,49 +20,70 @@ Treat the measurements as a per-machine comparison within one run. Runtime versi ## Measured comparison -These measurements used Node 26.7.0 on Linux x64. Payload sizes are exact; throughput is machine-local. +These measurements used Node 26.7.0 on Linux x64. Payload sizes are exact; throughput is the median of three full runs and is machine-local. -| Case | SchemaBinary raw/gzip/zstd | JSON raw/gzip/zstd | Msgpack raw/gzip/zstd | -| ---------------------- | -------------------------: | ------------------: | --------------------: | -| small record | 58 / 78 / 67 | 89 / 100 / 92 | 69 / 88 / 78 | -| nested payload | 318 / 305 / 300 | 453 / 303 / 309 | 385 / 304 / 299 | -| collections | 1452 / 792 / 776 | 1828 / 671 / 660 | 1462 / 788 / 805 | -| index signatures / 128 | 2251 / 689 / 656 | 2235 / 573 / 559 | 2203 / 676 / 629 | -| index signatures / 512 | 9355 / 2373 / 2189 | 9531 / 2266 / 2074 | 9287 / 2544 / 2304 | -| large repeated records | 27646 / 3065 / 3175 | 57529 / 3230 / 3008 | 51283 / 3516 / 3320 | +| Case | SchemaBinary raw/gzip/zstd | Fingerprint raw/gzip/zstd | JSON raw/gzip/zstd | Msgpack raw/gzip/zstd | +| ---------------------- | -------------------------: | ------------------------: | ------------------: | --------------------: | +| small record | 58 / 78 / 67 | 35 / 53 / 44 | 89 / 100 / 92 | 69 / 88 / 78 | +| nested payload | 318 / 305 / 300 | 206 / 209 / 203 | 453 / 303 / 309 | 385 / 304 / 299 | +| collections | 1452 / 792 / 776 | 1438 / 776 / 758 | 1828 / 671 / 660 | 1462 / 788 / 805 | +| index signatures / 128 | 2251 / 689 / 656 | 2258 / 697 / 665 | 2235 / 573 / 559 | 2203 / 676 / 629 | +| index signatures / 512 | 9355 / 2373 / 2189 | 9362 / 2383 / 2197 | 9531 / 2266 / 2074 | 9287 / 2544 / 2304 | +| large repeated records | 27646 / 3065 / 3175 | 19454 / 2766 / 2701 | 57529 / 3230 / 3008 | 51283 / 3516 / 3320 | Compression narrows or reverses the raw-size advantage on several cases. Repeated field ids compress well, so raw transport size and compressed transport size should be treated as separate results. -One representative run of the combined tree produced the following one-shot encode rates in operations per second: +Encode rates in operations per second: -| Case | SchemaBinary arena | SchemaBinary copy | Msgpack | -| ---------------------- | -----------------: | ----------------: | ------: | -| small record | 491,619 | 500,578 | 488,432 | -| nested payload | 232,943 | 220,297 | 208,758 | -| collections | 34,078 | 33,545 | 21,706 | -| index signatures / 128 | 14,286 | 14,191 | 33,791 | -| index signatures / 512 | 2,883 | 2,887 | 6,333 | -| large repeated records | 6,156 | 6,086 | 3,521 | +| Case | SchemaBinary arena | SchemaBinary copy | Fingerprint | Msgpack | +| ---------------------- | -----------------: | ----------------: | ----------: | ------: | +| small record | 501,284 | 488,131 | 517,385 | 480,915 | +| nested payload | 233,724 | 220,297 | 249,978 | 207,419 | +| collections | 34,342 | 33,780 | 34,180 | 21,994 | +| index signatures / 128 | 14,096 | 14,048 | 14,214 | 34,134 | +| index signatures / 512 | 2,830 | 2,852 | 2,864 | 6,218 | +| large repeated records | 5,908 | 5,719 | 6,688 | 3,445 | -The corresponding one-shot decode rates were: +Decode rates: -| Case | SchemaBinary | Msgpack | -| ---------------------- | -----------: | ------: | -| small record | 496,806 | 470,364 | -| nested payload | 221,214 | 201,178 | -| collections | 37,080 | 21,081 | -| index signatures / 128 | 20,494 | 29,208 | -| index signatures / 512 | 4,979 | 4,456 | -| large repeated records | 5,876 | 4,000 | +| Case | SchemaBinary | Fingerprint | Msgpack | +| ---------------------- | -----------: | ----------: | ------: | +| small record | 507,538 | 518,477 | 494,179 | +| nested payload | 225,371 | 242,223 | 201,306 | +| collections | 38,060 | 37,874 | 21,327 | +| index signatures / 128 | 20,750 | 20,387 | 29,064 | +| index signatures / 512 | 5,031 | 4,943 | 4,511 | +| large repeated records | 6,089 | 6,638 | 3,972 | The small cases are dominated by fixed per-call cost, so their arena and ownership-copy rows can trade places between runs. The larger cases are more stable; all throughput numbers remain machine-local rather than portable scores. -The per-frame repeated-record stream makes that distinction concrete: +The per-frame repeated-record stream makes the framing cost concrete: -| Format | Frames | Raw bytes | gzip -6 | zstd | -| ------------ | -----: | --------: | ------: | ---: | -| SchemaBinary | 200 | 27840 | 3109 | 3174 | -| Msgpack | 200 | 51520 | 2956 | 2888 | +| Format | Frames | Raw bytes | gzip -6 | zstd | +| ------------------------ | -----: | --------: | ------: | ---: | +| SchemaBinary | 200 | 27840 | 3109 | 3174 | +| SchemaBinary fingerprint | 200 | 21240 | 2876 | 2733 | +| Msgpack | 200 | 51520 | 2956 | 2888 | + +## Fingerprint mode + +Fingerprint mode is measured against the optimized default mode in the same run, not against any earlier tree. Sizes are exact; rates are medians of three runs. + +| Case | Raw bytes | Encode | Decode | Stream single | Stream batch | Stream fragmented | +| ------------------------- | --------: | -----: | -----: | ------------: | -----------: | ----------------: | +| small record | -39.7% | +3.2% | +2.2% | +4.9% | +7.0% | +14.7% | +| nested payload | -35.2% | +7.0% | +7.5% | +11.4% | +6.7% | +6.3% | +| collections | -1.0% | -0.5% | -0.5% | +1.4% | -0.3% | -0.5% | +| index signatures / 128 | +0.3% | +0.8% | -1.7% | +0.8% | +0.5% | +1.0% | +| index signatures / 512 | +0.1% | +1.2% | -1.7% | +2.7% | -1.4% | -0.9% | +| large repeated records | -29.6% | +13.2% | +9.0% | +10.2% | +10.9% | +9.7% | +| per-frame repeated record | -23.7% | | | +8.1% | +5.3% | +2.2% | + +The size result splits by shape. Struct-heavy payloads drop the 5-byte field id and, for fixed-size leaves, the length byte too, which is where the 24% to 40% raw savings come from. Index-signature records carry almost no named fields, so they pay the 8-byte fingerprint and save nothing: those two cases get marginally larger. Collections sit in between. + +Compression narrows the gap without erasing it. On the 200-frame stream, gzip goes from 3109 to 2876 bytes (-7.5%) and zstd from 3174 to 2733 (-13.9%), against -23.7% raw. Fingerprint mode is strongest on uncompressed transports, but the mismatch detection it adds is independent of compression. + +The decode gain comes from dropping the field-id varint, the sorted-field cursor and map fallback, the duplicate-id bookkeeping, and the per-field length prefix on fixed-size leaves. It is reported here as a measurement against the optimized parser. The pre-optimization estimate that field varints were 24% of decode time predates the unrolled reader and is not a forecast for this change. ## Parser optimization effects diff --git a/packages/effect/benchmark/schema/SchemaBinary.ts b/packages/effect/benchmark/schema/SchemaBinary.ts index d15b414d7f1..f82d1e24ef8 100644 --- a/packages/effect/benchmark/schema/SchemaBinary.ts +++ b/packages/effect/benchmark/schema/SchemaBinary.ts @@ -195,11 +195,14 @@ const prepare = >( } => { const jsonSchema = Schema.toCodecJson(schema) const binaryCodec = SchemaBinary.toCodec(schema) + const fingerprintCodec = SchemaBinary.toCodec(schema, { fingerprint: true }) const jsonCodec = Schema.fromJsonString(jsonSchema) const msgpackCodec = Msgpack.schema(jsonSchema) const binaryEncode = Schema.encodeUnknownSync(binaryCodec) const binaryDecode = Schema.decodeUnknownSync(binaryCodec) + const fingerprintEncode = Schema.encodeUnknownSync(fingerprintCodec) + const fingerprintDecode = Schema.decodeUnknownSync(fingerprintCodec) const jsonEncode = Schema.encodeUnknownSync(jsonCodec) const jsonDecode = Schema.decodeUnknownSync(jsonCodec) const msgpackEncode = Schema.encodeUnknownSync(msgpackCodec) @@ -207,11 +210,13 @@ const prepare = >( const binary = binaryEncode(value) const binaryCopy = binary.slice() + const fingerprint = fingerprintEncode(value).slice() const json = jsonEncode(value) const jsonBytes = textEncoder.encode(json) const msgpack = msgpackEncode(value) assert.deepStrictEqual(binaryDecode(binary), value) + assert.deepStrictEqual(fingerprintDecode(fingerprint), value) assert.deepStrictEqual(jsonDecode(json), value) assert.deepStrictEqual(msgpackDecode(msgpack), value) @@ -229,6 +234,12 @@ const prepare = >( encode: () => binaryEncode(value).slice(), decode: () => binaryDecode(binaryCopy) }, + { + name: "SchemaBinary fingerprint", + ...sizes(fingerprint), + encode: () => fingerprintEncode(value), + decode: () => fingerprintDecode(fingerprint) + }, { name: "JSON", ...sizes(jsonBytes), @@ -253,12 +264,19 @@ const prepareStream = >( ): { readonly formats: ReadonlyArray; readonly sizes: ReadonlyArray } => { const binaryCodec = SchemaBinary.toCodec(schema) const binaryEncode = Schema.encodeUnknownSync(binaryCodec) - const binaryFrames = values.map((value) => binaryEncode(value)) + const binaryFrames = values.map((value) => binaryEncode(value).slice()) const binaryStream = concatFrames(binaryFrames) const binaryFragments = binaryFrames.map((frame) => { return [frame.subarray(0, 1), frame.subarray(1)] as const }) + const fingerprintEncode = Schema.encodeUnknownSync(SchemaBinary.toCodec(schema, { fingerprint: true })) + const fingerprintFrames = values.map((value) => fingerprintEncode(value).slice()) + const fingerprintStream = concatFrames(fingerprintFrames) + const fingerprintFragments = fingerprintFrames.map((frame) => { + return [frame.subarray(0, 1), frame.subarray(1)] as const + }) + const jsonSchema = Schema.toCodecJson(schema) const encodeMsgpackValue = Schema.encodeUnknownSync(jsonSchema) const decodeMsgpackValue = Schema.decodeUnknownSync(jsonSchema) @@ -266,25 +284,41 @@ const prepareStream = >( const msgpackStream = concatFrames(values.map((value) => msgpackPackr.pack(encodeMsgpackValue(value)).slice())) const msgpackUnpackr = new Unpackr() - const singleParser = SchemaBinary.parser(schema) - const fragmentedParser = SchemaBinary.parser(schema) - const batchParser = SchemaBinary.parser(schema) - let singleIndex = 0 - let fragmentedIndex = 0 - const decodeSingle = () => singleParser.feedSync(binaryFrames[singleIndex++ % binaryFrames.length]) - const decodeFragmented = () => { - const fragments = binaryFragments[fragmentedIndex++ % binaryFragments.length] - const first = fragmentedParser.feedSync(fragments[0]) - const second = fragmentedParser.feedSync(fragments[1]) - return first.length === 0 ? second : [...first, ...second] + const feedShapes = (options?: { readonly fingerprint: true }) => { + const frames = options === undefined ? binaryFrames : fingerprintFrames + const fragments = options === undefined ? binaryFragments : fingerprintFragments + const stream = options === undefined ? binaryStream : fingerprintStream + const singleParser = SchemaBinary.parser(schema, options) + const fragmentedParser = SchemaBinary.parser(schema, options) + const batchParser = SchemaBinary.parser(schema, options) + let singleIndex = 0 + let fragmentedIndex = 0 + return { + single: () => singleParser.feedSync(frames[singleIndex++ % frames.length]), + fragmented: () => { + const pair = fragments[fragmentedIndex++ % fragments.length] + const first = fragmentedParser.feedSync(pair[0]) + const second = fragmentedParser.feedSync(pair[1]) + return first.length === 0 ? second : [...first, ...second] + }, + batch: () => batchParser.feedSync(stream) + } } - const decodeBatch = () => batchParser.feedSync(binaryStream) + + const defaultFeeds = feedShapes() + const fingerprintFeeds = feedShapes({ fingerprint: true }) + const decodeSingle = defaultFeeds.single + const decodeFragmented = defaultFeeds.fragmented + const decodeBatch = defaultFeeds.batch const decodeMsgpackStream = () => msgpackUnpackr.unpackMultiple(msgpackStream).map((value) => decodeMsgpackValue(value)) assert.deepStrictEqual(decodeSingle(), [values[0]]) assert.deepStrictEqual(decodeFragmented(), [values[0]]) assert.deepStrictEqual(decodeBatch(), values) + assert.deepStrictEqual(fingerprintFeeds.single(), [values[0]]) + assert.deepStrictEqual(fingerprintFeeds.fragmented(), [values[0]]) + assert.deepStrictEqual(fingerprintFeeds.batch(), values) assert.deepStrictEqual(decodeMsgpackStream(), values) return { @@ -292,10 +326,14 @@ const prepareStream = >( { name: "SchemaBinary parser / single frame", framesPerOp: 1, decode: decodeSingle }, { name: "SchemaBinary parser / batch", framesPerOp: values.length, decode: decodeBatch }, { name: "SchemaBinary parser / fragmented", framesPerOp: 1, decode: decodeFragmented }, + { name: "SchemaBinary fingerprint / single frame", framesPerOp: 1, decode: fingerprintFeeds.single }, + { name: "SchemaBinary fingerprint / batch", framesPerOp: values.length, decode: fingerprintFeeds.batch }, + { name: "SchemaBinary fingerprint / fragmented", framesPerOp: 1, decode: fingerprintFeeds.fragmented }, { name: "Msgpack unpackMultiple / batch", framesPerOp: values.length, decode: decodeMsgpackStream } ], sizes: [ { name: "SchemaBinary", frames: values.length, ...sizes(binaryStream) }, + { name: "SchemaBinary fingerprint", frames: values.length, ...sizes(fingerprintStream) }, { name: "Msgpack", frames: values.length, ...sizes(msgpackStream) } ] } diff --git a/packages/effect/src/unstable/encoding/SchemaBinary.ts b/packages/effect/src/unstable/encoding/SchemaBinary.ts index 6209d1e6265..1a498cf4a63 100644 --- a/packages/effect/src/unstable/encoding/SchemaBinary.ts +++ b/packages/effect/src/unstable/encoding/SchemaBinary.ts @@ -12,6 +12,14 @@ * a bare varint, so a checked number and an unchecked one are different wire * layouts for the same value. * + * The opt-in fingerprint mode (`{ fingerprint: true }`) trades that tolerance + * for a smaller frame. Every frame carries an 8-byte 64-bit FNV-1a hash of the + * compiled wire layout; structs are written positionally, with a presence + * bitmap instead of field ids and no length prefix on fixed-size leaves, and + * union members are addressed by a canonical index. A reader whose layout + * hashes differently rejects the frame instead of guessing it. The two modes + * are selected by envelope flag bit 0 and are never interchangeable. + * * @since 4.0.0 */ import * as BigDecimal from "../../BigDecimal.ts" @@ -35,7 +43,10 @@ import * as SchemaTransformation from "../../SchemaTransformation.ts" const FIELD_ID_ANNOTATION_KEY = "~effect/encoding/SchemaBinary/fieldId" +// Envelope flags select the wire mode. Bit 0 is the opt-in fingerprint / +// positional mode; every other bit stays reserved and fails closed. const ENVELOPE = 0x10 // version nibble 1, flags 0 +const ENVELOPE_FINGERPRINT = 0x11 // version nibble 1, flag bit 0 const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER) const BIGINT_ZERO = BigInt(0) @@ -44,6 +55,9 @@ const BIGINT_TWO = BigInt(2) const BIGINT_SEVEN = BigInt(7) const BIGINT_VARINT_MASK = BigInt(0x7F) const BIGINT_NANOS_PER_MILLI = BigInt(1_000_000) +const BIGINT_BYTE_MASK = BigInt(0xFF) +const BIGINT_U32_MASK = BigInt(0xFFFFFFFF) +const BIGINT_THIRTY_TWO = BigInt(32) const utf8Encode = new TextEncoder() const utf8DecodeFatal = new TextDecoder("utf-8", { fatal: true }) @@ -126,6 +140,18 @@ function fnv32(bytes: Uint8Array): number { return hash >>> 0 } +const FNV64_OFFSET_BASIS = BigInt("14695981039346656037") +const FNV64_PRIME = BigInt("1099511628211") +const FNV64_MASK = BigInt("18446744073709551615") + +function fnv64(bytes: ArrayLike): bigint { + let hash = FNV64_OFFSET_BASIS + for (let i = 0; i < bytes.length; i++) { + hash = ((hash ^ BigInt(bytes[i])) * FNV64_PRIME) & FNV64_MASK + } + return hash +} + function compareBytes(a: Uint8Array, b: Uint8Array): number { const len = Math.min(a.length, b.length) for (let i = 0; i < len; i++) { @@ -482,12 +508,14 @@ class Reader { end = 0 options: SchemaAST.ParseOptions = EMPTY_PARSE_OPTIONS indexSignatures: IndexSignatureCache | undefined + positional = false reset( buf: Uint8Array, start: number, end: number, options: SchemaAST.ParseOptions, - indexSignatures: IndexSignatureCache + indexSignatures: IndexSignatureCache, + positional: boolean ) { if ( this.buf.buffer !== buf.buffer || @@ -501,6 +529,7 @@ class Reader { this.end = end this.options = options this.indexSignatures = indexSignatures + this.positional = positional } release() { this.pos = this.end = 0 @@ -508,6 +537,7 @@ class Reader { this.view = EMPTY_READER_VIEW this.options = EMPTY_PARSE_OPTIONS this.indexSignatures = undefined + this.positional = false } get remaining(): number { return this.end - this.pos @@ -710,6 +740,8 @@ interface Field { readonly optional: boolean readonly annotations: Schema.Annotations.Key | undefined layout: Layout + // fingerprint mode: true when the field is written without a length prefix + inline: boolean } interface ExtraSignature { @@ -724,6 +756,8 @@ interface StructLayout { readonly byId: Map readonly extra: Array readonly names: Set + // fingerprint mode: one presence bit per optional field, in field order + optionalCount: number } interface Slot { @@ -754,6 +788,23 @@ interface VariantRow { readonly sentinels: ReadonlyArray readonly tuple: boolean payload: Layout + // fingerprint mode: index into `byPos` + position: number +} + +interface UnionMember { + readonly kind: number + readonly layout: Layout + // fingerprint mode: index into `byPos` + position: number +} + +// A row of the canonical member order fingerprint mode writes as a varint. +// `variant` is set for sentinel-discriminated members, whose sentinel +// properties decode restores. +interface UnionPosition { + readonly variant: VariantRow | undefined + readonly layout: Layout } interface UnionLayout { @@ -761,56 +812,11 @@ interface UnionLayout { readonly ast: SchemaAST.AST readonly variants: Array readonly byTag: Map - readonly others: Array + readonly others: Array readonly byKind: Map -} - -function kindByte(layout: Layout): number { - switch (layout._) { - case "bool": - return K.bool - case "null": - return K.null - case "undefined": - return K.undefined - case "number": - case "int": - return K.number - case "string": - case "symbol": - return K.string - case "bytes": - return K.bytes - case "bigint": - return K.bigint - case "int64": - return K.int64 - case "struct": - return K.struct - case "array": - return K.array - case "option": - return K.option - case "result": - return K.result - case "duration": - return K.duration - case "bigDecimal": - return K.bigDecimal - case "dateTimeZoned": - return K.dateTimeZoned - case "json": - return K.json - case "exit": - return K.exit - case "cause": - return K.cause - case "causeReason": - return K.causeReason - case "union": - case "never": - throw new Error("Binary layout: union members are not uniquely identifiable") - } + // fingerprint mode: variants by ascending tag, then the remaining members by + // ascending kind, so declaration order never reaches the wire. + readonly byPos: Array } // A slot whose encoding delimits itself, so it needs no length prefix even @@ -830,6 +836,13 @@ function packedSize(layout: Layout): number | undefined { } } +// A slot the layout alone can delimit, so no length prefix is written: a +// fixed-size leaf, a zero-width leaf, or a self-delimiting varint. +function isInlineSlot(layout: Layout): boolean { + return packedSize(layout) !== undefined || isSelfDelimiting(layout) || + layout._ === "null" || layout._ === "undefined" +} + // ----------------------------------------------------------------------------- // declaration rewrite: attach `toCodecJson ?? toCodec` links to non-native // declarations so the existing Schema machinery runs them at encode/decode time @@ -1171,7 +1184,8 @@ function compileLayout(root: SchemaAST.AST): CompiledLayout { index: 0, optional: ps.type.context?.isOptional === true, annotations, - layout: undefined as unknown as Layout + layout: undefined as unknown as Layout, + inline: false }) types.push(ps.type) } @@ -1193,12 +1207,18 @@ function compileLayout(root: SchemaAST.AST): CompiledLayout { fields, byId: new Map(), extra, - names: new Set(fields.map((f) => f.name)) + names: new Set(fields.map((f) => f.name)), + optionalCount: 0 } memo.set(ast, layout) for (let i = 0; i < fields.length; i++) { - fields[i].layout = compile(types[i]) - layout.byId.set(fields[i].id, fields[i]) + const field = fields[i] + field.layout = compile(types[i]) + // A recursive field sees a partially filled placeholder here, but its + // discriminant is set at construction, which is all inlining depends on. + field.inline = isInlineSlot(field.layout) + if (field.optional) layout.optionalCount++ + layout.byId.set(field.id, field) } fields.sort((a, b) => a.id - b.id) for (let i = 0; i < fields.length; i++) fields[i].index = i @@ -1371,7 +1391,15 @@ function compileLayout(root: SchemaAST.AST): CompiledLayout { } tags.set(tag, sentinels) } - const layout: UnionLayout = { _: "union", ast, variants: [], byTag: new Map(), others: [], byKind: new Map() } + const layout: UnionLayout = { + _: "union", + ast, + variants: [], + byTag: new Map(), + others: [], + byKind: new Map(), + byPos: [] + } memo.set(ast, layout) for (const { member, sentinels } of variantMembers) { const tag = sentinelSetHash(sentinels) @@ -1388,27 +1416,40 @@ function compileLayout(root: SchemaAST.AST): CompiledLayout { fields, byId: new Map(fields.map((f) => [f.id, f])), extra: struct.extra, - names: struct.names + names: struct.names, + optionalCount: fields.reduce((count, f) => f.optional ? count + 1 : count, 0) } tuple = false } else { payload = full tuple = true } - const row: VariantRow = { tag, sentinels, tuple, payload } + const row: VariantRow = { tag, sentinels, tuple, payload, position: 0 } layout.variants.push(row) layout.byTag.set(tag, row) } for (const [kind, row] of literalRows) { - layout.others.push(row) + layout.others.push({ kind, layout: row, position: 0 }) layout.byKind.set(kind, row) } for (const { kind, member } of rowMembers) { const row = compile(member) - layout.others.push(row) + layout.others.push({ kind, layout: row, position: 0 }) layout.byKind.set(kind, row) } - layout.others.sort((a, b) => matchRank(a) - matchRank(b)) + // Fingerprint mode addresses members by position, so that order is derived + // from tags and kinds rather than from how the union was written. + for (const row of [...layout.variants].sort((a, b) => a.tag - b.tag)) { + row.position = layout.byPos.length + layout.byPos.push({ variant: row, layout: row.payload }) + } + for (const member of [...layout.others].sort((a, b) => a.kind - b.kind)) { + member.position = layout.byPos.length + layout.byPos.push({ variant: undefined, layout: member.layout }) + } + // Encode probes members in match order: specific runtime guards first, + // `json` (which matches anything) last. + layout.others.sort((a, b) => matchRank(a.layout) - matchRank(b.layout)) return layout } @@ -1416,6 +1457,223 @@ function compileLayout(root: SchemaAST.AST): CompiledLayout { return { layout, recursive } } +// ----------------------------------------------------------------------------- +// layout fingerprint +// ----------------------------------------------------------------------------- + +// Structural tags for the fingerprint walk. They are deliberately separate +// from the wire kinds `K`, because layouts that share a wire kind can still +// differ on the wire or in the value they produce (`number` vs `int`, +// `string` vs `symbol`, `Date` vs `DateTimeUtc`). +const F = { + backEdge: 0, + bool: 1, + null: 2, + undefined: 3, + number: 4, + int: 5, + string: 6, + symbol: 7, + bytes: 8, + bigint: 9, + json: 10, + duration: 11, + bigDecimal: 12, + dateTimeZoned: 13, + date: 14, + dateTimeUtc: 15, + never: 16, + struct: 17, + array: 18, + union: 19, + option: 20, + result: 21, + exit: 22, + cause: 23, + causeReason: 24 +} as const + +function pushUvarint(out: Array, n: number) { + while (n > 0x7F) { + out.push((n & 0x7F) | 0x80) + n = Math.floor(n / 128) + } + out.push(n) +} + +function pushU32(out: Array, n: number) { + out.push(n & 0xFF, (n >>> 8) & 0xFF, (n >>> 16) & 0xFF, (n >>> 24) & 0xFF) +} + +function pushU64(out: Array, n: bigint) { + for (let i = 0; i < 8; i++) { + out.push(Number((n >> BigInt(i * 8)) & BIGINT_BYTE_MASK)) + } +} + +/** + * Hashes the wire-relevant structure of a compiled layout with 64-bit FNV-1a. + * + * The hash is a Merkle walk: every node mixes its own structure plus the + * 64-bit hash of each child, so two structurally identical layouts hash the + * same whether or not they share a compiled node. Field and property names, + * checks, and annotations are never mixed in; field ids, optionality, wire + * kinds, variant tags, and array shape are. Cycles terminate on a back edge + * carrying the number of levels back to the repeated node, which keeps the + * hash independent of where the cycle was entered. + */ +function layoutFingerprint(root: Layout): bigint { + const cache = new Map() + const stack: Array = [] + // Shallowest stack index the subtree currently being hashed reached back to. + // A subtree that never reached above its own root is a closed unit, so its + // hash can be reused wherever that layout appears. + let escape = Number.MAX_SAFE_INTEGER + + function go(layout: Layout): bigint { + const at = stack.lastIndexOf(layout) + if (at >= 0) { + if (at < escape) escape = at + const out: Array = [F.backEdge] + pushUvarint(out, stack.length - at) + return fnv64(out) + } + const cached = cache.get(layout) + if (cached !== undefined) return cached + const self = stack.length + const outerEscape = escape + escape = Number.MAX_SAFE_INTEGER + stack.push(layout) + const hash = fnv64(structure(layout)) + stack.pop() + const closed = escape >= self + if (closed) cache.set(layout, hash) + escape = closed ? outerEscape : Math.min(outerEscape, escape) + return hash + } + + function structure(layout: Layout): Array { + const out: Array = [] + switch (layout._) { + case "bool": + case "null": + case "undefined": + case "number": + case "int": + case "string": + case "symbol": + case "bytes": + case "bigint": + case "json": + case "duration": + case "bigDecimal": + case "dateTimeZoned": + out.push(F[layout._]) + return out + case "int64": + out.push(layout.flavor === "date" ? F.date : F.dateTimeUtc) + return out + case "never": + out.push(F.never) + return out + case "struct": { + out.push(F.struct) + pushUvarint(out, layout.fields.length) + for (const field of layout.fields) { + pushU32(out, field.id) + out.push(field.optional ? 1 : 0) + pushU64(out, go(field.layout)) + } + pushUvarint(out, layout.extra.length) + for (const signature of layout.extra) pushU64(out, go(signature.layout)) + return out + } + case "array": { + out.push(F.array) + pushUvarint(out, layout.elements.length) + for (const slot of layout.elements) { + out.push(slot.optional ? 1 : 0) + pushU64(out, go(slot.layout)) + } + pushUvarint(out, layout.rest.length) + for (const rest of layout.rest) pushU64(out, go(rest)) + return out + } + case "union": { + out.push(F.union) + pushUvarint(out, layout.byPos.length) + for (const position of layout.byPos) { + const variant = position.variant + if (variant === undefined) out.push(0) + else { + out.push(1, variant.tuple ? 1 : 0) + pushU32(out, variant.tag) + } + pushU64(out, go(position.layout)) + } + return out + } + case "option": + out.push(F.option) + pushU64(out, go(layout.value)) + return out + case "result": + out.push(F.result) + pushU64(out, go(layout.success)) + pushU64(out, go(layout.failure)) + return out + case "exit": + out.push(F.exit) + pushU64(out, go(layout.value)) + pushU64(out, go(layout.error)) + pushU64(out, go(layout.defect)) + return out + case "cause": + case "causeReason": + out.push(layout._ === "cause" ? F.cause : F.causeReason) + pushU64(out, go(layout.error)) + pushU64(out, go(layout.defect)) + return out + } + } + + return go(root) +} + +// ----------------------------------------------------------------------------- +// wire modes +// ----------------------------------------------------------------------------- + +// A codec speaks exactly one wire mode. The default mode carries field ids and +// tolerates unknown fields and members; fingerprint mode drops both in favour +// of a per-frame layout hash that fails closed on any mismatch. +interface Mode { + readonly positional: boolean + readonly envelope: number + readonly expectedEnvelope: string + readonly fingerprintLo: number + readonly fingerprintHi: number +} + +const defaultMode: Mode = { + positional: false, + envelope: ENVELOPE, + expectedEnvelope: "version 1 envelope, flags 0", + fingerprintLo: 0, + fingerprintHi: 0 +} + +function fingerprintMode(layout: Layout): Mode { + const fingerprint = layoutFingerprint(layout) + return { + positional: true, + envelope: ENVELOPE_FINGERPRINT, + expectedEnvelope: "version 1 envelope, flags 1", + fingerprintLo: Number(fingerprint & BIGINT_U32_MASK), + fingerprintHi: Number((fingerprint >> BIGINT_THIRTY_TWO) & BIGINT_U32_MASK) + } +} + // specific runtime guards first, `json` (which matches anything) last function matchRank(layout: Layout): number { switch (layout._) { @@ -1485,6 +1743,7 @@ function matchesLayout(layout: Layout, value: unknown): boolean { interface EncodeContext { readonly options: SchemaAST.ParseOptions + readonly positional: boolean indexSignatures: IndexSignatureCache | undefined } @@ -1611,19 +1870,28 @@ function encodeReason(ctx: EncodeContext, layout: { error: Layout; defect: Layou } } +type ExtraPair = [keyBytes: Uint8Array, key: string, signature: ExtraSignature] + +// Extra keys sorted by raw UTF-8, so a record encodes the same bytes whatever +// order its keys were inserted in. +function extraPairs(ctx: EncodeContext, layout: StructLayout, obj: Record): Array { + const named = layout.names + const pairs: Array = [] + for (const key of Object.keys(obj)) { + if (named.has(key)) continue + const signature = (ctx.indexSignatures ??= new IndexSignatureCache(ctx.options)).find(layout, key) + if (signature === undefined) continue + pairs.push([utf8Encode.encode(key), key, signature]) + } + pairs.sort((a, b) => compareBytes(a[0], b[0])) + return pairs +} + function encodeStructFields(ctx: EncodeContext, layout: StructLayout, value: object, w: Writer) { const obj = value as Record if (layout.extra.length > 0) { - const named = layout.names - const pairs: Array<[Uint8Array, string, ExtraSignature]> = [] - for (const key of Object.keys(obj)) { - if (named.has(key)) continue - const signature = (ctx.indexSignatures ??= new IndexSignatureCache(ctx.options)).find(layout, key) - if (signature === undefined) continue - pairs.push([utf8Encode.encode(key), key, signature]) - } + const pairs = extraPairs(ctx, layout, obj) if (pairs.length > 0) { - pairs.sort((a, b) => compareBytes(a[0], b[0])) w.uvarint(0) const mark = w.beginSized() for (const [keyBytes, key, signature] of pairs) { @@ -1652,6 +1920,52 @@ function encodeStructFields(ctx: EncodeContext, layout: StructLayout, value: obj } } +// Fingerprint mode struct: both sides compiled the same layout, so a field +// needs neither an id nor a length the layout already implies. Optional fields +// announce themselves in a leading bitmap, one bit per optional field in field +// order, and extra keys follow the named fields behind their own count. +function encodeStructPositional(ctx: EncodeContext, layout: StructLayout, value: object, w: Writer) { + const obj = value as Record + const bitmapBytes = (layout.optionalCount + 7) >> 3 + let bitmap = 0 + if (bitmapBytes > 0) { + bitmap = w.len + for (let i = 0; i < bitmapBytes; i++) w.byte(0) + } + const fields = layout.fields + let optionalIndex = 0 + for (let i = 0; i < fields.length; i++) { + const field = fields[i] + const name = field.name + const present = Object.hasOwn(obj, name) + if (field.optional) { + // `w.buf` and `w.start` move when the arena grows, so the bitmap byte is + // addressed from the current base every time. + if (present) w.buf[w.start + bitmap + (optionalIndex >> 3)] |= 1 << (optionalIndex & 7) + optionalIndex++ + if (!present) continue + } else if (!present) { + issuePath[issuePathLen++] = name + throw issueError(new SchemaIssue.MissingKey(field.annotations)) + } + issuePath[issuePathLen++] = name + if (field.inline) encodeValue(ctx, field.layout, obj[name], w) + else encodeSized(ctx, field.layout, obj[name], w) + issuePathLen-- + } + if (layout.extra.length > 0) { + const pairs = extraPairs(ctx, layout, obj) + w.uvarint(pairs.length) + for (const [keyBytes, key, signature] of pairs) { + w.uvarint(keyBytes.length) + w.bytes(keyBytes) + issuePath[issuePathLen++] = key + encodeSized(ctx, signature.layout, obj[key], w) + issuePathLen-- + } + } +} + function arraySlot(layout: ArrayLayout, index: number, count: number): Layout { const elementLen = layout.elements.length if (index < elementLen) return layout.elements[index].layout @@ -1694,9 +2008,7 @@ function encodeArray(ctx: EncodeContext, layout: ArrayLayout, value: unknown, w: for (let i = 0; i < count; i++) { const slot = arraySlot(layout, i, count) issuePath[issuePathLen++] = i - if ( - packedSize(slot) !== undefined || isSelfDelimiting(slot) || slot._ === "null" || slot._ === "undefined" - ) { + if (isInlineSlot(slot)) { encodeValue(ctx, slot, arr[i], w) } else { encodeSized(ctx, slot, arr[i], w) @@ -1725,13 +2037,16 @@ function encodeNumberRun(arr: ReadonlyArray, count: number, w: Writer) } } +function matchesVariant(variant: VariantRow, value: unknown): boolean { + return variant.tuple + ? Array.isArray(value) && variant.sentinels.every((s) => value[s.key as number] === s.literal) + : Predicate.isObject(value) && !Array.isArray(value) && + variant.sentinels.every((s) => (value as Record)[s.key] === s.literal) +} + function encodeUnion(ctx: EncodeContext, layout: UnionLayout, value: unknown, w: Writer) { for (const variant of layout.variants) { - const matches = variant.tuple - ? Array.isArray(value) && variant.sentinels.every((s) => value[s.key as number] === s.literal) - : Predicate.isObject(value) && !Array.isArray(value) && - variant.sentinels.every((s) => (value as Record)[s.key] === s.literal) - if (matches) { + if (matchesVariant(variant, value)) { w.byte(K.variant) w.u32le(variant.tag) encodeValue(ctx, variant.payload, value, w) @@ -1739,9 +2054,30 @@ function encodeUnion(ctx: EncodeContext, layout: UnionLayout, value: unknown, w: } } for (const member of layout.others) { - if (matchesLayout(member, value)) { - w.byte(kindByte(member)) - encodeValue(ctx, member, value, w) + if (matchesLayout(member.layout, value)) { + w.byte(member.kind) + encodeValue(ctx, member.layout, value, w) + return + } + } + throw issueError(new SchemaIssue.InvalidType(layout.ast, value, ctx.options)) +} + +// Fingerprint mode union: both sides share the member table, so one varint +// index into the canonical order replaces the kind byte and the 32-bit +// sentinel tag. +function encodeUnionPositional(ctx: EncodeContext, layout: UnionLayout, value: unknown, w: Writer) { + for (const variant of layout.variants) { + if (matchesVariant(variant, value)) { + w.uvarint(variant.position) + encodeValue(ctx, variant.payload, value, w) + return + } + } + for (const member of layout.others) { + if (matchesLayout(member.layout, value)) { + w.uvarint(member.position) + encodeValue(ctx, member.layout, value, w) return } } @@ -1885,7 +2221,8 @@ function encodeValue(ctx: EncodeContext, layout: Layout, value: unknown, w: Writ return } case "struct": { - encodeStructFields(ctx, layout, value as object, w) + if (ctx.positional) encodeStructPositional(ctx, layout, value as object, w) + else encodeStructFields(ctx, layout, value as object, w) return } case "array": { @@ -1893,7 +2230,8 @@ function encodeValue(ctx: EncodeContext, layout: Layout, value: unknown, w: Writ return } case "union": - encodeUnion(ctx, layout, value, w) + if (ctx.positional) encodeUnionPositional(ctx, layout, value, w) + else encodeUnion(ctx, layout, value, w) return case "never": throw issueError(new SchemaIssue.InvalidType(layout.ast, value, ctx.options)) @@ -1905,9 +2243,15 @@ function encodeValue(ctx: EncodeContext, layout: Layout, value: unknown, w: Writ // (an inner `toCodec` used as a `Uint8Array` field) simply allocates its own. let pooledWriter: Writer | undefined = new Writer() -function encodeFrame(layout: Layout, value: unknown, options: SchemaAST.ParseOptions): Uint8Array { +function encodeFrame( + layout: Layout, + value: unknown, + options: SchemaAST.ParseOptions, + mode: Mode +): Uint8Array { const ctx: EncodeContext = { options, + positional: mode.positional, indexSignatures: undefined } const w = pooledWriter ?? new Writer() @@ -1918,7 +2262,11 @@ function encodeFrame(layout: Layout, value: unknown, options: SchemaAST.ParseOpt issuePathLen = 0 try { const mark = w.beginSized() - w.byte(ENVELOPE) + w.byte(mode.envelope) + if (mode.positional) { + w.u32le(mode.fingerprintLo) + w.u32le(mode.fingerprintHi) + } encodeValue(ctx, layout, value, w) w.endSized(mark) return w.out() @@ -2028,6 +2376,86 @@ function decodeStruct(layout: StructLayout, r: Reader): unknown { return out } +// Mirror of `encodeStructPositional`. Field order is the layout, so there is +// no id to read, no cursor to advance, and no duplicate-id bookkeeping; a +// field is either announced by the presence bitmap or unconditionally there. +function decodeStructPositional(layout: StructLayout, r: Reader): unknown { + const out: Record = {} + const bitmapBytes = (layout.optionalCount + 7) >> 3 + let bitmap = 0 + if (bitmapBytes > 0) { + if (r.pos + bitmapBytes > r.end) invalid("complete value", undefined, r.options) + bitmap = r.pos + r.pos += bitmapBytes + } + const buf = r.buf + const fields = layout.fields + let optionalIndex = 0 + let issues: Array | undefined + for (let i = 0; i < fields.length; i++) { + const field = fields[i] + if (field.optional) { + const present = (buf[bitmap + (optionalIndex >> 3)] & (1 << (optionalIndex & 7))) !== 0 + optionalIndex++ + if (!present) continue + } + issuePath[issuePathLen++] = field.name + let value: unknown + if (field.inline) { + if (isSelfDelimiting(field.layout)) { + value = decodeValue(field.layout, r) + } else { + // a fixed-size leaf, or a zero-width `null` / `undefined` + const saved = r.enter(packedSize(field.layout) ?? 0) + value = decodeChecked(field.layout, r) + r.exit(saved) + } + } else { + const saved = r.enter(r.uvarint()) + value = decodeChecked(field.layout, r) + r.exit(saved) + } + issuePathLen-- + // Only a newer writer's unknown `CauseReason` tag reaches this, since + // fingerprint mode has no unknown fields or union members. + if (value !== ABSENT) out[field.name] = value + else if (!field.optional) { + const issue = new SchemaIssue.Pointer([field.name], new SchemaIssue.MissingKey(field.annotations)) + if (issues === undefined) issues = [issue] + else issues.push(issue) + if (r.options.errors !== "all") break + } + } + if (issues !== undefined) { + throw issueError( + new SchemaIssue.Composite(layout.ast, issues as [SchemaIssue.Issue, ...Array]) + ) + } + if (layout.extra.length > 0) { + const count = r.uvarint() + if (count > r.remaining) invalid("complete value", undefined, r.options) + let seen: Set | undefined + for (let i = 0; i < count; i++) { + const key = r.readUtf8(r.uvarint()) + if (seen === undefined) seen = new Set([key]) + else { + if (seen.has(key)) invalid("unique extra keys", undefined, r.options) + seen.add(key) + } + const saved = r.enter(r.uvarint()) + const signature = r.indexSignatures!.find(layout, key) + if (signature !== undefined) { + issuePath[issuePathLen++] = key + const value = decodeChecked(signature.layout, r) + issuePathLen-- + if (value !== ABSENT) out[key] = value + } + r.exit(saved) + } + } + return out +} + function decodeExtraPairs(layout: StructLayout, r: Reader, out: Record) { let seen: Set | undefined while (r.pos < r.end) { @@ -2173,6 +2601,22 @@ function decodeUnion(layout: UnionLayout, r: Reader): unknown { return decodeChecked(member, r) } +// Mirror of `encodeUnionPositional`. An index outside the table means the +// frame does not match the layout its fingerprint claimed, so it fails rather +// than resolving to absent. +function decodeUnionPositional(layout: UnionLayout, r: Reader): unknown { + const position = layout.byPos[r.uvarint()] + if (position === undefined) invalid("known union member", undefined, r.options) + const payload = decodeChecked(position.layout, r) + const variant = position.variant + if (variant !== undefined && !variant.tuple && payload !== ABSENT) { + for (const sentinel of variant.sentinels) { + ;(payload as Record)[sentinel.key] = sentinel.literal + } + } + return payload +} + function decodeReason(layout: { error: Layout; defect: Layout }, r: Reader): unknown { const tag = r.byte() switch (tag) { @@ -2320,19 +2764,25 @@ function decodeValue(layout: Layout, r: Reader): unknown { case "causeReason": return decodeReason(layout, r) case "struct": - return decodeStruct(layout, r) + return r.positional ? decodeStructPositional(layout, r) : decodeStruct(layout, r) case "array": return decodeArray(layout, r) case "union": - return decodeUnion(layout, r) + return r.positional ? decodeUnionPositional(layout, r) : decodeUnion(layout, r) case "never": throw issueError(new SchemaIssue.InvalidType(layout.ast, undefined, r.options)) } } -function decodeFrameBody(layout: Layout, r: Reader): unknown { +function decodeFrameBody(layout: Layout, r: Reader, mode: Mode): unknown { const envelope = r.byte() - if (envelope !== ENVELOPE) invalid("version 1 envelope, flags 0", envelope, r.options) + if (envelope !== mode.envelope) invalid(mode.expectedEnvelope, envelope, r.options) + if (mode.positional) { + if (r.remaining < 8) invalid("complete value", undefined, r.options) + if (r.u32le() !== mode.fingerprintLo || r.u32le() !== mode.fingerprintHi) { + invalid("matching layout fingerprint", undefined, r.options) + } + } const value = decodeChecked(layout, r) if (value === ABSENT) throw issueError(new SchemaIssue.MissingKey(undefined)) return value @@ -2343,18 +2793,23 @@ function decodeFrameBody(layout: Layout, r: Reader): unknown { // is returned even when decoding fails. let pooledReader: Reader | undefined = new Reader() -function decodeOneShot(layout: Layout, bytes: Uint8Array, options: SchemaAST.ParseOptions): unknown { +function decodeOneShot( + layout: Layout, + bytes: Uint8Array, + options: SchemaAST.ParseOptions, + mode: Mode +): unknown { const r = pooledReader ?? new Reader() const pooled = r === pooledReader if (pooled) pooledReader = undefined - r.reset(bytes, 0, bytes.length, options, new IndexSignatureCache(options)) + r.reset(bytes, 0, bytes.length, options, new IndexSignatureCache(options), mode.positional) const savedPathLen = issuePathLen issuePathLen = 0 try { const n = r.uvarint() if (n === 0) invalid("nonzero frame length", undefined, options) const saved = r.enter(n) - const value = decodeFrameBody(layout, r) + const value = decodeFrameBody(layout, r, mode) r.exit(saved) if (r.pos !== bytes.length) invalid("no leftover bytes", undefined, options) return value @@ -2369,12 +2824,15 @@ function decodeOneShot(layout: Layout, bytes: Uint8Array, options: SchemaAST.Par // user API // ----------------------------------------------------------------------------- -function makeTransformation(layout: Layout): SchemaTransformation.Transformation> { +function makeTransformation( + layout: Layout, + mode: Mode +): SchemaTransformation.Transformation> { return SchemaTransformation.transformOrFail({ decode: (bytes: Uint8Array, options) => Effect.suspend(() => { try { - return Effect.succeed(decodeOneShot(layout, bytes, options)) + return Effect.succeed(decodeOneShot(layout, bytes, options, mode)) } catch (e) { if (e instanceof IssueError) return Effect.fail(e.issue) throw e @@ -2383,7 +2841,7 @@ function makeTransformation(layout: Layout): SchemaTransformation.Transformation encode: (value: unknown, options) => Effect.suspend(() => { try { - return Effect.succeed(encodeFrame(layout, value, options)) + return Effect.succeed(encodeFrame(layout, value, options, mode)) } catch (e) { if (e instanceof IssueError) return Effect.fail(e.issue) throw e @@ -2392,6 +2850,10 @@ function makeTransformation(layout: Layout): SchemaTransformation.Transformation }) } +function compileMode(layout: Layout, fingerprint: boolean | undefined): Mode { + return fingerprint === true ? fingerprintMode(layout) : defaultMode +} + function compileTarget(schema: Schema.Constraint): { target: Schema.Constraint; layout: Layout } { const raw = Schema.make(toBinaryAST(schema.ast)) const { layout, recursive } = compileLayout(SchemaAST.toEncoded(raw.ast)) @@ -2432,6 +2894,32 @@ function withCycleGuard(target: Schema.Constraint): Schema.Constraint { )(target) } +/** + * Selects the wire mode. + * + * The default mode is evolution friendly: every struct field carries its wire + * id, unknown fields and union members are skipped, and a reader compiled from + * a different but compatible schema still decodes the frame. + * + * `fingerprint: true` trades that tolerance for size and speed. Each frame + * carries an 8-byte hash of the compiled wire layout, structs are written + * positionally without field ids, fixed-size leaves drop their length prefix, + * and union members are addressed by a canonical index instead of a kind byte + * and a 32-bit sentinel tag. A reader whose layout hashes differently rejects + * the frame rather than guessing, so both sides must ship the same wire + * layout. Non-wire schema changes (checks, annotations, decoded-side + * transformations, field declaration order) leave the hash alone. + * + * @category models + * @since 4.0.0 + */ +export interface Options { + /** + * @since 4.0.0 + */ + readonly fingerprint?: boolean | undefined +} + /** * The codec type returned by {@link toCodec}. * @@ -2458,6 +2946,10 @@ export interface toCodec extends * One-shot encode/decode reuse the existing runners: exactly one frame, * leftover bytes are malformed. Use {@link parser} for concatenated frames. * + * Pass `{ fingerprint: true }` for the positional wire mode described on + * {@link Options}. The two modes are not interchangeable: a frame written in + * one is rejected by a codec built for the other. + * * Encoded results are stable views into a bump-allocated arena. A result's * `byteLength` covers exactly one frame, but its `.buffer` may be larger, * `byteOffset` may be non-zero, and unrelated encoded results may share the @@ -2480,10 +2972,10 @@ export interface toCodec extends * @category constructors * @since 4.0.0 */ -export function toCodec(schema: S): toCodec { +export function toCodec(schema: S, options?: Options): toCodec { const { layout, target } = compileTarget(schema) return (Schema.Uint8Array as Schema.instanceOf>).pipe( - Schema.decodeTo(target, makeTransformation(layout)) + Schema.decodeTo(target, makeTransformation(layout, compileMode(layout, options?.fingerprint))) ) as unknown as toCodec } @@ -2520,14 +3012,18 @@ export interface Parser { * stay observable; after a failure the parser is spent and rejects further * calls. * + * `fingerprint` selects the wire mode and must match the writer; see + * {@link Options}. + * * @category constructors * @since 4.0.0 */ export function parser( schema: S, - options?: SchemaAST.ParseOptions & { readonly maxFrameSize?: number | undefined } + options?: SchemaAST.ParseOptions & Options & { readonly maxFrameSize?: number | undefined } ): Parser { const { layout, target } = compileTarget(schema) + const mode = compileMode(layout, options?.fingerprint) const parseOptions: SchemaAST.ParseOptions = options ?? {} const maxFrameSize = options?.maxFrameSize const decodeEncoded = Schema.decodeUnknownSync(target as Schema.ConstraintDecoder, parseOptions) @@ -2632,8 +3128,8 @@ export function parser( issuePathLen = 0 const bodyStart = bufferStart + headerLen indexSignatures.beginFrame() - body.reset(buffer, bodyStart, bodyStart + frameLen, parseOptions, indexSignatures) - out.push(decodeEncoded(decodeFrameBody(layout, body))) + body.reset(buffer, bodyStart, bodyStart + frameLen, parseOptions, indexSignatures, mode.positional) + out.push(decodeEncoded(decodeFrameBody(layout, body, mode))) } catch (e) { spent = true release() diff --git a/packages/effect/test/unstable/encoding/SchemaBinary.test.ts b/packages/effect/test/unstable/encoding/SchemaBinary.test.ts index e986e9b0067..cb08c77887a 100644 --- a/packages/effect/test/unstable/encoding/SchemaBinary.test.ts +++ b/packages/effect/test/unstable/encoding/SchemaBinary.test.ts @@ -43,6 +43,24 @@ const sameNumber = (actual: unknown, expected: number) => { ) } +const encodeFingerprint = (schema: Schema.Codec, value: A): Uint8Array => + Schema.encodeUnknownSync(SchemaBinary.toCodec(schema, { fingerprint: true }))(value) + +const roundtripFingerprint = (schema: Schema.Codec, value: A): A => { + const codec = SchemaBinary.toCodec(schema, { fingerprint: true }) + return Schema.decodeUnknownSync(codec)(Schema.encodeUnknownSync(codec)(value)) +} + +// The 8 bytes that follow the fingerprint envelope, as a comparable string. +const fingerprintOf = (schema: Schema.Codec, value: A): string => { + const bytes = encodeFingerprint(schema, value) + let offset = 0 + while ((bytes[offset] & 0x80) !== 0) offset++ + offset++ + assert.strictEqual(bytes[offset], 0x11) + return Array.from(bytes.subarray(offset + 1, offset + 9)).join(",") +} + const schemaError = (f: () => unknown): Schema.SchemaError => { try { f() @@ -1037,4 +1055,480 @@ describe("SchemaBinary", () => { assert.match(schemaError(() => encode(Schema.Symbol, Symbol("local"))).message, /registered symbol/) }) }) + + describe("fingerprint mode", () => { + const Person = Schema.Struct({ + name: Schema.String, + age: Schema.Number.check(Schema.isInt()), + active: Schema.Boolean, + nickname: Schema.optional(Schema.String) + }) + const person = { name: "Ada", age: 36, active: true } + + it("writes a fingerprint envelope, a presence bitmap, and no field ids", () => { + // length | envelope 0x11 | fingerprint | bitmap | age varint | name | active + assert.deepStrictEqual([...encodeFingerprint(Person, person)], [ + 16, + 0x11, + 184, + 213, + 85, + 38, + 67, + 231, + 116, + 53, + 0, + 72, + 3, + 65, + 100, + 97, + 1 + ]) + // the same frame with the optional field present: its bit is set and it + // moves ahead of `age`, which is where its wire id sorts it + assert.deepStrictEqual([...encodeFingerprint(Person, { ...person, nickname: "A" })], [ + 19, + 0x11, + 184, + 213, + 85, + 38, + 67, + 231, + 116, + 53, + 1, + 2, + 1, + 65, + 72, + 3, + 65, + 100, + 97, + 1 + ]) + }) + + it("leaves the default mode untouched", () => { + const expected = [...encode(Person, person)] + assert.deepStrictEqual(expected[1], 0x10) + assert.deepStrictEqual([...Schema.encodeUnknownSync(SchemaBinary.toCodec(Person, {}))(person)], expected) + assert.deepStrictEqual( + [...Schema.encodeUnknownSync(SchemaBinary.toCodec(Person, { fingerprint: false }))(person)], + expected + ) + }) + + it("addresses union members by canonical position", () => { + const Click = Schema.Struct({ _tag: Schema.Literal("click"), x: Schema.Number.check(Schema.isInt()) }) + const Key = Schema.Struct({ _tag: Schema.Literal("key"), code: Schema.String }) + const Event = Schema.Union([Click, Key]) + assert.deepStrictEqual([...encodeFingerprint(Event, { _tag: "click", x: 3 })], [ + 11, + 0x11, + 237, + 163, + 78, + 151, + 96, + 43, + 76, + 0, + 0, + 6 + ]) + const key = { _tag: "key", code: "Esc" } as const + const keyFrame = [...encodeFingerprint(Event, key)] + assert.deepStrictEqual(keyFrame, [14, 0x11, 237, 163, 78, 151, 96, 43, 76, 0, 1, 3, 69, 115, 99]) + // declaration order never reaches the wire + assert.deepStrictEqual([...encodeFingerprint(Schema.Union([Key, Click]), key)], keyFrame) + assert.deepStrictEqual(roundtripFingerprint(Event, key), key) + }) + + it("counts the extra-key map instead of reserving field zero", () => { + const schema = Schema.Record(Schema.String, Schema.Number) + assert.deepStrictEqual([...encodeFingerprint(schema, { b: 2, a: 1 })], [ + 18, + 0x11, + 189, + 249, + 115, + 23, + 65, + 225, + 229, + 6, + 2, + 1, + 97, + 1, + 2, + 1, + 98, + 1, + 4 + ]) + assert.deepStrictEqual(roundtripFingerprint(schema, { z: 1, a: 2 }), { z: 1, a: 2 }) + assert.deepStrictEqual(roundtripFingerprint(schema, {}), {}) + }) + + it("keeps the default tuple and array layout", () => { + assert.deepStrictEqual([...encodeFingerprint(Schema.Tuple([Schema.Boolean, Schema.String]), [true, "x"])], [ + 12, + 0x11, + 3, + 44, + 71, + 25, + 31, + 66, + 141, + 82, + 1, + 1, + 120 + ]) + assert.deepStrictEqual([...encodeFingerprint(Schema.Array(Schema.Number.check(Schema.isInt())), [1, -2, 3])], [ + 13, + 0x11, + 206, + 94, + 13, + 113, + 158, + 175, + 76, + 32, + 3, + 2, + 5, + 6 + ]) + }) + + it("inlines fixed-size and zero-width leaves", () => { + const schema = Schema.Struct({ + nothing: Schema.Null, + flag: Schema.Boolean, + missing: Schema.Undefined, + when: Schema.Date, + count: Schema.Number.check(Schema.isInt()), + label: Schema.String + }) + const value = { nothing: null, flag: true, missing: undefined, when: new Date(1000), count: -7, label: "hi" } + // 1 envelope + 8 fingerprint + 1 varint + 8 int64 + 3 string + 1 bool + assert.strictEqual(encodeFingerprint(schema, value).length, 23) + assert.deepStrictEqual(roundtripFingerprint(schema, value), value) + }) + + it("spans a presence bitmap across several bytes", () => { + const schema = Schema.Struct({ + a: Schema.optional(Schema.Number), + b: Schema.optional(Schema.Number), + c: Schema.optional(Schema.Number), + d: Schema.optional(Schema.Number), + e: Schema.optional(Schema.Number), + f: Schema.optional(Schema.Number), + g: Schema.optional(Schema.Number), + h: Schema.optional(Schema.Number), + i: Schema.optional(Schema.Number), + j: Schema.optional(Schema.Number) + }) + assert.deepStrictEqual(roundtripFingerprint(schema, {}), {}) + assert.deepStrictEqual(roundtripFingerprint(schema, { a: 1, j: 2 }), { a: 1, j: 2 }) + const full = { a: 1, b: 2, c: 3, d: 4, e: 5, f: 6, g: 7, h: 8, i: 9, j: 10 } + assert.deepStrictEqual(roundtripFingerprint(schema, full), full) + }) + + it("round-trips native declarations, recursion, and mixed unions", () => { + const natives = Schema.Struct({ + option: Schema.Option(Schema.String), + result: Schema.Result(Schema.Number, Schema.String), + big: Schema.BigInt, + bytes: Schema.Uint8Array, + duration: Schema.Duration, + decimal: Schema.BigDecimal + }) + const nativeValue = { + option: Option.some("x"), + result: Result.fail("boom"), + big: 2n ** 70n, + bytes: new Uint8Array([1, 2, 3]), + duration: Duration.nanos(1_500_000_000n), + decimal: BigDecimal.make(123n, 2) + } + const natived = roundtripFingerprint(natives, nativeValue) + assert.deepStrictEqual(natived.option, nativeValue.option) + assert.deepStrictEqual(natived.result, nativeValue.result) + assert.strictEqual(natived.big, nativeValue.big) + assert.deepStrictEqual([...natived.bytes], [1, 2, 3]) + assert.strictEqual(Duration.toNanosUnsafe(natived.duration), 1_500_000_000n) + assert.strictEqual(natived.decimal.value, 123n) + assert.strictEqual(natived.decimal.scale, 2) + + interface Tree { + readonly value: number + readonly children: ReadonlyArray + } + const Tree: Schema.Codec = Schema.Struct({ + value: Schema.Number, + children: Schema.Array(Schema.suspend((): Schema.Codec => Tree)) + }) + const tree = { value: 1, children: [{ value: 2, children: [] }, { value: 3, children: [] }] } + assert.deepStrictEqual(roundtripFingerprint(Tree, tree), tree) + + const mixed = Schema.Union([Schema.String, Schema.Number, Schema.Struct({ n: Schema.Boolean })]) + assert.deepStrictEqual(roundtripFingerprint(mixed, "x"), "x") + assert.deepStrictEqual(roundtripFingerprint(mixed, 1.5), 1.5) + assert.deepStrictEqual(roundtripFingerprint(mixed, { n: true }), { n: true }) + }) + }) + + describe("layout fingerprint", () => { + const base = Schema.Struct({ name: Schema.String, age: Schema.Number }) + const value = { name: "a", age: 1 } + const baseline = fingerprintOf(base, value) + + it("ignores schema changes that do not reach the wire", () => { + assert.strictEqual( + fingerprintOf( + Schema.Struct({ + name: Schema.String.check(Schema.isMinLength(1)).annotate({ description: "the name" }), + age: Schema.Number + }), + value + ), + baseline + ) + // property declaration order: encode sorts by wire id + assert.strictEqual(fingerprintOf(Schema.Struct({ age: Schema.Number, name: Schema.String }), value), baseline) + // a decoded-side transformation leaves the encoded layout alone + assert.strictEqual( + fingerprintOf( + Schema.Struct({ name: Schema.String, age: Schema.Number.pipe(Schema.decodeTo(Schema.Number)) }), + value + ), + baseline + ) + }) + + it("does not depend on whether a sub-schema is shared or repeated", () => { + const Point = Schema.Struct({ x: Schema.Number, y: Schema.Number }) + const pair = { a: { x: 1, y: 2 }, b: { x: 3, y: 4 } } + const shared = fingerprintOf(Schema.Struct({ a: Point, b: Point }), pair) + const repeated = fingerprintOf( + Schema.Struct({ + a: Schema.Struct({ x: Schema.Number, y: Schema.Number }), + b: Schema.Struct({ x: Schema.Number, y: Schema.Number }) + }), + pair + ) + assert.strictEqual(shared, repeated) + + interface Tree { + readonly value: number + readonly children: ReadonlyArray + } + const makeTree = (): Schema.Codec => { + const self: Schema.Codec = Schema.Struct({ + value: Schema.Number, + children: Schema.Array(Schema.suspend((): Schema.Codec => self)) + }) + return self + } + const leaf = { value: 1, children: [] } + assert.strictEqual(fingerprintOf(makeTree(), leaf), fingerprintOf(makeTree(), leaf)) + // one recursive node reached from two fields, versus two of them + const both = { left: leaf, right: leaf } + const one = makeTree() + assert.strictEqual( + fingerprintOf(Schema.Struct({ left: one, right: one }), both), + fingerprintOf(Schema.Struct({ left: makeTree(), right: makeTree() }), both) + ) + }) + + it("changes whenever the wire layout changes", () => { + const changed = [ + fingerprintOf(Schema.Struct({ label: Schema.String, age: Schema.Number }), { label: "a", age: 1 }), + fingerprintOf( + Schema.Struct({ name: Schema.String, age: Schema.Number, extra: Schema.Boolean }), + { name: "a", age: 1, extra: true } + ), + fingerprintOf(Schema.Struct({ name: Schema.String, age: Schema.optional(Schema.Number) }), value), + fingerprintOf(Schema.Struct({ name: Schema.String, age: Schema.Number.check(Schema.isInt()) }), value), + fingerprintOf( + Schema.Struct({ name: Schema.String.pipe(SchemaBinary.fieldId(1)), age: Schema.Number }), + value + ), + fingerprintOf(Schema.Struct({ name: Schema.String, age: Schema.String }), { name: "a", age: "1" }) + ] + assert.strictEqual(new Set([baseline, ...changed]).size, changed.length + 1) + + assert.notStrictEqual( + fingerprintOf(Schema.Tuple([Schema.Number, Schema.Number]), [1, 2]), + fingerprintOf(Schema.Tuple([Schema.Number, Schema.Number, Schema.Number]), [1, 2, 3]) + ) + assert.notStrictEqual( + fingerprintOf(Schema.Array(Schema.Number), [1]), + fingerprintOf(Schema.Tuple([Schema.Number]), [1]) + ) + assert.notStrictEqual( + fingerprintOf(Schema.Date, new Date(0)), + fingerprintOf(Schema.DateTimeUtc, DateTime.makeUnsafe(0)) + ) + + const A = Schema.Struct({ _tag: Schema.Literal("a"), n: Schema.Number }) + const B = Schema.Struct({ _tag: Schema.Literal("b"), s: Schema.String }) + const C = Schema.Struct({ _tag: Schema.Literal("c"), s: Schema.String }) + const a = { _tag: "a", n: 1 } as const + assert.strictEqual(fingerprintOf(Schema.Union([A, B]), a), fingerprintOf(Schema.Union([B, A]), a)) + assert.notStrictEqual(fingerprintOf(Schema.Union([A, B]), a), fingerprintOf(Schema.Union([A, B, C]), a)) + }) + }) + + describe("fingerprint mode failures", () => { + const Person = Schema.Struct({ name: Schema.String, age: Schema.Number }) + const person = { name: "Ada", age: 36 } + + it("fails closed when the layout fingerprint differs", () => { + const frame = encodeFingerprint(Person, person) + const other = SchemaBinary.toCodec(Schema.Struct({ label: Schema.String, age: Schema.Number }), { + fingerprint: true + }) + assert.include( + schemaError(() => Schema.decodeUnknownSync(other)(frame)).message, + "Expected matching layout fingerprint" + ) + }) + + it("rejects the other mode's frames in both directions", () => { + const fingerprintFrame = encodeFingerprint(Person, person) + const defaultFrame = encode(Person, person) + assert.include( + schemaError(() => Schema.decodeUnknownSync(SchemaBinary.toCodec(Person))(fingerprintFrame)).message, + "Expected version 1 envelope, flags 0" + ) + assert.include( + schemaError(() => Schema.decodeUnknownSync(SchemaBinary.toCodec(Person, { fingerprint: true }))(defaultFrame)) + .message, + "Expected version 1 envelope, flags 1" + ) + }) + + it("rejects truncated and oversized frames", () => { + const codec = SchemaBinary.toCodec(Person, { fingerprint: true }) + const frame = encodeFingerprint(Person, person) + // the fingerprint itself does not fit + const shortFrame = concat(new Uint8Array([5, 0x11]), frame.subarray(2, 6)) + assert.include( + schemaError(() => Schema.decodeUnknownSync(codec)(shortFrame)).message, + "Expected complete value" + ) + assert.include( + schemaError(() => Schema.decodeUnknownSync(codec)(frame.subarray(0, frame.length - 1))).message, + "Expected complete value" + ) + assert.include( + schemaError(() => Schema.decodeUnknownSync(codec)(concat(frame, new Uint8Array([0])))).message, + "Expected no leftover bytes" + ) + const withLeftover = new Uint8Array(frame) + withLeftover[0] = frame[0] + 1 + assert.include( + schemaError(() => Schema.decodeUnknownSync(codec)(concat(withLeftover, new Uint8Array([0])))).message, + "Expected no leftover bytes" + ) + }) + + it("rejects a union position outside the member table", () => { + const Event = Schema.Union([ + Schema.Struct({ _tag: Schema.Literal("a"), n: Schema.Number }), + Schema.Struct({ _tag: Schema.Literal("b"), s: Schema.String }) + ]) + const codec = SchemaBinary.toCodec(Event, { fingerprint: true }) + const frame = new Uint8Array(encodeFingerprint(Event, { _tag: "b", s: "x" })) + // the byte after the envelope and fingerprint is the member position + frame[10] = 7 + assert.include( + schemaError(() => Schema.decodeUnknownSync(codec)(frame)).message, + "Expected known union member" + ) + }) + + it("reports a required field a newer writer left unreadable as missing", () => { + const schema = Schema.Struct({ reason: Schema.CauseReason(Schema.String, Schema.String) }) + const frame = new Uint8Array(encodeFingerprint(schema, { reason: Cause.makeFailReason("boom") })) + // the reason payload is length-prefixed; its first byte is the tag + frame[11] = 9 + const error = schemaError(() => + Schema.decodeUnknownSync(SchemaBinary.toCodec(schema, { fingerprint: true }))(frame) + ) + assert.include(error.message, "reason") + assert.include(error.message, "Missing key") + }) + + it("rejects duplicate extra keys", () => { + const schema = Schema.Record(Schema.String, Schema.Number) + const frame = encodeFingerprint(schema, { a: 1, b: 2 }) + const duplicated = new Uint8Array(frame) + // rewrite the second key so both pairs claim "a" + duplicated[duplicated.length - 3] = 97 + assert.include( + schemaError(() => Schema.decodeUnknownSync(SchemaBinary.toCodec(schema, { fingerprint: true }))(duplicated)) + .message, + "Expected unique extra keys" + ) + }) + }) + + describe("fingerprint mode parser", () => { + const Person = Schema.Struct({ name: Schema.String, age: Schema.Number }) + const first = { name: "Ada", age: 36 } + const second = { name: "Grace", age: 45 } + + it("parses concatenated frames, one byte at a time, and mid-frame splits", () => { + const frames = concat(encodeFingerprint(Person, first), encodeFingerprint(Person, second)) + assert.deepStrictEqual( + SchemaBinary.parser(Person, { fingerprint: true }).feedSync(frames), + [first, second] + ) + + const byteParser = SchemaBinary.parser(Person, { fingerprint: true }) + const out: Array = [] + for (const byte of frames) out.push(...byteParser.feedSync(new Uint8Array([byte]))) + byteParser.endSync() + assert.deepStrictEqual(out, [first, second]) + + // a split inside the fingerprint must wait rather than fail + const split = SchemaBinary.parser(Person, { fingerprint: true }) + assert.deepStrictEqual(split.feedSync(frames.subarray(0, 6)), []) + assert.deepStrictEqual(split.feedSync(frames.subarray(6)), [first, second]) + }) + + it("fails closed on a default-mode frame in a fingerprint stream", () => { + const parser = SchemaBinary.parser(Person, { fingerprint: true }) + const mixed = concat(encodeFingerprint(Person, first), encode(Person, second)) + assert.deepStrictEqual(parser.feedSync(mixed), [first]) + assert.include( + schemaError(() => parser.feedSync(new Uint8Array(0))).message, + "Expected version 1 envelope, flags 1" + ) + assert.include(schemaError(() => parser.endSync()).message, "Expected parser is spent") + }) + + it("honours maxFrameSize and reports truncation at end", () => { + const bounded = SchemaBinary.parser(Person, { fingerprint: true, maxFrameSize: 4 }) + assert.include( + schemaError(() => bounded.feedSync(encodeFingerprint(Person, first))).message, + "Expected frame within maxFrameSize" + ) + const truncated = SchemaBinary.parser(Person, { fingerprint: true }) + const frame = encodeFingerprint(Person, first) + assert.deepStrictEqual(truncated.feedSync(frame.subarray(0, frame.length - 2)), []) + assert.include(schemaError(() => truncated.endSync()).message, "Expected complete value") + }) + }) }) From 5bb27c05f770c7fe8f70e9f296f055c6627cb450 Mon Sep 17 00:00:00 2001 From: Tim Smart Date: Thu, 20 Aug 2026 10:38:42 +0000 Subject: [PATCH 2/2] Document SchemaBinary layout-graph fingerprint semantics The fingerprint hashes the compiled layout graph, not the infinite wire shape it denotes. Those coincide for acyclic layouts but not once a cycle is involved, so re-factoring a recursive schema moves the hash without changing a byte on the wire. Say so instead of promising canonicalisation, and pin the alias-level case with a test. Co-Authored-By: Claude Opus 5 --- .changeset/schema-binary-fingerprint-mode.md | 4 +- .../src/unstable/encoding/SchemaBinary.ts | 41 +++++++--- .../unstable/encoding/SchemaBinary.test.ts | 80 +++++++++++++++++-- 3 files changed, 107 insertions(+), 18 deletions(-) diff --git a/.changeset/schema-binary-fingerprint-mode.md b/.changeset/schema-binary-fingerprint-mode.md index c91bf350693..7ab6281ff07 100644 --- a/.changeset/schema-binary-fingerprint-mode.md +++ b/.changeset/schema-binary-fingerprint-mode.md @@ -6,6 +6,8 @@ Add an opt-in `SchemaBinary` fingerprint / positional wire mode. `SchemaBinary.toCodec(schema, { fingerprint: true })` and `SchemaBinary.parser(schema, { fingerprint: true })` select a second wire mode, chosen by envelope flag bit 0. Every frame carries an 8-byte 64-bit FNV-1a hash of the compiled wire layout, and a reader whose layout hashes differently rejects the frame instead of guessing. In exchange, structs are written positionally: no field ids, a presence bitmap for optional fields, no length prefix on fixed-size leaves, and a canonical varint index in place of the union kind byte and 32-bit sentinel tag. -The hash covers wire-relevant structure only. Checks, annotations, decoded-side transformations, property declaration order, and whether a sub-schema is shared or repeated leave it unchanged; renames, added or removed fields, optionality, leaf types, tuple shape, and union membership change it. +The hash covers wire-relevant structure only. Checks, annotations, decoded-side transformations, property declaration order, and repeating an acyclic sub-schema instead of sharing one leave it unchanged; renames, added or removed fields, optionality, leaf types, tuple shape, and union membership change it. + +What is hashed is the compiled layout graph, not the infinite wire shape that graph denotes. The two coincide for acyclic layouts. They do not once a cycle is involved: a self-recursive schema and the same schema behind one extra non-recursive alias produce byte-identical frames but different hashes, and reject each other. Peers must ship the same schema definition, not merely the same wire shape. The default mode is unchanged and remains the default. The two modes are not interchangeable: a frame written in one is rejected by a codec built for the other. On the benchmark payloads, fingerprint mode is 1% to 40% smaller raw depending on the case, with the largest wins on per-frame streams of repeated records and no win on index-signature records, where the fingerprint costs more than the field ids it removes. diff --git a/packages/effect/src/unstable/encoding/SchemaBinary.ts b/packages/effect/src/unstable/encoding/SchemaBinary.ts index 1a498cf4a63..09a74627f50 100644 --- a/packages/effect/src/unstable/encoding/SchemaBinary.ts +++ b/packages/effect/src/unstable/encoding/SchemaBinary.ts @@ -17,8 +17,9 @@ * compiled wire layout; structs are written positionally, with a presence * bitmap instead of field ids and no length prefix on fixed-size leaves, and * union members are addressed by a canonical index. A reader whose layout - * hashes differently rejects the frame instead of guessing it. The two modes - * are selected by envelope flag bit 0 and are never interchangeable. + * hashes differently rejects the frame instead of guessing it, so peers must + * ship the same schema definition rather than merely a compatible one. The two + * modes are selected by envelope flag bit 0 and are never interchangeable. * * @since 4.0.0 */ @@ -1512,15 +1513,25 @@ function pushU64(out: Array, n: bigint) { } /** - * Hashes the wire-relevant structure of a compiled layout with 64-bit FNV-1a. + * Hashes the compiled layout graph with 64-bit FNV-1a. * * The hash is a Merkle walk: every node mixes its own structure plus the - * 64-bit hash of each child, so two structurally identical layouts hash the - * same whether or not they share a compiled node. Field and property names, - * checks, and annotations are never mixed in; field ids, optionality, wire - * kinds, variant tags, and array shape are. Cycles terminate on a back edge - * carrying the number of levels back to the repeated node, which keeps the - * hash independent of where the cycle was entered. + * 64-bit hash of each child. Field and property names, checks, and annotations + * are never mixed in; field ids, optionality, wire kinds, variant tags, and + * array shape are. Cycles terminate on a back edge carrying the number of + * levels back to the repeated node, so the hash does not depend on where a + * given cycle is entered, and an acyclic sub-layout hashes the same whether it + * is a shared compiled node or written out twice. + * + * What this hashes is the compiled layout *graph*, not the infinite wire shape + * that graph denotes. Those coincide for acyclic layouts. They do not once a + * cycle is involved: the same unfolding has many finite cyclic representations + * and each encodes differently, so a self-recursive `Tree` and the same schema + * with one extra non-recursive node in front of the cycle produce identical + * frames but different hashes and reject each other. Closing that gap needs + * bisimulation minimisation over the layout graph, which is a lot of machinery + * for a case that already fails closed. Peers must ship the same schema + * definition, not merely the same wire shape. */ function layoutFingerprint(root: Layout): bigint { const cache = new Map() @@ -2906,9 +2917,15 @@ function withCycleGuard(target: Schema.Constraint): Schema.Constraint { * positionally without field ids, fixed-size leaves drop their length prefix, * and union members are addressed by a canonical index instead of a kind byte * and a 32-bit sentinel tag. A reader whose layout hashes differently rejects - * the frame rather than guessing, so both sides must ship the same wire - * layout. Non-wire schema changes (checks, annotations, decoded-side - * transformations, field declaration order) leave the hash alone. + * the frame rather than guessing. Non-wire schema changes (checks, + * annotations, decoded-side transformations, field declaration order, and + * repeating an acyclic sub-schema instead of sharing one) leave the hash + * alone. + * + * The hash covers the compiled layout graph rather than the wire shape it + * denotes, so peers must ship the same schema definition. Re-factoring a + * recursive schema without changing a byte of its output still moves the + * hash; see {@link layoutFingerprint}. * * @category models * @since 4.0.0 diff --git a/packages/effect/test/unstable/encoding/SchemaBinary.test.ts b/packages/effect/test/unstable/encoding/SchemaBinary.test.ts index cb08c77887a..6b2bf34a83e 100644 --- a/packages/effect/test/unstable/encoding/SchemaBinary.test.ts +++ b/packages/effect/test/unstable/encoding/SchemaBinary.test.ts @@ -51,16 +51,26 @@ const roundtripFingerprint = (schema: Schema.Codec, value: A): A => return Schema.decodeUnknownSync(codec)(Schema.encodeUnknownSync(codec)(value)) } -// The 8 bytes that follow the fingerprint envelope, as a comparable string. -const fingerprintOf = (schema: Schema.Codec, value: A): string => { +// Splits a fingerprint frame into its layout hash and the payload after it, +// both as comparable strings. +const fingerprintFrame = ( + schema: Schema.Codec, + value: A +): { readonly fingerprint: string; readonly payload: string } => { const bytes = encodeFingerprint(schema, value) let offset = 0 while ((bytes[offset] & 0x80) !== 0) offset++ offset++ assert.strictEqual(bytes[offset], 0x11) - return Array.from(bytes.subarray(offset + 1, offset + 9)).join(",") + return { + fingerprint: Array.from(bytes.subarray(offset + 1, offset + 9)).join(","), + payload: Array.from(bytes.subarray(offset + 9)).join(",") + } } +const fingerprintOf = (schema: Schema.Codec, value: A): string => + fingerprintFrame(schema, value).fingerprint + const schemaError = (f: () => unknown): Schema.SchemaError => { try { f() @@ -1315,7 +1325,7 @@ describe("SchemaBinary", () => { ) }) - it("does not depend on whether a sub-schema is shared or repeated", () => { + it("does not depend on whether an acyclic sub-schema is shared or repeated", () => { const Point = Schema.Struct({ x: Schema.Number, y: Schema.Number }) const pair = { a: { x: 1, y: 2 }, b: { x: 3, y: 4 } } const shared = fingerprintOf(Schema.Struct({ a: Point, b: Point }), pair) @@ -1341,7 +1351,9 @@ describe("SchemaBinary", () => { } const leaf = { value: 1, children: [] } assert.strictEqual(fingerprintOf(makeTree(), leaf), fingerprintOf(makeTree(), leaf)) - // one recursive node reached from two fields, versus two of them + // one recursive node reached from two fields, versus two of them. Both + // sides have the same cycle structure, which is the only recursive case + // the hash canonicalises; see the factoring test below for the limit. const both = { left: leaf, right: leaf } const one = makeTree() assert.strictEqual( @@ -1350,6 +1362,64 @@ describe("SchemaBinary", () => { ) }) + it("hashes the layout graph, so re-factoring a recursive schema moves it", () => { + interface Tree { + readonly value: number + readonly children: ReadonlyArray + } + const makeTree = (): Schema.Codec => { + const self: Schema.Codec = Schema.Struct({ + value: Schema.Number, + children: Schema.Array(Schema.suspend((): Schema.Codec => self)) + }) + return self + } + // Three finite graphs denoting the same infinite wire shape: the cycle + // itself, the cycle behind one non-recursive alias, and a two-node + // mutual recursion of the same shape. + const direct = makeTree() + const aliased = Schema.Struct({ + value: Schema.Number, + children: Schema.Array(makeTree()) + }) as unknown as Schema.Codec + const mutualA: Schema.Codec = Schema.Struct({ + value: Schema.Number, + children: Schema.Array(Schema.suspend((): Schema.Codec => mutualB)) + }) + const mutualB: Schema.Codec = Schema.Struct({ + value: Schema.Number, + children: Schema.Array(Schema.suspend((): Schema.Codec => mutualA)) + }) + + const tree = { value: 1, children: [{ value: 2, children: [] }] } + const frames = [direct, aliased, mutualA].map((schema) => fingerprintFrame(schema, tree)) + + // identical bytes on the wire + assert.strictEqual(new Set(frames.map((frame) => frame.payload)).size, 1) + // and identical in the tolerant default mode, in both directions + const directDefault = SchemaBinary.toCodec(direct) + const aliasedDefault = SchemaBinary.toCodec(aliased) + assert.deepStrictEqual( + Schema.decodeUnknownSync(aliasedDefault)(Schema.encodeUnknownSync(directDefault)(tree)), + tree + ) + assert.deepStrictEqual( + Schema.decodeUnknownSync(directDefault)(Schema.encodeUnknownSync(aliasedDefault)(tree)), + tree + ) + + // but each factoring hashes differently, and they reject each other + assert.strictEqual(new Set(frames.map((frame) => frame.fingerprint)).size, 3) + assert.include( + schemaError(() => + Schema.decodeUnknownSync(SchemaBinary.toCodec(aliased, { fingerprint: true }))( + encodeFingerprint(direct, tree) + ) + ).message, + "Expected matching layout fingerprint" + ) + }) + it("changes whenever the wire layout changes", () => { const changed = [ fingerprintOf(Schema.Struct({ label: Schema.String, age: Schema.Number }), { label: "a", age: 1 }),