diff --git a/.changeset/schema-binary-codec.md b/.changeset/schema-binary-codec.md new file mode 100644 index 00000000000..7c906ca5de6 --- /dev/null +++ b/.changeset/schema-binary-codec.md @@ -0,0 +1,5 @@ +--- +"effect": patch +--- + +Add `SchemaBinary`, a compact Schema-derived codec with streaming parsing and optional fingerprinted layouts, plus a fingerprinted `RpcSerialization` layer. diff --git a/packages/effect/benchmark/rpc/RpcSerialization.ts b/packages/effect/benchmark/rpc/RpcSerialization.ts new file mode 100644 index 00000000000..d50e49d46c7 --- /dev/null +++ b/packages/effect/benchmark/rpc/RpcSerialization.ts @@ -0,0 +1,279 @@ +import { Effect, Exit, Schema } from "effect" +import { Rpc, RpcSerialization } from "effect/unstable/rpc" +import assert from "node:assert/strict" +import { cpus } from "node:os" +import { Bench } from "tinybench" + +const User = Schema.Struct({ + id: Schema.String, + displayName: Schema.String, + email: Schema.String, + active: Schema.Boolean, + roles: Schema.Array(Schema.String) +}) + +const SearchUsers = Rpc.make("SearchUsers", { + payload: Schema.Struct({ + organizationId: Schema.String, + query: Schema.String, + page: Schema.Number, + pageSize: Schema.Number, + filters: Schema.Struct({ + active: Schema.optional(Schema.Boolean), + roles: Schema.Array(Schema.String) + }) + }), + success: Schema.Array(User), + error: Schema.Struct({ + code: Schema.String, + message: Schema.String + }) +}) + +const Event = Schema.Struct({ + sequence: Schema.Number, + timestamp: Schema.Date, + level: Schema.Literals(["info", "warning", "error"]), + message: Schema.String, + attributes: Schema.Record(Schema.String, Schema.String) +}) + +const searchPayload = { + organizationId: "org-effect", + query: "schema binary", + page: 3, + pageSize: 25, + filters: { + active: true, + roles: ["maintainer", "contributor"] + } +} + +const users = Array.from({ length: 12 }, (_, index) => ({ + id: `user-${index.toString().padStart(3, "0")}`, + displayName: `Benchmark User ${index}`, + email: `benchmark-${index}@example.com`, + active: index % 4 !== 0, + roles: index % 3 === 0 ? ["maintainer", "contributor"] : ["contributor"] +})) + +const makeEvent = (index: number) => ({ + sequence: index + 1, + timestamp: new Date(1_756_000_000_000 + index * 1_000), + level: index % 11 === 0 ? "warning" as const : "info" as const, + message: `Processed RPC event ${index + 1}`, + attributes: { + region: ["eu-west-1", "us-east-1", "ap-southeast-2"][index % 3]!, + worker: `worker-${index % 8}`, + attempt: String(index % 3 + 1) + } +}) + +const events = [ + makeEvent(0), + ...Array.from({ length: 31 }, (_, index) => makeEvent(index + 1)) +] satisfies Schema.NonEmptyArray["Type"] + +interface BenchmarkCase { + readonly name: string + readonly schema: Schema.Codec + readonly value: unknown + readonly envelope: (hole: unknown) => unknown + readonly hole: (envelope: unknown) => unknown +} + +const getHole = (key: PropertyKey) => (envelope: unknown): unknown => { + assert(typeof envelope === "object" && envelope !== null && key in envelope) + return (envelope as Record)[key] +} + +const cases: ReadonlyArray = [ + { + name: "request / nested search payload", + schema: SearchUsers.payloadSchema, + value: searchPayload, + envelope: (payload) => ({ + _tag: "Request", + id: 1, + tag: SearchUsers._tag, + payload, + headers: [ + ["authorization", "Bearer benchmark-token"], + ["x-request-id", "benchmark-request-0001"] + ], + traceId: "0123456789abcdef0123456789abcdef", + spanId: "0123456789abcdef", + sampled: true + }), + hole: getHole("payload") + }, + { + name: "exit / 12-user success", + schema: Rpc.exitSchema(SearchUsers), + value: Exit.succeed(users), + envelope: (exit) => ({ + _tag: "Exit", + requestId: 1, + exit + }), + hole: getHole("exit") + }, + { + name: "chunk / 32 events", + schema: Schema.NonEmptyArray(Event), + value: events, + envelope: (values) => ({ + _tag: "Chunk", + requestId: 1, + values + }), + hole: getHole("values") + } +] + +const schemaBinary = Effect.runSync( + RpcSerialization.RpcSerialization.pipe(Effect.provide(RpcSerialization.layerSchemaBinary)) +) + +const formats = [ + { name: "Msgpack", serialization: RpcSerialization.msgPack }, + { name: "SchemaBinary", serialization: schemaBinary } +] as const + +const warmupIterations = 500 +const iterations = 5_000 + +interface Prepared { + readonly caseName: string + readonly formatName: string + readonly firstFrameSize: number + readonly steadyFrameSize: number + readonly encode: () => unknown + readonly decode: () => unknown +} + +const toBytes = (encoded: Uint8Array | string | undefined): Uint8Array => { + assert(encoded instanceof Uint8Array) + return encoded.slice() +} + +const prepared = cases.flatMap((testCase): ReadonlyArray => + formats.map(({ name, serialization }) => { + const encodeHole = Schema.encodeUnknownSync(serialization.codecFor(testCase.schema)) + const decodeHole = Schema.decodeUnknownSync(serialization.codecFor(testCase.schema)) + const encoder = serialization.makeUnsafe() + const firstFrame = toBytes(encoder.encode(testCase.envelope(encodeHole(testCase.value)))) + const steadyFrame = toBytes(encoder.encode(testCase.envelope(encodeHole(testCase.value)))) + const decoder = serialization.makeUnsafe() + + const decode = (frame: Uint8Array) => { + const envelopes = decoder.decode(frame) + assert.strictEqual(envelopes.length, 1) + return decodeHole(testCase.hole(envelopes[0])) + } + + assert.deepStrictEqual(decode(firstFrame), testCase.value) + assert.deepStrictEqual(decode(steadyFrame), testCase.value) + + return { + caseName: testCase.name, + formatName: name, + firstFrameSize: firstFrame.length, + steadyFrameSize: steadyFrame.length, + encode: () => encoder.encode(testCase.envelope(encodeHole(testCase.value))), + decode: () => decode(steadyFrame) + } + }) +) + +console.log(`${process.platform} ${process.arch}; ${cpus()[0]?.model ?? "unknown CPU"}; Node ${process.version}`) +console.log( + "End-to-end operations include the payload codec plus RPC envelope framing; codec construction is excluded." +) +console.log( + "Msgpack uses RpcSerialization.msgPack defaults, including records. SchemaBinary fingerprints envelopes only." +) +console.log( + "First-frame sizes use a fresh parser; steady sizes and throughput reuse one parser as on a long-lived connection." +) +console.log( + `${warmupIterations.toLocaleString()} warmup operations and ${iterations.toLocaleString()} measured operations per case, format, and direction.` +) + +console.table(prepared.map((entry) => ({ + Case: entry.caseName, + Format: entry.formatName, + "First frame bytes": entry.firstFrameSize, + "Steady frame bytes": entry.steadyFrameSize +}))) + +const bench = new Bench({ + iterations, + time: 0, + warmupIterations, + warmupTime: 0, + timestampProvider: "hrtimeNow" +}) +const labels = new Map() +let sink: unknown + +for (const entry of prepared) { + for (const [direction, run] of [["encode", entry.encode], ["decode", entry.decode]] as const) { + const name = `${entry.caseName} / ${entry.formatName} / ${direction}` + labels.set(name, { + caseName: entry.caseName, + formatName: entry.formatName, + direction + }) + bench.add(name, () => { + sink = run() + }) + } +} + +await bench.run() +assert.notStrictEqual(sink, undefined) + +const msgpackThroughput = new Map() +for (const task of bench.tasks) { + const label = labels.get(task.name)! + if (label.formatName === "Msgpack" && task.result?.state === "completed") { + msgpackThroughput.set(`${label.caseName}/${label.direction}`, task.result.throughput.mean) + } +} + +console.table(bench.tasks.map((task) => { + const label = labels.get(task.name)! + const result = task.result + if (result?.state === "errored") { + return { + Case: label.caseName, + Format: label.formatName, + Direction: label.direction, + Error: result.error.message + } + } + if (result?.state !== "completed") { + return { + Case: label.caseName, + Format: label.formatName, + Direction: label.direction, + State: result?.state ?? "missing result" + } + } + const baseline = msgpackThroughput.get(`${label.caseName}/${label.direction}`)! + return { + Case: label.caseName, + Format: label.formatName, + Direction: label.direction, + "Throughput avg (ops/s)": Math.round(result.throughput.mean), + "vs Msgpack": `${(result.throughput.mean / baseline).toFixed(2)}x`, + "Latency med (us/op)": (result.latency.p50 * 1_000).toFixed(2), + "Latency RME": `${result.latency.rme.toFixed(2)}%`, + Samples: result.latency.samplesCount + } +})) diff --git a/packages/effect/benchmark/schema/SchemaBinary.md b/packages/effect/benchmark/schema/SchemaBinary.md new file mode 100644 index 00000000000..13e5ad26aad --- /dev/null +++ b/packages/effect/benchmark/schema/SchemaBinary.md @@ -0,0 +1,98 @@ +# SchemaBinary benchmark + +Run from the repository root: + +```sh +nix develop -c pnpm --dir packages/effect exec node benchmark/schema/SchemaBinary.ts +``` + +These results are from one full run at commit `ef9b90a9b` on Linux x86_64 with Node 26.7.0. Codec and schema construction are excluded. One-shot tasks use 100 warmups and 1,000 measured samples; streaming tasks use 25 warmups and 250 measured samples. Throughput is machine-local and should only be compared within this run. + +Fingerprint mode omits field identifiers when the schema fingerprint matches. JSON, Msgpack, and NDJSON use the same `Schema.toCodecJson` representation. + +The streaming setup compares the closest public decode paths. SchemaBinary reuses one synchronous parser for each feed shape. Msgpack synchronously calls `unpackMultiple` on the batch, then validates each value. NDJSON runs `Ndjson.decodeSchema` through Effect Stream and Channel for every operation, including UTF-8 decoding, line splitting, `JSON.parse`, schema validation, and runtime scheduling. A batch is one Channel run over 32 lines, or 200 lines for the per-frame case. Single and fragmented measurements each run a complete Channel for one line, so their scheduling cost is not amortized. This makes batch the closest throughput comparison while preserving the cost of each public API. + +SchemaBinary uses length-prefixed frames, NDJSON includes one newline byte per frame, and Msgpack concatenates self-delimiting values. Fragmented inputs split after the first byte. Stream compression is applied to the complete concatenated stream. + +The `200-row array payload` case uses `Schema.Array(LargeRow)`, so one value is an array containing 200 rows. A one-shot operation encodes or decodes that entire array. Its streaming batch contains 32 frames with the same 200-row array, or 6,400 row occurrences in total. The `200 single-row frames` case uses `LargeRow` directly and sends the 200 distinct rows as 200 frames. Streaming throughput is decoded values per second: arrays per second for the first case and rows per second for the second. Multiply the array rate by 200 to compare decoded row throughput. + +## Payload size + +Cells contain raw / gzip -6 / zstd bytes. + +| Case | SchemaBinary | Fingerprint | JSON | Msgpack | +| ---------------------- | ------------------: | ------------------: | ------------------: | ------------------: | +| small record | 58 / 78 / 67 | 35 / 53 / 44 | 89 / 100 / 92 | 69 / 88 / 78 | +| nested payload | 318 / 305 / 300 | 206 / 209 / 203 | 453 / 303 / 309 | 385 / 304 / 299 | +| collections | 1452 / 792 / 776 | 1438 / 776 / 758 | 1828 / 671 / 660 | 1462 / 788 / 805 | +| index signatures / 128 | 2251 / 689 / 656 | 2258 / 697 / 665 | 2235 / 573 / 559 | 2203 / 676 / 629 | +| index signatures / 512 | 9355 / 2373 / 2189 | 9362 / 2383 / 2197 | 9531 / 2266 / 2074 | 9287 / 2544 / 2304 | +| 200-row array payload | 27646 / 3065 / 3175 | 19454 / 2766 / 2701 | 57529 / 3230 / 3008 | 51283 / 3516 / 3320 | + +Streaming cells contain total raw / gzip -6 / zstd bytes for the complete stream. + +| Case | Frames | SchemaBinary | Fingerprint | Msgpack | NDJSON | +| ---------------------- | -----: | --------------------: | --------------------: | --------------------: | ---------------------: | +| small record | 32 | 1856 / 95 / 75 | 1120 / 65 / 51 | 2240 / 111 / 87 | 2880 / 123 / 98 | +| nested payload | 32 | 10176 / 391 / 305 | 6592 / 259 / 208 | 10880 / 369 / 306 | 14528 / 399 / 315 | +| collections | 32 | 46464 / 1122 / 793 | 46016 / 1102 / 776 | 47424 / 1130 / 813 | 58528 / 1079 / 676 | +| index signatures / 128 | 32 | 72032 / 1229 / 669 | 72256 / 1270 / 678 | 71008 / 1252 / 649 | 71552 / 1073 / 540 | +| index signatures / 512 | 32 | 299360 / 5528 / 2227 | 299584 / 5528 / 2235 | 297696 / 5775 / 2037 | 305024 / 5831 / 1829 | +| 200-row array payload | 32 | 884672 / 14110 / 3259 | 622528 / 12524 / 2765 | 623488 / 12335 / 2736 | 1840960 / 87481 / 3226 | +| 200 single-row frames | 200 | 27840 / 3109 / 3174 | 21240 / 2876 / 2733 | 51520 / 2956 / 2888 | 57528 / 3230 / 3019 | + +## One-shot throughput + +Average encode operations per second: + +| Case | Default | Fingerprint | JSON | Msgpack | +| ---------------------- | --------: | ----------: | ------: | ------: | +| small record | 1,138,225 | 1,147,081 | 452,215 | 715,571 | +| nested payload | 395,240 | 408,163 | 263,498 | 281,540 | +| collections | 35,376 | 35,651 | 21,778 | 22,134 | +| index signatures / 128 | 14,096 | 14,167 | 35,928 | 35,077 | +| index signatures / 512 | 2,871 | 2,867 | 6,508 | 6,314 | +| 200-row array payload | 6,995 | 8,128 | 6,125 | 4,038 | + +Average decode operations per second: + +| Case | Default | Fingerprint | JSON | Msgpack | +| ---------------------- | --------: | ----------: | ------: | ------: | +| small record | 1,121,650 | 1,115,901 | 418,359 | 687,335 | +| nested payload | 354,482 | 376,201 | 217,858 | 266,371 | +| collections | 39,697 | 39,708 | 20,723 | 21,943 | +| index signatures / 128 | 20,589 | 20,675 | 28,624 | 29,884 | +| index signatures / 512 | 4,988 | 4,969 | 5,411 | 4,562 | +| 200-row array payload | 7,233 | 7,934 | 5,630 | 4,761 | + +## Streaming decode throughput + +Average decoded values per second for batched input: + +| Case | Default | Fingerprint | Msgpack | NDJSON | +| ---------------------- | --------: | ----------: | ------: | ------: | +| small record | 4,385,709 | 5,493,837 | 901,253 | 849,385 | +| nested payload | 819,281 | 977,777 | 284,386 | 306,687 | +| collections | 118,938 | 119,806 | 21,189 | 20,364 | +| index signatures / 128 | 58,715 | 58,699 | 27,863 | 28,431 | +| index signatures / 512 | 8,374 | 8,322 | 4,055 | 5,064 | +| 200-row array payload | 11,147 | 12,996 | 6,345 | 4,953 | +| 200 single-row frames | 2,127,598 | 2,382,278 | 856,723 | 946,703 | + +Average decoded values per second for single and first-byte-fragmented input: + +| Case | Default single | Default fragmented | Fingerprint single | Fingerprint fragmented | NDJSON single | NDJSON fragmented | +| ---------------------- | -------------: | -----------------: | -----------------: | ---------------------: | ------------: | ----------------: | +| small record | 2,585,851 | 2,296,255 | 3,131,403 | 2,871,336 | 137,242 | 142,477 | +| nested payload | 710,222 | 699,114 | 836,484 | 826,233 | 109,985 | 108,842 | +| collections | 116,964 | 116,132 | 117,837 | 116,882 | 18,463 | 18,438 | +| index signatures / 128 | 57,343 | 57,974 | 58,448 | 58,763 | 24,688 | 24,643 | +| index signatures / 512 | 8,414 | 8,704 | 8,793 | 8,619 | 5,218 | 5,195 | +| 200-row array payload | 11,419 | 11,474 | 13,312 | 13,295 | 5,206 | 5,127 | +| 200 single-row frames | 1,390,397 | 1,369,513 | 1,594,590 | 1,512,635 | 130,684 | 127,909 | + +## Analysis + +- SchemaBinary leads JSON and Msgpack on struct- and collection-heavy one-shot cases. JSON and Msgpack remain faster on large index-signature encodes, where field lookup dominates and the binary layout offers little structural advantage. +- SchemaBinary is faster than both text and Msgpack streaming in every batch case. NDJSON and Msgpack batch throughput are close and trade places by shape, but NDJSON single-frame throughput is dominated by running a complete Effect Channel for each value. +- NDJSON is the largest raw stream in every case. Compression changes the ranking for repeated keys: NDJSON has the smallest zstd output for collections and both index-signature cases. Fingerprint SchemaBinary remains the smallest raw format for struct-heavy values. diff --git a/packages/effect/benchmark/schema/SchemaBinary.ts b/packages/effect/benchmark/schema/SchemaBinary.ts new file mode 100644 index 00000000000..178dd9480f2 --- /dev/null +++ b/packages/effect/benchmark/schema/SchemaBinary.ts @@ -0,0 +1,519 @@ +import { Effect, Schema, Stream } from "effect" +import { Msgpack, Ndjson, SchemaBinary } from "effect/unstable/encoding" +import { Packr, Unpackr } from "msgpackr" +import assert from "node:assert/strict" +import { gzipSync, zstdCompressSync } from "node:zlib" +import { Bench } from "tinybench" + +const streamBatchSize = 32 +const repeatedRecordStreamSize = 200 + +const SmallRecord = Schema.Struct({ + id: Schema.Number, + active: Schema.Boolean, + score: Schema.Number, + retryCount: Schema.Number, + region: Schema.String, + verified: Schema.Boolean +}) + +const LineItem = Schema.Struct({ + sku: Schema.String, + quantity: Schema.Number, + unitPrice: Schema.Number +}) + +const NestedPayload = Schema.Struct({ + orderId: Schema.String, + customer: Schema.Struct({ + id: Schema.String, + name: Schema.String, + email: Schema.String + }), + shipping: Schema.Struct({ + street: Schema.String, + city: Schema.String, + postalCode: Schema.String, + country: Schema.String + }), + lines: Schema.Array(LineItem), + metadata: Schema.Struct({ + source: Schema.String, + campaign: Schema.String, + priority: Schema.Boolean + }) +}) + +const Collections = Schema.Struct({ + tags: Schema.Array(Schema.String), + metrics: Schema.Record(Schema.String, Schema.Number), + samples: Schema.Array(Schema.Tuple([Schema.Number, Schema.Number, Schema.Boolean])), + buckets: Schema.Array(Schema.Array(Schema.Number)) +}) + +const LargeRow = Schema.Struct({ + transactionIdentifier: Schema.String, + customerIdentifier: Schema.String, + productDescription: Schema.String, + fulfillmentLocation: Schema.String, + quantityPurchased: Schema.Number, + unitPriceInCents: Schema.Number, + discountInBasisPoints: Schema.Number, + requiresManualReview: Schema.Boolean +}) + +const LargePayload = Schema.Array(LargeRow) + +const largeRows = Array.from({ length: repeatedRecordStreamSize }, (_, index) => ({ + transactionIdentifier: `transaction-${index.toString().padStart(4, "0")}`, + customerIdentifier: `customer-${index % 37}`, + productDescription: `Product ${index % 19} with a repeated descriptive field value`, + fulfillmentLocation: ["London", "New York", "Singapore", "Sydney"][index % 4]!, + quantityPurchased: index % 9 + 1, + unitPriceInCents: 500 + index % 73 * 25, + discountInBasisPoints: index % 5 * 125, + requiresManualReview: index % 17 === 0 +})) + +const metrics = (count: number) => + Object.fromEntries(Array.from({ length: count }, (_, index) => [`metric-${index}`, index * 1.25])) + +const cases = [ + { + name: "small record", + schema: SmallRecord, + value: { + id: 42, + active: true, + score: 98.5, + retryCount: 2, + region: "eu-west-1", + verified: false + } + }, + { + name: "nested payload", + schema: NestedPayload, + value: { + orderId: "order-2026-000184", + customer: { + id: "customer-91", + name: "Ada Lovelace", + email: "ada@example.com" + }, + shipping: { + street: "12 Analytical Engine Way", + city: "London", + postalCode: "SW1A 1AA", + country: "GB" + }, + lines: [ + { sku: "widget-blue", quantity: 2, unitPrice: 12.5 }, + { sku: "adapter-pro", quantity: 1, unitPrice: 48 }, + { sku: "cable-2m", quantity: 3, unitPrice: 8.25 } + ], + metadata: { + source: "partner-api", + campaign: "summer-2026", + priority: true + } + } + }, + { + name: "collections", + schema: Collections, + value: { + tags: Array.from({ length: 24 }, (_, index) => `tag-${index}`), + metrics: Object.fromEntries(Array.from({ length: 24 }, (_, index) => [`metric-${index}`, index * 1.25])), + samples: Array.from({ length: 48 }, (_, index) => [index, index / 10, index % 3 === 0] as const), + buckets: Array.from({ length: 8 }, (_, bucket) => Array.from({ length: 16 }, (_, index) => bucket * 100 + index)) + } + }, + { + name: "index signatures / 128 keys", + schema: Schema.Record(Schema.String, Schema.Number), + value: metrics(128) + }, + { + name: "index signatures / 512 keys", + schema: Schema.Record(Schema.String, Schema.Number), + value: metrics(512) + }, + { + name: "200-row array payload", + schema: LargePayload, + value: largeRows + } +] as const + +interface Format { + readonly name: string + readonly encodedSize: number + readonly gzipSize: number + readonly zstdSize: number + readonly encode: () => unknown + readonly decode: () => unknown +} + +interface StreamFormat { + readonly name: string + readonly framesPerOp: number + readonly decode: () => ReadonlyArray | Promise> +} + +interface StreamSize { + readonly name: string + readonly frames: number + readonly encodedSize: number + readonly gzipSize: number + readonly zstdSize: number +} + +const textEncoder = new TextEncoder() + +const concatFrames = (frames: ReadonlyArray): Uint8Array => { + const out = new Uint8Array(frames.reduce((length, frame) => length + frame.length, 0)) + let offset = 0 + for (const frame of frames) { + out.set(frame, offset) + offset += frame.length + } + return out +} + +const sizes = (encoded: Uint8Array): Pick => ({ + encodedSize: encoded.length, + gzipSize: gzipSync(encoded).length, + zstdSize: zstdCompressSync(encoded).length +}) + +const prepare = >( + schema: S, + value: S["Type"] +): { + readonly formats: ReadonlyArray +} => { + const jsonSchema = Schema.toCodecJson(schema) + const binaryCodec = SchemaBinary.toCodec(schema) + const fingerprintCodec = SchemaBinary.toCodec(schema, { fingerprint: true }) + const jsonCodec = Schema.fromJsonString(jsonSchema) + const msgpackCodec = Msgpack.schema(jsonSchema) + + const binaryEncode = Schema.encodeUnknownSync(binaryCodec) + const binaryDecode = Schema.decodeUnknownSync(binaryCodec) + const fingerprintEncode = Schema.encodeUnknownSync(fingerprintCodec) + const fingerprintDecode = Schema.decodeUnknownSync(fingerprintCodec) + const jsonEncode = Schema.encodeUnknownSync(jsonCodec) + const jsonDecode = Schema.decodeUnknownSync(jsonCodec) + const msgpackEncode = Schema.encodeUnknownSync(msgpackCodec) + const msgpackDecode = Schema.decodeUnknownSync(msgpackCodec) + + const binary = binaryEncode(value) + const fingerprint = fingerprintEncode(value).slice() + const json = jsonEncode(value) + const jsonBytes = textEncoder.encode(json) + const msgpack = msgpackEncode(value) + + assert.deepStrictEqual(binaryDecode(binary), value) + assert.deepStrictEqual(fingerprintDecode(fingerprint), value) + assert.deepStrictEqual(jsonDecode(json), value) + assert.deepStrictEqual(msgpackDecode(msgpack), value) + + return { + formats: [ + { + name: "SchemaBinary", + ...sizes(binary), + encode: () => binaryEncode(value), + decode: () => binaryDecode(binary) + }, + { + name: "SchemaBinary fingerprint", + ...sizes(fingerprint), + encode: () => fingerprintEncode(value), + decode: () => fingerprintDecode(fingerprint) + }, + { + name: "JSON", + ...sizes(jsonBytes), + encode: () => jsonEncode(value), + decode: () => jsonDecode(json) + }, + { + name: "Msgpack", + ...sizes(msgpack), + encode: () => msgpackEncode(value), + decode: () => msgpackDecode(msgpack) + } + ] + } +} + +const prepared = cases.map((testCase) => ({ name: testCase.name, ...prepare(testCase.schema, testCase.value) })) + +const prepareStream = async >( + schema: S, + values: ReadonlyArray +): Promise<{ readonly formats: ReadonlyArray; readonly sizes: ReadonlyArray }> => { + const binaryCodec = SchemaBinary.toCodec(schema) + const binaryEncode = Schema.encodeUnknownSync(binaryCodec) + const binaryFrames = values.map((value) => binaryEncode(value).slice()) + const binaryStream = concatFrames(binaryFrames) + const binaryFragments = binaryFrames.map((frame) => { + return [frame.subarray(0, 1), frame.subarray(1)] as const + }) + + const fingerprintEncode = Schema.encodeUnknownSync(SchemaBinary.toCodec(schema, { fingerprint: true })) + const fingerprintFrames = values.map((value) => fingerprintEncode(value).slice()) + const fingerprintStream = concatFrames(fingerprintFrames) + const fingerprintFragments = fingerprintFrames.map((frame) => { + return [frame.subarray(0, 1), frame.subarray(1)] as const + }) + + const jsonSchema = Schema.toCodecJson(schema) + const encodeMsgpackValue = Schema.encodeUnknownSync(jsonSchema) + const decodeMsgpackValue = Schema.decodeUnknownSync(jsonSchema) + const msgpackPackr = new Packr() + const msgpackStream = concatFrames(values.map((value) => msgpackPackr.pack(encodeMsgpackValue(value)).slice())) + const msgpackUnpackr = new Unpackr() + + const ndjsonFrames = values.map((value) => textEncoder.encode(`${JSON.stringify(encodeMsgpackValue(value))}\n`)) + const ndjsonStream = concatFrames(ndjsonFrames) + const ndjsonFragments = ndjsonFrames.map((frame) => { + return [frame.subarray(0, 1), frame.subarray(1)] as const + }) + const ndjsonDecoder = Ndjson.decodeSchema(jsonSchema)() + const runNdjson = (chunks: ReadonlyArray) => + Effect.runPromise( + Stream.fromIterable(chunks).pipe( + Stream.pipeThroughChannel(ndjsonDecoder), + Stream.runCollect, + Effect.map((chunk) => Array.from(chunk)) + ) + ) + let ndjsonSingleIndex = 0 + let ndjsonFragmentedIndex = 0 + const decodeNdjsonSingle = () => runNdjson([ndjsonFrames[ndjsonSingleIndex++ % ndjsonFrames.length]!]) + const decodeNdjsonBatch = () => runNdjson([ndjsonStream]) + const decodeNdjsonFragmented = () => runNdjson(ndjsonFragments[ndjsonFragmentedIndex++ % ndjsonFragments.length]!) + + const feedShapes = (options?: { readonly fingerprint: true }) => { + const frames = options === undefined ? binaryFrames : fingerprintFrames + const fragments = options === undefined ? binaryFragments : fingerprintFragments + const stream = options === undefined ? binaryStream : fingerprintStream + const singleParser = SchemaBinary.parser(schema, options) + const fragmentedParser = SchemaBinary.parser(schema, options) + const batchParser = SchemaBinary.parser(schema, options) + let singleIndex = 0 + let fragmentedIndex = 0 + return { + single: () => singleParser.feedSync(frames[singleIndex++ % frames.length]), + fragmented: () => { + const pair = fragments[fragmentedIndex++ % fragments.length] + const first = fragmentedParser.feedSync(pair[0]) + const second = fragmentedParser.feedSync(pair[1]) + return first.length === 0 ? second : [...first, ...second] + }, + batch: () => batchParser.feedSync(stream) + } + } + + const defaultFeeds = feedShapes() + const fingerprintFeeds = feedShapes({ fingerprint: true }) + const decodeSingle = defaultFeeds.single + const decodeFragmented = defaultFeeds.fragmented + const decodeBatch = defaultFeeds.batch + const decodeMsgpackStream = () => + msgpackUnpackr.unpackMultiple(msgpackStream).map((value) => decodeMsgpackValue(value)) + + assert.deepStrictEqual(decodeSingle(), [values[0]]) + assert.deepStrictEqual(decodeFragmented(), [values[0]]) + assert.deepStrictEqual(decodeBatch(), values) + assert.deepStrictEqual(fingerprintFeeds.single(), [values[0]]) + assert.deepStrictEqual(fingerprintFeeds.fragmented(), [values[0]]) + assert.deepStrictEqual(fingerprintFeeds.batch(), values) + assert.deepStrictEqual(decodeMsgpackStream(), values) + assert.deepStrictEqual(await decodeNdjsonSingle(), [values[0]]) + assert.deepStrictEqual(await decodeNdjsonFragmented(), [values[0]]) + assert.deepStrictEqual(await decodeNdjsonBatch(), values) + + return { + formats: [ + { name: "SchemaBinary parser / single frame", framesPerOp: 1, decode: decodeSingle }, + { name: "SchemaBinary parser / batch", framesPerOp: values.length, decode: decodeBatch }, + { name: "SchemaBinary parser / fragmented", framesPerOp: 1, decode: decodeFragmented }, + { name: "SchemaBinary fingerprint / single frame", framesPerOp: 1, decode: fingerprintFeeds.single }, + { name: "SchemaBinary fingerprint / batch", framesPerOp: values.length, decode: fingerprintFeeds.batch }, + { name: "SchemaBinary fingerprint / fragmented", framesPerOp: 1, decode: fingerprintFeeds.fragmented }, + { name: "Msgpack unpackMultiple / batch", framesPerOp: values.length, decode: decodeMsgpackStream }, + { name: "NDJSON Channel / single frame", framesPerOp: 1, decode: decodeNdjsonSingle }, + { name: "NDJSON Channel / batch", framesPerOp: values.length, decode: decodeNdjsonBatch }, + { name: "NDJSON Channel / fragmented", framesPerOp: 1, decode: decodeNdjsonFragmented } + ], + sizes: [ + { name: "SchemaBinary", frames: values.length, ...sizes(binaryStream) }, + { name: "SchemaBinary fingerprint", frames: values.length, ...sizes(fingerprintStream) }, + { name: "Msgpack", frames: values.length, ...sizes(msgpackStream) }, + { name: "NDJSON", frames: values.length, ...sizes(ndjsonStream) } + ] + } +} + +const preparedStreams = await Promise.all([ + ...cases.map((testCase) => ({ + name: testCase.name, + prepared: prepareStream(testCase.schema, Array.from({ length: streamBatchSize }, () => testCase.value)) + })), + { name: "200 single-row frames", prepared: prepareStream(LargeRow, largeRows) } +].map(async ({ name, prepared }) => ({ name, ...await prepared }))) + +console.log(`Node ${process.version}; codec and schema construction excluded from timings.`) +console.log("JSON and Msgpack use the same Schema.toCodecJson representation; JSON sizes are UTF-8 bytes.") +console.log( + "Compare formats within a case and direction in the same run; absolute rates vary with the machine and runtime." +) + +console.table(prepared.flatMap((testCase) => + testCase.formats.map((format) => ({ + Case: testCase.name, + Format: format.name, + "Raw bytes": format.encodedSize, + "gzip -6 bytes": format.gzipSize, + "zstd bytes": format.zstdSize + })) +)) + +console.log( + "SchemaBinary streaming reuses one parser per feed shape; NDJSON runs its Channel per operation. Fragmented frames split after the first byte." +) +console.table(preparedStreams.flatMap((testCase) => + testCase.sizes.map((format) => ({ + Case: testCase.name, + Format: format.name, + Frames: format.frames, + "Raw bytes": format.encodedSize, + "Bytes / frame": format.encodedSize / format.frames, + "gzip -6 bytes": format.gzipSize, + "zstd bytes": format.zstdSize + })) +)) + +const bench = new Bench({ + iterations: 1_000, + time: 0, + warmupIterations: 100, + warmupTime: 0, + timestampProvider: "hrtimeNow" +}) +const tasks = new Map() +const sinkSentinel = Symbol("benchmark did not run") +let sink: unknown = sinkSentinel + +for (const testCase of prepared) { + for (const format of testCase.formats) { + for (const [direction, run] of [["encode", format.encode], ["decode", format.decode]] as const) { + const name = `${testCase.name} / ${format.name} / ${direction}` + tasks.set(name, { caseName: testCase.name, formatName: format.name, direction }) + bench.add(name, () => { + sink = run() + }) + } + } +} + +await bench.run() + +if (sink === sinkSentinel) { + throw new Error("Benchmark did not run") +} + +console.table(bench.tasks.map((task) => { + const labels = tasks.get(task.name)! + const result = task.result + if (result?.state === "errored") { + return { + Case: labels.caseName, + Format: labels.formatName, + Direction: labels.direction, + Error: result.error.message + } + } + if (result?.state !== "completed") { + return { + Case: labels.caseName, + Format: labels.formatName, + Direction: labels.direction, + State: result?.state ?? "missing result" + } + } + return { + Case: labels.caseName, + Format: labels.formatName, + Direction: labels.direction, + "Throughput avg (ops/s)": Math.round(result.throughput.mean), + "Latency med (us/op)": (result.latency.p50 * 1_000).toFixed(2), + "Latency RME": `${result.latency.rme.toFixed(2)}%`, + Samples: result.latency.samplesCount + } +})) + +const streamBench = new Bench({ + iterations: 250, + time: 0, + warmupIterations: 25, + warmupTime: 0, + timestampProvider: "hrtimeNow" +}) +const streamTasks = new Map< + string, + { readonly caseName: string; readonly formatName: string; readonly framesPerOp: number } +>() + +for (const testCase of preparedStreams) { + for (const format of testCase.formats) { + const name = `${testCase.name} / ${format.name} / stream decode` + streamTasks.set(name, { + caseName: testCase.name, + formatName: format.name, + framesPerOp: format.framesPerOp + }) + streamBench.add(name, () => { + const decoded = format.decode() + if (decoded instanceof Promise) { + return decoded.then((value) => { + sink = value + }) + } + sink = decoded + }) + } +} + +await streamBench.run() + +console.table(streamBench.tasks.map((task) => { + const labels = streamTasks.get(task.name)! + const result = task.result + if (result?.state === "errored") { + return { + Case: labels.caseName, + Format: labels.formatName, + Error: result.error.message + } + } + if (result?.state !== "completed") { + return { + Case: labels.caseName, + Format: labels.formatName, + State: result?.state ?? "missing result" + } + } + return { + Case: labels.caseName, + Format: labels.formatName, + "Throughput avg (values/s)": Math.round(result.throughput.mean * labels.framesPerOp), + "Latency med (us/value)": (result.latency.p50 * 1_000 / labels.framesPerOp).toFixed(2), + "Latency RME": `${result.latency.rme.toFixed(2)}%`, + Samples: result.latency.samplesCount + } +})) diff --git a/packages/effect/src/unstable/cluster/Envelope.ts b/packages/effect/src/unstable/cluster/Envelope.ts index 68ba6577681..f77459d4efe 100644 --- a/packages/effect/src/unstable/cluster/Envelope.ts +++ b/packages/effect/src/unstable/cluster/Envelope.ts @@ -47,7 +47,12 @@ export const OpaqueHole: Schema.declare = Schema.declare( (_: unknown): _ is any => true, { expected: "an already-encoded value", - toCodecJson: () => undefined + toCodecJson: () => undefined, + toCodec: () => + Schema.link()( + Schema.Uint8Array, + SchemaTransformation.passthrough() + ) } ) diff --git a/packages/effect/src/unstable/encoding/SchemaBinary.ts b/packages/effect/src/unstable/encoding/SchemaBinary.ts new file mode 100644 index 00000000000..463134a8ebe --- /dev/null +++ b/packages/effect/src/unstable/encoding/SchemaBinary.ts @@ -0,0 +1,3049 @@ +/** + * A compact binary codec derived from the encoded side of a Schema. + * + * The default wire format supports compatible schema evolution. Fingerprint + * mode uses positional layouts and rejects mismatches for smaller frames. + * Encoded results are arena-backed views; see {@link toCodec} for ownership + * details. + * + * @since 4.0.0 + */ +import * as BigDecimal from "../../BigDecimal.ts" +import * as Cause from "../../Cause.ts" +import * as Chunk from "../../Chunk.ts" +import * as DateTime from "../../DateTime.ts" +import * as Duration from "../../Duration.ts" +import * as Effect from "../../Effect.ts" +import * as Exit from "../../Exit.ts" +import * as HashMap from "../../HashMap.ts" +import * as HashSet from "../../HashSet.ts" +import * as InternalRecord from "../../internal/record.ts" +import * as Option from "../../Option.ts" +import * as Predicate from "../../Predicate.ts" +import * as Redacted from "../../Redacted.ts" +import * as Result from "../../Result.ts" +import * as Schema from "../../Schema.ts" +import * as SchemaAST from "../../SchemaAST.ts" +import * as SchemaIssue from "../../SchemaIssue.ts" +import * as SchemaParser from "../../SchemaParser.ts" +import * as SchemaTransformation from "../../SchemaTransformation.ts" + +const FIELD_ID_ANNOTATION_KEY = "~effect/encoding/SchemaBinary/fieldId" + +// Bit 0 selects fingerprint mode; all other flag bits are reserved. +const ENVELOPE = 0x10 // version nibble 1, flags 0 +const ENVELOPE_FINGERPRINT = 0x11 // version nibble 1, flag bit 0 + +const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER) +const BIGINT_ZERO = BigInt(0) +const BIGINT_ONE = BigInt(1) +const BIGINT_TWO = BigInt(2) +const BIGINT_SEVEN = BigInt(7) +const BIGINT_VARINT_MASK = BigInt(0x7F) +const BIGINT_NANOS_PER_MILLI = BigInt(1_000_000) +const BIGINT_BYTE_MASK = BigInt(0xFF) +const BIGINT_U32_MASK = BigInt(0xFFFFFFFF) +const BIGINT_THIRTY_TWO = BigInt(32) + +const utf8Encode = new TextEncoder() +const utf8DecodeFatal = new TextDecoder("utf-8", { fatal: true }) + +// General numbers use up to seven varint bytes or an eight-byte f64. +const NUMBER_VARINT_MAX_BYTES = 7 + +const NUMBER_VARINT_MAX_MAGNITUDE = 281_474_976_710_655 // 2 ** 48 - 1 + +// Above this magnitude, build the sign-magnitude code with bigint. +const EXACT_MAGNITUDE_MAX = 4_503_599_627_370_495 // 2 ** 52 - 1 + +const NUMBER_RUN_F64 = 0 +const NUMBER_RUN_VARINT = 1 + +function isVarintNumber(value: unknown): boolean { + return typeof value === "number" && Number.isInteger(value) && + value >= -NUMBER_VARINT_MAX_MAGNITUDE && value <= NUMBER_VARINT_MAX_MAGNITUDE +} + +// Sign-magnitude preserves `-0`, unlike zigzag. +function decodeSignMagnitude(code: number): number { + const magnitude = Math.floor(code / 2) + return code % 2 === 1 ? -magnitude : magnitude +} + +const K = { + bool: 1, + null: 2, + undefined: 3, + number: 4, + string: 5, + bytes: 6, + bigint: 7, + int64: 8, + struct: 9, + variant: 10, + array: 11, + option: 12, + result: 13, + duration: 14, + bigDecimal: 15, + dateTimeZoned: 16, + json: 17, + exit: 18, + cause: 19, + causeReason: 20 +} as const + +function fnv32(bytes: ArrayLike): number { + let hash = 0x811C9DC5 + for (let i = 0; i < bytes.length; i++) { + hash = Math.imul(hash ^ bytes[i], 0x01000193) + } + return hash >>> 0 +} + +const FNV64_OFFSET_BASIS = BigInt("14695981039346656037") +const FNV64_PRIME = BigInt("1099511628211") +const FNV64_MASK = BigInt("18446744073709551615") + +function fnv64(bytes: ArrayLike): bigint { + let hash = FNV64_OFFSET_BASIS + for (let i = 0; i < bytes.length; i++) { + hash = ((hash ^ BigInt(bytes[i])) * FNV64_PRIME) & FNV64_MASK + } + return hash +} + +function pushBytes(out: Array, bytes: ArrayLike) { + for (let i = 0; i < bytes.length; i++) out.push(bytes[i]) +} + +function pushUvarint(out: Array, n: number) { + while (n > 0x7F) { + out.push((n & 0x7F) | 0x80) + n = Math.floor(n / 128) + } + out.push(n) +} + +function pushU32(out: Array, n: number) { + out.push(n & 0xFF, (n >>> 8) & 0xFF, (n >>> 16) & 0xFF, (n >>> 24) & 0xFF) +} + +function pushU64(out: Array, n: bigint) { + for (let i = 0; i < 8; i++) { + out.push(Number((n >> BigInt(i * 8)) & BIGINT_BYTE_MASK)) + } +} + +function compareBytes(a: Uint8Array, b: Uint8Array): number { + const len = Math.min(a.length, b.length) + for (let i = 0; i < len; i++) { + if (a[i] !== b[i]) return a[i] - b[i] + } + return a.length - b.length +} + +class IssueError extends Error { + readonly issue: SchemaIssue.Issue + constructor(issue: SchemaIssue.Issue) { + super("SchemaBinary failure") + this.issue = issue + } +} + +function invalid(expected: string, input?: unknown, options?: SchemaAST.ParseOptions): never { + throw issueError(new SchemaIssue.InvalidValue({ expected }, input, options)) +} + +// Materialize the ambient path only when raising an issue. +const issuePath: Array = [] +let issuePathLen = 0 + +function issueError(issue: SchemaIssue.Issue): IssueError { + return new IssueError( + issuePathLen === 0 ? issue : new SchemaIssue.Pointer(issuePath.slice(0, issuePathLen), issue) + ) +} + +function uvarintBytes(n: number): Uint8Array { + const out = new Uint8Array(uvarintSize(n)) + let p = 0 + while (n > 0x7F) { + out[p++] = (n & 0x7F) | 0x80 + n = Math.floor(n / 128) + } + out[p] = n + return out +} + +function uvarintSize(n: number): number { + let size = 1 + while (n > 0x7F) { + n = Math.floor(n / 128) + size++ + } + return size +} + +// Avoid TextDecoder's fixed cost for short strings. +const UTF8_INLINE_LIMIT = 32 + +function decodeUtf8( + buf: Uint8Array, + start: number, + end: number, + options: SchemaAST.ParseOptions | undefined +): string { + const len = end - start + if (len === 0) return "" + if (len <= UTF8_INLINE_LIMIT) { + let ascii = true + for (let i = start; i < end; i++) { + if (buf[i] > 0x7F) { + ascii = false + break + } + } + if (ascii) { + let out = "" + let i = start + for (; i + 4 <= end; i += 4) out += String.fromCharCode(buf[i], buf[i + 1], buf[i + 2], buf[i + 3]) + for (; i < end; i++) out += String.fromCharCode(buf[i]) + return out + } + } + try { + return utf8DecodeFatal.decode(buf.subarray(start, end)) + } catch { + invalid("utf-8", undefined, options) + } +} + +const OUTPUT_ARENA_SIZE = 8 * 1024 + +// Bound attacker-controlled keys retained between parser feeds. +const PARSER_INDEX_SIGNATURE_CACHE_SIZE = 256 + +interface OutputArena { + readonly buf: Uint8Array + readonly view: DataView + offset: number + writing: boolean +} + +function makeOutputArena(size: number): OutputArena { + const buf: Uint8Array = new Uint8Array(size) + return { buf, view: new DataView(buf.buffer), offset: 0, writing: false } +} + +let outputArena = makeOutputArena(OUTPUT_ARENA_SIZE) + +class Writer { + buf: Uint8Array = new Uint8Array(0) + view = new DataView(this.buf.buffer) + arena: OutputArena | undefined + start = 0 + len = 0 + reset() { + let arena = outputArena + // Nested codecs cannot share the active arena tail. + if (arena.writing || arena.offset >= arena.buf.length) { + arena = outputArena = makeOutputArena(OUTPUT_ARENA_SIZE) + } + arena.writing = true + this.arena = arena + this.buf = arena.buf + this.view = arena.view + this.start = arena.offset + this.len = 0 + } + ensure(n: number) { + if (this.start + this.len + n > this.buf.length) { + const previous = this.arena! + const required = this.len + n + let size = OUTPUT_ARENA_SIZE + while (size < required) size *= 2 + const next = makeOutputArena(size) + next.writing = true + next.buf.set(this.buf.subarray(this.start, this.start + this.len)) + previous.writing = false + this.arena = outputArena = next + this.buf = next.buf + this.view = next.view + this.start = 0 + } + } + byte(b: number) { + this.ensure(1) + this.buf[this.start + this.len++] = b + } + bytes(b: Uint8Array) { + this.ensure(b.length) + this.buf.set(b, this.start + this.len) + this.len += b.length + } + uvarint(n: number) { + this.ensure(10) + const buf = this.buf + let p = this.start + this.len + while (n > 0x7F) { + buf[p++] = (n & 0x7F) | 0x80 + n = n < 0x80000000 ? n >>> 7 : Math.floor(n / 128) + } + buf[p++] = n + this.len = p - this.start + } + uvarintBig(n: bigint) { + while (n > BIGINT_VARINT_MASK) { + this.byte(Number(n & BIGINT_VARINT_MASK) | 0x80) + n >>= BIGINT_SEVEN + } + this.byte(Number(n)) + } + zigzag(n: bigint) { + this.uvarintBig(n >= BIGINT_ZERO ? n << BIGINT_ONE : (-n << BIGINT_ONE) - BIGINT_ONE) + } + numberVarint(n: number) { + const negative = n < 0 || (n === 0 && 1 / n < 0) + const magnitude = negative ? -n : n + if (magnitude <= EXACT_MAGNITUDE_MAX) { + this.uvarint(magnitude * 2 + (negative ? 1 : 0)) + } else { + this.uvarintBig(BigInt(magnitude) * BIGINT_TWO + (negative ? BIGINT_ONE : BIGINT_ZERO)) + } + } + f64(n: number) { + this.ensure(8) + this.view.setFloat64(this.start + this.len, n, true) + this.len += 8 + } + i64(n: bigint) { + this.ensure(8) + this.view.setBigInt64(this.start + this.len, n, true) + this.len += 8 + } + u32le(n: number) { + this.ensure(4) + this.view.setUint32(this.start + this.len, n >>> 0, true) + this.len += 4 + } + i32le(n: number) { + this.ensure(4) + this.view.setInt32(this.start + this.len, n | 0, true) + this.len += 4 + } + // Writes a field id and reserves its length byte under one bounds check. + idAndMark(idBytes: Uint8Array): number { + const n = idBytes.length + this.ensure(n + 1) + const buf = this.buf + let p = this.start + this.len + for (let i = 0; i < n; i++) buf[p++] = idBytes[i] + const mark = p - this.start + this.len = mark + 1 + return mark + } + // Reserve one byte, then expand and backfill the length prefix if needed. + beginSized(): number { + this.ensure(1) + return this.len++ + } + endSized(mark: number) { + const payload = this.len - mark - 1 + if (payload < 0x80) { + this.buf[this.start + mark] = payload + return + } + const size = uvarintSize(payload) + const extra = size - 1 + this.ensure(extra) + const absoluteMark = this.start + mark + this.buf.copyWithin(absoluteMark + size, absoluteMark + 1, this.start + this.len) + this.len += extra + let n = payload + let p = absoluteMark + while (n > 0x7F) { + this.buf[p++] = (n & 0x7F) | 0x80 + n = Math.floor(n / 128) + } + this.buf[p] = n + } + string(s: string) { + const n = s.length + if (n === 0) return + this.ensure(n * 3) + const buf = this.buf + let p = this.start + this.len + for (let i = 0; i < n; i++) { + const c = s.charCodeAt(i) + if (c > 0x7F) { + this.len += utf8Encode.encodeInto(s, buf.subarray(this.start + this.len)).written + return + } + buf[p++] = c + } + this.len = p - this.start + } + out(): Uint8Array { + const arena = this.arena! + const out = new Uint8Array(arena.buf.buffer, this.start, this.len) + if (arena === outputArena) arena.offset = this.start + this.len + arena.writing = false + this.arena = undefined + return out + } + abort() { + const arena = this.arena + if (arena === undefined) return + if (arena === outputArena) arena.offset = this.start + arena.writing = false + this.arena = undefined + } +} + +// Index-signature predicates are not part of the fingerprint, so unmatched +// keys are dropped in both modes. +function matchIndexSignature( + layout: StructLayout, + key: string, + options: SchemaAST.ParseOptions +): ExtraSignature | undefined { + return layout.extra.find((s) => SchemaAST.getIndexSignatureKeys({ [key]: null }, s.parameter, options).length > 0) +} + +class IndexSignatureCache { + entries = new WeakMap>() + readonly orderLayouts: Array | undefined + readonly orderKeys: Array | undefined + readonly options: SchemaAST.ParseOptions + size = 0 + next = 0 + replace = false + // Only the parser cache outlives one operation, and it is always bounded. + constructor(options: SchemaAST.ParseOptions, capacity?: number) { + this.options = options + this.orderLayouts = capacity === undefined ? undefined : new Array(capacity) + this.orderKeys = capacity === undefined ? undefined : new Array(capacity) + } + beginFrame() { + this.replace = true + } + find(layout: StructLayout, key: string): ExtraSignature | undefined { + let entries = this.entries.get(layout) + if (entries === undefined) { + entries = new Map() + this.entries.set(layout, entries) + } + if (entries.has(key)) return entries.get(key) + const signature = matchIndexSignature(layout, key, this.options) + const orderLayouts = this.orderLayouts + if (orderLayouts === undefined) { + entries.set(key, signature) + return signature + } + const orderKeys = this.orderKeys! + if (this.size < orderLayouts.length) { + orderLayouts[this.size] = layout + orderKeys[this.size] = key + this.size++ + entries.set(key, signature) + } else if (this.replace) { + this.replace = false + const evictedLayout = orderLayouts[this.next]! + const evictedEntries = evictedLayout === layout ? entries : this.entries.get(evictedLayout) + evictedEntries?.delete(orderKeys[this.next]!) + orderLayouts[this.next] = layout + orderKeys[this.next] = key + this.next = (this.next + 1) % orderLayouts.length + entries.set(key, signature) + } + return signature + } + clear() { + this.entries = new WeakMap() + this.orderLayouts?.fill(undefined) + this.orderKeys?.fill(undefined) + this.size = this.next = 0 + this.replace = false + } +} + +// A named buffer preserves the build's pure annotation for these placeholders. +const EMPTY_READER_ARRAY_BUFFER = new ArrayBuffer(0) +const EMPTY_READER_BUFFER = new Uint8Array(EMPTY_READER_ARRAY_BUFFER) +const EMPTY_READER_VIEW = new DataView(EMPTY_READER_ARRAY_BUFFER) +const EMPTY_PARSE_OPTIONS: SchemaAST.ParseOptions = {} + +let lastReaderBuffer: ArrayBufferLike = EMPTY_READER_ARRAY_BUFFER +let lastReaderOffset = 0 +let lastReaderLength = 0 +let lastReaderView: DataView = EMPTY_READER_VIEW + +function readerView(buf: Uint8Array): DataView { + if ( + buf.buffer !== lastReaderBuffer || + buf.byteOffset !== lastReaderOffset || + buf.byteLength !== lastReaderLength + ) { + lastReaderBuffer = buf.buffer + lastReaderOffset = buf.byteOffset + lastReaderLength = buf.byteLength + lastReaderView = new DataView(buf.buffer, buf.byteOffset, buf.byteLength) + } + return lastReaderView +} + +class Reader { + pos = 0 + buf: Uint8Array = EMPTY_READER_BUFFER + view: DataView = EMPTY_READER_VIEW + end = 0 + options: SchemaAST.ParseOptions = EMPTY_PARSE_OPTIONS + indexSignatures: IndexSignatureCache | undefined + positional = false + reset( + buf: Uint8Array, + start: number, + end: number, + options: SchemaAST.ParseOptions, + indexSignatures: IndexSignatureCache | undefined, + positional: boolean + ) { + this.view = readerView(buf) + this.buf = buf + this.pos = start + this.end = end + this.options = options + this.indexSignatures = indexSignatures + this.positional = positional + } + release() { + this.pos = this.end = 0 + this.buf = EMPTY_READER_BUFFER + this.view = EMPTY_READER_VIEW + this.options = EMPTY_PARSE_OPTIONS + this.indexSignatures = undefined + this.positional = false + } + get remaining(): number { + return this.end - this.pos + } + byte(): number { + if (this.pos >= this.end) invalid("complete value", undefined, this.options) + return this.buf[this.pos++] + } + take(n: number): Uint8Array { + if (this.pos + n > this.end) invalid("complete value", undefined, this.options) + const out = this.buf.subarray(this.pos, this.pos + n) + this.pos += n + return out + } + // Decode a bounded child value without allocating another reader. + enter(len: number): number { + if (this.pos + len > this.end) invalid("complete value", undefined, this.options) + const saved = this.end + this.end = this.pos + len + return saved + } + exit(saved: number) { + this.pos = this.end + this.end = saved + } + readUtf8(n: number): string { + if (this.pos + n > this.end) invalid("complete value", undefined, this.options) + const start = this.pos + this.pos += n + return decodeUtf8(this.buf, start, start + n, this.options) + } + // Use bitwise arithmetic for the first four varint groups. + uvarint(): number { + const buf = this.buf + const end = this.end + let pos = this.pos + if (pos >= end) invalid("complete value", undefined, this.options) + let b = buf[pos++] + if (b < 0x80) { + this.pos = pos + return b + } + let value = b & 0x7F + if (pos >= end) invalid("complete value", undefined, this.options) + b = buf[pos++] + value |= (b & 0x7F) << 7 + if (b < 0x80) { + this.pos = pos + return value + } + if (pos >= end) invalid("complete value", undefined, this.options) + b = buf[pos++] + value |= (b & 0x7F) << 14 + if (b < 0x80) { + this.pos = pos + return value + } + if (pos >= end) invalid("complete value", undefined, this.options) + b = buf[pos++] + value |= (b & 0x7F) << 21 + if (b < 0x80) { + this.pos = pos + return value + } + let scale = 268435456 // 2 ** 28 + for (let i = 4; i < 10; i++) { + if (pos >= end) { + this.pos = pos + invalid("complete value", undefined, this.options) + } + b = buf[pos++] + const chunk = (b & 0x7F) * scale + if (chunk > Number.MAX_SAFE_INTEGER - value) { + this.pos = pos + invalid("safe integer length", undefined, this.options) + } + value += chunk + if (b < 0x80) { + this.pos = pos + return value + } + scale *= 128 + } + this.pos = pos + invalid("uvarint", undefined, this.options) + } + uvarintBig(): bigint { + let value = BIGINT_ZERO + let shift = BIGINT_ZERO + while (true) { + const b = this.byte() + value |= BigInt(b & 0x7F) << shift + if ((b & 0x80) === 0) return value + shift += BIGINT_SEVEN + } + } + // Wider schema-proven integers switch to bigint arithmetic. + numberVarint(): number { + const buf = this.buf + const end = this.end + let pos = this.pos + let value = 0 + let scale = 1 + for (let i = 0; i < NUMBER_VARINT_MAX_BYTES; i++) { + if (pos >= end) { + this.pos = pos + invalid("complete value", undefined, this.options) + } + const b = buf[pos++] + value += (b & 0x7F) * scale + if (b < 0x80) { + this.pos = pos + return decodeSignMagnitude(value) + } + scale *= 128 + } + const code = this.uvarintBig() + const magnitude = Number(code >> BIGINT_ONE) + if (!Number.isFinite(magnitude)) invalid("safe integer length", undefined, this.options) + return (code & BIGINT_ONE) === BIGINT_ONE ? -magnitude : magnitude + } + zigzag(): bigint { + const u = this.uvarintBig() + return (u & BIGINT_ONE) === BIGINT_ONE + ? -((u + BIGINT_ONE) >> BIGINT_ONE) + : u >> BIGINT_ONE + } + f64(): number { + if (this.pos + 8 > this.end) invalid("complete value", undefined, this.options) + const value = this.view.getFloat64(this.pos, true) + this.pos += 8 + return value + } + i64(): bigint { + if (this.pos + 8 > this.end) invalid("complete value", undefined, this.options) + const value = this.view.getBigInt64(this.pos, true) + this.pos += 8 + return value + } + u32le(): number { + if (this.pos + 4 > this.end) invalid("complete value", undefined, this.options) + const value = this.view.getUint32(this.pos, true) + this.pos += 4 + return value + } + i32le(): number { + if (this.pos + 4 > this.end) invalid("complete value", undefined, this.options) + const value = this.view.getInt32(this.pos, true) + this.pos += 4 + return value + } +} + +type LeafKind = + | "bool" + | "null" + | "undefined" + | "number" + | "int" + | "string" + | "symbol" + | "bytes" + | "bigint" + | "json" + | "duration" + | "bigDecimal" + | "dateTimeZoned" + +type Layout = + | { readonly _: LeafKind } + | { readonly _: "int64"; readonly flavor: "date" | "utc" } + | { readonly _: "never"; readonly ast: SchemaAST.AST } + | StructLayout + | ArrayLayout + | UnionLayout + | { readonly _: "option"; value: Layout } + | { readonly _: "result"; success: Layout; failure: Layout } + | { readonly _: "exit"; value: Layout; cause: ReasonLayout } + | ReasonLayout + +interface ReasonLayout { + readonly _: "cause" | "causeReason" + error: Layout + defect: Layout +} + +interface Field { + readonly name: string + readonly id: number + readonly idBytes: Uint8Array + index: number + readonly optional: boolean + readonly annotations: Schema.Annotations.Key | undefined + layout: Layout + inline: boolean +} + +interface ExtraSignature { + readonly parameter: SchemaAST.IndexSignature["parameter"] + layout: Layout +} + +interface StructLayout { + readonly _: "struct" + readonly ast: SchemaAST.AST + readonly fields: Array + readonly byId: Map + readonly extra: Array + readonly names: Set + optionalCount: number +} + +interface Slot { + readonly optional: boolean + layout: Layout +} + +interface ArrayLayout { + readonly _: "array" + readonly ast: SchemaAST.AST + readonly elements: Array + readonly rest: Array + readonly hasCount: boolean + readonly minCount: number + // Shared layout for `Schema.Array(S)`. + uniform: Layout | undefined + uniformInline: boolean + uniformPacked: number | undefined + uniformNumbers: boolean +} + +interface VariantRow { + readonly tag: number + readonly sentinels: ReadonlyArray + readonly tuple: boolean + payload: Layout + position: number +} + +interface UnionMember { + readonly kind: number + readonly layout: Layout + position: number +} + +interface UnionPosition { + readonly variant: VariantRow | undefined + readonly layout: Layout +} + +interface UnionLayout { + readonly _: "union" + readonly ast: SchemaAST.AST + readonly variants: Array + readonly byTag: Map + readonly others: Array + readonly byKind: Map + // Canonical order used by fingerprint mode. + readonly byPos: Array +} + +function isSelfDelimiting(layout: Layout): boolean { + return layout._ === "int" +} + +function packedSize(layout: Layout): number | undefined { + switch (layout._) { + case "bool": + return 1 + case "int64": + return 8 + default: + return undefined + } +} + +function isInlineSlot(layout: Layout): boolean { + return packedSize(layout) !== undefined || isSelfDelimiting(layout) || + layout._ === "null" || layout._ === "undefined" +} + +const nativeKinds: Record = { + "effect/schema/Date": K.int64, + "effect/schema/DateTimeUtc": K.int64, + "effect/schema/DateTimeZoned": K.dateTimeZoned, + "effect/schema/Duration": K.duration, + "effect/schema/BigDecimal": K.bigDecimal, + "effect/schema/Uint8Array": K.bytes, + "effect/schema/Option": K.option, + "effect/schema/Result": K.result, + "effect/schema/Exit": K.exit, + "effect/schema/Cause": K.cause, + "effect/schema/CauseReason": K.causeReason +} + +function representationId(ast: SchemaAST.AST): string | undefined { + const representation = ast.annotations?.representation + return Predicate.isObject(representation) && typeof (representation as { id?: unknown }).id === "string" + ? (representation as { id: string }).id + : undefined +} + +const toBinaryAST = SchemaAST.applyToSelfOrLastLinkEncodingIdempotent((ast) => { + const out = toBinaryASTStep(ast) + const context = ast.context + if (out === ast || context === undefined) return out + return SchemaAST.replaceContextLastLink( + out, + new SchemaAST.Context(context.isOptional, context.isMutable, undefined, context.annotations) + ) +}) + +function toBinaryASTStep(ast: SchemaAST.AST): SchemaAST.AST { + switch (ast._tag) { + case "Declaration": { + const id = representationId(ast) + if ( + id !== undefined && (id in nativeKinds || id === "effect/schema/Json" || id === "effect/schema/MutableJson") + ) { + return ast.recur(toBinaryAST) + } + const getJson = ast.annotations?.toCodecJson + const getCodec = ast.annotations?.toCodec + if (!Predicate.isFunction(getJson) && !Predicate.isFunction(getCodec)) { + return ast + } + const typeParameters = ast.typeParameters.map((tp) => Schema.make(SchemaAST.toEncoded(tp))) + const link = (Predicate.isFunction(getJson) ? getJson(typeParameters) : undefined) ?? + (Predicate.isFunction(getCodec) ? getCodec(typeParameters) : undefined) + return link === undefined ? ast : SchemaAST.replaceEncoding(ast, [SchemaAST.mapLink(link, toBinaryAST)]) + } + case "Arrays": + case "Objects": + case "Union": + case "Suspend": + return ast.recur(toBinaryAST) + default: + return ast + } +} + +function sentinelSetHash(sentinels: ReadonlyArray): number { + const sorted = [...sentinels].sort((a, b) => { + const an = typeof a.key === "number" + const bn = typeof b.key === "number" + if (an !== bn) return an ? -1 : 1 + if (an) return (a.key as number) - (b.key as number) + return compareBytes(utf8Encode.encode(a.key as string), utf8Encode.encode(b.key as string)) + }) + const out: Array = [] + for (const sentinel of sorted) { + const keyBytes = utf8Encode.encode(String(sentinel.key)) + out.push(typeof sentinel.key === "number" ? 0 : 1) + pushU32(out, keyBytes.length) + pushBytes(out, keyBytes) + const literal = sentinel.literal + const valueBytes = utf8Encode.encode(sentinelLiteralString(literal)) + out.push(sentinelLiteralKind(literal)) + pushU32(out, valueBytes.length) + pushBytes(out, valueBytes) + } + return fnv32(out) +} + +function sentinelLiteralKind(literal: SchemaAST.LiteralValue | symbol): number { + switch (typeof literal) { + case "string": + return 1 + case "number": + return 2 + case "boolean": + return 3 + case "bigint": + return 4 + default: + return 5 + } +} + +function sentinelLiteralString(literal: SchemaAST.LiteralValue | symbol): string { + if (typeof literal === "symbol") { + const key = globalThis.Symbol.keyFor(literal) + if (key === undefined) { + throw new Error("Binary layout: unregistered unique symbol (Symbol.keyFor)") + } + return key + } + return String(literal) +} + +function parameterHasSymbol(parameter: SchemaAST.AST): boolean { + switch (parameter._tag) { + case "Symbol": + return true + case "Union": + return parameter.types.some(parameterHasSymbol) + default: + return false + } +} + +function isJsonDeclaration(ast: SchemaAST.Declaration): boolean { + const id = representationId(ast) + if (id === "effect/schema/Json" || id === "effect/schema/MutableJson") return true + return Predicate.isFunction(ast.annotations?.toCodecJson) +} + +// Detect integer checks nested inside filter groups. +function provesInteger(ast: SchemaAST.AST): boolean { + const checks = ast.checks + if (checks === undefined) return false + const go = (check: SchemaAST.Check): boolean => { + const id = check.annotations?.representation?.id + if (id === "effect/schema/isInt") return true + return check._tag === "FilterGroup" && check.checks.some(go) + } + return checks.some(go) +} + +function resolveSuspend(ast: SchemaAST.AST): SchemaAST.AST { + while (ast._tag === "Suspend") { + ast = ast.thunk() + } + return ast +} + +function flattenMembers(union: SchemaAST.Union): Array { + const out: Array = [] + const seen = new Set() + const go = (ast: SchemaAST.AST) => { + const resolved = resolveSuspend(ast) + if (seen.has(resolved)) return + seen.add(resolved) + switch (resolved._tag) { + case "Never": + return + case "Union": + resolved.types.forEach(go) + return + case "Enum": + SchemaAST.enumsToLiterals(resolved).types.forEach(go) + return + default: + out.push(resolved) + } + } + union.types.forEach(go) + return out +} + +function literalKind(literal: SchemaAST.LiteralValue): LeafKind { + switch (typeof literal) { + case "string": + return "string" + case "number": + return "number" + case "boolean": + return "bool" + default: + return "bigint" + } +} + +// Classify union members without eagerly compiling recursive members. +function astKind(ast: SchemaAST.AST): number { + switch (ast._tag) { + case "String": + case "TemplateLiteral": + case "Symbol": + return K.string + case "UniqueSymbol": + if (globalThis.Symbol.keyFor(ast.symbol) === undefined) { + throw new Error("Binary layout: unregistered unique symbol (Symbol.keyFor)") + } + return K.string + case "Boolean": + return K.bool + case "Null": + return K.null + case "Undefined": + case "Void": + return K.undefined + case "Number": + return K.number + case "BigInt": + return K.bigint + case "Literal": + switch (typeof ast.literal) { + case "string": + return K.string + case "number": + return K.number + case "boolean": + return K.bool + default: + return K.bigint + } + case "Unknown": + case "Any": + case "ObjectKeyword": + return K.json + case "Objects": + return K.struct + case "Arrays": + return K.array + case "Declaration": { + const id = representationId(ast) + if (id !== undefined && id in nativeKinds) return nativeKinds[id] + if (isJsonDeclaration(ast)) return K.json + throw new Error(`Binary layout: declaration ${id ?? ""} has no toCodecJson or toCodec`) + } + case "Suspend": + return astKind(resolveSuspend(ast)) + default: + throw new Error("Binary layout: union members are not uniquely identifiable") + } +} + +interface CompiledLayout { + readonly layout: Layout + readonly recursive: boolean + readonly decodeExact: boolean +} + +function compileLayout(root: SchemaAST.AST): CompiledLayout { + const decodeExact = isDecodeExact(root) + root = SchemaAST.toEncoded(root) + const memo = new Map() + let recursive = false + + function compile(ast: SchemaAST.AST): Layout { + const hit = memo.get(ast) + if (hit !== undefined) return hit + const layout = go(ast) + memo.set(ast, layout) + return layout + } + + function go(ast: SchemaAST.AST): Layout { + switch (ast._tag) { + case "String": + case "TemplateLiteral": + return { _: "string" } + case "Symbol": + case "UniqueSymbol": + return { _: "symbol" } + case "Boolean": + return { _: "bool" } + case "Null": + return { _: "null" } + case "Undefined": + case "Void": + return { _: "undefined" } + case "Number": + return provesInteger(ast) ? { _: "int" } : { _: "number" } + case "BigInt": + return { _: "bigint" } + case "Literal": + return { _: literalKind(ast.literal) } + case "Unknown": + case "Any": + case "ObjectKeyword": + return { _: "json" } + case "Never": + return { _: "never", ast } + case "Enum": + return compileUnion(ast, flattenMembers(SchemaAST.enumsToLiterals(ast))) + case "Suspend": { + recursive = true + const layout = compile(ast.thunk()) + memo.set(ast, layout) + return layout + } + case "Objects": + return compileStruct(ast) + case "Arrays": + return compileArray(ast) + case "Union": + return compileUnion(ast, flattenMembers(ast)) + case "Declaration": + return compileDeclaration(ast) + } + } + + function compileStruct(ast: SchemaAST.Objects): StructLayout { + const fields: Array = [] + const types: Array = [] + const idNames = new Map>() + for (const ps of ast.propertySignatures) { + if (typeof ps.name === "symbol") { + throw new Error("Binary layout: symbol property names are illegal") + } + const name = String(ps.name) + const annotations = ps.type.context?.annotations + const explicit = annotations?.[FIELD_ID_ANNOTATION_KEY] + const id = typeof explicit === "number" ? explicit : fnv32(utf8Encode.encode(name)) + if (!Number.isInteger(id) || id < 0 || id > 0xFFFFFFFF) { + throw new Error(`Binary layout: illegal field id for ${name}`) + } + const names = idNames.get(id) + if (names === undefined) idNames.set(id, [name]) + else names.push(name) + fields.push({ + name, + id, + idBytes: uvarintBytes(id), + index: 0, + optional: ps.type.context?.isOptional === true, + annotations, + layout: undefined as unknown as Layout, + inline: false + }) + types.push(ps.type) + } + for (const [id, names] of idNames) { + if (id === 0 || names.length > 1) { + throw new Error(`Binary layout field id collision: ${id} (${names.join(", ")})`) + } + } + const extra: Array = [] + for (const is of ast.indexSignatures) { + if (parameterHasSymbol(is.parameter)) { + throw new Error("Binary layout: symbol property names are illegal") + } + extra.push({ parameter: is.parameter, layout: undefined as unknown as Layout }) + } + const layout: StructLayout = { + _: "struct", + ast, + fields, + byId: new Map(), + extra, + names: new Set(fields.map((f) => f.name)), + optionalCount: 0 + } + memo.set(ast, layout) + for (let i = 0; i < fields.length; i++) { + const field = fields[i] + field.layout = compile(types[i]) + // Recursive placeholders already have the discriminant used here. + field.inline = isInlineSlot(field.layout) + if (field.optional) layout.optionalCount++ + layout.byId.set(field.id, field) + } + fields.sort((a, b) => a.id - b.id) + for (let i = 0; i < fields.length; i++) fields[i].index = i + for (let i = 0; i < extra.length; i++) { + extra[i].layout = compile(ast.indexSignatures[i].type) + } + return layout + } + + function compileArray(ast: SchemaAST.Arrays): ArrayLayout { + const hasCount = ast.rest.length > 0 || ast.elements.some((e) => e.context?.isOptional === true) + const requiredElements = ast.elements.filter((e) => e.context?.isOptional !== true).length + const tailLen = Math.max(0, ast.rest.length - 1) + const layout: ArrayLayout = { + _: "array", + ast, + elements: [], + rest: [], + hasCount, + minCount: requiredElements + tailLen, + uniform: undefined, + uniformInline: false, + uniformPacked: undefined, + uniformNumbers: false + } + memo.set(ast, layout) + for (const element of ast.elements) { + layout.elements.push({ + optional: element.context?.isOptional === true, + layout: compile(element) + }) + } + for (const rest of ast.rest) { + layout.rest.push(compile(rest)) + } + if (layout.elements.length === 0 && layout.rest.length === 1) { + const slot = layout.rest[0] + layout.uniform = slot + layout.uniformPacked = packedSize(slot) + layout.uniformNumbers = slot._ === "number" + layout.uniformInline = isInlineSlot(slot) + } + return layout + } + + function compileDeclaration(ast: SchemaAST.Declaration): Layout { + const id = representationId(ast) + switch (id) { + case "effect/schema/Date": + return { _: "int64", flavor: "date" } + case "effect/schema/DateTimeUtc": + return { _: "int64", flavor: "utc" } + case "effect/schema/DateTimeZoned": + return { _: "dateTimeZoned" } + case "effect/schema/Duration": + return { _: "duration" } + case "effect/schema/BigDecimal": + return { _: "bigDecimal" } + case "effect/schema/Uint8Array": + return { _: "bytes" } + } + const tps = ast.typeParameters + switch (id) { + case "effect/schema/Option": { + const layout = { _: "option" as const, value: undefined as unknown as Layout } + memo.set(ast, layout) + layout.value = compile(tps[0]) + return layout + } + case "effect/schema/Result": { + const layout = { + _: "result" as const, + success: undefined as unknown as Layout, + failure: undefined as unknown as Layout + } + memo.set(ast, layout) + layout.success = compile(tps[0]) + layout.failure = compile(tps[1]) + return layout + } + case "effect/schema/Exit": { + const cause: ReasonLayout = { + _: "cause", + error: undefined as unknown as Layout, + defect: undefined as unknown as Layout + } + const layout = { _: "exit" as const, value: undefined as unknown as Layout, cause } + memo.set(ast, layout) + layout.value = compile(tps[0]) + cause.error = compile(tps[1]) + cause.defect = compile(tps[2]) + return layout + } + case "effect/schema/Cause": + case "effect/schema/CauseReason": { + const layout: ReasonLayout = { + _: id === "effect/schema/Cause" ? "cause" : "causeReason", + error: undefined as unknown as Layout, + defect: undefined as unknown as Layout + } + memo.set(ast, layout) + layout.error = compile(tps[0]) + layout.defect = compile(tps[1]) + return layout + } + } + if (isJsonDeclaration(ast)) return { _: "json" } + throw new Error(`Binary layout: declaration ${id ?? ""} has no toCodecJson or toCodec`) + } + + function compileUnion(ast: SchemaAST.AST, members: Array): Layout { + if (members.length === 0) return { _: "never", ast } + const variantMembers: Array<{ member: SchemaAST.AST; sentinels: ReadonlyArray }> = [] + const rowMembers: Array<{ member: SchemaAST.AST; kind: number }> = [] + const literalRows = new Map() + const addLiteralRow = (kind: number, row: Layout) => { + const existing = literalRows.get(kind) + if (existing !== undefined && existing._ !== row._) { + throw new Error("Binary layout: union members are not uniquely identifiable") + } + literalRows.set(kind, row) + } + for (const member of members) { + if (member._tag === "Literal") { + addLiteralRow(astKind(member), { _: literalKind(member.literal) }) + continue + } + if (member._tag === "UniqueSymbol") { + astKind(member) // validates the symbol is registered + addLiteralRow(K.string, { _: "symbol" }) + continue + } + if (member._tag === "Objects" || member._tag === "Arrays") { + const sentinels = SchemaAST.collectSentinels(member) + if (sentinels !== undefined && sentinels.length > 0) { + for (const sentinel of sentinels) { + if (typeof sentinel.key === "symbol") { + throw new Error("Binary layout: symbol property names are illegal") + } + } + variantMembers.push({ member, sentinels }) + continue + } + } + rowMembers.push({ member, kind: astKind(member) }) + } + if (variantMembers.length === 0 && rowMembers.length === 0 && literalRows.size === 1) { + return literalRows.values().next().value! + } + const kinds = new Set(literalRows.keys()) + for (const { kind } of rowMembers) { + if (kinds.has(kind)) { + throw new Error("Binary layout: union members are not uniquely identifiable") + } + kinds.add(kind) + } + if (variantMembers.length === 0 && literalRows.size === 0 && rowMembers.length === 1) { + const layout = compile(rowMembers[0].member) + memo.set(ast, layout) + return layout + } + const tags = new Map>() + for (const { sentinels } of variantMembers) { + const tag = sentinelSetHash(sentinels) + if (tags.has(tag)) { + throw new Error(`Binary layout sentinel collision: ${tag}`) + } + tags.set(tag, sentinels) + } + const layout: UnionLayout = { + _: "union", + ast, + variants: [], + byTag: new Map(), + others: [], + byKind: new Map(), + byPos: [] + } + memo.set(ast, layout) + for (const { member, sentinels } of variantMembers) { + const tag = sentinelSetHash(sentinels) + const full = compile(member) + let payload: Layout + let tuple: boolean + if (member._tag === "Objects") { + const struct = full as StructLayout + const sentinelNames = new Set(sentinels.map((s) => String(s.key))) + const fields = struct.fields.filter((f) => !sentinelNames.has(f.name)) + payload = { + _: "struct", + ast: struct.ast, + fields, + byId: new Map(fields.map((f) => [f.id, f])), + extra: struct.extra, + names: struct.names, + optionalCount: fields.reduce((count, f) => f.optional ? count + 1 : count, 0) + } + tuple = false + } else { + payload = full + tuple = true + } + const row: VariantRow = { tag, sentinels, tuple, payload, position: 0 } + layout.variants.push(row) + layout.byTag.set(tag, row) + } + for (const [kind, row] of literalRows) { + layout.others.push({ kind, layout: row, position: 0 }) + layout.byKind.set(kind, row) + } + for (const { kind, member } of rowMembers) { + const row = compile(member) + layout.others.push({ kind, layout: row, position: 0 }) + layout.byKind.set(kind, row) + } + // Fingerprint positions are independent of declaration order. + for (const row of [...layout.variants].sort((a, b) => a.tag - b.tag)) { + row.position = layout.byPos.length + layout.byPos.push({ variant: row, layout: row.payload }) + } + for (const member of [...layout.others].sort((a, b) => a.kind - b.kind)) { + member.position = layout.byPos.length + layout.byPos.push({ variant: undefined, layout: member.layout }) + } + // Probe specific runtime guards before `json`, which matches anything. + layout.others.sort((a, b) => matchRank(a.layout) - matchRank(b.layout)) + return layout + } + + const layout = compile(root) + return { layout, recursive, decodeExact } +} + +// The parser may return the binary decoder's value directly only when that +// decoder proves every predicate and no Schema parser behavior can change it. +function isDecodeExact(root: SchemaAST.AST): boolean { + const exact = (ast: SchemaAST.AST): boolean => { + if ( + ast.encoding !== undefined || ast.checks !== undefined || + (ast as { readonly encodingChecks?: SchemaAST.Checks }).encodingChecks !== undefined || + ast.annotations?.parseOptions !== undefined || ast.context?.constructorDefault !== undefined + ) { + return false + } + switch (ast._tag) { + case "String": + case "Symbol": + case "Boolean": + case "Null": + case "Undefined": + case "Void": + case "Number": + case "BigInt": + return true + case "Arrays": + return ast.elements.every(exact) && ast.rest.every(exact) + case "Objects": + return ast.propertySignatures.every((property) => exact(property.type)) && + ast.indexSignatures.every((signature) => + signature.parameter._tag === "String" && exact(signature.parameter) && exact(signature.type) + ) + default: + return false + } + } + return exact(root) +} + +// Layouts sharing a wire kind can still require different fingerprints. +const F = { + backEdge: 0, + bool: 1, + null: 2, + undefined: 3, + number: 4, + int: 5, + string: 6, + symbol: 7, + bytes: 8, + bigint: 9, + json: 10, + duration: 11, + bigDecimal: 12, + dateTimeZoned: 13, + date: 14, + dateTimeUtc: 15, + never: 16, + struct: 17, + array: 18, + union: 19, + option: 20, + result: 21, + exit: 22, + cause: 23, + causeReason: 24 +} as const + +// Hash the compiled layout graph, including cycle back-edge distances. +function layoutFingerprint(root: Layout): bigint { + const cache = new Map() + const stack: Array = [] + // Cache only subtrees that do not escape above their own root. + let escape = Number.MAX_SAFE_INTEGER + + function go(layout: Layout): bigint { + const at = stack.lastIndexOf(layout) + if (at >= 0) { + if (at < escape) escape = at + const out: Array = [F.backEdge] + pushUvarint(out, stack.length - at) + return fnv64(out) + } + const cached = cache.get(layout) + if (cached !== undefined) return cached + const self = stack.length + const outerEscape = escape + escape = Number.MAX_SAFE_INTEGER + stack.push(layout) + const hash = fnv64(structure(layout)) + stack.pop() + const closed = escape >= self + if (closed) cache.set(layout, hash) + escape = closed ? outerEscape : Math.min(outerEscape, escape) + return hash + } + + function structure(layout: Layout): Array { + const out: Array = [] + switch (layout._) { + case "bool": + case "null": + case "undefined": + case "number": + case "int": + case "string": + case "symbol": + case "bytes": + case "bigint": + case "json": + case "duration": + case "bigDecimal": + case "dateTimeZoned": + out.push(F[layout._]) + return out + case "int64": + out.push(layout.flavor === "date" ? F.date : F.dateTimeUtc) + return out + case "never": + out.push(F.never) + return out + case "struct": { + out.push(F.struct) + pushUvarint(out, layout.fields.length) + for (const field of layout.fields) { + pushU32(out, field.id) + out.push(field.optional ? 1 : 0) + pushU64(out, go(field.layout)) + } + pushUvarint(out, layout.extra.length) + for (const signature of layout.extra) pushU64(out, go(signature.layout)) + return out + } + case "array": { + out.push(F.array) + pushUvarint(out, layout.elements.length) + for (const slot of layout.elements) { + out.push(slot.optional ? 1 : 0) + pushU64(out, go(slot.layout)) + } + pushUvarint(out, layout.rest.length) + for (const rest of layout.rest) pushU64(out, go(rest)) + return out + } + case "union": { + out.push(F.union) + pushUvarint(out, layout.byPos.length) + for (const position of layout.byPos) { + const variant = position.variant + if (variant === undefined) out.push(0) + else { + out.push(1, variant.tuple ? 1 : 0) + pushU32(out, variant.tag) + } + pushU64(out, go(position.layout)) + } + return out + } + case "option": + out.push(F.option) + pushU64(out, go(layout.value)) + return out + case "result": + out.push(F.result) + pushU64(out, go(layout.success)) + pushU64(out, go(layout.failure)) + return out + case "exit": + out.push(F.exit) + pushU64(out, go(layout.value)) + pushU64(out, go(layout.cause.error)) + pushU64(out, go(layout.cause.defect)) + return out + case "cause": + case "causeReason": + out.push(layout._ === "cause" ? F.cause : F.causeReason) + pushU64(out, go(layout.error)) + pushU64(out, go(layout.defect)) + return out + } + } + + return go(root) +} + +interface Mode { + readonly positional: boolean + readonly envelope: number + readonly expectedEnvelope: string + readonly fingerprintLo: number + readonly fingerprintHi: number +} + +const defaultMode: Mode = { + positional: false, + envelope: ENVELOPE, + expectedEnvelope: "version 1 envelope, flags 0", + fingerprintLo: 0, + fingerprintHi: 0 +} + +function fingerprintMode(layout: Layout): Mode { + const fingerprint = layoutFingerprint(layout) + return { + positional: true, + envelope: ENVELOPE_FINGERPRINT, + expectedEnvelope: "version 1 envelope, flags 1", + fingerprintLo: Number(fingerprint & BIGINT_U32_MASK), + fingerprintHi: Number((fingerprint >> BIGINT_THIRTY_TWO) & BIGINT_U32_MASK) + } +} + +function matchRank(layout: Layout): number { + switch (layout._) { + case "json": + return 2 + case "struct": + return 1 + default: + return 0 + } +} + +function matchesLayout(layout: Layout, value: unknown): boolean { + switch (layout._) { + case "bool": + return typeof value === "boolean" + case "null": + return value === null + case "undefined": + return value === undefined + case "number": + case "int": + return typeof value === "number" + case "string": + return typeof value === "string" + case "symbol": + return typeof value === "symbol" + case "bigint": + return typeof value === "bigint" + case "bytes": + return value instanceof Uint8Array + case "int64": + return layout.flavor === "date" + ? value instanceof Date + : DateTime.isDateTime(value) && value._tag === "Utc" + case "dateTimeZoned": + return DateTime.isDateTime(value) && value._tag === "Zoned" + case "duration": + return Duration.isDuration(value) + case "bigDecimal": + return BigDecimal.isBigDecimal(value) + case "option": + return Option.isOption(value) + case "result": + return Result.isResult(value) + case "exit": + return Exit.isExit(value) + case "cause": + return Cause.isCause(value) + case "causeReason": + return Cause.isReason(value) + case "array": + return Array.isArray(value) + case "struct": + return Predicate.isObject(value) && !Array.isArray(value) + case "json": + return true + case "union": + case "never": + return false + } +} + +interface EncodeContext { + readonly options: SchemaAST.ParseOptions + readonly positional: boolean + indexSignatures: IndexSignatureCache | undefined +} + +function encodeFail(expected: string, input: unknown, options: SchemaAST.ParseOptions): never { + throw issueError(new SchemaIssue.InvalidValue({ expected }, input, options)) +} + +function isCyclic(value: unknown, stack = new Set()): boolean { + if (!Predicate.isObjectOrArray(value)) return false + const isArray = Array.isArray(value) + const prototype = Object.getPrototypeOf(value) + if (isArray || prototype === Object.prototype || prototype === null) { + if (stack.has(value)) return true + stack.add(value) + try { + if (isArray) { + for (let i = 0; i < value.length; i++) { + if (isCyclic(value[i], stack)) return true + } + } else { + for (const key of Object.keys(value)) { + if (isCyclic((value as Record)[key], stack)) return true + } + } + } finally { + stack.delete(value) + } + return false + } + if ( + value instanceof Date || + value instanceof Uint8Array || + DateTime.isDateTime(value) || + Duration.isDuration(value) || + BigDecimal.isBigDecimal(value) + ) return false + if (stack.has(value)) return true + stack.add(value) + try { + if (Option.isOption(value)) { + return Option.isSome(value) && isCyclic(value.value, stack) + } + if (Result.isResult(value)) { + return isCyclic(Result.isSuccess(value) ? value.success : value.failure, stack) + } + if (Exit.isExit(value)) { + return isCyclic(Exit.isSuccess(value) ? value.value : value.cause, stack) + } + if (Cause.isCause(value)) { + return value.reasons.some((reason) => isCyclic(reason, stack)) + } + if (Cause.isReason(value)) { + switch (value._tag) { + case "Fail": + return isCyclic(value.error, stack) + case "Die": + return isCyclic(value.defect, stack) + case "Interrupt": + return false + } + } + if (Chunk.isChunk(value)) { + for (const item of value) { + if (isCyclic(item, stack)) return true + } + return false + } + if (HashMap.isHashMap(value)) { + for (const [key, item] of value) { + if (isCyclic(key, stack) || isCyclic(item, stack)) return true + } + return false + } + if (HashSet.isHashSet(value)) { + for (const item of value) { + if (isCyclic(item, stack)) return true + } + return false + } + if (Redacted.isRedacted(value)) { + return isCyclic(Redacted.value(value), stack) + } + for (const key of Object.keys(value)) { + if (isCyclic((value as Record)[key], stack)) return true + } + } finally { + stack.delete(value) + } + return false +} + +function encodeSized(ctx: EncodeContext, layout: Layout, value: unknown, w: Writer) { + const mark = w.beginSized() + encodeValue(ctx, layout, value, w) + w.endSized(mark) +} + +function encodeSymbol(ctx: EncodeContext, value: unknown, w: Writer) { + const key = globalThis.Symbol.keyFor(value as symbol) + if (key === undefined) encodeFail("registered symbol", value, ctx.options) + w.string(key) +} + +function encodeReason(ctx: EncodeContext, layout: ReasonLayout, value: unknown, w: Writer) { + const reason = value as Cause.Reason + switch (reason._tag) { + case "Fail": + w.byte(0) + encodeValue(ctx, layout.error, reason.error, w) + return + case "Die": + w.byte(1) + encodeValue(ctx, layout.defect, reason.defect, w) + return + case "Interrupt": + if (reason.fiberId === undefined) { + w.byte(2) + } else { + w.byte(3) + w.f64(reason.fiberId) + } + } +} + +type ExtraPair = [keyBytes: Uint8Array, key: string, signature: ExtraSignature] + +// Sort extra keys by raw UTF-8 for deterministic output. +function extraPairs(ctx: EncodeContext, layout: StructLayout, obj: Record): Array { + const named = layout.names + const pairs: Array = [] + for (const key of Object.keys(obj)) { + if (named.has(key)) continue + const signature = (ctx.indexSignatures ??= new IndexSignatureCache(ctx.options)).find(layout, key) + if (signature === undefined) continue + pairs.push([utf8Encode.encode(key), key, signature]) + } + pairs.sort((a, b) => compareBytes(a[0], b[0])) + return pairs +} + +function encodeExtraPairs( + ctx: EncodeContext, + pairs: Array, + obj: Record, + w: Writer +) { + for (const [keyBytes, key, signature] of pairs) { + w.uvarint(keyBytes.length) + w.bytes(keyBytes) + issuePath[issuePathLen++] = key + encodeSized(ctx, signature.layout, obj[key], w) + issuePathLen-- + } +} + +function encodeStructFields(ctx: EncodeContext, layout: StructLayout, value: object, w: Writer) { + const obj = value as Record + if (layout.extra.length > 0) { + const pairs = extraPairs(ctx, layout, obj) + if (pairs.length > 0) { + w.uvarint(0) + const mark = w.beginSized() + encodeExtraPairs(ctx, pairs, obj, w) + w.endSized(mark) + } + } + const fields = layout.fields + for (let i = 0; i < fields.length; i++) { + const field = fields[i] + const name = field.name + if (!Object.hasOwn(obj, name)) { + if (field.optional) continue + issuePath[issuePathLen++] = name + throw issueError(new SchemaIssue.MissingKey(field.annotations)) + } + const mark = w.idAndMark(field.idBytes) + issuePath[issuePathLen++] = name + encodeValue(ctx, field.layout, obj[name], w) + w.endSized(mark) + issuePathLen-- + } +} + +// Fingerprint structs use a presence bitmap followed by positional fields. +function encodeStructPositional(ctx: EncodeContext, layout: StructLayout, value: object, w: Writer) { + const obj = value as Record + const bitmapBytes = (layout.optionalCount + 7) >> 3 + let bitmap = 0 + if (bitmapBytes > 0) { + bitmap = w.len + for (let i = 0; i < bitmapBytes; i++) w.byte(0) + } + const fields = layout.fields + let optionalIndex = 0 + for (let i = 0; i < fields.length; i++) { + const field = fields[i] + const name = field.name + const present = Object.hasOwn(obj, name) + if (field.optional) { + // Resolve the bitmap address again after any arena growth. + if (present) w.buf[w.start + bitmap + (optionalIndex >> 3)] |= 1 << (optionalIndex & 7) + optionalIndex++ + if (!present) continue + } else if (!present) { + issuePath[issuePathLen++] = name + throw issueError(new SchemaIssue.MissingKey(field.annotations)) + } + issuePath[issuePathLen++] = name + if (field.inline) encodeValue(ctx, field.layout, obj[name], w) + else encodeSized(ctx, field.layout, obj[name], w) + issuePathLen-- + } + if (layout.extra.length > 0) { + const pairs = extraPairs(ctx, layout, obj) + w.uvarint(pairs.length) + encodeExtraPairs(ctx, pairs, obj, w) + } +} + +function arraySlot(layout: ArrayLayout, index: number, count: number): Layout { + const elementLen = layout.elements.length + if (index < elementLen) return layout.elements[index].layout + const tailLen = Math.max(0, layout.rest.length - 1) + const tailThreshold = Math.max(elementLen, count - tailLen) + return index >= tailThreshold ? layout.rest[index - tailThreshold + 1] : layout.rest[0] +} + +function encodeArray(ctx: EncodeContext, layout: ArrayLayout, value: unknown, w: Writer) { + const arr = value as ReadonlyArray + const count = arr.length + if (layout.rest.length === 0 && count > layout.elements.length) { + issuePath[issuePathLen++] = layout.elements.length + throw issueError( + new SchemaIssue.UnexpectedKey(layout.ast, arr[layout.elements.length], ctx.options) + ) + } + if (count < layout.minCount) { + issuePath[issuePathLen++] = count + throw issueError(new SchemaIssue.MissingKey(undefined)) + } + if (layout.hasCount) w.uvarint(count) + const uniform = layout.uniform + if (uniform !== undefined) { + if (layout.uniformNumbers) { + encodeNumberRun(arr, count, w) + return + } + const inline = layout.uniformInline + for (let i = 0; i < count; i++) { + issuePath[issuePathLen++] = i + if (inline) encodeValue(ctx, uniform, arr[i], w) + else encodeSized(ctx, uniform, arr[i], w) + issuePathLen-- + } + return + } + for (let i = 0; i < count; i++) { + const slot = arraySlot(layout, i, count) + issuePath[issuePathLen++] = i + if (isInlineSlot(slot)) { + encodeValue(ctx, slot, arr[i], w) + } else { + encodeSized(ctx, slot, arr[i], w) + } + issuePathLen-- + } +} + +// Uniform number arrays share one varint-or-f64 mode byte. +function encodeNumberRun(arr: ReadonlyArray, count: number, w: Writer) { + let varint = true + for (let i = 0; i < count; i++) { + if (!isVarintNumber(arr[i])) { + varint = false + break + } + } + if (varint) { + w.byte(NUMBER_RUN_VARINT) + for (let i = 0; i < count; i++) w.numberVarint(arr[i] as number) + } else { + w.byte(NUMBER_RUN_F64) + for (let i = 0; i < count; i++) w.f64(arr[i] as number) + } +} + +function matchesVariant(variant: VariantRow, value: unknown): boolean { + return variant.tuple + ? Array.isArray(value) && variant.sentinels.every((s) => value[s.key as number] === s.literal) + : Predicate.isObject(value) && !Array.isArray(value) && + variant.sentinels.every((s) => (value as Record)[s.key] === s.literal) +} + +function encodeUnion(ctx: EncodeContext, layout: UnionLayout, value: unknown, w: Writer) { + for (const variant of layout.variants) { + if (matchesVariant(variant, value)) { + if (ctx.positional) { + w.uvarint(variant.position) + } else { + w.byte(K.variant) + w.u32le(variant.tag) + } + encodeValue(ctx, variant.payload, value, w) + return + } + } + for (const member of layout.others) { + if (matchesLayout(member.layout, value)) { + if (ctx.positional) w.uvarint(member.position) + else w.byte(member.kind) + encodeValue(ctx, member.layout, value, w) + return + } + } + throw issueError(new SchemaIssue.InvalidType(layout.ast, value, ctx.options)) +} + +function encodeValue(ctx: EncodeContext, layout: Layout, value: unknown, w: Writer): void { + switch (layout._) { + case "bool": + w.byte(value === true ? 1 : 0) + return + case "null": + case "undefined": + return + case "number": + if (isVarintNumber(value)) w.numberVarint(value as number) + else w.f64(value as number) + return + case "int": { + // `disableChecks` can bypass the integer check used to select this layout. + if (!Number.isSafeInteger(value)) encodeFail("an integer", value, ctx.options) + w.numberVarint(value as number) + return + } + case "string": + w.string(value as string) + return + case "symbol": + encodeSymbol(ctx, value, w) + return + case "bytes": + w.bytes(value as Uint8Array) + return + case "bigint": + w.zigzag(value as bigint) + return + case "int64": { + const millis = layout.flavor === "date" ? (value as Date).getTime() : (value as DateTime.Utc).epochMilliseconds + if (Number.isNaN(millis)) encodeFail("a valid Date", value, ctx.options) + w.i64(BigInt(millis)) + return + } + case "dateTimeZoned": { + const zoned = value as DateTime.Zoned + w.i64(BigInt(zoned.epochMilliseconds)) + if (zoned.zone._tag === "Offset") { + w.byte(0) + w.i32le(zoned.zone.offset) + } else { + w.byte(1) + w.string(zoned.zone.id) + } + return + } + case "duration": { + const duration = (value as Duration.Duration).value + switch (duration._tag) { + case "Infinity": + w.byte(1) + return + case "NegativeInfinity": + w.byte(2) + return + case "Nanos": + w.byte(0) + w.zigzag(duration.nanos) + return + case "Millis": + w.byte(0) + w.zigzag(BigInt(duration.millis) * BIGINT_NANOS_PER_MILLI) + return + } + return + } + case "bigDecimal": { + const normalized = BigDecimal.normalize(value as BigDecimal.BigDecimal) + w.zigzag(normalized.value) + w.zigzag(BigInt(normalized.scale)) + return + } + case "json": { + let text: string | undefined + try { + text = JSON.stringify(value) + } catch { + text = undefined + } + if (text === undefined) { + if (isCyclic(value)) encodeFail("acyclic value", value, ctx.options) + encodeFail("a JSON-serializable value", value, ctx.options) + } + w.string(text) + return + } + case "option": { + const option = value as Option.Option + if (option._tag === "None") { + w.byte(0) + } else { + w.byte(1) + encodeValue(ctx, layout.value, option.value, w) + } + return + } + case "result": { + const result = value as Result.Result + if (result._tag === "Success") { + w.byte(0) + encodeValue(ctx, layout.success, result.success, w) + } else { + w.byte(1) + encodeValue(ctx, layout.failure, result.failure, w) + } + return + } + case "exit": { + const exit = value as Exit.Exit + if (exit._tag === "Success") { + w.byte(0) + encodeValue(ctx, layout.value, exit.value, w) + } else { + w.byte(1) + encodeValue(ctx, layout.cause, exit.cause, w) + } + return + } + case "cause": { + const reasons = (value as Cause.Cause).reasons + w.uvarint(reasons.length) + for (const reason of reasons) { + const mark = w.beginSized() + encodeReason(ctx, layout, reason, w) + w.endSized(mark) + } + return + } + case "causeReason": { + encodeReason(ctx, layout, value, w) + return + } + case "struct": { + if (ctx.positional) encodeStructPositional(ctx, layout, value as object, w) + else encodeStructFields(ctx, layout, value as object, w) + return + } + case "array": { + encodeArray(ctx, layout, value, w) + return + } + case "union": + encodeUnion(ctx, layout, value, w) + return + case "never": + throw issueError(new SchemaIssue.InvalidType(layout.ast, value, ctx.options)) + } +} + +// Reuse one top-level writer while allowing nested codecs to allocate their own. +let pooledWriter: Writer | undefined = new Writer() + +function encodeFrame( + layout: Layout, + value: unknown, + options: SchemaAST.ParseOptions, + mode: Mode +): Uint8Array { + const ctx: EncodeContext = { + options, + positional: mode.positional, + indexSignatures: undefined + } + const w = pooledWriter ?? new Writer() + const pooled = w === pooledWriter + if (pooled) pooledWriter = undefined + w.reset() + const savedPathLen = issuePathLen + issuePathLen = 0 + try { + const mark = w.beginSized() + w.byte(mode.envelope) + if (mode.positional) { + w.u32le(mode.fingerprintLo) + w.u32le(mode.fingerprintHi) + } + encodeValue(ctx, layout, value, w) + w.endSized(mark) + return w.out() + } finally { + w.abort() + issuePathLen = savedPathLen + if (pooled) pooledWriter = w + } +} + +// Unknown union members resolve to this sentinel. +const ABSENT = globalThis.Symbol.for("~effect/encoding/SchemaBinary/absent") + +function decodeChecked(layout: Layout, r: Reader): unknown { + const value = decodeValue(layout, r) + if (value !== ABSENT && r.pos !== r.end) invalid("no leftover bytes", undefined, r.options) + return value +} + +function decodeSized(layout: Layout, r: Reader): unknown { + const saved = r.enter(r.uvarint()) + const value = decodeChecked(layout, r) + r.exit(saved) + return value +} + +function decodeInline(layout: Layout, r: Reader): unknown { + if (isSelfDelimiting(layout)) return decodeValue(layout, r) + const saved = r.enter(packedSize(layout) ?? 0) + const value = decodeChecked(layout, r) + r.exit(saved) + return value +} + +function decodeSlot(layout: Layout, r: Reader): unknown { + if (isSelfDelimiting(layout)) return decodeValue(layout, r) + const size = packedSize(layout) + const saved = r.enter( + size !== undefined ? size : layout._ === "null" || layout._ === "undefined" ? 0 : r.uvarint() + ) + const value = decodeChecked(layout, r) + r.exit(saved) + return value +} + +function decodeExtraPair(layout: StructLayout, r: Reader, out: Record, seen: Set) { + const key = r.readUtf8(r.uvarint()) + if (seen.has(key)) invalid("unique extra keys", undefined, r.options) + seen.add(key) + const saved = r.enter(r.uvarint()) + const signature = (r.indexSignatures ??= new IndexSignatureCache(r.options)).find(layout, key) + if (signature !== undefined) { + issuePath[issuePathLen++] = key + const value = decodeChecked(signature.layout, r) + issuePathLen-- + if (value !== ABSENT) InternalRecord.assignProperty(out, key, value) + } + r.exit(saved) +} + +function missingKeyIssue(field: Field): SchemaIssue.Issue { + return new SchemaIssue.Pointer([field.name], new SchemaIssue.MissingKey(field.annotations)) +} + +function throwMissingKeys(layout: StructLayout, issues: Array): never { + throw issueError( + new SchemaIssue.Composite(layout.ast, issues as [SchemaIssue.Issue, ...Array]) + ) +} + +function decodeStruct(layout: StructLayout, r: Reader): unknown { + const out: Record = {} + // Use bitmasks for common structs and allocate sets only when needed. + let seenMask = 0 + let presentMask = 0 + let seenWide: Set | undefined + let presentWide: Set | undefined + let seenUnknown: Set | undefined + let seenExtra = false + // Fast-path fields in encoder order and fall back to the id map. + const fields = layout.fields + let cursor = 0 + while (r.pos < r.end) { + const id = r.uvarint() + const len = r.uvarint() + const saved = r.enter(len) + if (id === 0) { + if (seenExtra) invalid("unique field ids", undefined, r.options) + seenExtra = true + decodeExtraPairs(layout, r, out) + r.exit(saved) + continue + } + let field: Field | undefined + if (cursor < fields.length && fields[cursor].id === id) { + field = fields[cursor] + cursor++ + } else { + field = layout.byId.get(id) + if (field !== undefined) cursor = field.index + 1 + } + if (field === undefined) { + if (seenUnknown === undefined) seenUnknown = new Set() + else if (seenUnknown.has(id)) invalid("unique field ids", undefined, r.options) + seenUnknown.add(id) + r.exit(saved) + continue + } + const index = field.index + if (index < 32) { + const bit = 1 << index + if ((seenMask & bit) !== 0) invalid("unique field ids", undefined, r.options) + seenMask |= bit + } else if (seenWide === undefined) { + seenWide = new Set([index]) + } else { + if (seenWide.has(index)) invalid("unique field ids", undefined, r.options) + seenWide.add(index) + } + issuePath[issuePathLen++] = field.name + const value = decodeChecked(field.layout, r) + issuePathLen-- + if (value !== ABSENT) { + InternalRecord.assignProperty(out, field.name, value) + if (index < 32) presentMask |= 1 << index + else (presentWide ??= new Set()).add(index) + } + r.exit(saved) + } + let issues: Array | undefined + for (let i = 0; i < fields.length; i++) { + const field = fields[i] + const index = field.index + const present = index < 32 ? (presentMask & (1 << index)) !== 0 : presentWide?.has(index) === true + if (!field.optional && !present) { + ;(issues ??= []).push(missingKeyIssue(field)) + if (r.options.errors !== "all") break + } + } + if (issues !== undefined) throwMissingKeys(layout, issues) + return out +} + +function decodeStructPositional(layout: StructLayout, r: Reader): unknown { + const out: Record = {} + const bitmapBytes = (layout.optionalCount + 7) >> 3 + let bitmap = 0 + if (bitmapBytes > 0) { + if (r.pos + bitmapBytes > r.end) invalid("complete value", undefined, r.options) + bitmap = r.pos + r.pos += bitmapBytes + } + const buf = r.buf + const fields = layout.fields + let optionalIndex = 0 + let issues: Array | undefined + for (let i = 0; i < fields.length; i++) { + const field = fields[i] + if (field.optional) { + const present = (buf[bitmap + (optionalIndex >> 3)] & (1 << (optionalIndex & 7))) !== 0 + optionalIndex++ + if (!present) continue + } + issuePath[issuePathLen++] = field.name + const value = field.inline ? decodeInline(field.layout, r) : decodeSized(field.layout, r) + issuePathLen-- + if (value !== ABSENT) InternalRecord.assignProperty(out, field.name, value) + else if (!field.optional) { + ;(issues ??= []).push(missingKeyIssue(field)) + if (r.options.errors !== "all") break + } + } + if (issues !== undefined) throwMissingKeys(layout, issues) + if (layout.extra.length > 0) { + const count = r.uvarint() + if (count > r.remaining) invalid("complete value", undefined, r.options) + if (count > 0) { + const seen = new Set() + for (let i = 0; i < count; i++) decodeExtraPair(layout, r, out, seen) + } + } + return out +} + +function decodeExtraPairs(layout: StructLayout, r: Reader, out: Record) { + const seen = new Set() + while (r.pos < r.end) decodeExtraPair(layout, r, out, seen) +} + +function decodeArray(layout: ArrayLayout, r: Reader): unknown { + const elementLen = layout.elements.length + const count = layout.hasCount ? r.uvarint() : elementLen + // Cap allocation amplification from zero-width slots. + if (layout.hasCount && count > r.remaining + 1_048_576) { + invalid("array count within allocation limit", count, r.options) + } + if (layout.rest.length === 0 && count > elementLen) { + issuePath[issuePathLen++] = elementLen + throw issueError(new SchemaIssue.UnexpectedKey(layout.ast, undefined, r.options)) + } + if (count < layout.minCount) { + issuePath[issuePathLen++] = count + throw issueError(new SchemaIssue.MissingKey(undefined)) + } + const out: Array = new Array(count) + const uniform = layout.uniform + if (uniform !== undefined) { + if (layout.uniformNumbers) return decodeNumberRun(out, count, r) + if (isSelfDelimiting(uniform)) { + for (let i = 0; i < count; i++) { + issuePath[issuePathLen++] = i + out[i] = decodeValue(uniform, r) + issuePathLen-- + } + return out + } + const packed = layout.uniformPacked + const inline = layout.uniformInline + for (let i = 0; i < count; i++) { + issuePath[issuePathLen++] = i + const saved = r.enter(inline ? packed ?? 0 : r.uvarint()) + const value = decodeChecked(uniform, r) + r.exit(saved) + if (value === ABSENT) throw issueError(new SchemaIssue.MissingKey(undefined)) + issuePathLen-- + out[i] = value + } + return out + } + for (let i = 0; i < count; i++) { + const slot = arraySlot(layout, i, count) + const optional = i < elementLen && layout.elements[i].optional + if (!layout.hasCount && r.remaining === 0 && !optional && slot._ !== "null" && slot._ !== "undefined") { + issuePath[issuePathLen++] = i + throw issueError(new SchemaIssue.MissingKey(undefined)) + } + issuePath[issuePathLen++] = i + const value = decodeSlot(slot, r) + if (value === ABSENT) { + if (optional) invalid("known union member", undefined, r.options) + throw issueError(new SchemaIssue.MissingKey(undefined)) + } + issuePathLen-- + out[i] = value + } + if (!layout.hasCount && r.pos < r.end) { + issuePath[issuePathLen++] = elementLen + throw issueError(new SchemaIssue.UnexpectedKey(layout.ast, undefined, r.options)) + } + return out +} + +function decodeNumberRun(out: Array, count: number, r: Reader): Array { + const mode = r.byte() + if (mode === NUMBER_RUN_F64) { + if (r.remaining !== count * 8) invalid("f64", undefined, r.options) + for (let i = 0; i < count; i++) out[i] = r.f64() + return out + } + if (mode !== NUMBER_RUN_VARINT) invalid("f64", undefined, r.options) + for (let i = 0; i < count; i++) { + issuePath[issuePathLen++] = i + out[i] = r.numberVarint() + issuePathLen-- + } + return out +} + +function decodeUnion(layout: UnionLayout, r: Reader): unknown { + const kind = r.byte() + if (kind === K.variant) { + const tag = r.u32le() + const variant = layout.byTag.get(tag) + if (variant === undefined) { + r.take(r.remaining) + return ABSENT + } + const payload = decodeChecked(variant.payload, r) + if (payload === ABSENT) return ABSENT + if (!variant.tuple) { + for (const sentinel of variant.sentinels) { + InternalRecord.assignProperty(payload as object, sentinel.key, sentinel.literal) + } + } + return payload + } + const member = layout.byKind.get(kind) + if (member === undefined) { + r.take(r.remaining) + return ABSENT + } + return decodeChecked(member, r) +} + +// Fingerprint mode rejects selectors outside its canonical member table. +function decodeUnionPositional(layout: UnionLayout, r: Reader): unknown { + const position = layout.byPos[r.uvarint()] + if (position === undefined) invalid("known union member", undefined, r.options) + const payload = decodeChecked(position.layout, r) + const variant = position.variant + if (variant !== undefined && !variant.tuple && payload !== ABSENT) { + for (const sentinel of variant.sentinels) { + InternalRecord.assignProperty(payload as object, sentinel.key, sentinel.literal) + } + } + return payload +} + +function decodeReason(layout: ReasonLayout, r: Reader): unknown { + const tag = r.byte() + switch (tag) { + case 0: + return Cause.makeFailReason(requirePresent(decodeChecked(layout.error, r))) + case 1: + return Cause.makeDieReason(requirePresent(decodeChecked(layout.defect, r))) + case 2: + if (r.remaining !== 0) invalid("empty", undefined, r.options) + return Cause.makeInterruptReason() + case 3: { + if (r.remaining !== 8) invalid("f64", undefined, r.options) + return Cause.makeInterruptReason(r.f64()) + } + default: + r.take(r.remaining) + return ABSENT + } +} + +function requirePresent(value: unknown): unknown { + if (value === ABSENT) throw issueError(new SchemaIssue.MissingKey(undefined)) + return value +} + +function decodeValue(layout: Layout, r: Reader): unknown { + switch (layout._) { + case "bool": { + if (r.remaining !== 1) invalid("bool", undefined, r.options) + const b = r.byte() + if (b > 1) invalid("bool", undefined, r.options) + return b === 1 + } + case "null": + if (r.remaining !== 0) invalid("empty", undefined, r.options) + return null + case "undefined": + if (r.remaining !== 0) invalid("empty", undefined, r.options) + return undefined + case "number": { + const len = r.remaining + if (len === 8) return r.f64() + if (len === 0 || len > NUMBER_VARINT_MAX_BYTES) invalid("f64", undefined, r.options) + return r.numberVarint() + } + case "int": + return r.numberVarint() + case "string": + return r.readUtf8(r.end - r.pos) + case "symbol": + return globalThis.Symbol.for(r.readUtf8(r.end - r.pos)) + case "bytes": + return r.take(r.remaining).slice() + case "bigint": + return r.zigzag() + case "int64": { + if (r.remaining !== 8) invalid("int64", undefined, r.options) + const millis = Number(r.i64()) + return layout.flavor === "date" ? new Date(millis) : DateTime.makeUnsafe(millis) + } + case "dateTimeZoned": { + const millis = Number(r.i64()) + const tag = r.byte() + let timeZone: number | string + if (tag === 0) { + if (r.remaining !== 4) invalid("time zone", undefined, r.options) + timeZone = r.i32le() + } else if (tag === 1) { + timeZone = r.readUtf8(r.end - r.pos) + } else { + return invalid("time zone", undefined, r.options) + } + try { + return DateTime.makeZonedUnsafe(millis, { timeZone }) + } catch { + return invalid("time zone", undefined, r.options) + } + } + case "duration": { + const tag = r.byte() + switch (tag) { + case 0: + return Duration.nanos(r.zigzag()) + case 1: + return Duration.infinity + case 2: + return Duration.negativeInfinity + default: + return invalid("duration", undefined, r.options) + } + } + case "bigDecimal": { + const value = r.zigzag() + const scale = r.zigzag() + if (scale > MAX_SAFE_BIGINT || -scale > MAX_SAFE_BIGINT) invalid("safe integer length", undefined, r.options) + return BigDecimal.make(value, Number(scale)) + } + case "json": { + const text = r.readUtf8(r.end - r.pos) + try { + return JSON.parse(text) + } catch { + return invalid("json", undefined, r.options) + } + } + case "option": { + const tag = r.byte() + if (tag === 0) { + if (r.remaining !== 0) invalid("empty", undefined, r.options) + return Option.none() + } + if (tag !== 1) invalid("bool", undefined, r.options) + return Option.some(requirePresent(decodeChecked(layout.value, r))) + } + case "result": { + const tag = r.byte() + if (tag === 0) return Result.succeed(requirePresent(decodeChecked(layout.success, r))) + if (tag !== 1) invalid("bool", undefined, r.options) + return Result.fail(requirePresent(decodeChecked(layout.failure, r))) + } + case "exit": { + const tag = r.byte() + if (tag === 0) return Exit.succeed(requirePresent(decodeChecked(layout.value, r))) + if (tag !== 1) invalid("bool", undefined, r.options) + const cause = decodeValue(layout.cause, r) + return Exit.failCause(cause as Cause.Cause) + } + case "cause": { + const count = r.uvarint() + const reasons: Array> = [] + for (let i = 0; i < count; i++) { + issuePath[issuePathLen++] = i + const saved = r.enter(r.uvarint()) + const reason = decodeReason(layout, r) + r.exit(saved) + issuePathLen-- + if (reason !== ABSENT) reasons.push(reason as Cause.Reason) + } + if (r.pos !== r.end) invalid("no leftover bytes", undefined, r.options) + return Cause.fromReasons(reasons) + } + case "causeReason": + return decodeReason(layout, r) + case "struct": + return r.positional ? decodeStructPositional(layout, r) : decodeStruct(layout, r) + case "array": + return decodeArray(layout, r) + case "union": + return r.positional ? decodeUnionPositional(layout, r) : decodeUnion(layout, r) + case "never": + throw issueError(new SchemaIssue.InvalidType(layout.ast, undefined, r.options)) + } +} + +function decodeFrameBody(layout: Layout, r: Reader, mode: Mode): unknown { + const envelope = r.byte() + if (envelope !== mode.envelope) invalid(mode.expectedEnvelope, envelope, r.options) + if (mode.positional) { + if (r.remaining < 8) invalid("complete value", undefined, r.options) + if (r.u32le() !== mode.fingerprintLo || r.u32le() !== mode.fingerprintHi) { + invalid("matching layout fingerprint", undefined, r.options) + } + } + const value = decodeChecked(layout, r) + if (value === ABSENT) throw issueError(new SchemaIssue.MissingKey(undefined)) + return value +} + +// Reuse one top-level reader while allowing nested codecs to allocate their own. +let pooledReader: Reader | undefined = new Reader() + +function decodeOneShot( + layout: Layout, + bytes: Uint8Array, + options: SchemaAST.ParseOptions, + mode: Mode +): unknown { + const r = pooledReader ?? new Reader() + const pooled = r === pooledReader + if (pooled) pooledReader = undefined + r.reset(bytes, 0, bytes.length, options, undefined, mode.positional) + const savedPathLen = issuePathLen + issuePathLen = 0 + try { + const n = r.uvarint() + if (n === 0) invalid("nonzero frame length", undefined, options) + const saved = r.enter(n) + const value = decodeFrameBody(layout, r, mode) + r.exit(saved) + if (r.pos !== bytes.length) invalid("no leftover bytes", undefined, options) + return value + } finally { + issuePathLen = savedPathLen + r.release() + if (pooled) pooledReader = r + } +} + +function makeTransformation( + layout: Layout, + mode: Mode +): SchemaTransformation.Transformation> { + return SchemaTransformation.transformOrFail({ + decode: (bytes: Uint8Array, options) => { + try { + return Effect.succeed(decodeOneShot(layout, bytes, options, mode)) + } catch (e) { + return e instanceof IssueError ? Effect.fail(e.issue) : Effect.die(e) + } + }, + encode: (value: unknown, options) => { + try { + return Effect.succeed(encodeFrame(layout, value, options, mode)) + } catch (e) { + return e instanceof IssueError ? Effect.fail(e.issue) : Effect.die(e) + } + } + }) +} + +const fingerprintModeCache = new WeakMap() + +function compileMode(layout: Layout, fingerprint: boolean | undefined): Mode { + if (fingerprint !== true) return defaultMode + let mode = fingerprintModeCache.get(layout) + if (mode === undefined) { + mode = fingerprintMode(layout) + fingerprintModeCache.set(layout, mode) + } + return mode +} + +interface CompiledTarget { + readonly target: Schema.Constraint + readonly layout: Layout + readonly decodeExact: boolean +} + +const compileTargetCache = new WeakMap() + +function compileTarget( + schema: Schema.Constraint +): CompiledTarget { + const cached = compileTargetCache.get(schema.ast) + if (cached !== undefined) return cached + const raw = Schema.make(toBinaryAST(schema.ast)) + const { decodeExact, layout, recursive } = compileLayout(raw.ast) + // Only recursive schemas need the cycle walk. + const compiled = { target: recursive ? withCycleGuard(raw) : raw, layout, decodeExact } + compileTargetCache.set(schema.ast, compiled) + return compiled +} + +// Skip the cycle walk once for structurally decoded values. +function withCycleGuard(target: Schema.Constraint): Schema.Constraint { + const type = Schema.make(SchemaAST.toType(target.ast)) + const decoded = new WeakSet() + const guard = Schema.declareConstructor()( + [type], + ([type]) => (input, ast, options) => { + if (Predicate.isObjectOrArray(input) && decoded.delete(input)) { + return Effect.succeed(input) + } + return isCyclic(input) + ? Effect.fail(new SchemaIssue.InvalidType(ast, input, options)) + : SchemaParser.decodeUnknownEffect(type)(input, options) + }, + { identifier: "acyclic value" } + ) + return Schema.decodeTo( + guard, + SchemaTransformation.transform({ + decode: (value) => { + if (Predicate.isObjectOrArray(value)) decoded.add(value) + return value + }, + encode: (value) => value + }) + )(target) +} + +/** + * Selects the wire mode. + * + * The default mode supports compatible schema evolution. `fingerprint: true` + * uses positional layouts and an 8-byte layout hash for smaller frames, but + * requires peers to use the same schema definition. + * + * @category models + * @since 4.0.0 + */ +export interface Options { + /** + * @since 4.0.0 + */ + readonly fingerprint?: boolean | undefined +} + +/** + * The codec type returned by {@link toCodec}. + * + * @category models + * @since 4.0.0 + */ +export interface toCodec extends + Schema.Codec< + S["Type"], + Uint8Array, + S["DecodingServices"], + S["EncodingServices"] + > +{} + +/** + * Derives a compact binary codec from a schema. + * + * The wire layout is compiled from the encoded side of the schema. Each + * encode/decode handles exactly one frame; use {@link parser} for streams. + * + * Encoded results are arena-backed views and may share a larger buffer. Use + * `bytes.slice()` when independent ownership is required. + * + * **Example** + * + * ```ts + * import { Schema } from "effect" + * import { SchemaBinary } from "effect/unstable/encoding" + * + * const Person = Schema.Struct({ name: Schema.String, age: Schema.Number }) + * const codec = SchemaBinary.toCodec(Person) + * + * const bytes = Schema.encodeUnknownSync(codec)({ name: "Ada", age: 36 }) + * const person = Schema.decodeUnknownSync(codec)(bytes) + * ``` + * + * @category constructors + * @since 4.0.0 + */ +export function toCodec(schema: S, options?: Options): toCodec { + const { layout, target } = compileTarget(schema) + return (Schema.Uint8Array as Schema.instanceOf>).pipe( + Schema.decodeTo(target, makeTransformation(layout, compileMode(layout, options?.fingerprint))) + ) as unknown as toCodec +} + +/** + * A stateful frame parser for concatenated {@link toCodec} outputs. + * + * @category models + * @since 4.0.0 + */ +export interface Parser { + /** + * @since 4.0.0 + */ + feed(chunk: Uint8Array): Effect.Effect, Schema.SchemaError> + /** + * @since 4.0.0 + */ + feedSync(chunk: Uint8Array): ReadonlyArray + /** + * @since 4.0.0 + */ + end(): Effect.Effect + /** + * @since 4.0.0 + */ + endSync(): void +} + +/** + * Creates a stateful parser for a stream of concatenated frames. + * + * Values completed before a failure remain observable. After a failure, the + * parser rejects further calls. Use `maxFrameSize` to limit buffered frames. + * + * @category constructors + * @since 4.0.0 + */ +export function parser( + schema: S, + options?: SchemaAST.ParseOptions & Options & { readonly maxFrameSize?: number | undefined } +): Parser { + const { decodeExact, layout, target } = compileTarget(schema) + const mode = compileMode(layout, options?.fingerprint) + const parseOptions: SchemaAST.ParseOptions = options ?? {} + const maxFrameSize = options?.maxFrameSize + const decodeEncoded = decodeExact + ? undefined + : Schema.decodeUnknownSync(target as Schema.ConstraintDecoder, parseOptions) + let buffer = new Uint8Array(0) + let bufferStart = 0 + let bufferEnd = 0 + let stashed: Schema.SchemaError | undefined + let spent = false + const body = new Reader() + const indexSignatures = new IndexSignatureCache(parseOptions, PARSER_INDEX_SIGNATURE_CACHE_SIZE) + + const release = () => { + body.release() + indexSignatures.clear() + buffer = EMPTY_READER_BUFFER + bufferStart = bufferEnd = 0 + } + + const failSync = (expected: string, input?: unknown): never => { + throw new Schema.SchemaError(new SchemaIssue.InvalidValue({ expected }, input, parseOptions)) + } + + const takeStashed = (): never => { + const error = stashed! + stashed = undefined + throw error + } + + const self: Parser = { + feedSync(chunk) { + if (stashed !== undefined) return takeStashed() + if (spent) return failSync("parser is spent") + if (chunk.length > 0) { + const remaining = bufferEnd - bufferStart + const required = remaining + chunk.length + if (required > buffer.length) { + const next = new Uint8Array(Math.max(256, buffer.length * 2, required)) + next.set(buffer.subarray(bufferStart, bufferEnd)) + buffer = next + bufferStart = 0 + bufferEnd = remaining + } else if (bufferStart > 0) { + buffer.copyWithin(0, bufferStart, bufferEnd) + bufferStart = 0 + bufferEnd = remaining + } + buffer.set(chunk, bufferEnd) + bufferEnd += chunk.length + } + const out: Array = [] + const fail = (expected: string, input?: unknown): Array => { + spent = true + const error = new Schema.SchemaError(new SchemaIssue.InvalidValue({ expected }, input, parseOptions)) + release() + if (out.length === 0) throw error + stashed = error + return out + } + while (true) { + // Use bigint only for frame lengths wider than six varint groups. + let frameLen = 0 + let headerLen = -1 + const buffered = bufferEnd - bufferStart + let scale = 1 + for (let i = 0; i < Math.min(6, buffered); i++) { + const b = buffer[bufferStart + i] + frameLen += (b & 0x7F) * scale + if ((b & 0x80) === 0) { + headerLen = i + 1 + break + } + scale *= 128 + } + if (headerLen === -1) { + if (buffered < 6) return out + let n = BigInt(frameLen) + for (let i = 6; i < Math.min(10, buffered); i++) { + const b = buffer[bufferStart + i] + n |= BigInt(b & 0x7F) << BigInt(i * 7) + if ((b & 0x80) === 0) { + headerLen = i + 1 + if (n > MAX_SAFE_BIGINT) return fail("safe integer length", n) + frameLen = Number(n) + break + } + } + if (headerLen === -1) { + if (buffered >= 10) return fail("uvarint", buffer.subarray(bufferStart, bufferStart + 10)) + return out + } + } + if (frameLen === 0) return fail("nonzero frame length", frameLen) + if (maxFrameSize !== undefined && frameLen > maxFrameSize) { + return fail("frame within maxFrameSize", frameLen) + } + if (headerLen + frameLen > buffered) return out + const savedPathLen = issuePathLen + try { + issuePathLen = 0 + const bodyStart = bufferStart + headerLen + indexSignatures.beginFrame() + body.reset(buffer, bodyStart, bodyStart + frameLen, parseOptions, indexSignatures, mode.positional) + const value = decodeFrameBody(layout, body, mode) + out.push((decodeEncoded === undefined ? value : decodeEncoded(value)) as S["Type"]) + } catch (e) { + spent = true + release() + const error = e instanceof IssueError + ? new Schema.SchemaError(e.issue) + : Schema.isSchemaError(e) + ? e + : (() => { + throw e + })() + if (out.length === 0) throw error + stashed = error + return out + } finally { + issuePathLen = savedPathLen + } + bufferStart += headerLen + frameLen + if (bufferStart === bufferEnd) bufferStart = bufferEnd = 0 + } + }, + endSync() { + if (stashed !== undefined) return takeStashed() + if (spent) return failSync("parser is spent") + spent = true + const input = bufferStart < bufferEnd ? buffer.subarray(bufferStart, bufferEnd) : undefined + release() + if (input !== undefined) return failSync("complete value", input) + }, + feed: (chunk) => + Effect.suspend(() => { + try { + return Effect.succeed(self.feedSync(chunk)) + } catch (e) { + if (Schema.isSchemaError(e)) return Effect.fail(e) + throw e + } + }), + end: () => + Effect.suspend(() => { + try { + self.endSync() + return Effect.void + } catch (e) { + if (Schema.isSchemaError(e)) return Effect.fail(e) + throw e + } + }) + } + return self +} + +/** + * Assigns an explicit wire field id to a struct property. + * + * Use this to preserve the wire id across a rename or resolve a hash collision. + * Valid ids are integers from 1 through 4294967295. + * + * **Example** + * + * ```ts + * import { Schema } from "effect" + * import { SchemaBinary } from "effect/unstable/encoding" + * + * const Person = Schema.Struct({ + * id: Schema.String.pipe(SchemaBinary.fieldId(1)) + * }) + * ``` + * + * @category annotations + * @since 4.0.0 + */ +export function fieldId(id: number) { + if (!Number.isInteger(id) || id <= 0 || id > 0xFFFFFFFF) { + throw new Error(`Binary layout field id must be an integer in [1, 4294967295], got ${id}`) + } + const annotations = { [FIELD_ID_ANNOTATION_KEY]: id } + const annotateLastLink = SchemaAST.applyToLastLink((ast) => SchemaAST.annotateKey(ast, annotations)) + return (self: S): S["Rebuild"] => + self.rebuild(annotateLastLink(SchemaAST.annotateKey(self.ast, annotations))) +} diff --git a/packages/effect/src/unstable/encoding/index.ts b/packages/effect/src/unstable/encoding/index.ts index da43977da52..7c5a50a3fba 100644 --- a/packages/effect/src/unstable/encoding/index.ts +++ b/packages/effect/src/unstable/encoding/index.ts @@ -19,6 +19,11 @@ export * as Msgpack from "./Msgpack.ts" */ export * as Ndjson from "./Ndjson.ts" +/** + * @since 4.0.0 + */ +export * as SchemaBinary from "./SchemaBinary.ts" + /** * @since 4.0.0 */ diff --git a/packages/effect/src/unstable/rpc/RpcMessage.ts b/packages/effect/src/unstable/rpc/RpcMessage.ts index 63465917bb2..90e67c9ca81 100644 --- a/packages/effect/src/unstable/rpc/RpcMessage.ts +++ b/packages/effect/src/unstable/rpc/RpcMessage.ts @@ -12,6 +12,7 @@ */ import type { NonEmptyReadonlyArray } from "../../Array.ts" import type { Branded } from "../../Brand.ts" +import * as Schema from "../../Schema.ts" import type { Headers } from "../http/Headers.ts" import type * as Rpc from "./Rpc.ts" import type { RpcClientError } from "./RpcClientError.ts" @@ -336,6 +337,10 @@ export interface ResponseDefectEncoded { * hole belongs to the protocol. Encode it with * `protocol.codecFor(Schema.Defect())` first. * + * This constructor produces the structured exit used by JSON-compatible + * protocols. Serializations whose `codecFor` returns bytes must encode the + * complete RPC exit before placing it in the response envelope instead. + * * @category constructors * @since 4.0.0 */ @@ -403,6 +408,65 @@ export interface Pong { readonly _tag: "Pong" } +const RequestIdSchema = Schema.Union([Schema.String, Schema.Number]) + +/** + * Schema for transport-encoded RPC requests whose payload hole has already + * been filled by the active serialization. + * + * @category schemas + * @since 4.0.0 + */ +const RequestEncodedSchema = Schema.Struct({ + _tag: Schema.tag("Request"), + id: RequestIdSchema, + tag: Schema.String, + payload: Schema.Uint8Array, + headers: Schema.Array(Schema.Tuple([Schema.String, Schema.String])), + isNotification: Schema.optional(Schema.Literal(true)), + traceId: Schema.optional(Schema.String), + spanId: Schema.optional(Schema.String), + sampled: Schema.optional(Schema.Boolean) +}) + +/** + * Schema for every transport-encoded RPC envelope that crosses the wire. + * Binary serializers use it only after each schema-dependent hole has been + * encoded as bytes. `ClientProtocolError` is excluded because clients create + * it locally rather than receiving it from a server. + * + * @category schemas + * @since 4.0.0 + */ +export const EncodedSchema = Schema.Union([ + RequestEncodedSchema, + Schema.Struct({ + _tag: Schema.tag("Ack"), + requestId: RequestIdSchema + }), + Schema.Struct({ + _tag: Schema.tag("Interrupt"), + requestId: RequestIdSchema + }), + Schema.Struct({ _tag: Schema.tag("Ping") }), + Schema.Struct({ _tag: Schema.tag("Eof") }), + Schema.Struct({ + _tag: Schema.tag("Chunk"), + requestId: RequestIdSchema, + values: Schema.Uint8Array + }), + Schema.Struct({ + _tag: Schema.tag("Exit"), + requestId: RequestIdSchema, + exit: Schema.Uint8Array + }), + Schema.Struct({ + _tag: Schema.tag("Defect"), + defect: Schema.Uint8Array + }), + Schema.Struct({ _tag: Schema.tag("Pong") }) +]) + /** * Represents the reusable `Pong` message value. * diff --git a/packages/effect/src/unstable/rpc/RpcSerialization.ts b/packages/effect/src/unstable/rpc/RpcSerialization.ts index dd874b3512a..6d0dd28eccd 100644 --- a/packages/effect/src/unstable/rpc/RpcSerialization.ts +++ b/packages/effect/src/unstable/rpc/RpcSerialization.ts @@ -3,9 +3,9 @@ * * `RpcSerialization` is the boundary between `RpcMessage` envelopes and the * bytes or strings carried by a transport. This module provides built-in - * serializers for JSON, newline-delimited JSON, JSON-RPC 2.0, and MessagePack, - * including framed formats that can decode multiple messages from streaming - * chunks. + * serializers for JSON, newline-delimited JSON, JSON-RPC 2.0, MessagePack, and + * SchemaBinary, including framed formats that can decode multiple messages + * from streaming chunks. * * @since 4.0.0 */ @@ -16,7 +16,8 @@ import * as Layer from "../../Layer.ts" import * as Predicate from "../../Predicate.ts" import { hasProperty } from "../../Predicate.ts" import * as Schema from "../../Schema.ts" -import type * as RpcMessage from "./RpcMessage.ts" +import * as SchemaBinary from "../encoding/SchemaBinary.ts" +import * as RpcMessage from "./RpcMessage.ts" /** * Builds the codec used to fill the `unknown` holes of RPC protocol messages, @@ -590,6 +591,48 @@ export const makeMsgPack = ( */ export const msgPack: RpcSerialization["Service"] = makeMsgPack({ useRecords: true }) +const defaultSchemaBinaryMaxFrameSize = 16 * 1024 * 1024 + +const makeSchemaBinary = (options?: { + readonly maxFrameSize?: number | undefined +}): RpcSerialization["Service"] => { + const maxFrameSize = options?.maxFrameSize ?? defaultSchemaBinaryMaxFrameSize + const envelopeCodec = SchemaBinary.toCodec(RpcMessage.EncodedSchema, { fingerprint: true }) + const encodeEnvelope = Schema.encodeUnknownSync(envelopeCodec) + return RpcSerialization.of({ + contentType: "application/vnd.effect.rpc+schema-binary", + includesFraming: true, + codecFor: SchemaBinary.toCodec as CodecFor, + makeUnsafe: () => { + const parser = SchemaBinary.parser(RpcMessage.EncodedSchema, { fingerprint: true, maxFrameSize }) + const encoder = new TextEncoder() + return { + decode: (data) => parser.feedSync(typeof data === "string" ? encoder.encode(data) : data), + encode: (response) => { + if (!Array.isArray(response)) { + return encodeEnvelope(response) + } + if (response.length === 0) return undefined + const frames = new Array(response.length) + let length = 0 + for (let i = 0; i < response.length; i++) { + const frame = encodeEnvelope(response[i]) + frames[i] = frame + length += frame.length + } + const encoded = new Uint8Array(length) + let offset = 0 + for (let i = 0; i < frames.length; i++) { + encoded.set(frames[i], offset) + offset += frames[i].length + } + return encoded + } + } + } + }) +} + /** * RPC serialization layer that uses JSON for serialization. * @@ -671,3 +714,23 @@ export const layerMsgPack: Layer.Layer = Layer.succeed(RpcSeri export const layerMsgPackWith = ( options?: (Msgpackr.Options & StreamOptions) | undefined ): Layer.Layer => Layer.succeed(RpcSerialization)(makeMsgPack(options)) + +/** + * RPC serialization layer that uses SchemaBinary for payloads and fingerprinted + * RPC envelopes. It includes framing with a 16 MiB maximum frame size. + * + * @category layers + * @since 4.0.0 + */ +export const layerSchemaBinary: Layer.Layer = Layer.sync(RpcSerialization)(makeSchemaBinary) + +/** + * RPC serialization layer that uses SchemaBinary with a custom maximum frame + * size. RPC envelope fingerprints are always enabled. + * + * @category layers + * @since 4.0.0 + */ +export const layerSchemaBinaryWith = (options: { + readonly maxFrameSize: number +}): Layer.Layer => Layer.sync(RpcSerialization)(() => makeSchemaBinary(options)) diff --git a/packages/effect/src/unstable/rpc/RpcServer.ts b/packages/effect/src/unstable/rpc/RpcServer.ts index 977995958d7..d179d570f76 100644 --- a/packages/effect/src/unstable/rpc/RpcServer.ts +++ b/packages/effect/src/unstable/rpc/RpcServer.ts @@ -591,7 +591,7 @@ export const make: ( type Schemas = { readonly decode: (u: unknown) => Effect.Effect, Schema.SchemaError> readonly encodeChunk: ( - u: ReadonlyArray + u: NonEmptyReadonlyArray ) => Effect.Effect, Schema.SchemaError> readonly encodeExit: (u: unknown) => Effect.Effect readonly encodeDefect: (u: unknown) => Effect.Effect @@ -609,7 +609,7 @@ export const make: ( decode: Schema.decodeUnknownEffect(codecFor(rpc.payloadSchema)) as any, encodeChunk: Schema.encodeUnknownEffect( codecFor( - Schema.Array(Option.isSome(streamSchemas) ? streamSchemas.value.success : Schema.Any) + Schema.NonEmptyArray(Option.isSome(streamSchemas) ? streamSchemas.value.success : Schema.Any) ) ) as any, encodeExit: Schema.encodeUnknownEffect(codecFor(Rpc.exitSchema(rpc as any))) as any, diff --git a/packages/effect/test/cluster/Envelope.test.ts b/packages/effect/test/cluster/Envelope.test.ts index 84c036f1e7f..46f535636f0 100644 --- a/packages/effect/test/cluster/Envelope.test.ts +++ b/packages/effect/test/cluster/Envelope.test.ts @@ -1,6 +1,7 @@ import { assert, describe, it } from "@effect/vitest" import { Schema } from "effect" -import { EntityAddress, EntityId, EntityType, Envelope, ShardId, Snowflake } from "effect/unstable/cluster" +import { EntityAddress, EntityId, EntityType, Envelope, Reply, ShardId, Snowflake } from "effect/unstable/cluster" +import { SchemaBinary } from "effect/unstable/encoding" import { Headers } from "effect/unstable/http" const request = { @@ -33,4 +34,25 @@ describe("Envelope.OpaqueHole", () => { ) as Envelope.PartialRequestEncoded assert.strictEqual(encoded.payload, bytes) }) + + it("compiles the hole and Reply.Encoded as a bytes leaf under SchemaBinary", () => { + const bytes = Uint8Array.of(1, 2, 3) + const encodedBytes = Schema.encodeSync(SchemaBinary.toCodec(Schema.Uint8Array))(bytes) + const encodedHole = Schema.encodeSync(SchemaBinary.toCodec(Envelope.OpaqueHole))(bytes) + const encodedReply = Schema.encodeSync(SchemaBinary.toCodec(Reply.Encoded))(bytes as unknown as Reply.Encoded) + + assert.deepStrictEqual(encodedHole, encodedBytes) + assert.deepStrictEqual(encodedReply, encodedBytes) + assert.deepStrictEqual(Schema.decodeSync(SchemaBinary.toCodec(Envelope.OpaqueHole))(encodedHole), bytes) + assert.deepStrictEqual( + Schema.decodeSync(SchemaBinary.toCodec(Reply.Encoded))(encodedReply) as unknown, + bytes + ) + }) + + it("reports non-byte OpaqueHole inputs as schema failures", () => { + const encode = Schema.encodeUnknownSync(SchemaBinary.toCodec(Envelope.OpaqueHole)) + + assert.throws(() => encode({ id: 1 }), /Uint8Array/) + }) }) diff --git a/packages/effect/test/rpc/RpcSerialization.test.ts b/packages/effect/test/rpc/RpcSerialization.test.ts index 36301594d55..bf933d6d913 100644 --- a/packages/effect/test/rpc/RpcSerialization.test.ts +++ b/packages/effect/test/rpc/RpcSerialization.test.ts @@ -1,9 +1,10 @@ import { afterEach, assert, describe, it } from "@effect/vitest" import { Effect, Exit, Layer, Schema, Stream } from "effect" +import { SchemaBinary } from "effect/unstable/encoding" import { HttpRouter } from "effect/unstable/http" import * as HttpClient from "effect/unstable/http/HttpClient" import * as HttpClientResponse from "effect/unstable/http/HttpClientResponse" -import { Rpc, RpcClient, RpcGroup, RpcSchema, RpcSerialization, RpcServer } from "effect/unstable/rpc" +import { Rpc, RpcClient, RpcGroup, type RpcMessage, RpcSchema, RpcSerialization, RpcServer } from "effect/unstable/rpc" const responseExitSuccess = (requestId: string | number, value: unknown) => ({ _tag: "Exit", @@ -115,6 +116,49 @@ const makeClient = Effect.fnUntraced(function*() { return { client, requests } as const }) +const BinaryServer = RpcServer.layerHttp({ + group: Rpcs, + path: "/rpc", + protocol: "http" +}).pipe( + Layer.provide(Handlers), + Layer.provide(RpcSerialization.layerSchemaBinary) +) + +const makeBinaryClient = Effect.fnUntraced(function*() { + const { dispose, handler } = HttpRouter.toWebHandler(BinaryServer) + yield* Effect.addFinalizer(() => Effect.promise(dispose)) + + const httpClient = HttpClient.make((request) => { + const body = (request.body as any).body as Uint8Array + return Effect.map( + Effect.promise(() => + handler(new Request("http://test/rpc", { method: "POST", body: new Uint8Array(body).buffer })) + ), + (response) => HttpClientResponse.fromWeb(request, response) + ) + }) + + return yield* RpcClient.make(Rpcs).pipe( + Effect.provide( + RpcClient.layerProtocolHttp({ url: "http://test/rpc" }).pipe( + Layer.provide(RpcSerialization.layerSchemaBinary), + Layer.provide(Layer.succeed(HttpClient.HttpClient)(httpClient)) + ) + ) + ) +}) + +const uvarint = (value: number): Uint8Array => { + const bytes: Array = [] + while (value >= 0x80) { + bytes.push((value & 0x7f) | 0x80) + value = Math.floor(value / 0x80) + } + bytes.push(value) + return Uint8Array.from(bytes) +} + describe("RpcSerialization", () => { describe.sequential("jsonRpc inherited properties", () => { afterEach(() => { @@ -413,6 +457,123 @@ describe("RpcSerialization", () => { Effect.provide(RpcSerialization.layerMsgPackWith({ maxBufferSize: 2 })) )) + describe("SchemaBinary", () => { + it.effect("roundtrips requests and streamed responses over HTTP", () => + Effect.gen(function*() { + const client = yield* makeBinaryClient() + + assert.strictEqual(yield* client.Echo({ value: "hi" }), "hi!") + assert.deepStrictEqual(yield* Stream.runCollect(client.Counts({ to: 3 })), [1, 2, 3]) + + const error = yield* Effect.flip(client.Fail({})) + assert.instanceOf(error, EchoError) + assert.strictEqual(error.at.getTime(), failedAt.getTime()) + + const exit = yield* Effect.exit(client.Boom({})) + assert(Exit.isFailure(exit)) + assert.include(String(exit.cause), "boom") + })) + + it.effect("fingerprints envelopes", () => + Effect.gen(function*() { + const serialization = yield* RpcSerialization.RpcSerialization + const parser = serialization.makeUnsafe() + const incompatibleEnvelope = Schema.Struct({ + _tag: Schema.tag("Request"), + id: Schema.Union([Schema.String, Schema.Number]), + tag: Schema.String, + payload: Schema.Uint8Array, + headers: Schema.Array(Schema.Tuple([Schema.String, Schema.String])), + added: Schema.optional(Schema.String) + }) + const frame = Schema.encodeSync(SchemaBinary.toCodec(incompatibleEnvelope, { fingerprint: true }))({ + _tag: "Request", + id: 1, + tag: "Echo", + payload: Uint8Array.of(1), + headers: [] + }) + + assert.throws(() => parser.decode(frame), /matching layout fingerprint/) + }).pipe(Effect.provide(RpcSerialization.layerSchemaBinary))) + + it.effect("keeps payload schemas evolution-friendly", () => + Effect.gen(function*() { + const serialization = yield* RpcSerialization.RpcSerialization + const Writer = Schema.Struct({ + value: Schema.String, + added: Schema.optional(Schema.String) + }) + const Reader = Schema.Struct({ value: Schema.String }) + const encoded = Schema.encodeSync(serialization.codecFor(Writer))({ value: "ok", added: "new" }) + + assert.instanceOf(encoded, Uint8Array) + assert.deepStrictEqual(Schema.decodeSync(serialization.codecFor(Reader))(encoded), { value: "ok" }) + }).pipe(Effect.provide(RpcSerialization.layerSchemaBinary))) + + it.effect("uses fingerprinted envelope framing and the binary content type", () => + Effect.gen(function*() { + const serialization = yield* RpcSerialization.RpcSerialization + const parser = serialization.makeUnsafe() + const request: RpcMessage.RequestEncoded = { + _tag: "Request", + id: 1, + tag: "Echo", + payload: Uint8Array.of(1, 2, 3), + headers: [] + } + const frame = parser.encode(request) + + assert.strictEqual(serialization.contentType, "application/vnd.effect.rpc+schema-binary") + assert.strictEqual(serialization.includesFraming, true) + assert.instanceOf(frame, Uint8Array) + assert.deepStrictEqual(parser.decode(frame), [request]) + }).pipe(Effect.provide(RpcSerialization.layerSchemaBinary))) + + it.effect("owns encoded frames without copying envelope holes", () => + Effect.gen(function*() { + const serialization = yield* RpcSerialization.RpcSerialization + const encoder = serialization.makeUnsafe() + const payload = Uint8Array.of(1, 2, 3) + const request: RpcMessage.RequestEncoded = { + _tag: "Request", + id: 1, + tag: "Echo", + payload, + headers: [] + } + const frame = encoder.encode(request) + assert(frame instanceof Uint8Array) + const expected = frame.slice() + + payload.fill(9) + for (let i = 0; i < 1_000; i++) encoder.encode({ ...request, id: i }) + + assert.deepStrictEqual(frame, expected) + assert.deepStrictEqual(serialization.makeUnsafe().decode(frame), [{ + ...request, + payload: Uint8Array.of(1, 2, 3) + }]) + }).pipe(Effect.provide(RpcSerialization.layerSchemaBinary))) + + it.effect("defaults maxFrameSize to 16 MiB", () => + Effect.gen(function*() { + const serialization = yield* RpcSerialization.RpcSerialization + assert.deepStrictEqual(serialization.makeUnsafe().decode(uvarint(16 * 1024 * 1024)), []) + assert.throws( + () => serialization.makeUnsafe().decode(uvarint(16 * 1024 * 1024 + 1)), + /frame within maxFrameSize/ + ) + }).pipe(Effect.provide(RpcSerialization.layerSchemaBinary))) + + it.effect("layerSchemaBinaryWith overrides maxFrameSize", () => + Effect.gen(function*() { + const serialization = yield* RpcSerialization.RpcSerialization + assert.deepStrictEqual(serialization.makeUnsafe().decode(uvarint(4)), []) + assert.throws(() => serialization.makeUnsafe().decode(uvarint(5)), /frame within maxFrameSize/) + }).pipe(Effect.provide(RpcSerialization.layerSchemaBinaryWith({ maxFrameSize: 4 })))) + }) + describe("codecFor", () => { it("built-in serializations JSON-lower the hole", () => { const encode = Schema.encodeSync(RpcSerialization.json.codecFor(Schema.Date)) diff --git a/packages/effect/test/unstable/encoding/SchemaBinary.test.ts b/packages/effect/test/unstable/encoding/SchemaBinary.test.ts new file mode 100644 index 00000000000..2153878433d --- /dev/null +++ b/packages/effect/test/unstable/encoding/SchemaBinary.test.ts @@ -0,0 +1,1785 @@ +import { assert, describe, it } from "@effect/vitest" +import { + BigDecimal, + Cause, + Chunk, + DateTime, + Duration, + Effect, + Exit, + HashMap, + HashSet, + Option, + Redacted, + Result, + Schema, + SchemaIssue, + SchemaParser, + SchemaTransformation +} from "effect" +import * as SchemaBinary from "effect/unstable/encoding/SchemaBinary" + +const encode = (schema: Schema.Codec, value: A): Uint8Array => + Schema.encodeUnknownSync(SchemaBinary.toCodec(schema))(value) + +const roundtrip = (schema: Schema.Codec, value: A): A => { + const codec = SchemaBinary.toCodec(schema) + return Schema.decodeUnknownSync(codec)(Schema.encodeUnknownSync(codec)(value)) +} + +const concat = (...chunks: ReadonlyArray): Uint8Array => { + const out = new Uint8Array(chunks.reduce((length, chunk) => length + chunk.length, 0)) + let offset = 0 + for (const chunk of chunks) { + out.set(chunk, offset) + offset += chunk.length + } + return out +} + +const sameNumber = (actual: unknown, expected: number) => { + assert.isTrue( + Object.is(actual, expected), + `expected ${globalThis.String(actual)} to be exactly ${globalThis.String(expected)}` + ) +} + +const encodeFingerprint = (schema: Schema.Codec, value: A): Uint8Array => + Schema.encodeUnknownSync(SchemaBinary.toCodec(schema, { fingerprint: true }))(value) + +const roundtripFingerprint = (schema: Schema.Codec, value: A): A => { + const codec = SchemaBinary.toCodec(schema, { fingerprint: true }) + return Schema.decodeUnknownSync(codec)(Schema.encodeUnknownSync(codec)(value)) +} + +// Splits a fingerprint frame into its layout hash and the payload after it, +// both as comparable strings. +const fingerprintFrame = ( + schema: Schema.Codec, + value: A +): { readonly fingerprint: string; readonly payload: string } => { + const bytes = encodeFingerprint(schema, value) + let offset = 0 + while ((bytes[offset] & 0x80) !== 0) offset++ + offset++ + assert.strictEqual(bytes[offset], 0x11) + return { + fingerprint: Array.from(bytes.subarray(offset + 1, offset + 9)).join(","), + payload: Array.from(bytes.subarray(offset + 9)).join(",") + } +} + +const fingerprintOf = (schema: Schema.Codec, value: A): string => + fingerprintFrame(schema, value).fingerprint + +const schemaError = (f: () => unknown): Schema.SchemaError => { + try { + f() + } catch (error) { + if (Schema.isSchemaError(error)) return error + throw error + } + throw new Error("expected SchemaError") +} + +describe("SchemaBinary", () => { + describe("wire layout", () => { + it("packs fixed-size array elements", () => { + const numbers = [1.5, Number.NaN, -0, Number.POSITIVE_INFINITY] + const numberBytes = encode(Schema.Array(Schema.Number), numbers) + const decoded = roundtrip(Schema.Array(Schema.Number), numbers) + // length, envelope, count, mode byte, four f64 elements + assert.strictEqual(numberBytes.length, 36) + assert.strictEqual(decoded[0], 1.5) + assert.isTrue(Number.isNaN(decoded[1])) + assert.isTrue(Object.is(decoded[2], -0)) + assert.strictEqual(decoded[3], Number.POSITIVE_INFINITY) + + const boolBytes = encode(Schema.Array(Schema.Boolean), [true, false, true]) + assert.strictEqual(boolBytes.length, 6) + assert.deepStrictEqual(roundtrip(Schema.Array(Schema.Boolean), [true, false, true]), [true, false, true]) + }) + + it("length-prefixes variable-size array elements and nested arrays", () => { + assert.deepStrictEqual( + roundtrip(Schema.Array(Schema.String), ["ab", "", "cdé"]), + ["ab", "", "cdé"] + ) + assert.deepStrictEqual( + roundtrip(Schema.Array(Schema.Array(Schema.Number)), [[1, 2], [], [3]]), + [[1, 2], [], [3]] + ) + }) + + it("round-trips zero-width array elements", () => { + assert.deepStrictEqual(roundtrip(Schema.Array(Schema.Null), [null, null, null]), [null, null, null]) + assert.deepStrictEqual( + roundtrip(Schema.Array(Schema.Undefined), [undefined, undefined]), + [undefined, undefined] + ) + }) + + it("omits the count for fixed tuples and includes it for optional tuples", () => { + const pair = Schema.Tuple([Schema.String, Schema.Number]) + // length, envelope, then a length-prefixed slot each: "key" and varint 42 + assert.strictEqual(encode(pair, ["key", 42]).length, 8) + assert.deepStrictEqual(roundtrip(pair, ["key", 42]), ["key", 42]) + + const optional = Schema.Tuple([Schema.Number, Schema.optionalKey(Schema.Number)]) + assert.deepStrictEqual(roundtrip(optional, [1]), [1]) + assert.deepStrictEqual(roundtrip(optional, [1, 2]), [1, 2]) + }) + + it("supports tuple rest and trailing slots", () => { + const schema = Schema.TupleWithRest(Schema.Tuple([Schema.String]), [Schema.Number, Schema.Boolean]) + assert.deepStrictEqual( + roundtrip(schema, ["head", 1.5, 2.5, 3.5, true]), + ["head", 1.5, 2.5, 3.5, true] + ) + assert.deepStrictEqual(roundtrip(schema, ["head", false]), ["head", false]) + }) + + it("writes a length-first frame and rejects envelope or leftovers", () => { + const codec = SchemaBinary.toCodec(Schema.Number) + const bytes = Schema.encodeUnknownSync(codec)(1) + assert.deepStrictEqual(Array.from(bytes), [2, 0x10, 0x02]) + + const flags = bytes.slice() + flags[1] = 0x11 + assert.match(schemaError(() => Schema.decodeUnknownSync(codec)(flags)).message, /version 1 envelope, flags 0/) + + const version = bytes.slice() + version[1] = 0x20 + assert.match(schemaError(() => Schema.decodeUnknownSync(codec)(version)).message, /version 1 envelope, flags 0/) + assert.match( + schemaError(() => Schema.decodeUnknownSync(codec)(concat(bytes, bytes))).message, + /no leftover bytes/ + ) + }) + }) + + describe("output arena", () => { + it("keeps an earlier result stable through later encodes and arena rollover", () => { + const codec = SchemaBinary.toCodec(Schema.String) + const encode = Schema.encodeUnknownSync(codec) + const first = encode("first") + const expected = first.slice() + let rolledOver = false + + for (let i = 0; i < 500; i++) { + const later = encode(`later-${i}-${"x".repeat(32)}`) + if (later.buffer !== first.buffer) rolledOver = true + assert.deepStrictEqual(first, expected) + } + + assert.isTrue(rolledOver) + assert.strictEqual(Schema.decodeUnknownSync(codec)(first), "first") + }) + + it("keeps different results independent inside a shared arena", () => { + const codec = SchemaBinary.toCodec(Schema.String) + const encode = Schema.encodeUnknownSync(codec) + const decode = Schema.decodeUnknownSync(codec) + const results = [encode("alpha"), encode("bravo"), encode("charlie")] + + assert.deepStrictEqual(results.map((bytes) => decode(bytes)), ["alpha", "bravo", "charlie"]) + const shared = results.flatMap((left, index) => results.slice(index + 1).map((right) => [left, right] as const)) + .find(([left, right]) => left.buffer === right.buffer) + assert.isDefined(shared) + assert.notStrictEqual(shared![0].byteOffset, shared![1].byteOffset) + }) + + it("preserves nested two-phase codec composition", () => { + const Inner = Schema.Struct({ id: Schema.Number, label: Schema.String }) + const Outer = Schema.Struct({ id: Schema.String, inner: SchemaBinary.toCodec(Inner) }) + const codec = SchemaBinary.toCodec(Outer) + const value = { id: "outer", inner: { id: 1, label: "inner" } } + + assert.deepStrictEqual(Schema.decodeUnknownSync(codec)(Schema.encodeUnknownSync(codec)(value)), value) + }) + + it.effect("keeps concurrent fiber results readable after their producing turn", () => { + const Value = Schema.Struct({ id: Schema.Number, value: Schema.String }) + const codec = SchemaBinary.toCodec(Value) + const encode = Schema.encodeUnknownEffect(codec) + const decode = Schema.decodeUnknownSync(codec) + const values = Array.from({ length: 100 }, (_, id) => ({ id, value: `value-${id}` })) + + return Effect.gen(function*() { + const encoded = yield* Effect.forEach( + values, + (value) => encode(value).pipe(Effect.flatMap((bytes) => Effect.yieldNow.pipe(Effect.as(bytes)))), + { concurrency: "unbounded" } + ) + yield* Effect.yieldNow + assert.deepStrictEqual(encoded.map((bytes) => decode(bytes)), values) + }) + }) + }) + + describe("numbers", () => { + it("encodes integral values as sign-magnitude varints", () => { + assert.deepStrictEqual(Array.from(encode(Schema.Number, 0)), [2, 0x10, 0]) + assert.deepStrictEqual(Array.from(encode(Schema.Number, -0)), [2, 0x10, 1]) + assert.deepStrictEqual(Array.from(encode(Schema.Number, 1)), [2, 0x10, 2]) + assert.deepStrictEqual(Array.from(encode(Schema.Number, -1)), [2, 0x10, 3]) + for (const value of [0, -0, 1, -1, 42, -42, 127, -128]) { + sameNumber(roundtrip(Schema.Number, value), value) + } + }) + + it("covers every varint width up to the seven byte cap, both signs", () => { + for (let width = 1; width <= 7; width++) { + // the widest magnitude this many varint bytes can hold, and the first + // magnitude that needs one more + const last = 2 ** (7 * width - 1) - 1 + const next = 2 ** (7 * width - 1) + for (const sign of [1, -1]) { + const inside = sign * last + const outside = sign * next + assert.strictEqual(encode(Schema.Number, inside).length, 2 + width, `${inside}`) + sameNumber(roundtrip(Schema.Number, inside), inside) + // past the cap the value takes the f64 form instead of an eighth byte + assert.strictEqual(encode(Schema.Number, outside).length, width === 7 ? 10 : 3 + width, `${outside}`) + sameNumber(roundtrip(Schema.Number, outside), outside) + } + } + }) + + it("keeps f64 for values no capped varint can hold", () => { + const values = [ + 1.5, + -1.5, + Number.EPSILON, + Number.NaN, + Number.POSITIVE_INFINITY, + Number.NEGATIVE_INFINITY, + Number.MAX_SAFE_INTEGER, + Number.MIN_SAFE_INTEGER, + Number.MAX_VALUE, + 2 ** 48, + -(2 ** 48) + ] + for (const value of values) { + assert.strictEqual(encode(Schema.Number, value).length, 10, `${value}`) + sameNumber(roundtrip(Schema.Number, value), value) + } + }) + + it("discriminates the two forms by the enclosing field length", () => { + const schema = Schema.Struct({ n: Schema.Number }) + // length, envelope, five-byte field id, field length, payload + assert.strictEqual(encode(schema, { n: 7 }).length, 9) + assert.strictEqual(encode(schema, { n: 7.5 }).length, 16) + assert.deepStrictEqual(roundtrip(schema, { n: 7 }), { n: 7 }) + assert.deepStrictEqual(roundtrip(schema, { n: 7.5 }), { n: 7.5 }) + sameNumber(roundtrip(schema, { n: -0 }).n, -0) + }) + + it("packs a uniform number array behind one mode byte", () => { + const integers = [0, -0, 1, -1, 1000, -1000] + // length, envelope, count, mode, four one-byte and two two-byte varints + assert.strictEqual(encode(Schema.Array(Schema.Number), integers).length, 12) + const decoded = roundtrip(Schema.Array(Schema.Number), integers) + integers.forEach((value, index) => sameNumber(decoded[index], value)) + + // one value outside the varint form moves the whole run to f64 + const mixed = [1, 2.5] + assert.strictEqual(encode(Schema.Array(Schema.Number), mixed).length, 20) + assert.deepStrictEqual(roundtrip(Schema.Array(Schema.Number), mixed), mixed) + assert.deepStrictEqual(roundtrip(Schema.Array(Schema.Number), []), []) + assert.deepStrictEqual( + roundtrip(Schema.Array(Schema.Array(Schema.Number)), [[1, 2], [], [3.5]]), + [[1, 2], [], [3.5]] + ) + }) + + it("length-prefixes each number slot of a tuple", () => { + const pair = Schema.Tuple([Schema.Number, Schema.Number]) + assert.strictEqual(encode(pair, [1, 2]).length, 6) + assert.strictEqual(encode(pair, [1, 2.5]).length, 13) + assert.deepStrictEqual(roundtrip(pair, [1, 2.5]), [1, 2.5]) + const rest = Schema.TupleWithRest(Schema.Tuple([Schema.String]), [Schema.Number]) + assert.deepStrictEqual(roundtrip(rest, ["head", 1, 2.5, 3]), ["head", 1, 2.5, 3]) + }) + + it("carries both forms behind the union kind byte", () => { + const schema = Schema.Union([Schema.Number, Schema.String]) + assert.strictEqual(encode(schema, 5).length, 4) + assert.strictEqual(encode(schema, 5.5).length, 11) + assert.strictEqual(roundtrip(schema, 5), 5) + assert.strictEqual(roundtrip(schema, 5.5), 5.5) + assert.strictEqual(roundtrip(schema, "x"), "x") + sameNumber(roundtrip(schema, -0), -0) + + const tagged = Schema.Union([ + Schema.Struct({ _tag: Schema.Literal("A"), n: Schema.Number }), + Schema.Struct({ _tag: Schema.Literal("B"), n: Schema.Int }) + ]) + assert.deepStrictEqual(roundtrip(tagged, { _tag: "A", n: 1.5 }), { _tag: "A", n: 1.5 }) + assert.deepStrictEqual(roundtrip(tagged, { _tag: "B", n: -7 }), { _tag: "B", n: -7 }) + }) + + it("parses a stream whose numbers change width", () => { + const values = [0, -0, 1, -1, 63, 64, 1_000_000, -1_000_000, 1.5, Number.NaN, Number.MAX_SAFE_INTEGER] + const bytes = concat(...values.map((value) => encode(Schema.Number, value))) + const parser = SchemaBinary.parser(Schema.Number) + const out: Array = [] + for (const byte of bytes) out.push(...parser.feedSync(Uint8Array.of(byte))) + parser.endSync() + assert.strictEqual(out.length, values.length) + values.forEach((value, index) => sameNumber(out[index], value)) + }) + + it("emits a bare varint when the schema proves the value is an integer", () => { + assert.deepStrictEqual(Array.from(encode(Schema.Int, 1)), [2, 0x10, 2]) + assert.deepStrictEqual(Array.from(encode(Schema.Natural, 1)), [2, 0x10, 2]) + assert.deepStrictEqual(Array.from(encode(Schema.Number.check(Schema.isInt32()), 1)), [2, 0x10, 2]) + + // no mode byte for an array, no length prefix for a tuple slot + assert.strictEqual(encode(Schema.Array(Schema.Int), [1, 2, 3]).length, 6) + assert.strictEqual(encode(Schema.Array(Schema.Number), [1, 2, 3]).length, 7) + assert.strictEqual(encode(Schema.Tuple([Schema.Int, Schema.Int]), [1, 2]).length, 4) + assert.strictEqual(encode(Schema.Tuple([Schema.Number, Schema.Number]), [1, 2]).length, 6) + + for (const value of [0, -0, 1, -1, 2 ** 48, -(2 ** 48), Number.MAX_SAFE_INTEGER, Number.MIN_SAFE_INTEGER]) { + sameNumber(roundtrip(Schema.Int, value), value) + sameNumber(roundtrip(Schema.Array(Schema.Int), [value])[0], value) + sameNumber(roundtrip(Schema.Struct({ n: Schema.Int }), { n: value }).n, value) + } + assert.deepStrictEqual( + roundtrip(Schema.Array(Schema.Int), [1, -2, 3]), + [1, -2, 3] + ) + }) + + it("rejects a non-integer that reaches the integer layout", () => { + const codec = SchemaBinary.toCodec(Schema.Int) + assert.match( + schemaError(() => Schema.encodeUnknownSync(codec, { disableChecks: true })(1.5)).message, + /an integer/ + ) + }) + + it("rejects malformed number payloads", () => { + const codec = SchemaBinary.toCodec(Schema.Number) + // nine payload bytes are neither the varint nor the f64 form + const wide = new Uint8Array([10, 0x10, 0, 0, 0, 0, 0, 0, 0, 0, 0]) + assert.match(schemaError(() => Schema.decodeUnknownSync(codec)(wide)).message, /Expected f64/) + // a varint that never terminates inside its extent + const truncated = new Uint8Array([3, 0x10, 0x80, 0x80]) + assert.match(schemaError(() => Schema.decodeUnknownSync(codec)(truncated)).message, /complete value/) + + const arrayCodec = SchemaBinary.toCodec(Schema.Array(Schema.Number)) + const run = Schema.encodeUnknownSync(arrayCodec)([1, 2, 3]) + const unknownMode = run.slice() + unknownMode[3] = 2 + assert.match(schemaError(() => Schema.decodeUnknownSync(arrayCodec)(unknownMode)).message, /Expected f64/) + const shortRun = run.slice(0, run.length - 1) + shortRun[0] -= 1 + assert.match(schemaError(() => Schema.decodeUnknownSync(arrayCodec)(shortRun)).message, /complete value/) + }) + }) + + describe("schema evolution", () => { + it("skips unknown fields, accepts field reorder, and leaves missing optionals absent", () => { + const Writer = Schema.Struct({ a: Schema.Number, extra: Schema.String, b: Schema.String }) + const Reader = Schema.Struct({ b: Schema.String, a: Schema.Number, optional: Schema.optionalKey(Schema.Boolean) }) + const bytes = encode(Writer, { a: 1, extra: "drop", b: "keep" }) + assert.deepStrictEqual(Schema.decodeUnknownSync(SchemaBinary.toCodec(Reader))(bytes), { a: 1, b: "keep" }) + + const OrderedA = Schema.Struct({ a: Schema.Number, b: Schema.String }) + const OrderedB = Schema.Struct({ b: Schema.String, a: Schema.Number }) + assert.deepStrictEqual([...encode(OrderedA, { a: 1, b: "x" })], [...encode(OrderedB, { a: 1, b: "x" })]) + }) + + it("uses fieldId as the encoded-side field identity", () => { + const Before = Schema.Struct({ oldName: Schema.String.pipe(SchemaBinary.fieldId(1)) }) + const After = Schema.Struct({ newName: Schema.String.pipe(SchemaBinary.fieldId(1)) }) + const bytes = encode(Before, { oldName: "value" }) + assert.deepStrictEqual(Schema.decodeUnknownSync(SchemaBinary.toCodec(After))(bytes), { newName: "value" }) + assert.throws(() => SchemaBinary.fieldId(0)) + assert.throws(() => SchemaBinary.fieldId(1.5)) + }) + + it("skips an unknown union member on an optional struct field", () => { + const A = Schema.Struct({ _tag: Schema.Literal("A"), n: Schema.Number }) + const B = Schema.Struct({ _tag: Schema.Literal("B"), text: Schema.String }) + const Writer = Schema.Struct({ event: Schema.optionalKey(Schema.Union([A, B])) }) + const Reader = Schema.Struct({ event: Schema.optionalKey(Schema.Union([A])) }) + assert.deepStrictEqual( + Schema.decodeUnknownSync(SchemaBinary.toCodec(Reader))(encode(Writer, { event: { _tag: "B", text: "new" } })), + {} + ) + assert.deepStrictEqual( + Schema.decodeUnknownSync(SchemaBinary.toCodec(Reader))(encode(Writer, { event: { _tag: "A", n: 1 } })), + { event: { _tag: "A", n: 1 } } + ) + }) + + it("keeps tuple sentinels in variant payloads", () => { + const A = Schema.Tuple([Schema.Literal("A"), Schema.Number]) + const B = Schema.Tuple([Schema.Literal("B"), Schema.String]) + const schema = Schema.Union([A, B]) + assert.deepStrictEqual(roundtrip(schema, ["A", 1]), ["A", 1]) + assert.deepStrictEqual(roundtrip(schema, ["B", "value"]), ["B", "value"]) + }) + + it("rejects unknown union members in positional slots", () => { + const A = Schema.Struct({ _tag: Schema.Literal("A") }) + const B = Schema.Struct({ _tag: Schema.Literal("B") }) + const WriterArray = Schema.Array(Schema.Union([A, B])) + const ReaderArray = Schema.Array(Schema.Union([A])) + assert.match( + schemaError(() => + Schema.decodeUnknownSync(SchemaBinary.toCodec(ReaderArray))(encode(WriterArray, [{ _tag: "B" }])) + ).message, + /Missing key/ + ) + + const WriterTuple = Schema.Tuple([Schema.optionalKey(Schema.Union([A, B]))]) + const ReaderTuple = Schema.Tuple([Schema.optionalKey(Schema.Union([A]))]) + assert.match( + schemaError(() => + Schema.decodeUnknownSync(SchemaBinary.toCodec(ReaderTuple))(encode(WriterTuple, [{ _tag: "B" }])) + ).message, + /known union member/ + ) + }) + + it("reports the array or tuple index for an unknown positional union member", () => { + const A = Schema.Struct({ _tag: Schema.Literal("A") }) + const B = Schema.Struct({ _tag: Schema.Literal("B") }) + const WriterArray = Schema.Struct({ xs: Schema.Array(Schema.Union([A, B])) }) + const ReaderArray = Schema.Struct({ xs: Schema.Array(Schema.Union([A])) }) + assert.match( + schemaError(() => + Schema.decodeUnknownSync(SchemaBinary.toCodec(ReaderArray))( + encode(WriterArray, { xs: [{ _tag: "B" }] }) + ) + ).message, + /Missing key\n at \["xs"\]\[0\]/ + ) + + const WriterTuple = Schema.Struct({ xs: Schema.Tuple([Schema.String, Schema.Union([A, B])]) }) + const ReaderTuple = Schema.Struct({ xs: Schema.Tuple([Schema.String, Schema.Union([A])]) }) + assert.match( + schemaError(() => + Schema.decodeUnknownSync(SchemaBinary.toCodec(ReaderTuple))( + encode(WriterTuple, { xs: ["head", { _tag: "B" }] }) + ) + ).message, + /Missing key\n at \["xs"\]\[1\]/ + ) + }) + + it("uses kind tags for mixed enums", () => { + const Mixed = { Text: "text", Code: 1 } as const + const schema = Schema.Enum(Mixed) + assert.strictEqual(roundtrip(schema, Mixed.Text), "text") + assert.strictEqual(roundtrip(schema, Mixed.Code), 1) + }) + + it("encodes record entries in the reserved field zero map", () => { + const schema = Schema.Record(Schema.String, Schema.Number) + const value = { z: 1, a: 2 } + assert.deepStrictEqual(roundtrip(schema, value), value) + assert.deepStrictEqual([...encode(schema, value)], [...encode(schema, { a: 2, z: 1 })]) + }) + + it("round-trips __proto__ named fields without changing the prototype", () => { + const schema = Schema.Struct({ ["__proto__"]: Schema.String, ok: Schema.String }) + const value = { ["__proto__"]: "named", ok: "yes" } + + for (const result of [roundtrip(schema, value), roundtripFingerprint(schema, value)]) { + assert.strictEqual(Object.getPrototypeOf(result), Object.prototype) + assert.isTrue(Object.prototype.propertyIsEnumerable.call(result, "__proto__")) + assert.strictEqual(result.__proto__, "named") + assert.strictEqual(({} as Record).polluted, undefined) + } + }) + + it("round-trips __proto__ record entries with object values without changing the prototype", () => { + const schema = Schema.Record(Schema.String, Schema.Struct({ polluted: Schema.Boolean })) + const value = { ["__proto__"]: { polluted: true } } + + for (const result of [roundtrip(schema, value), roundtripFingerprint(schema, value)]) { + assert.strictEqual(Object.getPrototypeOf(result), Object.prototype) + assert.isTrue(Object.prototype.propertyIsEnumerable.call(result, "__proto__")) + assert.deepStrictEqual(result.__proto__, { polluted: true }) + assert.strictEqual(({} as Record).polluted, undefined) + } + }) + + it("round-trips __proto__ tagged-union sentinels without changing the prototype", () => { + const A = Schema.Struct({ ["__proto__"]: Schema.Literal("A"), value: Schema.String }) + const B = Schema.Struct({ ["__proto__"]: Schema.Literal("B"), value: Schema.String }) + const schema = Schema.Union([A, B]) + const value = { ["__proto__"]: "A" as const, value: "a" } + + for (const result of [roundtrip(schema, value), roundtripFingerprint(schema, value)]) { + assert.strictEqual(Object.getPrototypeOf(result), Object.prototype) + assert.isTrue(Object.prototype.propertyIsEnumerable.call(result, "__proto__")) + assert.strictEqual(result.__proto__, "A") + assert.strictEqual(({} as Record).polluted, undefined) + } + }) + }) + + describe("parser", () => { + it("parses concatenated frames split across chunks", () => { + const schema = Schema.Struct({ a: Schema.Number }) + const first = encode(schema, { a: 1 }) + const second = encode(schema, { a: 2 }) + const bytes = concat(first, second) + const parser = SchemaBinary.parser(schema) + assert.deepStrictEqual(parser.feedSync(bytes.slice(0, first.length + 3)), [{ a: 1 }]) + assert.deepStrictEqual(parser.feedSync(bytes.slice(first.length + 3)), [{ a: 2 }]) + parser.endSync() + assert.match(schemaError(() => parser.feedSync(new Uint8Array())).message, /parser is spent/) + }) + + it("parses exact nested structs, arrays, and string records", () => { + const schema = Schema.Struct({ + id: Schema.Number, + active: Schema.Boolean, + tags: Schema.Array(Schema.String), + metrics: Schema.Record(Schema.String, Schema.Number) + }) + const value = { id: 1, active: true, tags: ["a", "b"], metrics: { x: 1, y: 2 } } + const parser = SchemaBinary.parser(schema) + + assert.deepStrictEqual(parser.feedSync(encode(schema, value)), [value]) + parser.endSync() + }) + + it("keeps __proto__ safe on the exact parser path", () => { + const schema = Schema.Struct({ ["__proto__"]: Schema.String, value: Schema.Number }) + const value = { ["__proto__"]: "own", value: 1 } + const parser = SchemaBinary.parser(schema) + const [result] = parser.feedSync(encode(schema, value)) + + assert.strictEqual(Object.getPrototypeOf(result), Object.prototype) + assert.isTrue(Object.prototype.propertyIsEnumerable.call(result, "__proto__")) + assert.strictEqual(result.__proto__, "own") + assert.strictEqual(({} as Record).polluted, undefined) + parser.endSync() + }) + + it("retains Schema predicates erased by the binary layout", () => { + const rejects = ( + reader: Schema.Constraint, + writer: Schema.Codec, + value: unknown + ) => { + const parser = SchemaBinary.parser(reader) + assert.isTrue(Schema.isSchemaError(schemaError(() => parser.feedSync(encode(writer, value))))) + } + + rejects(Schema.TemplateLiteral(["a"]), Schema.String, "zzz") + rejects(Schema.Literal("a"), Schema.String, "zzz") + rejects(Schema.Enum({ A: "a", B: "b" }), Schema.String, "zzz") + rejects(Schema.UniqueSymbol(Symbol.for("expected")), Schema.Symbol, Symbol.for("other")) + rejects(Schema.ObjectKeyword, Schema.Unknown, 1) + }) + + it("retains Schema parsing for numeric record keys", () => { + const Writer = Schema.Record(Schema.String, Schema.Number) + const Reader = Schema.Record(Schema.Number, Schema.Number) + const parser = SchemaBinary.parser(Reader) + + assert.deepStrictEqual(parser.feedSync(encode(Writer, { "01": 1, "1e2": 2, "-0": 3 })), [{ + "0": 3, + "1": 1, + "100": 2 + }]) + parser.endSync() + }) + + it("retains transformations, checks, unions, declarations, and recursive validation", () => { + const transformed = SchemaBinary.parser(Schema.NumberFromString) + assert.deepStrictEqual(transformed.feedSync(encode(Schema.NumberFromString, 123)), [123]) + transformed.endSync() + + const NonNegative = Schema.Number.check(Schema.isGreaterThanOrEqualTo(0)) + assert.match( + schemaError(() => SchemaBinary.parser(NonNegative).feedSync(encode(Schema.Number, -1))).message, + /greater than or equal to 0/ + ) + + const CheckedUnion = Schema.Union([NonNegative, Schema.String]) + const BroadUnion = Schema.Union([Schema.Number, Schema.String]) + assert.match( + schemaError(() => SchemaBinary.parser(CheckedUnion).feedSync(encode(BroadUnion, -1))).message, + /greater than or equal to 0/ + ) + + const CheckedOption = Schema.Option(NonNegative) + const BroadOption = Schema.Option(Schema.Number) + assert.match( + schemaError(() => SchemaBinary.parser(CheckedOption).feedSync(encode(BroadOption, Option.some(-1)))).message, + /greater than or equal to 0/ + ) + + interface Node { + readonly value: number + readonly children: ReadonlyArray + } + let Writer: Schema.Codec + Writer = Schema.Struct({ + value: Schema.Number, + children: Schema.Array(Schema.suspend(() => Writer)) + }) + let Reader: Schema.Codec + Reader = Schema.Struct({ + value: NonNegative, + children: Schema.Array(Schema.suspend(() => Reader)) + }) + const invalid = { value: 1, children: [{ value: -1, children: [] }] } + assert.match( + schemaError(() => SchemaBinary.parser(Reader).feedSync(encode(Writer, invalid))).message, + /greater than or equal to 0/ + ) + }) + + it("retains partial frames across one-byte feeds", () => { + const bytes = concat(...Array.from({ length: 100 }, (_, i) => encode(Schema.Number, i))) + const parser = SchemaBinary.parser(Schema.Number) + const values: Array = [] + for (const byte of bytes) values.push(...parser.feedSync(Uint8Array.of(byte))) + parser.endSync() + assert.deepStrictEqual(values, Array.from({ length: 100 }, (_, i) => i)) + }) + + it("waits for a fragmented multi-byte frame header", () => { + const value = "x".repeat(300) + const bytes = encode(Schema.String, value) + const parser = SchemaBinary.parser(Schema.String) + + assert.deepStrictEqual(parser.feedSync(bytes.slice(0, 1)), []) + assert.deepStrictEqual(parser.feedSync(bytes.slice(1)), [value]) + parser.endSync() + }) + + it("uses the bigint header path for longer safe lengths", () => { + // Canonical uvarint(Number.MAX_SAFE_INTEGER): seven continuation groups + // followed by the final four bits. maxFrameSize rejects it before the + // parser waits for an impractically large body. + const bytes = Uint8Array.of(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0F) + const parser = SchemaBinary.parser(Schema.String, { maxFrameSize: 1 }) + + assert.deepStrictEqual(parser.feedSync(bytes.slice(0, 7)), []) + assert.match(schemaError(() => parser.feedSync(bytes.slice(7))).message, /frame within maxFrameSize/) + }) + + it("keeps parser index-signature caching bounded with FIFO eviction", () => { + let checks = 0 + const Key = Schema.String.check(Schema.makeFilter((_: string) => { + checks++ + return true + })) + const Writer = Schema.Record(Schema.String, Schema.Number) + const parser = SchemaBinary.parser(Schema.Record(Key, Schema.Number)) + const feed = (key: string) => { + const value = { [key]: 1 } + assert.deepStrictEqual(parser.feedSync(encode(Writer, value)), [value]) + } + + // The parser cache holds 256 attacker-controlled (layout, key) pairs. + // The 257th evicts key-0, while key-1 remains cached. + for (let i = 0; i < 257; i++) feed(`key-${i}`) + const beforeHit = checks + feed("key-1") + const hitChecks = checks - beforeHit + const beforeMiss = checks + feed("key-0") + const missChecks = checks - beforeMiss + + assert.strictEqual(missChecks, hitChecks + 1) + parser.endSync() + }) + + it("does not thrash the index-signature cache above its bound", () => { + let checks = 0 + const Key = Schema.String.check(Schema.makeFilter((_: string) => { + checks++ + return true + })) + const Writer = Schema.Record(Schema.String, Schema.Number) + const Reader = Schema.Record(Key, Schema.Number) + const allHitParser = SchemaBinary.parser(Reader) + const allHitValue = Object.fromEntries(Array.from({ length: 256 }, (_, index) => [`hit-${index}`, index])) + const allHitBytes = encode(Writer, allHitValue) + + assert.deepStrictEqual(allHitParser.feedSync(allHitBytes), [allHitValue]) + const beforeAllHit = checks + assert.deepStrictEqual(allHitParser.feedSync(allHitBytes), [allHitValue]) + const allHitChecksPerKey = (checks - beforeAllHit) / 256 + allHitParser.endSync() + + const parser = SchemaBinary.parser(Reader) + const value = Object.fromEntries(Array.from({ length: 300 }, (_, index) => [`key-${index}`, index])) + const bytes = encode(Writer, value) + + assert.deepStrictEqual(parser.feedSync(bytes), [value]) + const firstChecks = checks + assert.deepStrictEqual(parser.feedSync(bytes), [value]) + const secondChecks = checks - firstChecks + const misses = secondChecks - allHitChecksPerKey * 300 + + // Derive the decoder's per-key work from a fully warm parser. Each cache + // miss adds one classification check beyond that all-hit baseline. + assert.strictEqual(misses, 45) + parser.endSync() + }) + + it("isolates index-signature caches by parser options", () => { + const Key = Schema.String.check(Schema.makeFilter((key: string) => key.startsWith("allowed-"))) + const Writer = Schema.Record(Schema.String, Schema.Number) + const Reader = Schema.Record(Key, Schema.Number) + const bytes = encode(Writer, { denied: 1 }) + const strict = SchemaBinary.parser(Reader) + const unchecked = SchemaBinary.parser(Reader, { disableChecks: true }) + + assert.deepStrictEqual(strict.feedSync(bytes), [{}]) + assert.deepStrictEqual(unchecked.feedSync(bytes), [{ denied: 1 }]) + strict.endSync() + unchecked.endSync() + }) + + it("keeps nested SchemaBinary decodes independent from the outer reader", () => { + const Inner = Schema.Struct({ id: Schema.Number, label: Schema.String }) + const Outer = Schema.Struct({ id: Schema.String, inner: SchemaBinary.toCodec(Inner) }) + const first = { id: "first", inner: { id: 1, label: "one" } } + const second = { id: "second", inner: { id: 2, label: "two" } } + const parser = SchemaBinary.parser(Outer) + + assert.deepStrictEqual(parser.feedSync(concat(encode(Outer, first), encode(Outer, second))), [first, second]) + parser.endSync() + }) + + it("returns a checked-out one-shot reader after an exceptional decode", () => { + const codec = SchemaBinary.toCodec(Schema.String) + const decode = Schema.decodeUnknownSync(codec) + + assert.match(schemaError(() => decode(Uint8Array.of(2, 0x10, 0xFF))).message, /utf-8/) + assert.strictEqual(decode(encode(Schema.String, "after failure")), "after failure") + }) + + it("delivers completed values before reporting a later failure", () => { + const good = encode(Schema.Number, 1) + const bad = encode(Schema.Number, 2).slice(0, 2) + const parser = SchemaBinary.parser(Schema.Number) + assert.deepStrictEqual(parser.feedSync(concat(good, bad)), [1]) + assert.match(schemaError(() => parser.endSync()).message, /complete value/) + assert.match(schemaError(() => parser.feedSync(new Uint8Array())).message, /parser is spent/) + }) + + it("enforces maxFrameSize", () => { + const bytes = encode(Schema.String, "too large") + const parser = SchemaBinary.parser(Schema.String, { maxFrameSize: 2 }) + assert.match(schemaError(() => parser.feedSync(bytes)).message, /frame within maxFrameSize/) + }) + + it("fails a ten-byte unterminated length immediately", () => { + const bytes = new Uint8Array(10).fill(0x80) + const parser = SchemaBinary.parser(Schema.String, { reportInput: true }) + const error = schemaError(() => parser.feedSync(bytes)) + assert.match(error.message, /uvarint/) + assert.isTrue(SchemaIssue.hasInput(error.issue)) + }) + + it("rejects a terminated header above the safe-integer bound", () => { + const bytes = Uint8Array.of(0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x10) + const parser = SchemaBinary.parser(Schema.String) + + assert.match(schemaError(() => parser.feedSync(bytes)).message, /safe integer length/) + assert.match(schemaError(() => parser.feedSync(new Uint8Array())).message, /parser is spent/) + }) + + it.effect("wraps feed and end in SchemaError effects", () => + Effect.gen(function*() { + const parser = SchemaBinary.parser(Schema.Number) + const values = yield* parser.feed(encode(Schema.Number, 1)) + assert.deepStrictEqual(values, [1]) + yield* parser.end() + })) + }) + + describe("native declarations", () => { + it("round-trips bigint, Date, bytes, Option, and Result", () => { + assert.strictEqual(roundtrip(Schema.BigInt, -12345678901234567890n), -12345678901234567890n) + assert.strictEqual(roundtrip(Schema.Date, new Date(-123456789)).getTime(), -123456789) + assert.deepStrictEqual([...roundtrip(Schema.Uint8Array, Uint8Array.of(0, 1, 255))], [0, 1, 255]) + + const option = roundtrip(Schema.Option(Schema.String), Option.some("value")) + assert.isTrue(Option.isSome(option)) + if (Option.isSome(option)) assert.strictEqual(option.value, "value") + + const result = roundtrip(Schema.Result(Schema.Number, Schema.String), Result.fail("error")) + assert.isTrue(Result.isFailure(result)) + if (Result.isFailure(result)) assert.strictEqual(result.failure, "error") + }) + + it("round-trips Duration and normalized BigDecimal", () => { + const duration = roundtrip(Schema.Duration, Duration.millis(1.5)) + assert.strictEqual(Duration.toNanosUnsafe(duration), 1_500_000n) + assert.strictEqual(roundtrip(Schema.Duration, Duration.infinity), Duration.infinity) + + const decimal = roundtrip(Schema.BigDecimal, BigDecimal.make(100n, 2)) + assert.strictEqual(decimal.value, 1n) + assert.strictEqual(decimal.scale, 0) + }) + + it("round-trips UTC and zoned DateTime values", () => { + const utc = DateTime.makeUnsafe(-123456789) + assert.strictEqual(DateTime.toEpochMillis(roundtrip(Schema.DateTimeUtc, utc)), -123456789) + + for ( + const zoned of [ + DateTime.makeZonedUnsafe(123456789, { timeZone: 3_600_000 }), + DateTime.makeZonedUnsafe(123456789, { timeZone: "Europe/London" }) + ] + ) { + const decoded = roundtrip(Schema.DateTimeZoned, zoned) + assert.strictEqual(DateTime.toEpochMillis(decoded), DateTime.toEpochMillis(zoned)) + assert.strictEqual(DateTime.zoneToString(decoded.zone), DateTime.zoneToString(zoned.zone)) + } + }) + + it("round-trips Exit, Cause, and CauseReason", () => { + const causeSchema = Schema.Cause(Schema.String, Schema.Unknown) + const cause = Cause.fromReasons([ + Cause.makeFailReason("boom"), + Cause.makeDieReason({ defect: true }), + Cause.makeInterruptReason(0), + Cause.makeInterruptReason() + ]) + const decoded = roundtrip(causeSchema, cause) + assert.deepStrictEqual(decoded.reasons.map((reason) => reason._tag), ["Fail", "Die", "Interrupt", "Interrupt"]) + assert.strictEqual((decoded.reasons[2] as Cause.Interrupt).fiberId, 0) + assert.strictEqual((decoded.reasons[3] as Cause.Interrupt).fiberId, undefined) + + const reason = roundtrip( + Schema.CauseReason(Schema.String, Schema.Unknown), + Cause.makeFailReason("failure") + ) + assert.strictEqual(reason._tag, "Fail") + + const exit = roundtrip(Schema.Exit(Schema.Number, Schema.String, Schema.Unknown), Exit.failCause(cause)) + assert.isTrue(Exit.isFailure(exit)) + }) + + // An `Exit` layout carries the `Cause` node its failure branch writes, + // rather than rebuilding one per value. Both branches must still round-trip + // in fingerprint mode, where the failure branch has no length prefix to + // resynchronise on, and the shared node must not make the two schemas hash + // alike. + it("round-trips both Exit branches in fingerprint mode", () => { + const ExitSchema = Schema.Exit(Schema.Number, Schema.String, Schema.Unknown) + const cause = Cause.fromReasons([Cause.makeFailReason("boom"), Cause.makeDieReason({ defect: true })]) + + assert.isTrue(Exit.isSuccess(roundtripFingerprint(ExitSchema, Exit.succeed(7)))) + + const failure = roundtripFingerprint(ExitSchema, Exit.failCause(cause)) + assert.deepStrictEqual( + Exit.isFailure(failure) ? failure.cause.reasons.map((reason) => reason._tag) : [], + ["Fail", "Die"] + ) + + assert.notStrictEqual( + fingerprintFrame(ExitSchema, Exit.failCause(cause)).fingerprint, + fingerprintFrame(Schema.Cause(Schema.String, Schema.Unknown), cause).fingerprint + ) + }) + }) + + describe("generic declarations and recursion", () => { + it("uses declaration codec links for collections, Redacted, Class, and TaggedClass", () => { + const chunk = roundtrip(Schema.Chunk(Schema.Number), Chunk.make(1, 2, 3)) + assert.deepStrictEqual(Chunk.toReadonlyArray(chunk), [1, 2, 3]) + + const map = roundtrip(Schema.HashMap(Schema.String, Schema.Number), HashMap.make(["a", 1], ["b", 2])) + assert.strictEqual(HashMap.get(map, "a").pipe(Option.getOrUndefined), 1) + assert.strictEqual(HashMap.get(map, "b").pipe(Option.getOrUndefined), 2) + + const set = roundtrip(Schema.HashSet(Schema.String), HashSet.make("a", "b")) + assert.isTrue(HashSet.has(set, "a")) + assert.isTrue(HashSet.has(set, "b")) + + const redacted = roundtrip(Schema.Redacted(Schema.String), Redacted.make("secret")) + assert.strictEqual(Redacted.value(redacted), "secret") + + class Person extends Schema.Class("Person")({ name: Schema.String }) {} + const person = roundtrip(Person, new Person({ name: "Ada" })) + assert.instanceOf(person, Person) + assert.strictEqual(person.name, "Ada") + + class Event extends Schema.TaggedClass()("Event", { value: Schema.Number }) {} + const event = roundtrip(Event, new Event({ value: 1 })) + assert.instanceOf(event, Event) + assert.strictEqual(event._tag, "Event") + assert.strictEqual(event.value, 1) + }) + + it("compiles and round-trips recursive suspended schemas", () => { + interface Node { + readonly value: number + readonly children: ReadonlyArray + } + let Node: Schema.Codec + Node = Schema.Struct({ + value: Schema.Number, + children: Schema.Array(Schema.suspend(() => Node)) + }) + const value: Node = { value: 1, children: [{ value: 2, children: [] }] } + assert.deepStrictEqual(roundtrip(Node, value), value) + }) + + it("fails cyclic JSON values with the acyclic value issue", () => { + const value: Record = {} + value.self = value + assert.match(schemaError(() => encode(Schema.Unknown, value)).message, /acyclic value/) + }) + + it("fails cyclic recursive struct values with the acyclic value issue", () => { + interface Node { + readonly value: number + readonly next?: Node + } + let Node: Schema.Codec + Node = Schema.Struct({ + value: Schema.Number, + next: Schema.optionalKey(Schema.suspend(() => Node)) + }) + const value: { value: number; next?: Node } = { value: 1 } + value.next = value + assert.match(schemaError(() => encode(Node, value)).message, /acyclic value/) + }) + + it("fails array-mediated cycles with the acyclic value issue", () => { + interface Node { + readonly name: string + readonly children: ReadonlyArray + } + let Node: Schema.Codec + Node = Schema.Struct({ + name: Schema.String, + children: Schema.Array(Schema.suspend(() => Node)) + }) + const node: { name: string; children: Array } = { name: "root", children: [] } + node.children.push(node) + assert.match(schemaError(() => encode(Node, node)).message, /acyclic value/) + + const unknown: Array = [] + unknown.push(unknown) + assert.match(schemaError(() => encode(Schema.Unknown, unknown)).message, /acyclic value/) + }) + + it("runs user encoding links before the binary layer", () => { + const bytes = encode(Schema.NumberFromString, 123) + assert.strictEqual(bytes.length, 5) + assert.strictEqual(roundtrip(Schema.NumberFromString, 123), 123) + + const Before = Schema.Struct({ value: Schema.NumberFromString.pipe(SchemaBinary.fieldId(7)) }) + const After = Schema.Struct({ renamed: Schema.NumberFromString.pipe(SchemaBinary.fieldId(7)) }) + assert.deepStrictEqual( + Schema.decodeUnknownSync(SchemaBinary.toCodec(After))(encode(Before, { value: 123 })), + { renamed: 123 } + ) + }) + + it.effect("round-trips a yielding transformOrFail through SchemaParser Effect APIs", () => + Effect.gen(function*() { + const schema = Schema.String.pipe( + Schema.decodeTo( + Schema.Number, + SchemaTransformation.transformOrFail({ + decode: (s) => Effect.yieldNow.pipe(Effect.as(Number(s))), + encode: (n) => Effect.yieldNow.pipe(Effect.as(String(n))) + }) + ) + ) + const codec = SchemaBinary.toCodec(schema) + const bytes = yield* SchemaParser.encodeUnknownEffect(codec)(123) + assert.strictEqual(yield* SchemaParser.decodeUnknownEffect(codec)(bytes), 123) + })) + + it("keeps a sound runtime type guard on the derived codec", () => { + const codec = SchemaBinary.toCodec(Schema.Struct({ name: Schema.String, age: Schema.Number })) + assert.isTrue(Schema.is(codec)({ name: "Ada", age: 42 })) + assert.isFalse(Schema.is(codec)({ nope: 1 })) + }) + + it("keeps a sound runtime type guard on recursive derived codecs", () => { + interface Node { + readonly value: number + readonly children: ReadonlyArray + } + let Node: Schema.Codec + Node = Schema.Struct({ + value: Schema.Number, + children: Schema.Array(Schema.suspend(() => Node)) + }) + const codec = SchemaBinary.toCodec(Node) + + assert.isTrue(Schema.is(codec)({ value: 1, children: [] })) + assert.isFalse(Schema.is(codec)({ nope: 1 })) + assert.isFalse(Schema.is(Schema.Struct({ inner: codec }))({ inner: { nope: 1 } })) + }) + }) + + describe("parse options", () => { + it("honors checks and disableChecks", () => { + const NonNegative = Schema.Number.check(Schema.isGreaterThanOrEqualTo(0)) + const bytes = encode(Schema.Number, -1.5) + assert.match( + schemaError(() => Schema.decodeUnknownSync(SchemaBinary.toCodec(NonNegative))(bytes)).message, + /greater than or equal to 0/ + ) + + const parser = SchemaBinary.parser(NonNegative, { disableChecks: true }) + assert.deepStrictEqual(parser.feedSync(bytes), [-1.5]) + parser.endSync() + }) + + it("honors disableChecks for recursive schemas", () => { + interface Node { + readonly value: number + readonly children: ReadonlyArray + } + let Writer: Schema.Codec + Writer = Schema.Struct({ + value: Schema.Number, + children: Schema.Array(Schema.suspend(() => Writer)) + }) + let Reader: Schema.Codec + Reader = Schema.Struct({ + value: Schema.Number.check(Schema.isGreaterThanOrEqualTo(0)), + children: Schema.Array(Schema.suspend(() => Reader)) + }) + const value: Node = { value: -1.5, children: [] } + const parser = SchemaBinary.parser(Reader, { disableChecks: true }) + + assert.deepStrictEqual(parser.feedSync(encode(Writer, value)), [value]) + parser.endSync() + assert.deepStrictEqual( + Schema.decodeUnknownSync(SchemaBinary.toCodec(Reader), { disableChecks: true })(encode(Writer, value)), + value + ) + }) + + it("keeps one-shot index-signature caching within each options call", () => { + const Key = Schema.String.check(Schema.makeFilter((key: string) => key.startsWith("allowed-"))) + const Writer = Schema.Record(Schema.String, Schema.Number) + const codec = SchemaBinary.toCodec(Schema.Record(Key, Schema.Number)) + const bytes = encode(Writer, { denied: 1 }) + + assert.deepStrictEqual(Schema.decodeUnknownSync(codec)(bytes), {}) + assert.deepStrictEqual(Schema.decodeUnknownSync(codec, { disableChecks: true })(bytes), { denied: 1 }) + }) + + it("honors errors all for missing fields", () => { + const bytes = encode(Schema.Struct({}), {}) + const Reader = Schema.Struct({ a: Schema.String, b: Schema.Number }) + const error = schemaError(() => Schema.decodeUnknownSync(SchemaBinary.toCodec(Reader), { errors: "all" })(bytes)) + assert.strictEqual(error.message.match(/Missing key/g)?.length, 2) + }) + }) + + describe("layout and data errors", () => { + it("throws Error while compiling invalid layouts", () => { + const declaration = Schema.declare((_): _ is { readonly value: string } => true) + assert.throws( + () => SchemaBinary.toCodec(declaration), + /Binary layout: declaration has no toCodecJson or toCodec/ + ) + assert.throws( + () => + SchemaBinary.toCodec(Schema.Union([ + Schema.Struct({ a: Schema.String }), + Schema.Struct({ b: Schema.Number }) + ])), + /union members are not uniquely identifiable/ + ) + assert.throws( + () => + SchemaBinary.toCodec(Schema.Struct({ + a: Schema.String.pipe(SchemaBinary.fieldId(1)), + b: Schema.String.pipe(SchemaBinary.fieldId(1)) + })), + /Binary layout field id collision: 1/ + ) + assert.throws( + () => + SchemaBinary.toCodec(Schema.Union([ + Schema.Literal("a"), + Schema.UniqueSymbol(Symbol.for("SchemaBinary/symbol")) + ])), + /union members are not uniquely identifiable/ + ) + assert.throws( + () => SchemaBinary.toCodec(Schema.Struct({ [Symbol.for("SchemaBinary/key")]: Schema.String })), + /symbol property names are illegal/ + ) + assert.throws( + () => + SchemaBinary.toCodec(Schema.Union([ + Schema.Struct({ _tag: Schema.Literal("1a0a49t3mq") }), + Schema.Struct({ _tag: Schema.Literal("m7r4q02dm7") }) + ])), + /Binary layout sentinel collision: 3370793117/ + ) + }) + + it("uses SchemaIssue for binary failures", () => { + const codec = SchemaBinary.toCodec(Schema.Boolean) + const error = schemaError(() => Schema.decodeUnknownSync(codec)(Uint8Array.of(2, 0x10, 2))) + assert.isTrue(SchemaIssue.isIssue(error.issue)) + assert.match(error.message, /bool/) + }) + + it.effect("preserves the SchemaParser Issue and Schema SchemaError surfaces", () => + Effect.gen(function*() { + const codec = SchemaBinary.toCodec(Schema.Boolean) + const bytes = Uint8Array.of(2, 0x10, 2) + const issue = yield* SchemaParser.decodeUnknownEffect(codec)(bytes).pipe(Effect.flip) + assert.isTrue(SchemaIssue.isIssue(issue)) + const error = yield* Schema.decodeUnknownEffect(codec)(bytes).pipe(Effect.flip) + assert.isTrue(Schema.isSchemaError(error)) + })) + + it("reports a missing fixed tuple slot as MissingKey", () => { + const emptyTuple = encode(Schema.Tuple([]), []) + const error = schemaError(() => + Schema.decodeUnknownSync(SchemaBinary.toCodec(Schema.Tuple([Schema.String])))(emptyTuple) + ) + assert.match(error.message, /Missing key/) + }) + + it("rejects malformed lengths and text", () => { + const string = SchemaBinary.toCodec(Schema.String) + assert.match( + schemaError(() => Schema.decodeUnknownSync(string)(Uint8Array.of(0))).message, + /nonzero frame length/ + ) + assert.match( + schemaError(() => + Schema.decodeUnknownSync(string)(Uint8Array.of(0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x10)) + ).message, + /safe integer length/ + ) + assert.match( + schemaError(() => Schema.decodeUnknownSync(string)(Uint8Array.of(2, 0x10, 0xFF))).message, + /utf-8/ + ) + }) + + it("rejects attacker-sized zero-width array counts", () => { + const codec = SchemaBinary.toCodec(Schema.Array(Schema.Null)) + const bytes = Uint8Array.of(6, 0x10, 0x80, 0x80, 0x80, 0x80, 0x10) + assert.match( + schemaError(() => Schema.decodeUnknownSync(codec)(bytes)).message, + /array count within allocation limit/ + ) + }) + + it("rejects duplicate struct field ids and extra keys", () => { + const struct = Schema.Struct({ value: Schema.String }) + const encodedStruct = encode(struct, { value: "x" }) + const field = encodedStruct.slice(2) + const duplicateField = concat(Uint8Array.of(1 + field.length * 2, 0x10), field, field) + assert.match( + schemaError(() => Schema.decodeUnknownSync(SchemaBinary.toCodec(struct))(duplicateField)).message, + /unique field ids/ + ) + + const record = Schema.Record(Schema.String, Schema.Number) + const encodedRecord = encode(record, { a: 1 }) + const pair = encodedRecord.slice(4) + const duplicateKey = concat(Uint8Array.of(3 + pair.length * 2, 0x10, 0, pair.length * 2), pair, pair) + assert.match( + schemaError(() => Schema.decodeUnknownSync(SchemaBinary.toCodec(record))(duplicateKey)).message, + /unique extra keys/ + ) + }) + + it("rejects a duplicate field id after an unknown union decodes as absent", () => { + const A = Schema.Struct({ _tag: Schema.Literal("A") }) + const B = Schema.Struct({ _tag: Schema.Literal("B"), value: Schema.String }) + const Writer = Schema.Struct({ event: Schema.optionalKey(Schema.Union([A, B])) }) + const Reader = Schema.Struct({ event: Schema.optionalKey(Schema.Union([A])) }) + const encoded = encode(Writer, { event: { _tag: "B", value: "new" } }) + const field = encoded.slice(2) + const duplicateField = concat(Uint8Array.of(1 + field.length * 2, 0x10), field, field) + + assert.match( + schemaError(() => Schema.decodeUnknownSync(SchemaBinary.toCodec(Reader))(duplicateField)).message, + /unique field ids/ + ) + }) + + it("does not satisfy wide-field presence from the extra-key map", () => { + const fields: Record = {} + const value: Record = {} + for (let i = 0; i < 34; i++) { + const key = `field${i}` + fields[key] = Schema.String + value[key] = key + } + const record = Schema.Record(Schema.String, Schema.String) + const struct = Schema.StructWithRest(Schema.Struct(fields), [record]) + const error = schemaError(() => + Schema.decodeUnknownSync(SchemaBinary.toCodec(struct), { errors: "all" })(encode(record, value)) + ) + + assert.strictEqual(error.message.match(/Missing key/g)?.length, 34) + }) + + it("fails Never values and unregistered symbols through SchemaError", () => { + assert.isTrue(Schema.isSchemaError(schemaError(() => encode(Schema.Never, undefined)))) + assert.isTrue( + Schema.isSchemaError( + schemaError(() => Schema.decodeUnknownSync(SchemaBinary.toCodec(Schema.Never))(Uint8Array.of(1, 0x10))) + ) + ) + assert.match(schemaError(() => encode(Schema.Symbol, Symbol("local"))).message, /registered symbol/) + }) + }) + + describe("fingerprint mode", () => { + const Person = Schema.Struct({ + name: Schema.String, + age: Schema.Number.check(Schema.isInt()), + active: Schema.Boolean, + nickname: Schema.optional(Schema.String) + }) + const person = { name: "Ada", age: 36, active: true } + + it("writes a fingerprint envelope, a presence bitmap, and no field ids", () => { + // length | envelope 0x11 | fingerprint | bitmap | age varint | name | active + assert.deepStrictEqual([...encodeFingerprint(Person, person)], [ + 16, + 0x11, + 184, + 213, + 85, + 38, + 67, + 231, + 116, + 53, + 0, + 72, + 3, + 65, + 100, + 97, + 1 + ]) + // the same frame with the optional field present: its bit is set and it + // moves ahead of `age`, which is where its wire id sorts it + assert.deepStrictEqual([...encodeFingerprint(Person, { ...person, nickname: "A" })], [ + 19, + 0x11, + 184, + 213, + 85, + 38, + 67, + 231, + 116, + 53, + 1, + 2, + 1, + 65, + 72, + 3, + 65, + 100, + 97, + 1 + ]) + }) + + it("leaves the default mode untouched", () => { + const expected = [...encode(Person, person)] + assert.deepStrictEqual(expected[1], 0x10) + assert.deepStrictEqual([...Schema.encodeUnknownSync(SchemaBinary.toCodec(Person, {}))(person)], expected) + assert.deepStrictEqual( + [...Schema.encodeUnknownSync(SchemaBinary.toCodec(Person, { fingerprint: false }))(person)], + expected + ) + }) + + it("addresses union members by canonical position", () => { + const Click = Schema.Struct({ _tag: Schema.Literal("click"), x: Schema.Number.check(Schema.isInt()) }) + const Key = Schema.Struct({ _tag: Schema.Literal("key"), code: Schema.String }) + const Event = Schema.Union([Click, Key]) + assert.deepStrictEqual([...encodeFingerprint(Event, { _tag: "click", x: 3 })], [ + 11, + 0x11, + 237, + 163, + 78, + 151, + 96, + 43, + 76, + 0, + 0, + 6 + ]) + const key = { _tag: "key", code: "Esc" } as const + const keyFrame = [...encodeFingerprint(Event, key)] + assert.deepStrictEqual(keyFrame, [14, 0x11, 237, 163, 78, 151, 96, 43, 76, 0, 1, 3, 69, 115, 99]) + // declaration order never reaches the wire + assert.deepStrictEqual([...encodeFingerprint(Schema.Union([Key, Click]), key)], keyFrame) + assert.deepStrictEqual(roundtripFingerprint(Event, key), key) + }) + + it("counts the extra-key map instead of reserving field zero", () => { + const schema = Schema.Record(Schema.String, Schema.Number) + assert.deepStrictEqual([...encodeFingerprint(schema, { b: 2, a: 1 })], [ + 18, + 0x11, + 189, + 249, + 115, + 23, + 65, + 225, + 229, + 6, + 2, + 1, + 97, + 1, + 2, + 1, + 98, + 1, + 4 + ]) + assert.deepStrictEqual(roundtripFingerprint(schema, { z: 1, a: 2 }), { z: 1, a: 2 }) + assert.deepStrictEqual(roundtripFingerprint(schema, {}), {}) + }) + + it("keeps the default tuple and array layout", () => { + assert.deepStrictEqual([...encodeFingerprint(Schema.Tuple([Schema.Boolean, Schema.String]), [true, "x"])], [ + 12, + 0x11, + 3, + 44, + 71, + 25, + 31, + 66, + 141, + 82, + 1, + 1, + 120 + ]) + assert.deepStrictEqual([...encodeFingerprint(Schema.Array(Schema.Number.check(Schema.isInt())), [1, -2, 3])], [ + 13, + 0x11, + 206, + 94, + 13, + 113, + 158, + 175, + 76, + 32, + 3, + 2, + 5, + 6 + ]) + }) + + it("inlines fixed-size and zero-width leaves", () => { + const schema = Schema.Struct({ + nothing: Schema.Null, + flag: Schema.Boolean, + missing: Schema.Undefined, + when: Schema.Date, + count: Schema.Number.check(Schema.isInt()), + label: Schema.String + }) + const value = { nothing: null, flag: true, missing: undefined, when: new Date(1000), count: -7, label: "hi" } + // 1 envelope + 8 fingerprint + 1 varint + 8 int64 + 3 string + 1 bool + assert.strictEqual(encodeFingerprint(schema, value).length, 23) + assert.deepStrictEqual(roundtripFingerprint(schema, value), value) + }) + + it("spans a presence bitmap across several bytes", () => { + const schema = Schema.Struct({ + a: Schema.optional(Schema.Number), + b: Schema.optional(Schema.Number), + c: Schema.optional(Schema.Number), + d: Schema.optional(Schema.Number), + e: Schema.optional(Schema.Number), + f: Schema.optional(Schema.Number), + g: Schema.optional(Schema.Number), + h: Schema.optional(Schema.Number), + i: Schema.optional(Schema.Number), + j: Schema.optional(Schema.Number) + }) + assert.deepStrictEqual(roundtripFingerprint(schema, {}), {}) + assert.deepStrictEqual(roundtripFingerprint(schema, { a: 1, j: 2 }), { a: 1, j: 2 }) + const full = { a: 1, b: 2, c: 3, d: 4, e: 5, f: 6, g: 7, h: 8, i: 9, j: 10 } + assert.deepStrictEqual(roundtripFingerprint(schema, full), full) + }) + + it("round-trips native declarations, recursion, and mixed unions", () => { + const natives = Schema.Struct({ + option: Schema.Option(Schema.String), + result: Schema.Result(Schema.Number, Schema.String), + big: Schema.BigInt, + bytes: Schema.Uint8Array, + duration: Schema.Duration, + decimal: Schema.BigDecimal + }) + const nativeValue = { + option: Option.some("x"), + result: Result.fail("boom"), + big: 2n ** 70n, + bytes: new Uint8Array([1, 2, 3]), + duration: Duration.nanos(1_500_000_000n), + decimal: BigDecimal.make(123n, 2) + } + const natived = roundtripFingerprint(natives, nativeValue) + assert.deepStrictEqual(natived.option, nativeValue.option) + assert.deepStrictEqual(natived.result, nativeValue.result) + assert.strictEqual(natived.big, nativeValue.big) + assert.deepStrictEqual([...natived.bytes], [1, 2, 3]) + assert.strictEqual(Duration.toNanosUnsafe(natived.duration), 1_500_000_000n) + assert.strictEqual(natived.decimal.value, 123n) + assert.strictEqual(natived.decimal.scale, 2) + + interface Tree { + readonly value: number + readonly children: ReadonlyArray + } + const Tree: Schema.Codec = Schema.Struct({ + value: Schema.Number, + children: Schema.Array(Schema.suspend((): Schema.Codec => Tree)) + }) + const tree = { value: 1, children: [{ value: 2, children: [] }, { value: 3, children: [] }] } + assert.deepStrictEqual(roundtripFingerprint(Tree, tree), tree) + + const mixed = Schema.Union([Schema.String, Schema.Number, Schema.Struct({ n: Schema.Boolean })]) + assert.deepStrictEqual(roundtripFingerprint(mixed, "x"), "x") + assert.deepStrictEqual(roundtripFingerprint(mixed, 1.5), 1.5) + assert.deepStrictEqual(roundtripFingerprint(mixed, { n: true }), { n: true }) + }) + }) + + describe("layout fingerprint", () => { + const base = Schema.Struct({ name: Schema.String, age: Schema.Number }) + const value = { name: "a", age: 1 } + const baseline = fingerprintOf(base, value) + + it("ignores schema changes that do not reach the wire", () => { + assert.strictEqual( + fingerprintOf( + Schema.Struct({ + name: Schema.String.check(Schema.isMinLength(1)).annotate({ description: "the name" }), + age: Schema.Number + }), + value + ), + baseline + ) + // property declaration order: encode sorts by wire id + assert.strictEqual(fingerprintOf(Schema.Struct({ age: Schema.Number, name: Schema.String }), value), baseline) + // a decoded-side transformation leaves the encoded layout alone + assert.strictEqual( + fingerprintOf( + Schema.Struct({ name: Schema.String, age: Schema.Number.pipe(Schema.decodeTo(Schema.Number)) }), + value + ), + baseline + ) + }) + + it("does not depend on whether an acyclic sub-schema is shared or repeated", () => { + const Point = Schema.Struct({ x: Schema.Number, y: Schema.Number }) + const pair = { a: { x: 1, y: 2 }, b: { x: 3, y: 4 } } + const shared = fingerprintOf(Schema.Struct({ a: Point, b: Point }), pair) + const repeated = fingerprintOf( + Schema.Struct({ + a: Schema.Struct({ x: Schema.Number, y: Schema.Number }), + b: Schema.Struct({ x: Schema.Number, y: Schema.Number }) + }), + pair + ) + assert.strictEqual(shared, repeated) + + interface Tree { + readonly value: number + readonly children: ReadonlyArray + } + const makeTree = (): Schema.Codec => { + const self: Schema.Codec = Schema.Struct({ + value: Schema.Number, + children: Schema.Array(Schema.suspend((): Schema.Codec => self)) + }) + return self + } + const leaf = { value: 1, children: [] } + assert.strictEqual(fingerprintOf(makeTree(), leaf), fingerprintOf(makeTree(), leaf)) + // one recursive node reached from two fields, versus two of them. Both + // sides have the same cycle structure, which is the only recursive case + // the hash canonicalises; see the factoring test below for the limit. + const both = { left: leaf, right: leaf } + const one = makeTree() + assert.strictEqual( + fingerprintOf(Schema.Struct({ left: one, right: one }), both), + fingerprintOf(Schema.Struct({ left: makeTree(), right: makeTree() }), both) + ) + }) + + it("hashes the layout graph, so re-factoring a recursive schema moves it", () => { + interface Tree { + readonly value: number + readonly children: ReadonlyArray + } + const makeTree = (): Schema.Codec => { + const self: Schema.Codec = Schema.Struct({ + value: Schema.Number, + children: Schema.Array(Schema.suspend((): Schema.Codec => self)) + }) + return self + } + // Three finite graphs denoting the same infinite wire shape: the cycle + // itself, the cycle behind one non-recursive alias, and a two-node + // mutual recursion of the same shape. + const direct = makeTree() + const aliased = Schema.Struct({ + value: Schema.Number, + children: Schema.Array(makeTree()) + }) as unknown as Schema.Codec + const mutualA: Schema.Codec = Schema.Struct({ + value: Schema.Number, + children: Schema.Array(Schema.suspend((): Schema.Codec => mutualB)) + }) + const mutualB: Schema.Codec = Schema.Struct({ + value: Schema.Number, + children: Schema.Array(Schema.suspend((): Schema.Codec => mutualA)) + }) + + const tree = { value: 1, children: [{ value: 2, children: [] }] } + const frames = [direct, aliased, mutualA].map((schema) => fingerprintFrame(schema, tree)) + + // identical bytes on the wire + assert.strictEqual(new Set(frames.map((frame) => frame.payload)).size, 1) + // and identical in the tolerant default mode, in both directions + const directDefault = SchemaBinary.toCodec(direct) + const aliasedDefault = SchemaBinary.toCodec(aliased) + assert.deepStrictEqual( + Schema.decodeUnknownSync(aliasedDefault)(Schema.encodeUnknownSync(directDefault)(tree)), + tree + ) + assert.deepStrictEqual( + Schema.decodeUnknownSync(directDefault)(Schema.encodeUnknownSync(aliasedDefault)(tree)), + tree + ) + + // but each factoring hashes differently, and they reject each other + assert.strictEqual(new Set(frames.map((frame) => frame.fingerprint)).size, 3) + assert.include( + schemaError(() => + Schema.decodeUnknownSync(SchemaBinary.toCodec(aliased, { fingerprint: true }))( + encodeFingerprint(direct, tree) + ) + ).message, + "Expected matching layout fingerprint" + ) + }) + + it("changes whenever the wire layout changes", () => { + const changed = [ + fingerprintOf(Schema.Struct({ label: Schema.String, age: Schema.Number }), { label: "a", age: 1 }), + fingerprintOf( + Schema.Struct({ name: Schema.String, age: Schema.Number, extra: Schema.Boolean }), + { name: "a", age: 1, extra: true } + ), + fingerprintOf(Schema.Struct({ name: Schema.String, age: Schema.optional(Schema.Number) }), value), + fingerprintOf(Schema.Struct({ name: Schema.String, age: Schema.Number.check(Schema.isInt()) }), value), + fingerprintOf( + Schema.Struct({ name: Schema.String.pipe(SchemaBinary.fieldId(1)), age: Schema.Number }), + value + ), + fingerprintOf(Schema.Struct({ name: Schema.String, age: Schema.String }), { name: "a", age: "1" }) + ] + assert.strictEqual(new Set([baseline, ...changed]).size, changed.length + 1) + + assert.notStrictEqual( + fingerprintOf(Schema.Tuple([Schema.Number, Schema.Number]), [1, 2]), + fingerprintOf(Schema.Tuple([Schema.Number, Schema.Number, Schema.Number]), [1, 2, 3]) + ) + assert.notStrictEqual( + fingerprintOf(Schema.Array(Schema.Number), [1]), + fingerprintOf(Schema.Tuple([Schema.Number]), [1]) + ) + assert.notStrictEqual( + fingerprintOf(Schema.Date, new Date(0)), + fingerprintOf(Schema.DateTimeUtc, DateTime.makeUnsafe(0)) + ) + + const A = Schema.Struct({ _tag: Schema.Literal("a"), n: Schema.Number }) + const B = Schema.Struct({ _tag: Schema.Literal("b"), s: Schema.String }) + const C = Schema.Struct({ _tag: Schema.Literal("c"), s: Schema.String }) + const a = { _tag: "a", n: 1 } as const + assert.strictEqual(fingerprintOf(Schema.Union([A, B]), a), fingerprintOf(Schema.Union([B, A]), a)) + assert.notStrictEqual(fingerprintOf(Schema.Union([A, B]), a), fingerprintOf(Schema.Union([A, B, C]), a)) + }) + }) + + describe("fingerprint mode failures", () => { + const Person = Schema.Struct({ name: Schema.String, age: Schema.Number }) + const person = { name: "Ada", age: 36 } + + it("fails closed when the layout fingerprint differs", () => { + const frame = encodeFingerprint(Person, person) + const other = SchemaBinary.toCodec(Schema.Struct({ label: Schema.String, age: Schema.Number }), { + fingerprint: true + }) + assert.include( + schemaError(() => Schema.decodeUnknownSync(other)(frame)).message, + "Expected matching layout fingerprint" + ) + }) + + it("rejects the other mode's frames in both directions", () => { + const fingerprintFrame = encodeFingerprint(Person, person) + const defaultFrame = encode(Person, person) + assert.include( + schemaError(() => Schema.decodeUnknownSync(SchemaBinary.toCodec(Person))(fingerprintFrame)).message, + "Expected version 1 envelope, flags 0" + ) + assert.include( + schemaError(() => Schema.decodeUnknownSync(SchemaBinary.toCodec(Person, { fingerprint: true }))(defaultFrame)) + .message, + "Expected version 1 envelope, flags 1" + ) + }) + + it("rejects truncated and oversized frames", () => { + const codec = SchemaBinary.toCodec(Person, { fingerprint: true }) + const frame = encodeFingerprint(Person, person) + // the fingerprint itself does not fit + const shortFrame = concat(new Uint8Array([5, 0x11]), frame.subarray(2, 6)) + assert.include( + schemaError(() => Schema.decodeUnknownSync(codec)(shortFrame)).message, + "Expected complete value" + ) + assert.include( + schemaError(() => Schema.decodeUnknownSync(codec)(frame.subarray(0, frame.length - 1))).message, + "Expected complete value" + ) + assert.include( + schemaError(() => Schema.decodeUnknownSync(codec)(concat(frame, new Uint8Array([0])))).message, + "Expected no leftover bytes" + ) + const withLeftover = new Uint8Array(frame) + withLeftover[0] = frame[0] + 1 + assert.include( + schemaError(() => Schema.decodeUnknownSync(codec)(concat(withLeftover, new Uint8Array([0])))).message, + "Expected no leftover bytes" + ) + }) + + it("rejects a union position outside the member table", () => { + const Event = Schema.Union([ + Schema.Struct({ _tag: Schema.Literal("a"), n: Schema.Number }), + Schema.Struct({ _tag: Schema.Literal("b"), s: Schema.String }) + ]) + const codec = SchemaBinary.toCodec(Event, { fingerprint: true }) + const frame = new Uint8Array(encodeFingerprint(Event, { _tag: "b", s: "x" })) + // the byte after the envelope and fingerprint is the member position + frame[10] = 7 + assert.include( + schemaError(() => Schema.decodeUnknownSync(codec)(frame)).message, + "Expected known union member" + ) + }) + + it("reports a required field a newer writer left unreadable as missing", () => { + const schema = Schema.Struct({ reason: Schema.CauseReason(Schema.String, Schema.String) }) + const frame = new Uint8Array(encodeFingerprint(schema, { reason: Cause.makeFailReason("boom") })) + // the reason payload is length-prefixed; its first byte is the tag + frame[11] = 9 + const error = schemaError(() => + Schema.decodeUnknownSync(SchemaBinary.toCodec(schema, { fingerprint: true }))(frame) + ) + assert.include(error.message, "reason") + assert.include(error.message, "Missing key") + }) + + it("rejects duplicate extra keys", () => { + const schema = Schema.Record(Schema.String, Schema.Number) + const frame = encodeFingerprint(schema, { a: 1, b: 2 }) + const duplicated = new Uint8Array(frame) + // rewrite the second key so both pairs claim "a" + duplicated[duplicated.length - 3] = 97 + assert.include( + schemaError(() => Schema.decodeUnknownSync(SchemaBinary.toCodec(schema, { fingerprint: true }))(duplicated)) + .message, + "Expected unique extra keys" + ) + }) + }) + + describe("fingerprint mode parser", () => { + const Person = Schema.Struct({ name: Schema.String, age: Schema.Number }) + const first = { name: "Ada", age: 36 } + const second = { name: "Grace", age: 45 } + + it("parses concatenated frames, one byte at a time, and mid-frame splits", () => { + const frames = concat(encodeFingerprint(Person, first), encodeFingerprint(Person, second)) + assert.deepStrictEqual( + SchemaBinary.parser(Person, { fingerprint: true }).feedSync(frames), + [first, second] + ) + + const byteParser = SchemaBinary.parser(Person, { fingerprint: true }) + const out: Array = [] + for (const byte of frames) out.push(...byteParser.feedSync(new Uint8Array([byte]))) + byteParser.endSync() + assert.deepStrictEqual(out, [first, second]) + + // a split inside the fingerprint must wait rather than fail + const split = SchemaBinary.parser(Person, { fingerprint: true }) + assert.deepStrictEqual(split.feedSync(frames.subarray(0, 6)), []) + assert.deepStrictEqual(split.feedSync(frames.subarray(6)), [first, second]) + }) + + it("fails closed on a default-mode frame in a fingerprint stream", () => { + const parser = SchemaBinary.parser(Person, { fingerprint: true }) + const mixed = concat(encodeFingerprint(Person, first), encode(Person, second)) + assert.deepStrictEqual(parser.feedSync(mixed), [first]) + assert.include( + schemaError(() => parser.feedSync(new Uint8Array(0))).message, + "Expected version 1 envelope, flags 1" + ) + assert.include(schemaError(() => parser.endSync()).message, "Expected parser is spent") + }) + + it("honours maxFrameSize and reports truncation at end", () => { + const bounded = SchemaBinary.parser(Person, { fingerprint: true, maxFrameSize: 4 }) + assert.include( + schemaError(() => bounded.feedSync(encodeFingerprint(Person, first))).message, + "Expected frame within maxFrameSize" + ) + const truncated = SchemaBinary.parser(Person, { fingerprint: true }) + const frame = encodeFingerprint(Person, first) + assert.deepStrictEqual(truncated.feedSync(frame.subarray(0, frame.length - 2)), []) + assert.include(schemaError(() => truncated.endSync()).message, "Expected complete value") + }) + }) +}) diff --git a/packages/tools/bundle/fixtures/schema-binary.ts b/packages/tools/bundle/fixtures/schema-binary.ts new file mode 100644 index 00000000000..49a3dc84f92 --- /dev/null +++ b/packages/tools/bundle/fixtures/schema-binary.ts @@ -0,0 +1,10 @@ +import * as Schema from "effect/Schema" +import * as SchemaBinary from "effect/unstable/encoding/SchemaBinary" + +const schema = Schema.Struct({ + a: Schema.String, + b: Schema.optional(Schema.FiniteFromString), + c: Schema.Array(Schema.String) +}) + +export const codec = SchemaBinary.toCodec(schema)