diff --git a/LICENSE b/LICENSE index 5b19074..c99f1e9 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2024, Mapbox +Copyright (c) 2026, Mapbox All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/README.md b/README.md index 75ffb33..c87fd65 100644 --- a/README.md +++ b/README.md @@ -33,14 +33,14 @@ $ pbf example.proto > example.js Then read and write objects using the module like this: ```js -import Pbf from 'pbf'; +import {PbfReader, PbfWriter} from 'pbf'; import {readExample, writeExample} from './example.js'; // read -var obj = readExample(new Pbf(buffer)); +const obj = readExample(new PbfReader(buffer)); // write -const pbf = new Pbf(); +const pbf = new PbfWriter(); writeExample(obj, pbf); const buffer = pbf.finish(); ``` @@ -58,7 +58,7 @@ const {readExample, writeExample} = compile(proto); #### Custom Reading ```js -var data = new Pbf(buffer).readFields(readData, {}); +const data = new PbfReader(buffer).readFields(readData, {}); function readData(tag, data, pbf) { if (tag === 1) data.name = pbf.readString(); @@ -74,9 +74,9 @@ function readLayer(tag, layer, pbf) { #### Custom Writing ```js -var pbf = new Pbf(); +const pbf = new PbfWriter(); writeData(data, pbf); -var buffer = pbf.finish(); +const buffer = pbf.finish(); function writeData(data, pbf) { pbf.writeStringField(1, data.name); @@ -94,18 +94,18 @@ function writeLayer(layer, pbf) { Install using NPM with `npm install pbf`, then import as a module: ```js -import Pbf from 'pbf'; +import {PbfReader, PbfWriter} from 'pbf'; ``` Or use as a module directly in the browser with [jsDelivr](https://www.jsdelivr.com/esm): ```html ``` -Alternatively, there's a browser bundle with a `Pbf` global variable: +Alternatively, there's a browser bundle exposing a `Pbf` global with `PbfReader` and `PbfWriter` properties: ```html @@ -113,17 +113,19 @@ Alternatively, there's a browser bundle with a `Pbf` global variable: ## API -Create a `Pbf` object, optionally given a `Buffer` or `Uint8Array` as input data: +The library exposes two classes: `PbfReader` for decoding and `PbfWriter` for encoding. Splitting them lets bundlers tree-shake the half you don't use. + +Create a `PbfReader` from a `Buffer` or `Uint8Array`: ```js // parse a pbf file from disk in Node -const pbf = new Pbf(fs.readFileSync('data.pbf')); +const pbf = new PbfReader(fs.readFileSync('data.pbf')); // parse a pbf file in a browser after an ajax request with responseType="arraybuffer" -const pbf = new Pbf(new Uint8Array(xhr.response)); +const pbf = new PbfReader(new Uint8Array(xhr.response)); ``` -`Pbf` object properties: +Both classes expose the following properties: ```js pbf.length; // length of the underlying buffer @@ -143,7 +145,7 @@ pbf.readFields((tag) => { ``` It optionally accepts an object that will be passed to the reading function for easier construction of decoded data, -and also passes the `Pbf` object as a third argument: +and also passes the `PbfReader` object as a third argument: ```js const result = pbf.readFields(readField, {}) @@ -205,6 +207,12 @@ Packed reading methods: #### Writing +Create a `PbfWriter` (optionally with a pre-allocated `Buffer` or `Uint8Array`): + +```js +const pbf = new PbfWriter(); +``` + Write values: ```js @@ -287,7 +295,7 @@ The `--legacy` switch makes it generate a CommonJS module instead of ESM. `Pbf` will generate `read` and `write` functions for every message in the schema. For nested messages, their names will be concatenated — e.g. `Message` inside `Test` will produce `readTestMessage` and `writeTestMessage` functions. -* `read(pbf)` - decodes an object from the given `Pbf` instance. -* `write(obj, pbf)` - encodes an object into the given `Pbf` instance (usually empty). +* `read(pbf)` - decodes an object from the given `PbfReader` instance. +* `write(obj, pbf)` - encodes an object into the given `PbfWriter` instance (usually empty). The resulting code is clean and simple, so it's meant to be customized. diff --git a/bench/bench-tiles.js b/bench/bench-tiles.js index 1a512ee..2851c9a 100644 --- a/bench/bench-tiles.js +++ b/bench/bench-tiles.js @@ -4,7 +4,7 @@ import {mkdirSync, existsSync, readFileSync, writeFileSync} from 'fs'; import {join, dirname} from 'path'; import {fileURLToPath} from 'url'; import {readTile, writeTile} from '../test/fixtures/vector_tile.js'; -import Pbf from '../index.js'; +import {PbfReader, PbfWriter} from '../index.js'; const token = process.env.ACCESS_TOKEN; if (!token) throw new Error('Missing ACCESS_TOKEN environment variable (Mapbox access token).'); @@ -31,11 +31,11 @@ function processTile(body) { numTiles++; let now = clock(); - const tile = readTile(new Pbf(body)); + const tile = readTile(new PbfReader(body)); readTime += clock(now); now = clock(); - const pbf = new Pbf(); + const pbf = new PbfWriter(); writeTile(tile, pbf); const buf = pbf.finish(); writeTime += clock(now); diff --git a/bench/bench.js b/bench/bench.js index 589ca6c..06e4e1f 100644 --- a/bench/bench.js +++ b/bench/bench.js @@ -6,47 +6,47 @@ import protocolBuffers from 'protocol-buffers'; import protobufjs from 'protobufjs'; import {readTile, writeTile} from '../test/fixtures/vector_tile.js'; -import Pbf from '../index.js'; +import {PbfReader, PbfWriter} from '../index.js'; -var data = fs.readFileSync(new URL('../test/fixtures/12665.vector.pbf', import.meta.url)), +const data = fs.readFileSync(new URL('../test/fixtures/12665.vector.pbf', import.meta.url)), suite = new Benchmark.Suite(), vtProtoUrl = new URL('../test/fixtures/vector_tile.proto', import.meta.url), ProtocolBuffersTile = protocolBuffers(fs.readFileSync(vtProtoUrl)).Tile, ProtobufjsTile = protobufjs.loadSync(fileURLToPath(vtProtoUrl)).lookup('vector_tile.Tile'); -var pbfTile = readTile(new Pbf(data)), +const pbfTile = readTile(new PbfReader(data)), tileJSON = JSON.stringify(pbfTile), protocolBuffersTile = ProtocolBuffersTile.decode(data), protobufjsTile = ProtobufjsTile.decode(data); suite - .add('decode vector tile with pbf', function() { - readTile(new Pbf(data)); + .add('decode vector tile with pbf', () => { + readTile(new PbfReader(data)); }) - .add('encode vector tile with pbf', function() { - var pbf = new Pbf(); + .add('encode vector tile with pbf', () => { + const pbf = new PbfWriter(); writeTile(pbfTile, pbf); pbf.finish(); }) - .add('decode vector tile with protocol-buffers', function() { + .add('decode vector tile with protocol-buffers', () => { ProtocolBuffersTile.decode(data); }) - .add('encode vector tile with protocol-buffers', function() { + .add('encode vector tile with protocol-buffers', () => { ProtocolBuffersTile.encode(protocolBuffersTile); }) - .add('decode vector tile with protobuf.js', function() { + .add('decode vector tile with protobuf.js', () => { ProtobufjsTile.decode(data); }) - .add('encode vector tile with protobuf.js', function() { + .add('encode vector tile with protobuf.js', () => { ProtobufjsTile.encode(protobufjsTile); }) - .add('JSON.parse vector tile', function() { + .add('JSON.parse vector tile', () => { JSON.parse(tileJSON); }) - .add('JSON.stringify vector tile', function() { + .add('JSON.stringify vector tile', () => { JSON.stringify(pbfTile); }) - .on('cycle', function(event) { + .on('cycle', (event) => { console.log(String(event.target)); }) .run(); diff --git a/index.js b/index.js index fe75e64..2a623f7 100644 --- a/index.js +++ b/index.js @@ -12,11 +12,11 @@ const PBF_FIXED64 = 1; // 64-bit: double, fixed64, sfixed64 const PBF_BYTES = 2; // length-delimited: string, bytes, embedded messages, packed repeated fields const PBF_FIXED32 = 5; // 32-bit: float, fixed32, sfixed32 -export default class Pbf { +export class PbfReader { /** - * @param {Uint8Array | ArrayBuffer} [buf] + * @param {Uint8Array | ArrayBuffer} buf */ - constructor(buf = new Uint8Array(16)) { + constructor(buf) { this.buf = ArrayBuffer.isView(buf) ? buf : new Uint8Array(buf); this.dataView = new DataView(this.buf.buffer); this.pos = 0; @@ -24,11 +24,9 @@ export default class Pbf { this.length = this.buf.length; } - // === READING ================================================================= - /** * @template T - * @param {(tag: number, result: T, pbf: Pbf) => void} readField + * @param {(tag: number, result: T, pbf: PbfReader) => void} readField * @param {T} result * @param {number} [end] */ @@ -48,7 +46,7 @@ export default class Pbf { /** * @template T - * @param {(tag: number, result: T, pbf: Pbf) => void} readField + * @param {(tag: number, result: T, pbf: PbfReader) => void} readField * @param {T} result */ readMessage(readField, result) { @@ -98,9 +96,10 @@ export default class Pbf { */ readVarint(isSigned) { const buf = this.buf; - let val, b; + const b0 = buf[this.pos++]; + if (b0 < 0x80) return b0; - b = buf[this.pos++]; val = b & 0x7f; if (b < 0x80) return val; + let val = b0 & 0x7f, b; b = buf[this.pos++]; val |= (b & 0x7f) << 7; if (b < 0x80) return val; b = buf[this.pos++]; val |= (b & 0x7f) << 14; if (b < 0x80) return val; b = buf[this.pos++]; val |= (b & 0x7f) << 21; if (b < 0x80) return val; @@ -109,10 +108,6 @@ export default class Pbf { return readVarintRemainder(val, isSigned, this); } - readVarint64() { // for compatibility with v2.0.1 - return this.readVarint(true); - } - readSVarint() { const num = this.readVarint(); return num % 2 === 1 ? (num + 1) / -2 : num / 2; // zigzag encoding @@ -214,8 +209,18 @@ export default class Pbf { else if (type === PBF_FIXED64) this.pos += 8; else throw new Error(`Unimplemented type: ${type}`); } +} - // === WRITING ================================================================= +export class PbfWriter { + /** + * @param {Uint8Array | ArrayBuffer} [buf] + */ + constructor(buf = new Uint8Array(16)) { + this.buf = ArrayBuffer.isView(buf) ? buf : new Uint8Array(buf); + this.dataView = new DataView(this.buf.buffer); + this.pos = 0; + this.length = this.buf.length; + } /** * @param {number} tag @@ -280,6 +285,12 @@ export default class Pbf { writeVarint(val) { val = +val || 0; + if (val >= 0 && val < 0x80) { + if (this.pos >= this.length) this.realloc(1); + this.buf[this.pos++] = val; + return; + } + if (val > 0xfffffff || val < 0) { writeBigVarint(val, this); return; @@ -342,12 +353,13 @@ export default class Pbf { const len = buffer.length; this.writeVarint(len); this.realloc(len); - for (let i = 0; i < len; i++) this.buf[this.pos++] = buffer[i]; + this.buf.set(buffer, this.pos); + this.pos += len; } /** * @template T - * @param {(obj: T, pbf: Pbf) => void} fn + * @param {(obj: T, pbf: PbfWriter) => void} fn * @param {T} obj */ writeRawMessage(fn, obj) { @@ -369,7 +381,7 @@ export default class Pbf { /** * @template T * @param {number} tag - * @param {(obj: T, pbf: Pbf) => void} fn + * @param {(obj: T, pbf: PbfWriter) => void} fn * @param {T} obj */ writeMessage(tag, fn, obj) { @@ -528,12 +540,12 @@ export default class Pbf { writeBooleanField(tag, val) { this.writeVarintField(tag, +val); } -}; +} /** * @param {number} l * @param {boolean | undefined} s - * @param {Pbf} p + * @param {PbfReader} p */ function readVarintRemainder(l, s, p) { const buf = p.buf; @@ -560,7 +572,7 @@ function toNum(low, high, isSigned) { /** * @param {number} val - * @param {Pbf} pbf + * @param {PbfWriter} pbf */ function writeBigVarint(val, pbf) { let low, high; @@ -593,7 +605,7 @@ function writeBigVarint(val, pbf) { /** * @param {number} high * @param {number} low - * @param {Pbf} pbf + * @param {PbfWriter} pbf */ function writeBigVarintLow(low, high, pbf) { pbf.buf[pbf.pos++] = low & 0x7f | 0x80; low >>>= 7; @@ -605,7 +617,7 @@ function writeBigVarintLow(low, high, pbf) { /** * @param {number} high - * @param {Pbf} pbf + * @param {PbfWriter} pbf */ function writeBigVarintHigh(high, pbf) { const lsb = (high & 0x07) << 4; @@ -621,7 +633,7 @@ function writeBigVarintHigh(high, pbf) { /** * @param {number} startPos * @param {number} len - * @param {Pbf} pbf + * @param {PbfWriter} pbf */ function makeRoomForExtraLength(startPos, len, pbf) { const extraLen = @@ -631,68 +643,68 @@ function makeRoomForExtraLength(startPos, len, pbf) { // if 1 byte isn't enough for encoding message length, shift the data to the right pbf.realloc(extraLen); - for (let i = pbf.pos - 1; i >= startPos; i--) pbf.buf[i + extraLen] = pbf.buf[i]; + pbf.buf.copyWithin(startPos + extraLen, startPos, pbf.pos); } /** * @param {number[]} arr - * @param {Pbf} pbf + * @param {PbfWriter} pbf */ function writePackedVarint(arr, pbf) { for (let i = 0; i < arr.length; i++) pbf.writeVarint(arr[i]); } /** * @param {number[]} arr - * @param {Pbf} pbf + * @param {PbfWriter} pbf */ function writePackedSVarint(arr, pbf) { for (let i = 0; i < arr.length; i++) pbf.writeSVarint(arr[i]); } /** * @param {number[]} arr - * @param {Pbf} pbf + * @param {PbfWriter} pbf */ function writePackedFloat(arr, pbf) { for (let i = 0; i < arr.length; i++) pbf.writeFloat(arr[i]); } /** * @param {number[]} arr - * @param {Pbf} pbf + * @param {PbfWriter} pbf */ function writePackedDouble(arr, pbf) { for (let i = 0; i < arr.length; i++) pbf.writeDouble(arr[i]); } /** * @param {boolean[]} arr - * @param {Pbf} pbf + * @param {PbfWriter} pbf */ function writePackedBoolean(arr, pbf) { for (let i = 0; i < arr.length; i++) pbf.writeBoolean(arr[i]); } /** * @param {number[]} arr - * @param {Pbf} pbf + * @param {PbfWriter} pbf */ function writePackedFixed32(arr, pbf) { for (let i = 0; i < arr.length; i++) pbf.writeFixed32(arr[i]); } /** * @param {number[]} arr - * @param {Pbf} pbf + * @param {PbfWriter} pbf */ function writePackedSFixed32(arr, pbf) { for (let i = 0; i < arr.length; i++) pbf.writeSFixed32(arr[i]); } /** * @param {number[]} arr - * @param {Pbf} pbf + * @param {PbfWriter} pbf */ function writePackedFixed64(arr, pbf) { for (let i = 0; i < arr.length; i++) pbf.writeFixed64(arr[i]); } /** * @param {number[]} arr - * @param {Pbf} pbf + * @param {PbfWriter} pbf */ function writePackedSFixed64(arr, pbf) { for (let i = 0; i < arr.length; i++) pbf.writeSFixed64(arr[i]); diff --git a/package.json b/package.json index 1cb2a65..76aaacc 100644 --- a/package.json +++ b/package.json @@ -10,9 +10,9 @@ }, "scripts": { "bench": "node bench/bench.js", - "pretest": "eslint *.js compile.js test/*.js test/fixtures/*.js bin/pbf", - "test": "tsc && node --test", - "cov": "node --test --experimental-test-covetage", + "pretest": "eslint *.js compile.js test/*.js test/fixtures/*.js bench/*.js bin/pbf && tsc", + "test": "node --test test/*.test.js", + "cov": "node --test --experimental-test-coverage test/*.test.js", "build": "rollup -c", "prepublishOnly": "npm run test && npm run build" }, diff --git a/test/compile.test.js b/test/compile.test.js index e628cfb..7fa251e 100644 --- a/test/compile.test.js +++ b/test/compile.test.js @@ -3,7 +3,7 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import {sync as resolve} from 'resolve-protobuf-schema'; -import Pbf from '../index.js'; +import {PbfReader, PbfWriter} from '../index.js'; import {compile, compileRaw} from '../compile.js'; test('compiles all proto files to proper js', () => { @@ -27,10 +27,10 @@ test('compiles vector tile proto', () => { const tileBuf = fs.readFileSync(new URL('fixtures/12665.vector.pbf', import.meta.url)); const {readTile, writeTile} = compile(proto); - const tile = readTile(new Pbf(tileBuf)); + const tile = readTile(new PbfReader(tileBuf)); assert.equal(tile.layers.length, 11); - const pbf = new Pbf(); + const pbf = new PbfWriter(); writeTile(tile, pbf); const buf = pbf.finish(); assert.equal(buf.length, 124946); @@ -49,11 +49,11 @@ test('compiles packed proto', () => { types: [0, 1, 0, 1], value: [300, 400, 500] }; - const pbf = new Pbf(); + const pbf = new PbfWriter(); writeNotPacked(original, pbf); const buf = pbf.finish(); - const decompressed = readFalsePacked(new Pbf(buf)); + const decompressed = readFalsePacked(new PbfReader(buf)); assert.equal(buf.length, 17); assert.deepEqual(original, decompressed); }); @@ -66,11 +66,11 @@ test('reads packed with unpacked field', () => { types: [0, 1, 0, 1], value: [300, 400, 500] }; - const pbf = new Pbf(); + const pbf = new PbfWriter(); writePacked(original, pbf); const buf = pbf.finish(); - const decompressed = readFalsePacked(new Pbf(buf)); + const decompressed = readFalsePacked(new PbfReader(buf)); assert.equal(buf.length, 14); assert.deepEqual(original, decompressed); }); @@ -83,15 +83,15 @@ test('compiles packed proto3', () => { types: [0, 1, 0, 1], value: [300, 400, 500] }; - let pbf = new Pbf(); + let pbf = new PbfWriter(); writeFalsePacked(original, pbf); const falsePackedBuf = pbf.finish(); - pbf = new Pbf(); + pbf = new PbfWriter(); writeNotPacked(original, pbf); const notPackedBuf = pbf.finish(); - const decompressed = readNotPacked(new Pbf(falsePackedBuf)); + const decompressed = readNotPacked(new PbfReader(falsePackedBuf)); assert.deepEqual(original, decompressed); assert.equal(notPackedBuf.length, 14); assert.ok(falsePackedBuf.length > notPackedBuf.length, 'Did not respect [packed=false]'); @@ -104,11 +104,11 @@ test('compiles packed with multi-byte tags', () => { const original = { value: [300, 400, 500] }; - const pbf = new Pbf(); + const pbf = new PbfWriter(); writePacked(original, pbf); const buf = pbf.finish(); - const decompressed = readPacked(new Pbf(buf)); + const decompressed = readPacked(new PbfReader(buf)); assert.equal(buf.length, 9); assert.deepEqual(original, decompressed); }); @@ -116,12 +116,12 @@ test('compiles packed with multi-byte tags', () => { test('compiles defaults', () => { const proto = resolve(new URL('fixtures/defaults.proto', import.meta.url)); const {readEnvelope, writeEnvelope} = compile(proto); - const pbf = new Pbf(); + const pbf = new PbfWriter(); writeEnvelope({}, pbf); const buf = pbf.finish(); - const data = readEnvelope(new Pbf(buf)); + const data = readEnvelope(new PbfReader(buf)); assert.equal(buf.length, 0); assert.deepEqual(data, { @@ -136,12 +136,12 @@ test('compiles defaults', () => { test('compiles proto3 ignoring defaults', () => { const proto = resolve(new URL('fixtures/defaults_proto3.proto', import.meta.url)); const {readEnvelope, writeEnvelope} = compile(proto); - const pbf = new Pbf(); + const pbf = new PbfWriter(); writeEnvelope({}, pbf); const buf = pbf.finish(); - const data = readEnvelope(new Pbf(buf)); + const data = readEnvelope(new PbfReader(buf)); assert.equal(buf.length, 0); @@ -167,11 +167,11 @@ test('compiles maps', () => { } }; - const pbf = new Pbf(); + const pbf = new PbfWriter(); writeEnvelope(original, pbf); const buf = pbf.finish(); - const decompressed = readEnvelope(new Pbf(buf)); + const decompressed = readEnvelope(new PbfReader(buf)); assert.deepEqual(original, decompressed); }); @@ -179,7 +179,7 @@ test('compiles maps', () => { test('does not write undefined or null values', () => { const proto = resolve(new URL('fixtures/embedded_type.proto', import.meta.url)); const {writeEmbeddedType} = compile(proto); - const pbf = new Pbf(); + const pbf = new PbfWriter(); writeEmbeddedType({}, pbf); @@ -195,11 +195,11 @@ test('does not write undefined or null values', () => { test('handles all implicit default values', () => { const proto = resolve(new URL('fixtures/defaults_implicit.proto', import.meta.url)); const {readEnvelope, writeEnvelope} = compile(proto); - const pbf = new Pbf(); + const pbf = new PbfWriter(); writeEnvelope({}, pbf); const buf = pbf.finish(); - const data = readEnvelope(new Pbf(buf)); + const data = readEnvelope(new PbfReader(buf)); assert.equal(buf.length, 0); @@ -218,28 +218,28 @@ test('handles all implicit default values', () => { test('sets oneof field name', () => { const proto = resolve(new URL('fixtures/oneof.proto', import.meta.url)); const {readEnvelope, writeEnvelope} = compile(proto); - let pbf = new Pbf(); + let pbf = new PbfWriter(); writeEnvelope({}, pbf); - let data = readEnvelope(new Pbf(pbf.finish())); + let data = readEnvelope(new PbfReader(pbf.finish())); assert.equal(data.value, undefined); assert.equal(data.id, 0); - pbf = new Pbf(); + pbf = new PbfWriter(); writeEnvelope({ float: 1.5 }, pbf); - data = readEnvelope(new Pbf(pbf.finish())); + data = readEnvelope(new PbfReader(pbf.finish())); assert.equal(data.value, 'float'); assert.equal(data[data.value], 1.5); - pbf = new Pbf(); + pbf = new PbfWriter(); writeEnvelope({ float: 0 }, pbf); - data = readEnvelope(new Pbf(pbf.finish())); + data = readEnvelope(new PbfReader(pbf.finish())); assert.equal(data.value, 'float'); assert.equal(data[data.value], 0); @@ -248,7 +248,7 @@ test('sets oneof field name', () => { test('handles jstype=JS_STRING', () => { const proto = resolve(new URL('fixtures/type_string.proto', import.meta.url)); const {readTypeString, writeTypeString, readTypeNotString} = compile(proto); - const pbf = new Pbf(); + const pbf = new PbfWriter(); writeTypeString({ int: '-5', @@ -258,7 +258,7 @@ test('handles jstype=JS_STRING', () => { }, pbf); const buf = pbf.finish(); - let data = readTypeString(new Pbf(buf)); + let data = readTypeString(new PbfReader(buf)); assert.equal(data.int, '-5'); assert.equal(data.long, '10000'); @@ -267,7 +267,7 @@ test('handles jstype=JS_STRING', () => { assert.equal(data.default_implicit, '0'); assert.equal(data.default_explicit, '42'); - data = readTypeNotString(new Pbf(buf)); + data = readTypeNotString(new PbfReader(buf)); assert.equal(data.int, -5); assert.equal(data.long, 10000); assert.equal(data.boolVal, true); @@ -277,7 +277,7 @@ test('handles jstype=JS_STRING', () => { test('handles negative varint', () => { const proto = resolve(new URL('fixtures/varint.proto', import.meta.url)); const {readEnvelope, writeEnvelope} = compile(proto); - const pbf = new Pbf(); + const pbf = new PbfWriter(); writeEnvelope({ int: -5, @@ -285,7 +285,7 @@ test('handles negative varint', () => { }, pbf); const buf = pbf.finish(); - const data = readEnvelope(new Pbf(buf)); + const data = readEnvelope(new PbfReader(buf)); assert.equal(data.int, -5); assert.equal(data.long, -10); @@ -294,7 +294,7 @@ test('handles negative varint', () => { test('handles unsigned varint', () => { const proto = resolve(new URL('fixtures/varint.proto', import.meta.url)); const {readEnvelope, writeEnvelope} = compile(proto); - const pbf = new Pbf(); + const pbf = new PbfWriter(); writeEnvelope({ uint: Math.pow(2, 31), @@ -302,7 +302,7 @@ test('handles unsigned varint', () => { }, pbf); const buf = pbf.finish(); - const data = readEnvelope(new Pbf(buf)); + const data = readEnvelope(new PbfReader(buf)); assert.equal(data.uint, Math.pow(2, 31)); assert.equal(data.ulong, Math.pow(2, 63)); diff --git a/test/pbf.test.js b/test/pbf.test.js index e8c351a..ce4a020 100644 --- a/test/pbf.test.js +++ b/test/pbf.test.js @@ -1,5 +1,5 @@ -import Pbf from '../index.js'; +import {PbfReader, PbfWriter} from '../index.js'; import fs from 'fs'; import test from 'node:test'; import assert from 'node:assert/strict'; @@ -13,11 +13,12 @@ function toArray(buf) { } test('initialization', () => { - assert.doesNotThrow(() => new Pbf(Buffer.alloc(0))); + assert.doesNotThrow(() => new PbfReader(Buffer.alloc(0))); + assert.doesNotThrow(() => new PbfWriter()); }); test('realloc', () => { - const buf = new Pbf(Buffer.alloc(0)); + const buf = new PbfWriter(); buf.realloc(5); assert.ok(buf.length >= 5); buf.realloc(25); @@ -32,70 +33,64 @@ const testNumbers = [1, 0, 0, 4, 14, 23, 40, 86, 127, 141, 113, 925, 258, 1105, 301300890730496, 1310140661760000, 2883205519638528, 2690669862715392, 3319292539961344]; test('readVarint & writeVarint', () => { - const buf = new Pbf(Buffer.alloc(0)); + const writer = new PbfWriter(); for (let i = 0; i < testNumbers.length; i++) { - buf.writeVarint(testNumbers[i]); - if (testNumbers[i]) buf.writeVarint(-testNumbers[i]); + writer.writeVarint(testNumbers[i]); + if (testNumbers[i]) writer.writeVarint(-testNumbers[i]); } - const len = buf.finish().length; - assert.equal(len, 839); - buf.finish(); + const buf = writer.finish(); + assert.equal(buf.length, 839); + const reader = new PbfReader(buf); let i = 0; - while (buf.pos < len) { - assert.equal(buf.readVarint(), testNumbers[i]); - if (testNumbers[i]) assert.equal(buf.readVarint(true), -testNumbers[i]); + while (reader.pos < buf.length) { + assert.equal(reader.readVarint(), testNumbers[i]); + if (testNumbers[i]) assert.equal(reader.readVarint(true), -testNumbers[i]); i++; } }); test('writeVarint writes 0 for NaN', () => { const buf = Buffer.alloc(16); - const pbf = new Pbf(buf); + const writer = new PbfWriter(buf); // Initialize buffer to ensure consistent tests buf.write('0123456789abcdef', 0); - pbf.writeVarint('not a number'); - pbf.writeVarint(NaN); - pbf.writeVarint(50); - pbf.finish(); + writer.writeVarint('not a number'); + writer.writeVarint(NaN); + writer.writeVarint(50); - assert.equal(pbf.readVarint(), 0); - assert.equal(pbf.readVarint(), 0); - assert.equal(pbf.readVarint(), 50); + const reader = new PbfReader(writer.finish()); + assert.equal(reader.readVarint(), 0); + assert.equal(reader.readVarint(), 0); + assert.equal(reader.readVarint(), 50); }); test('readVarint signed', () => { let bytes = [0xc8, 0xe8, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01]; - let buf = new Pbf(Buffer.from(bytes)); + let buf = new PbfReader(Buffer.from(bytes)); assert.equal(buf.readVarint(true), -3000); bytes = [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01]; - buf = new Pbf(Buffer.from(bytes)); + buf = new PbfReader(Buffer.from(bytes)); assert.equal(buf.readVarint(true), -1); bytes = [0xc8, 0x01]; - buf = new Pbf(Buffer.from(bytes)); + buf = new PbfReader(Buffer.from(bytes)); assert.equal(buf.readVarint(true), 200); }); -test('readVarint64 (compatibility)', () => { - const bytes = [0xc8, 0xe8, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01]; - const buf = new Pbf(Buffer.from(bytes)); - assert.equal(buf.readVarint64(), -3000); -}); - test('readVarint & writeVarint handle really big numbers', () => { - const buf = new Pbf(); + const writer = new PbfWriter(); const bigNum1 = Math.pow(2, 60); const bigNum2 = Math.pow(2, 63); - buf.writeVarint(bigNum1); - buf.writeVarint(bigNum2); - buf.finish(); - assert.equal(buf.readVarint(), bigNum1); - assert.equal(buf.readVarint(), bigNum2); + writer.writeVarint(bigNum1); + writer.writeVarint(bigNum2); + const reader = new PbfReader(writer.finish()); + assert.equal(reader.readVarint(), bigNum1); + assert.equal(reader.readVarint(), bigNum2); }); const testSigned = [0, 1, 2, 0, 2, -1, 11, 18, -17, 145, 369, 891, -1859, -798, 2780, -13107, 12589, -16433, 21140, 148023, @@ -105,23 +100,23 @@ const testSigned = [0, 1, 2, 0, 2, -1, 11, 18, -17, 145, 369, 891, -1859, -798, 19583051038720, 83969719009280, 52578722775040, 416482297118720, 1981092523409408, -389256637841408]; test('readSVarint & writeSVarint', () => { - const buf = new Pbf(Buffer.alloc(0)); + const writer = new PbfWriter(); for (let i = 0; i < testSigned.length; i++) { - buf.writeSVarint(testSigned[i]); + writer.writeSVarint(testSigned[i]); } - const len = buf.finish().length; - assert.equal(len, 224); - buf.finish(); + const buf = writer.finish(); + assert.equal(buf.length, 224); + const reader = new PbfReader(buf); let i = 0; - while (buf.pos < len) { - assert.equal(buf.readSVarint(), testSigned[i++]); + while (reader.pos < buf.length) { + assert.equal(reader.readSVarint(), testSigned[i++]); } }); test('writeVarint throws error on a number that is too big', () => { - const buf = new Pbf(Buffer.alloc(0)); + const buf = new PbfWriter(); assert.throws(() => { buf.writeVarint(29234322996241367000012); // eslint-disable-line @@ -133,37 +128,37 @@ test('writeVarint throws error on a number that is too big', () => { }); test('readVarint throws error on a number that is longer than 10 bytes', () => { - const buf = new Pbf(Buffer.from([0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])); + const buf = new PbfReader(Buffer.from([0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff])); assert.throws(() => { buf.readVarint(); }); }); test('readBoolean & writeBoolean', () => { - const buf = new Pbf(); - buf.writeBoolean(true); - buf.writeBoolean(false); - buf.finish(); - assert.equal(buf.readBoolean(), true); - assert.equal(buf.readBoolean(), false); + const writer = new PbfWriter(); + writer.writeBoolean(true); + writer.writeBoolean(false); + const reader = new PbfReader(writer.finish()); + assert.equal(reader.readBoolean(), true); + assert.equal(reader.readBoolean(), false); }); test('readBytes', () => { - const buf = new Pbf([8, 1, 2, 3, 4, 5, 6, 7, 8]); + const buf = new PbfReader(new Uint8Array([8, 1, 2, 3, 4, 5, 6, 7, 8])); assert.deepEqual(toArray(buf.readBytes()), [1, 2, 3, 4, 5, 6, 7, 8]); }); test('writeBytes', () => { - const buf = new Pbf(); - buf.writeBytes([1, 2, 3, 4, 5, 6, 7, 8]); - const bytes = buf.finish(); + const writer = new PbfWriter(); + writer.writeBytes(new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8])); + const bytes = writer.finish(); assert.deepEqual(toArray(bytes), [8, 1, 2, 3, 4, 5, 6, 7, 8]); }); test('readDouble', () => { const buffer = Buffer.alloc(8); buffer.writeDoubleLE(12345.6789012345, 0); - const buf = new Pbf(buffer); + const buf = new PbfReader(buffer); assert.equal(Math.round(buf.readDouble() * 1e10) / 1e10, 12345.6789012345); }); @@ -171,28 +166,28 @@ test('readPacked and writePacked', () => { const testNumbers2 = testNumbers.slice(0, 10); function testPacked(type) { - const buf = new Pbf(); - buf[`writePacked${type}`](1, testNumbers2); - buf.finish(); - buf.readFields((tag) => { + const writer = new PbfWriter(); + writer[`writePacked${type}`](1, testNumbers2); + const reader = new PbfReader(writer.finish()); + reader.readFields((tag) => { const arr = []; - buf[`readPacked${type}`](arr); + reader[`readPacked${type}`](arr); if (tag === 1) assert.deepEqual(arr, testNumbers2, `packed ${type}`); else assert.fail(`wrong tag encountered: ${tag}`); }); } function testUnpacked(type) { - const buf = new Pbf(); + const writer = new PbfWriter(); const arr = []; testNumbers2.forEach((n) => { - buf[`write${type}Field`](1, n); + writer[`write${type}Field`](1, n); }); - buf.finish(); - buf.readFields(() => { - buf[`readPacked${type}`](arr); + const reader = new PbfReader(writer.finish()); + reader.readFields(() => { + reader[`readPacked${type}`](arr); }); assert.deepEqual(arr, testNumbers2, `packed ${type}`); @@ -203,12 +198,12 @@ test('readPacked and writePacked', () => { testUnpacked(type); }); - const buf = new Pbf(); - buf.writePackedBoolean(1, testNumbers2); - buf.finish(); - buf.readFields((tag) => { + const writer = new PbfWriter(); + writer.writePackedBoolean(1, testNumbers2); + const reader = new PbfReader(writer.finish()); + reader.readFields((tag) => { const arr = []; - buf.readPackedBoolean(arr); + reader.readPackedBoolean(arr); if (tag === 1) assert.deepEqual(arr, [true, false, false, true, true, true, true, true, true, true], 'packed Boolean'); else assert.fail(`wrong tag encountered: ${tag}`); @@ -216,60 +211,60 @@ test('readPacked and writePacked', () => { }); test('writePacked skips empty arrays', () => { - const pbf = new Pbf(); + const pbf = new PbfWriter(); pbf.writePackedBoolean(1, []); const buf = pbf.finish(); assert.equal(buf.length, 0); }); test('writeDouble', () => { - const buf = new Pbf(Buffer.alloc(8)); - buf.writeDouble(12345.6789012345); - buf.finish(); - assert.equal(Math.round(buf.readDouble() * 1e10) / 1e10, 12345.6789012345); + const writer = new PbfWriter(Buffer.alloc(8)); + writer.writeDouble(12345.6789012345); + const reader = new PbfReader(writer.finish()); + assert.equal(Math.round(reader.readDouble() * 1e10) / 1e10, 12345.6789012345); }); test('readFloat', () => { const buffer = Buffer.alloc(4); buffer.writeFloatLE(123.456, 0); - const buf = new Pbf(buffer); + const buf = new PbfReader(buffer); assert.equal(Math.round(1000 * buf.readFloat()) / 1000, 123.456); }); test('writeFloat', () => { - const buf = new Pbf(Buffer.alloc(4)); - buf.writeFloat(123.456); - buf.finish(); - assert.equal(Math.round(1000 * buf.readFloat()) / 1000, 123.456); + const writer = new PbfWriter(Buffer.alloc(4)); + writer.writeFloat(123.456); + const reader = new PbfReader(writer.finish()); + assert.equal(Math.round(1000 * reader.readFloat()) / 1000, 123.456); }); test('readFixed32', () => { const buffer = Buffer.alloc(16); buffer.writeUInt32LE(42, 0); buffer.writeUInt32LE(24, 4); - const buf = new Pbf(buffer); + const buf = new PbfReader(buffer); assert.equal(buf.readFixed32(), 42); assert.equal(buf.readFixed32(), 24); }); test('writeFixed32', () => { - const buf = new Pbf(Buffer.alloc(16)); - buf.writeFixed32(42); - buf.writeFixed32(24); - buf.finish(); - assert.equal(buf.readFixed32(), 42); - assert.equal(buf.readFixed32(), 24); + const writer = new PbfWriter(Buffer.alloc(16)); + writer.writeFixed32(42); + writer.writeFixed32(24); + const reader = new PbfReader(writer.finish()); + assert.equal(reader.readFixed32(), 42); + assert.equal(reader.readFixed32(), 24); }); test('readFixed64', () => { - const buf = new Pbf(Buffer.alloc(8)); - buf.writeFixed64(102451124123); - buf.finish(); - assert.deepEqual(buf.readFixed64(), 102451124123); + const writer = new PbfWriter(Buffer.alloc(8)); + writer.writeFixed64(102451124123); + const reader = new PbfReader(writer.finish()); + assert.deepEqual(reader.readFixed64(), 102451124123); }); test('writeFixed64', () => { - const buf = new Pbf(Buffer.alloc(8)); + const buf = new PbfWriter(Buffer.alloc(8)); buf.writeFixed64(102451124123); assert.deepEqual(toArray(buf.buf), [155, 23, 144, 218, 23, 0, 0, 0]); }); @@ -278,62 +273,60 @@ test('readSFixed32', () => { const buffer = Buffer.alloc(16); buffer.writeInt32LE(4223, 0); buffer.writeInt32LE(-1231, 4); - const buf = new Pbf(buffer); + const buf = new PbfReader(buffer); assert.equal(buf.readSFixed32(), 4223); assert.equal(buf.readSFixed32(), -1231); }); test('writeSFixed32', () => { - const buf = new Pbf(Buffer.alloc(16)); - buf.writeSFixed32(4223); - buf.writeSFixed32(-1231); - buf.finish(); - assert.equal(buf.readSFixed32(), 4223); - assert.equal(buf.readSFixed32(), -1231); + const writer = new PbfWriter(Buffer.alloc(16)); + writer.writeSFixed32(4223); + writer.writeSFixed32(-1231); + const reader = new PbfReader(writer.finish()); + assert.equal(reader.readSFixed32(), 4223); + assert.equal(reader.readSFixed32(), -1231); }); test('readSFixed64', () => { - const buf = new Pbf(Buffer.alloc(8)); - buf.writeSFixed64(-102451124123); - buf.finish(); - assert.deepEqual(buf.readSFixed64(), -102451124123); + const writer = new PbfWriter(Buffer.alloc(8)); + writer.writeSFixed64(-102451124123); + const reader = new PbfReader(writer.finish()); + assert.deepEqual(reader.readSFixed64(), -102451124123); }); test('writeSFixed64', () => { - const buf = new Pbf(Buffer.alloc(8)); + const buf = new PbfWriter(Buffer.alloc(8)); buf.writeSFixed64(-102451124123); assert.deepEqual(toArray(buf.buf), [101, 232, 111, 37, 232, 255, 255, 255]); }); test('writeString & readString', () => { - const buf = new Pbf(); - buf.writeString('Привет 李小龙'); - const bytes = buf.finish(); + const writer = new PbfWriter(); + writer.writeString('Привет 李小龙'); + const bytes = writer.finish(); assert.deepEqual(bytes, new Uint8Array([22, 208, 159, 209, 128, 208, 184, 208, 178, 208, 181, 209, 130, 32, 230, 157, 142, 229, 176, 143, 233, 190, 153])); - assert.equal(buf.readString(), 'Привет 李小龙'); + assert.equal(new PbfReader(bytes).readString(), 'Привет 李小龙'); }); test('writeString & readString longer', () => { const str = '{"Feature":"http://example.com/vocab#Feature","datetime":{"@id":"http://www.w3.org/2006/time#inXSDDateTime","@type":"http://www.w3.org/2001/XMLSchema#dateTime"},"when":"http://example.com/vocab#when"}'; - const buf = new Pbf(); - buf.writeString(str); - buf.finish(); - assert.equal(buf.readString(), str); + const writer = new PbfWriter(); + writer.writeString(str); + assert.equal(new PbfReader(writer.finish()).readString(), str); }); test('more complicated utf8', () => { - const buf = new Pbf(); + const writer = new PbfWriter(); // crazy test from github.com/mathiasbynens/utf8.js - const str = '\uDC00\uDC00\uDC00\uDC00A\uDC00\uD834\uDF06\uDC00\uDEEE\uDFFF\uD800\uDC00\uD800\uD800\uD800\uD800A' + - '\uD800\uD834\uDF06'; - buf.writeString(str); - buf.finish(); - const str2 = buf.readString(); + const str = '\uDC00\uDC00\uDC00\uDC00A\uDC00𝌆\uDC00\uDEEE\uDFFF𐀀\uD800\uD800\uD800\uD800A' + + '\uD800𝌆'; + writer.writeString(str); + const str2 = new PbfReader(writer.finish()).readString(); assert.deepEqual(new Uint8Array(str2), new Uint8Array(str)); }); test('readFields', () => { - const buf = new Pbf(fs.readFileSync(new URL('fixtures/12665.vector.pbf', import.meta.url))); + const buf = new PbfReader(fs.readFileSync(new URL('fixtures/12665.vector.pbf', import.meta.url))); const layerOffsets = []; const foo = {}; let res, buf2; @@ -353,7 +346,7 @@ test('readFields', () => { }); test('readMessage', () => { - const buf = new Pbf(fs.readFileSync(new URL('fixtures/12665.vector.pbf', import.meta.url))), + const buf = new PbfReader(fs.readFileSync(new URL('fixtures/12665.vector.pbf', import.meta.url))), layerNames = [], foo = {}; @@ -372,62 +365,62 @@ test('readMessage', () => { }); test('field writing methods', () => { - const buf = new Pbf(); - buf.writeFixed32Field(1, 100); - buf.writeFixed64Field(2, 200); - buf.writeVarintField(3, 1234); - buf.writeSVarintField(4, -599); - buf.writeStringField(5, 'Hello world'); - buf.writeFloatField(6, 123); - buf.writeDoubleField(7, 123); - buf.writeBooleanField(8, true); - buf.writeBytesField(9, [1, 2, 3]); - buf.writeMessage(10, () => { - buf.writeBooleanField(1, true); - buf.writePackedVarint(2, testNumbers); + const writer = new PbfWriter(); + writer.writeFixed32Field(1, 100); + writer.writeFixed64Field(2, 200); + writer.writeVarintField(3, 1234); + writer.writeSVarintField(4, -599); + writer.writeStringField(5, 'Hello world'); + writer.writeFloatField(6, 123); + writer.writeDoubleField(7, 123); + writer.writeBooleanField(8, true); + writer.writeBytesField(9, new Uint8Array([1, 2, 3])); + writer.writeMessage(10, () => { + writer.writeBooleanField(1, true); + writer.writePackedVarint(2, testNumbers); }); - buf.writeSFixed32Field(11, -123); - buf.writeSFixed64Field(12, -256); - - buf.finish(); - - buf.readFields((tag) => { - if (tag === 1) buf.readFixed32(); - else if (tag === 2) buf.readFixed64(); - else if (tag === 3) buf.readVarint(); - else if (tag === 4) buf.readSVarint(); - else if (tag === 5) buf.readString(); - else if (tag === 6) buf.readFloat(); - else if (tag === 7) buf.readDouble(); - else if (tag === 8) buf.readBoolean(); - else if (tag === 9) buf.readBytes(); - else if (tag === 10) buf.readMessage(() => { /* skip */ }); - else if (tag === 11) buf.readSFixed32(); - else if (tag === 12) buf.readSFixed64(); + writer.writeSFixed32Field(11, -123); + writer.writeSFixed64Field(12, -256); + + const reader = new PbfReader(writer.finish()); + + reader.readFields((tag) => { + if (tag === 1) reader.readFixed32(); + else if (tag === 2) reader.readFixed64(); + else if (tag === 3) reader.readVarint(); + else if (tag === 4) reader.readSVarint(); + else if (tag === 5) reader.readString(); + else if (tag === 6) reader.readFloat(); + else if (tag === 7) reader.readDouble(); + else if (tag === 8) reader.readBoolean(); + else if (tag === 9) reader.readBytes(); + else if (tag === 10) reader.readMessage(() => { /* skip */ }); + else if (tag === 11) reader.readSFixed32(); + else if (tag === 12) reader.readSFixed64(); else assert.fail('unknown tag'); }); }); test('skip', () => { - const buf = new Pbf(); - buf.writeFixed32Field(1, 100); - buf.writeFixed64Field(2, 200); - buf.writeVarintField(3, 1234); - buf.writeStringField(4, 'Hello world'); - buf.finish(); + const writer = new PbfWriter(); + writer.writeFixed32Field(1, 100); + writer.writeFixed64Field(2, 200); + writer.writeVarintField(3, 1234); + writer.writeStringField(4, 'Hello world'); - buf.readFields(() => { /* skip */ }); + const reader = new PbfReader(writer.finish()); + reader.readFields(() => { /* skip */ }); - assert.equal(buf.pos, buf.length); + assert.equal(reader.pos, reader.length); assert.throws(() => { - buf.skip(6); + reader.skip(6); }); }); test('write a raw message > 0x10000000', () => { - const buf = new Pbf(); + const buf = new PbfWriter(); const marker = 0xdeadbeef; const encodedMarker = new Uint8Array([0xef, 0xbe, 0xad, 0xde]); const markerSize = encodedMarker.length;