diff --git a/.cargo/config.toml.example b/.cargo/config.toml.example new file mode 100644 index 0000000..02add71 --- /dev/null +++ b/.cargo/config.toml.example @@ -0,0 +1,8 @@ +# Copy this file to `.cargo/config.toml` to tune the Rust code for your CPU. +# It is gitignored so machine-specific flags never get committed or published. +# +# cp .cargo/config.toml.example .cargo/config.toml +# +# For the C++ kernels, also pass `--features native` (see Cargo.toml). +[build] +rustflags = ["-C", "target-cpu=native"] diff --git a/.gitignore b/.gitignore index 01d5bdb..e5ca3a6 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,10 @@ Cargo.lock .vscode/ .idea/ .cache/ -build/ \ No newline at end of file +build/ + +compile_commands.json + +# Local-only cargo config that enables `-C target-cpu=native` for benching. +# Copy from `.cargo/config.toml.example`; never commit machine-specific flags. +/.cargo/config.toml diff --git a/CHANGELOG.md b/CHANGELOG.md index 963ec02..d0a120a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Bindings for [simdjson] v0.3.1. - Use [cxx] instead of [bindgen] because of too many generics. +- **Breaking:** restructured `tape::Error` and `serde::Error` into typed + variants carrying positions/lengths (`OutOfBounds`, `UnknownTapeType`, + `UnexpectedTapeType`, `StringOutOfBounds`, `MissingValueWord`, + `InvalidUtf8`) instead of a single `TapeCorrupted { message: String }`. + `serde::Error::TapeCorrupted` is removed; tape failures now surface as + `serde::Error::Tape { source: tape::Error }` (transparent). +- Build tuning: `lto = "fat"` + `codegen-units = 1` on release/bench + profiles. New opt-in `native` feature compiles the bundled simdjson C++ + with `-march=native`; copy `.cargo/config.toml.example` to + `.cargo/config.toml` for the matching Rust `-C target-cpu=native`. ## [0.1.0] - 2019-03-13 diff --git a/Cargo.toml b/Cargo.toml index 4762394..fd40ded 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,39 +1,58 @@ [package] name = "simdjson-rust" version = { workspace = true } -authors = ["SunDoge <384813529@qq.com>"] -edition = "2021" -license = "Apache-2.0" +edition = "2024" description = "Rust bindings for the simdjson project." -homepage = "https://crates.io/crates/simdjson-rust" documentation = "https://docs.rs/simdjson-rust" -repository = "https://github.com/SunDoge/simdjson-rust" readme = "README.md" +homepage = "https://crates.io/crates/simdjson-rust" +repository = "https://github.com/SunDoge/simdjson-rust" +license = "Apache-2.0" exclude = [".github/", "examples/"] - [workspace] resolver = "2" members = ["simdjson-sys"] [workspace.package] -version = "0.3.0-alpha.3" +version = "0.4.0-alpha.1" [workspace.dependencies] -simdjson-sys = { path = "simdjson-sys", version = "0.1.0-alpha.2" } +simdjson-sys = { path = "simdjson-sys", version = "0.2.0-alpha.1" } +[[bench]] +harness = false +name = "parser_bench" [dependencies] -thiserror = "1.0" +cxx = "1.0" simdjson-sys = { workspace = true } # serde compatibilty serde = { version = "1", features = ["derive"], optional = true } -serde_json = { version = "1", optional = true } +snafu = "0.9.1" +[dev-dependencies] +criterion = { version = "0.5", features = ["html_reports"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1.0" +simd-json = "0.14" [features] -default = [] +default = ["serde"] # serde compatibility -serde_impl = ["serde", "serde_json"] +serde = ["dep:serde"] + +# Compile the bundled simdjson C++ with `-march=native` for the building CPU. +# Off by default for portability. Pair with `RUSTFLAGS="-C target-cpu=native"` +# (or copy `.cargo/config.toml.example`) to also tune the Rust code. +native = ["simdjson-sys/native"] + +[profile.bench] +lto = "fat" +codegen-units = 1 + +[profile.release] +lto = "fat" +codegen-units = 1 diff --git a/README.md b/README.md index 0c4a78a..9f8ee0a 100644 --- a/README.md +++ b/README.md @@ -4,115 +4,167 @@ [![Crates.io](https://img.shields.io/crates/v/simdjson-rust?style=for-the-badge)](https://crates.io/crates/simdjson-rust) [![docs.rs](https://img.shields.io/docsrs/simdjson-rust/latest?style=for-the-badge)](https://docs.rs/simdjson-rust) -This crate currently uses [`simdjson 3.2.3`][simdjson]. You can have a try and give feedback. +Rust bindings for the [simdjson][simdjson] C++ library (currently **v4.6.4**), +exposing its DOM parser as a zero-copy tape and integrating with `serde` for +direct deserialization into Rust structs. -If you +## What this crate does -- find certain APIs are missing -- encounter memory errors -- experience performance degradation +simdjson parses a JSON document in two stages: **stage1** scans the input with +SIMD to find structural characters, **stage2** builds a flat _tape_ — an array +of 64-bit words encoding every value with its type tag and payload, plus a +side string buffer holding all (already-unescaped) string bytes. -Please submit an issue. +This crate hands that tape to Rust as two borrowed slices (`&[u64]` tape + +`&[u8]` string buffer) and lets you walk it with zero copy: -## Benchmark +- `dom::Parser` — parse a JSON string/bytes into a `TapeRef` cursor or a + `tape::Value` DOM tree. +- `tape::Value` — a zero-copy `enum` (`Null`/`Bool`/`Int64`/`Uint64`/`Double`/ + `String(&str)`/`Array`/`Object`) borrowing the parser's memory. +- `serde::from_str` / `from_tape` — deserialize any `serde::Deserialize` type + directly off the tape, with zero-copy `&'de str` for string fields. -Check [SunDoge/json-benchmark](https://github.com/SunDoge/json-benchmark/tree/simdjson-rust) - -``` - DOM STRUCT -======= serde_json ======= parse|stringify ===== parse|stringify ==== -data/canada.json 400 MB/s 460 MB/s 500 MB/s 390 MB/s -data/citm_catalog.json 540 MB/s 790 MB/s 1000 MB/s 1070 MB/s -data/twitter.json 390 MB/s 1020 MB/s 680 MB/s 1090 MB/s - -======= simd-json ======== parse|stringify ===== parse|stringify ==== -data/canada.json 400 MB/s 540 MB/s 530 MB/s -data/citm_catalog.json 1080 MB/s 1000 MB/s 1570 MB/s -data/twitter.json 1100 MB/s 1460 MB/s 1180 MB/s - -===== simdjson-rust ====== parse|stringify ===== parse|stringify ==== -data/canada.json 960 MB/s -data/citm_catalog.json 3110 MB/s -data/twitter.json 3160 MB/s -``` - -## Usage - -Add this to your `Cargo.toml` +## Quick start ```toml -# In the `[dependencies]` section -simdjson-rust = "0.3.0" +[dependencies] +simdjson-rust = "0.4.0-alpha.1" ``` -Then, get started. - ```rust -use simdjson_rust::prelude::*; -use simdjson_rust::{dom, ondemand}; - -fn main() -> simdjson_rust::Result<()> { - let ps = "[0,1,2,3]".to_padded_string(); - - // ondemand api. - { - let mut parser = ondemand::Parser::default(); - let mut doc = parser.iterate(&ps)?; - let mut array = doc.get_array()?; - for (index, value) in array.iter()?.enumerate() { - assert_eq!(index as u64, value?.get_uint64()?); - } - } - - // dom api. - { - let mut parser = dom::Parser::default(); - let elem = parser.parse(&ps)?; - let arr = elem.get_array()?; - for (index, value) in arr.iter().enumerate() { - assert_eq!(index as u64, value.get_uint64()?); - } - } - - Ok(()) -} -``` +use simdjson_rust::dom::Parser; -### `dom` and `ondemand` - -[`simdjson`][simdjson] now offer two kinds of API, `dom` and `ondemand`. -`dom` will parsed the whole string while `ondemand` only parse what you request. -Due to `ffi`, the overhead of `ondemand` API is relatively high. I have tested `lto` but it only improves a little :( +let mut parser = Parser::default(); +let value = parser.parse_to_value(r#"{"hello": "world", "n": 42}"#).unwrap(); +assert_eq!(value.get("n").unwrap().as_i64(), Some(42)); +``` -Thus it is suggestted that +With the default `serde` feature, deserialize straight into your own types: -- use `ondemand` if you only want to access a specific part of a large json, -- use `dom` if you want to parse the whole json. +```rust +use serde::Deserialize; +use simdjson_rust::serde::from_str; +#[derive(Deserialize, Debug, PartialEq)] +struct Point { x: f64, y: f64 } -### `padded_string` +let p: Point = from_str(r#"{"x": 1.0, "y": 2.0}"#).unwrap(); +assert_eq!(p, Point { x: 1.0, y: 2.0 }); +``` -[`simdjson`][simdjson] requires the input string to be padded. We must provide a string with `capacity = len + SIMDJSON_PADDING`. -We provide utils to do so. +For high-throughput work, reuse the `Parser` across documents and deserialize +from the tape cursor — this avoids re-allocating the parser's internal buffers +on every call: ```rust -use simdjson_rust::prelude::*; - -fn main() -> simdjson_rust::Result<()> { - let ps = make_padded_string("[0,1,2,3]"); - let ps = "[0,1,2,3]".to_padded_string(); - // or reuse a buffer. - let unpadded = String::from("[1,2,3,4]"); - let ps = unpadded.into_padded_string(); - // or load from file. - let ps = load_padded_string("test.json")?; - Ok(()) +use simdjson_rust::dom::Parser; +use simdjson_rust::serde::from_tape; + +let mut parser = Parser::default(); +for json in inputs { + let tape = parser.parse_str(json)?; + let item: MyStruct = from_tape(tape)?; + // ... } ``` -## Other interesting things - -There are also pure Rust port of [`simdjson`][simdjson] available here [`simd-json`](https://github.com/simd-lite/simd-json). - +## Performance + +Numbers below are from `cargo bench --bench parser_bench --features native` on +the author's machine (criterion, median). They compare three serde struct +deserialization paths — `serde_json`, the pure-Rust [`simd-json`][simd-json] +crate, and this crate — across three input sizes. Treat them as relative +indicators, not absolute claims; always benchmark on your own hardware and +your own data shape. + +| Input | `serde_json` | `simd-json` | `simdjson-rust` | +| --------------- | ------------ | ----------- | --------------- | +| Small (~100 B) | 84 ns | 286 ns | **110 ns** | +| Medium (~2 KB) | ~810 ns | 1210 ns | **638 ns** | +| Large (~100 KB) | 187 µs | 176 µs | **137 µs** | + +Where `simdjson-rust` wins: + +- **Large inputs**: the C++ stage1+stage2 pass is highly tuned, and the Rust + tape-walk that follows is pure sequential `&[u64]` reads. On 100 KB the + crate beats both `serde_json` and `simd-json`. +- **Repeated small parses**: `Parser` is reusable, so the per-call fixed cost + is low; `serde_json` is still a touch faster on a single 100 B document, but + `simdjson-rust` pulls ahead once the parser is reused and on medium inputs. +- **Zero-copy strings**: simdjson unescapes strings into `string_buf` during + stage2, so `&'de str` fields borrow directly with no re-validation or + allocation. + +Where it does _not_ win: + +- A single tiny JSON parsed once — `serde_json`'s scalar parser has the lowest + fixed cost. +- Anything that needs the C++ toolchain at build time (see below). + +### How the benchmark is run + +The bench lives in `benches/parser_bench.rs` and uses [criterion][criterion]. +Each of the three size groups deserializes the same struct shape via all three +parsers. To run it locally: + +```sh +# Copy the local tuning config (tunes the Rust code for your CPU). +cp .cargo/config.toml.example .cargo/config.toml +# Run with the C++ side tuned for your CPU too. +cargo bench --bench parser_bench --features native +# Or, portable build (no native tuning): +cargo bench --bench parser_bench +``` -[simdjson]: https://github.com/simdjson/simdjson \ No newline at end of file +Note: `simd-json`'s serde entry point rewrites its `&mut str` input in place, +so each iteration clones the input string for it — a constant tax specific to +that bench function, not the parser. Read its column as "parse + one +`String` clone". + +## Build requirements + +This crate builds simdjson from source via CMake, so the build host needs a +C++17 compiler and CMake. simdjson is fetched through CMake's `FetchContent` +on first build (no manual download), then cached. There is an opt-in `native` +cargo feature that compiles the C++ kernels with `-march=native` for the +building CPU; it is **off by default** so published binaries stay portable. +See `.cargo/config.toml.example` for the matching Rust `-C target-cpu=native`. + +## Why only DOM, not ondemand? + +simdjson offers two APIs: **DOM** (parse the whole document into a tape up +front) and **ondemand** (lazily parse only the fields you touch). This crate +currently binds only DOM. + +Ondemand would be attractive for "large document, few fields" workloads, but +binding it cleanly is hard: + +- simdjson's ondemand layer keeps its structural-index array and stage1 state + behind `internal::` and private members; there is no stable public surface + to hand that state to Rust without per-field FFI calls (which dominate the + cost for small reads). +- Doing it efficiently would mean exposing simdjson's internal stage1 + structural indexes to Rust and re-implementing stage2 (scope walking, + number parsing, string unescaping) in Rust — a large, internal-API-dependent + undertaking that would carry a real maintenance burden across simdjson + upgrades. + +For full-document deserialization — the common case and what `serde` is built +for — DOM is both simpler and faster (one C++ pass, then a pure-Rust tape +walk). Ondemand only wins when you deliberately skip most of a large document, +which is a narrower use case. + +This is not a closed door. If ondemand support would unblock a real workload +of yours, please open an issue describing your access pattern (document size, +how many fields you read). Concrete demand will re-prioritize the work, and +the likely path is a small patch to simdjson that exposes a stage1-only entry +point — feasible, but not worth building speculatively. + +## License + +Apache-2.0, same as [simdjson][simdjson]. + +[simdjson]: https://github.com/simdjson/simdjson +[simd-json]: https://github.com/simd-lite/simd-json +[criterion]: https://github.com/bheisler/criterion.rs diff --git a/benches/parser_bench.rs b/benches/parser_bench.rs new file mode 100644 index 0000000..4087163 --- /dev/null +++ b/benches/parser_bench.rs @@ -0,0 +1,196 @@ +use criterion::{Criterion, black_box, criterion_group, criterion_main}; +use serde::{Deserialize, Serialize}; +use simdjson_rust::{dom::Parser, serde::from_tape}; + +/// Deserialize `T` from `json` via the `simd-json` crate's serde path. +/// +/// `simd_json::serde::from_str` is unsafe: it rewrites its `&mut str` input +/// in place (using it as a padded scratch buffer), leaving it as non-UTF-8 +/// bytes, so each iteration must hand it a fresh owned copy. The clone cost +/// is identical for every size and is not specific to any parser; it is a +/// constant tax on this bench function alone, so read the simd-json numbers +/// as "parse + one String clone". +fn simd_json_crate_struct Deserialize<'de>>(json: &str) -> T { + let mut buf = json.to_string(); + // SAFETY: `buf` is a fresh owned copy not referenced elsewhere; the + // function leaves it as non-UTF-8, but we drop it immediately after. + unsafe { simd_json::serde::from_str(&mut buf).unwrap() } +} + +// --------------------------------------------------------------------------- +// Struct Definitions for Benchmarking +// --------------------------------------------------------------------------- + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] +struct SmallData { + x: f64, + y: f64, + label: String, + valid: bool, +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] +struct UserProfile { + id: u64, + name: String, + email: String, + active: bool, + friends: Vec, + score: f64, +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] +struct LargeData { + users: Vec, + version: String, + timestamp: u64, +} + +// --------------------------------------------------------------------------- +// Helper generators +// --------------------------------------------------------------------------- + +fn make_small_data() -> (SmallData, String) { + let data = SmallData { + x: 12.34, + y: -56.78, + label: "Benchmark Coordinates".to_string(), + valid: true, + }; + let json = serde_json::to_string(&data).unwrap(); + (data, json) +} + +fn make_medium_data() -> (UserProfile, String) { + let data = UserProfile { + id: 998877, + name: "John Doe".to_string(), + email: "john.doe@example.com".to_string(), + active: true, + friends: (0..50).map(|i| 1000 + i).collect(), + score: 98.76, + }; + let json = serde_json::to_string(&data).unwrap(); + (data, json) +} + +fn make_large_data() -> (LargeData, String) { + let users = (0..500) + .map(|id| UserProfile { + id, + name: format!("User Name {}", id), + email: format!("user.{}@example.com", id), + active: id % 2 == 0, + friends: (0..(id % 20)).map(|i| 10000 + i).collect(), + score: (id as f64) * 1.5, + }) + .collect(); + + let data = LargeData { + users, + version: "v1.0.0-benchmark".to_string(), + timestamp: 1719999999, + }; + let json = serde_json::to_string(&data).unwrap(); + (data, json) +} + +// --------------------------------------------------------------------------- +// Benchmarks +// --------------------------------------------------------------------------- + +fn bench_small(c: &mut Criterion) { + let (_, json) = make_small_data(); + let mut parser = Parser::default(); + + let mut group = c.benchmark_group("Small JSON (~100B)"); + + group.bench_function("serde_json_deserialize_struct", |b| { + b.iter(|| { + let res: SmallData = serde_json::from_str(black_box(&json)).unwrap(); + black_box(res); + }) + }); + + group.bench_function("simd_json_deserialize_struct", |b| { + b.iter(|| { + let res: SmallData = simd_json_crate_struct(black_box(&json)); + black_box(res); + }) + }); + + group.bench_function("simdjson_rust_deserialize_struct_reuse", |b| { + b.iter(|| { + let tape = parser.parse_str(black_box(&json)).unwrap(); + let res: SmallData = from_tape(tape).unwrap(); + black_box(res); + }) + }); + + group.finish(); +} + +fn bench_medium(c: &mut Criterion) { + let (_, json) = make_medium_data(); + let mut parser = Parser::default(); + + let mut group = c.benchmark_group("Medium JSON (~2KB)"); + + group.bench_function("serde_json_deserialize_struct", |b| { + b.iter(|| { + let res: UserProfile = serde_json::from_str(black_box(&json)).unwrap(); + black_box(res); + }) + }); + + group.bench_function("simd_json_deserialize_struct", |b| { + b.iter(|| { + let res: UserProfile = simd_json_crate_struct(black_box(&json)); + black_box(res); + }) + }); + + group.bench_function("simdjson_rust_deserialize_struct_reuse", |b| { + b.iter(|| { + let tape = parser.parse_str(black_box(&json)).unwrap(); + let res: UserProfile = from_tape(tape).unwrap(); + black_box(res); + }) + }); + + group.finish(); +} + +fn bench_large(c: &mut Criterion) { + let (_, json) = make_large_data(); + let mut parser = Parser::default(); + + let mut group = c.benchmark_group("Large JSON (~100KB)"); + + group.bench_function("serde_json_deserialize_struct", |b| { + b.iter(|| { + let res: LargeData = serde_json::from_str(black_box(&json)).unwrap(); + black_box(res); + }) + }); + + group.bench_function("simd_json_deserialize_struct", |b| { + b.iter(|| { + let res: LargeData = simd_json_crate_struct(black_box(&json)); + black_box(res); + }) + }); + + group.bench_function("simdjson_rust_deserialize_struct_reuse", |b| { + b.iter(|| { + let tape = parser.parse_str(black_box(&json)).unwrap(); + let res: LargeData = from_tape(tape).unwrap(); + black_box(res); + }) + }); + + group.finish(); +} + +criterion_group!(benches, bench_small, bench_medium, bench_large); +criterion_main!(benches); diff --git a/examples/issue_20.rs b/examples/issue_20.rs index 7fc5cee..da98462 100644 --- a/examples/issue_20.rs +++ b/examples/issue_20.rs @@ -1,23 +1,17 @@ -use simdjson_rust::{ondemand, prelude::*, Result}; +use simdjson_rust::dom::Parser; -fn main() -> Result<()> { - let data = r#"{"ingredients": [{"rose_wine": {"black_sesame_seed": 3}}, {"myrtleberry": {"black_sesame_seed": 5}}, {"grilled_beef": {"black_sesame_seed": 1}}, {"parmesan_cheese": {"black_sesame_seed": 2}}, {"strawberry_jam": {"black_sesame_seed": 1}}, {"tuna": {"black_sesame_seed": 3}}, {"black_tea": {"black_sesame_seed": 8}}, {"japanese_mint": {"black_sesame_seed": 5}}, {"pork": {"black_sesame_seed": 1}}, {"french_lavender": {"black_sesame_seed": 2}}, {"nutmeg": {"black_sesame_seed": 12}}, {"carob_fruit": {"black_sesame_seed": 2}}, {"mastic_gum_leaf_oil": {"black_sesame_seed": 2}}, {"california_pepper": {"black_sesame_seed": 5}}, {"orange_peel_oil": {"black_sesame_seed": 2}}, {"eucalyptus_oil": {"black_sesame_seed": 4}}, {"monarda_punctata": {"black_sesame_seed": 1}}, {"dried_green_tea": {"black_sesame_seed": 7}}, {"sake": {"black_sesame_seed": 2}}, {"currant": {"black_sesame_seed": 2}}, {"sweet_grass_oil": {"black_sesame_seed": 2}}, {"safflower_seed": {"black_sesame_seed": 17}}, {"sparkling_wine": {"black_sesame_seed": 3}}, {"enokidake": {"black_sesame_seed": 1}}, {"raw_peanut": {"black_sesame_seed": 1}}, {"cucumber": {"black_sesame_seed": 1}}, {"russian_cheese": {"black_sesame_seed": 1}}, {"catfish": {"black_sesame_seed": 3}}, {"origanum": {"black_sesame_seed": 2}}, {"roasted_beef": {"black_sesame_seed": 1}}, {"limburger_cheese": {"black_sesame_seed": 1}}, {"cantaloupe": {"black_sesame_seed": 3}}, {"corn_mint_oil": {"black_sesame_seed": 5}}, {"marjoram": {"black_sesame_seed": 4}}, {"herring": {"black_sesame_seed": 3}}, {"kumquat_peel_oil": {"black_sesame_seed": 1}}, {"cheese": {"black_sesame_seed": 1}}, {"lavender": {"black_sesame_seed": 1}}, {"cayenne": {"black_sesame_seed": 5}}, {"red_kidney_bean": {"black_sesame_seed": 4}}, {"mangosteen": {"black_sesame_seed": 2}}, {"lovage_root": {"black_sesame_seed": 1}}, {"mastic_gum": {"black_sesame_seed": 2}}, {"liver": {"black_sesame_seed": 1}}, {"thyme": {"black_sesame_seed": 3}}, {"oregano": {"black_sesame_seed": 1}}, {"lamb_liver": {"black_sesame_seed": 1}}, {"fruit": {"black_sesame_seed": 2}}, {"crab": {"black_sesame_seed": 1}}, {"french_peppermint": {"black_sesame_seed": 7}}, {"jamaican_rum": {"black_sesame_seed": 4}}, {"orange": {"black_sesame_seed": 7}}, {"romano_cheese": {"black_sesame_seed": 2}}, {"keta_salmon": {"black_sesame_seed": 3}}, {"juniper_berry": {"black_sesame_seed": 5}}, {"cheddar_cheese": {"black_sesame_seed": 1}}, {"calytrix_tetragona_oil": {"black_sesame_seed": 1}}, {"sea_bass": {"black_sesame_seed": 3}}, {"raw_pork": {"black_sesame_seed": 1}}, {"pear": {"black_sesame_seed": 2}}, {"pike": {"black_sesame_seed": 3}}, {"peanut": {"black_sesame_seed": 1}}, {"kumquat": {"black_sesame_seed": 2}}, {"european_cranberry": {"black_sesame_seed": 7}}, {"lemongrass": {"black_sesame_seed": 1}}, {"gin": {"black_sesame_seed": 1}}, {"orange_juice": {"black_sesame_seed": 1}}, {"dried_fig": {"black_sesame_seed": 2}}, {"sassafras": {"black_sesame_seed": 1}}, {"geranium": {"black_sesame_seed": 1}}, {"java_citronella": {"black_sesame_seed": 2}}, {"bourbon_whiskey": {"black_sesame_seed": 1}}, {"peanut_butter": {"black_sesame_seed": 1}}, {"smoked_fish": {"black_sesame_seed": 3}}, {"pennyroyal": {"black_sesame_seed": 1}}, {"roasted_shrimp": {"black_sesame_seed": 1}}, {"concord_grape": {"black_sesame_seed": 3}}, {"munster_cheese": {"black_sesame_seed": 1}}, {"sage": {"black_sesame_seed": 3}}, {"cocoa": {"black_sesame_seed": 3}}, {"champagne_wine": {"black_sesame_seed": 3}}, {"chicory_root": {"black_sesame_seed": 2}}, {"chamaecyparis_formosensis_oil": {"black_sesame_seed": 1}}, {"crowberry": {"black_sesame_seed": 5}}, {"water_apple": {"black_sesame_seed": 2}}, {"mint_oil": {"black_sesame_seed": 4}}, {"wheaten_bread": {"black_sesame_seed": 1}}, {"durian": {"black_sesame_seed": 2}}, {"cloudberry": {"black_sesame_seed": 7}}, {"japanese_star_anise": {"black_sesame_seed": 1}}, {"fried_cured_pork": {"black_sesame_seed": 1}}, {"satsuma": {"black_sesame_seed": 4}}, {"winter_savory": {"black_sesame_seed": 1}}, {"cowberry": {"black_sesame_seed": 5}}, {"snap_bean": {"black_sesame_seed": 4}}, {"cabernet_sauvignon_grape": {"black_sesame_seed": 3}}, {"port_wine": {"black_sesame_seed": 3}}, {"blackberry": {"black_sesame_seed": 7}}, {"tea_tree_oil": {"black_sesame_seed": 2}}, {"shiitake": {"black_sesame_seed": 1}}, {"cumin": {"black_sesame_seed": 6}}, {"prune": {"black_sesame_seed": 2}}, {"bog_blueberry": {"black_sesame_seed": 5}}, {"celery": {"black_sesame_seed": 4}}, {"boiled_pork": {"black_sesame_seed": 1}}, {"rooibus_tea": {"black_sesame_seed": 7}}, {"eucalyptus_globulus": {"black_sesame_seed": 1}}, {"tabasco_pepper": {"black_sesame_seed": 5}}, {"fermented_shrimp": {"black_sesame_seed": 1}}, {"jackfruit": {"black_sesame_seed": 2}}, {"gruyere_cheese": {"black_sesame_seed": 3}}, {"bread": {"black_sesame_seed": 1}}, {"pineapple": {"black_sesame_seed": 1}}, {"peppermint": {"black_sesame_seed": 7}}, {"cruciferae_seed": {"black_sesame_seed": 17}}, {"israeli_orange": {"black_sesame_seed": 7}}, {"monarda_punctata_oil": {"black_sesame_seed": 1}}, {"california_orange": {"black_sesame_seed": 7}}, {"peppermint_oil": {"black_sesame_seed": 5}}, {"ceylon_tea": {"black_sesame_seed": 7}}, {"cod": {"black_sesame_seed": 3}}, {"roasted_peanut": {"black_sesame_seed": 1}}, {"fish": {"black_sesame_seed": 3}}, {"plum": {"black_sesame_seed": 3}}, {"fennel": {"black_sesame_seed": 3}}, {"toasted_oat": {"black_sesame_seed": 1}}, {"cured_pork": {"black_sesame_seed": 4}}, {"pork_liver": {"black_sesame_seed": 1}}, {"mastic_gum_fruit_oil": {"black_sesame_seed": 1}}, {"passion_fruit": {"black_sesame_seed": 7}}, {"parsley": {"black_sesame_seed": 7}}, {"soybean": {"black_sesame_seed": 4}}, {"strawberry": {"black_sesame_seed": 6}}, {"scotch_spearmint_oil": {"black_sesame_seed": 4}}, {"pork_sausage": {"black_sesame_seed": 1}}, {"hernandia_peltata_oil": {"black_sesame_seed": 1}}, {"lemongrass_oil": {"black_sesame_seed": 1}}, {"roasted_filbert": {"black_sesame_seed": 3}}, {"wild_berry": {"black_sesame_seed": 5}}, {"wine": {"black_sesame_seed": 3}}, {"raw_lean_fish": {"black_sesame_seed": 3}}, {"roasted_lamb": {"black_sesame_seed": 2}}, {"pinto_bean": {"black_sesame_seed": 4}}, {"spearmint_oil": {"black_sesame_seed": 1}}, {"chicken": {"black_sesame_seed": 1}}, {"guarana": {"black_sesame_seed": 2}}, {"ocimum_viride": {"black_sesame_seed": 1}}, {"lantana_camara_oil": {"black_sesame_seed": 1}}, {"roman_chamomile": {"black_sesame_seed": 2}}, {"mate": {"black_sesame_seed": 1}}, {"roasted_mate": {"black_sesame_seed": 1}}, {"palmarosa": {"black_sesame_seed": 1}}, {"clary_sage": {"black_sesame_seed": 1}}, {"mint": {"black_sesame_seed": 5}}, {"myrtle": {"black_sesame_seed": 4}}, {"cabernet_sauvignon_wine": {"black_sesame_seed": 3}}, {"fermented_tea": {"black_sesame_seed": 7}}, {"brown_rice": {"black_sesame_seed": 1}}, {"mastic_gum_oil": {"black_sesame_seed": 2}}, {"vanilla": {"black_sesame_seed": 3}}, {"popcorn": {"black_sesame_seed": 2}}, {"eucalyptus_macarthurii": {"black_sesame_seed": 1}}, {"shrimp": {"black_sesame_seed": 1}}, {"malagueta_pepper": {"black_sesame_seed": 5}}, {"milk": {"black_sesame_seed": 4}}, {"muscadine_grape": {"black_sesame_seed": 3}}, {"mung_bean": {"black_sesame_seed": 4}}, {"kaffir_lime": {"black_sesame_seed": 5}}, {"huckleberry": {"black_sesame_seed": 5}}, {"tangerine_juice": {"black_sesame_seed": 1}}, {"muscat_grape": {"black_sesame_seed": 3}}, {"eucalyptus_bakeries_oil": {"black_sesame_seed": 1}}, {"smoked_pork_belly": {"black_sesame_seed": 1}}, {"camembert_cheese": {"black_sesame_seed": 1}}, {"haddock": {"black_sesame_seed": 3}}, {"cymbopogon_sennaarensis": {"black_sesame_seed": 1}}, {"whitefish": {"black_sesame_seed": 3}}, {"yellow_passion_fruit": {"black_sesame_seed": 1}}, {"calytrix_tetragona": {"black_sesame_seed": 1}}, {"roasted_malt": {"black_sesame_seed": 3}}, {"thymus": {"black_sesame_seed": 6}}, {"mandarin_peel": {"black_sesame_seed": 3}}, {"loganberry": {"black_sesame_seed": 5}}, {"tangerine_peel_oil": {"black_sesame_seed": 1}}, {"mango": {"black_sesame_seed": 6}}, {"roman_chamomile_oil": {"black_sesame_seed": 1}}, {"muskmelon": {"black_sesame_seed": 3}}, {"roasted_chicory_root": {"black_sesame_seed": 2}}, {"sherry": {"black_sesame_seed": 3}}, {"fatty_fish": {"black_sesame_seed": 3}}, {"lime_juice": {"black_sesame_seed": 1}}, {"dried_black_tea": {"black_sesame_seed": 7}}, {"malay_apple": {"black_sesame_seed": 2}}, {"navy_bean": {"black_sesame_seed": 4}}, {"smoked_pork": {"black_sesame_seed": 1}}, {"mutton_liver": {"black_sesame_seed": 1}}, {"seychelles_tea": {"black_sesame_seed": 7}}, {"lime": {"black_sesame_seed": 5}}, {"raw_fish": {"black_sesame_seed": 3}}, {"papaya": {"black_sesame_seed": 5}}, {"green_tea": {"black_sesame_seed": 7}}, {"citrus_peel_oil": {"black_sesame_seed": 7}}, {"seed": {"black_sesame_seed": 17}}, {"raw_fatty_fish": {"black_sesame_seed": 3}}, {"parsnip_fruit": {"black_sesame_seed": 2}}, {"parsnip": {"black_sesame_seed": 1}}, {"blenheim_apricot": {"black_sesame_seed": 2}}, {"buchu": {"black_sesame_seed": 4}}, {"blueberry": {"black_sesame_seed": 6}}, {"sauvignon_blanc_grape": {"black_sesame_seed": 3}}, {"kiwi": {"black_sesame_seed": 2}}, {"white_wine": {"black_sesame_seed": 4}}, {"long_pepper": {"black_sesame_seed": 5}}, {"fried_pork": {"black_sesame_seed": 1}}, {"kidney_bean": {"black_sesame_seed": 4}}, {"wild_raspberry": {"black_sesame_seed": 5}}, {"licorice": {"black_sesame_seed": 3}}, {"grapefruit": {"black_sesame_seed": 4}}, {"roasted_coconut": {"black_sesame_seed": 1}}, {"buchu_oil": {"black_sesame_seed": 4}}, {"guinea_pepper": {"black_sesame_seed": 5}}, {"burley_tobacco": {"black_sesame_seed": 1}}, {"monkey_orange": {"black_sesame_seed": 4}}, {"cooked_apple": {"black_sesame_seed": 2}}, {"roasted_cocoa": {"black_sesame_seed": 3}}, {"cream_cheese": {"black_sesame_seed": 1}}, {"smoked_fatty_fish": {"black_sesame_seed": 3}}, {"oatmeal": {"black_sesame_seed": 2}}, {"ocimum_gratissimum": {"black_sesame_seed": 1}}, {"coconut": {"black_sesame_seed": 1}}, {"roasted_pecan": {"black_sesame_seed": 3}}, {"horse_mackerel": {"black_sesame_seed": 3}}, {"peach": {"black_sesame_seed": 2}}, {"dwarf_quince": {"black_sesame_seed": 2}}, {"seed_oil": {"black_sesame_seed": 3}}, {"lingonberry": {"black_sesame_seed": 2}}, {"capsicum": {"black_sesame_seed": 5}}, {"leaf": {"black_sesame_seed": 6}}, {"jasmine_tea": {"black_sesame_seed": 7}}, {"elderberry": {"black_sesame_seed": 7}}, {"cape_gooseberry": {"black_sesame_seed": 6}}, {"roasted_spanish_peanut": {"black_sesame_seed": 1}}, {"lean_fish": {"black_sesame_seed": 3}}, {"comte_cheese": {"black_sesame_seed": 1}}, {"root": {"black_sesame_seed": 1}}, {"ginger": {"black_sesame_seed": 10}}, {"cherry": {"black_sesame_seed": 2}}, {"eucalyptus_dives": {"black_sesame_seed": 1}}, {"uncured_boiled_pork": {"black_sesame_seed": 1}}, {"raw_lamb": {"black_sesame_seed": 2}}, {"salmon": {"black_sesame_seed": 3}}, {"crownberry": {"black_sesame_seed": 5}}, {"rapeseed": {"black_sesame_seed": 17}}, {"dried_parsley": {"black_sesame_seed": 7}}, {"lemon_balm": {"black_sesame_seed": 3}}, {"roasted_barley": {"black_sesame_seed": 2}}, {"chicken_liver": {"black_sesame_seed": 1}}, {"tangerine": {"black_sesame_seed": 5}}, {"cilantro": {"black_sesame_seed": 1}}, {"fenugreek": {"black_sesame_seed": 1}}, {"swiss_cheese": {"black_sesame_seed": 1}}, {"raw_chicken": {"black_sesame_seed": 1}}, {"sheep_cheese": {"black_sesame_seed": 1}}, {"celery_seed": {"black_sesame_seed": 3}}, {"french_bean": {"black_sesame_seed": 4}}, {"whiskey": {"black_sesame_seed": 3}}, {"tuber": {"black_sesame_seed": 1}}, {"grape": {"black_sesame_seed": 3}}, {"coffee": {"black_sesame_seed": 3}}, {"filbert": {"black_sesame_seed": 2}}, {"peanut_oil": {"black_sesame_seed": 1}}, {"quince": {"black_sesame_seed": 2}}, {"spanish_sage": {"black_sesame_seed": 1}}, {"lemon_peel_oil": {"black_sesame_seed": 2}}, {"smoked_herring": {"black_sesame_seed": 3}}, {"coriander": {"black_sesame_seed": 6}}, {"rice": {"black_sesame_seed": 1}}, {"cinnamon": {"black_sesame_seed": 7}}, {"roasted_pork": {"black_sesame_seed": 1}}, {"chinese_quince": {"black_sesame_seed": 3}}, {"chive": {"black_sesame_seed": 3}}, {"grapefruit_juice": {"black_sesame_seed": 4}}, {"fried_chicken": {"black_sesame_seed": 2}}, {"emmental_cheese": {"black_sesame_seed": 1}}, {"melon": {"black_sesame_seed": 3}}, {"laurel": {"black_sesame_seed": 7}}, {"nectarine": {"black_sesame_seed": 4}}, {"wort": {"black_sesame_seed": 3}}, {"rum": {"black_sesame_seed": 4}}, {"caraway_seed": {"black_sesame_seed": 1}}, {"calamus": {"black_sesame_seed": 2}}, {"lemon_peel": {"black_sesame_seed": 2}}, {"watermelon": {"black_sesame_seed": 2}}, {"capsicum_annuum": {"black_sesame_seed": 5}}, {"hop": {"black_sesame_seed": 3}}, {"uncured_pork": {"black_sesame_seed": 1}}, {"provolone_cheese": {"black_sesame_seed": 1}}, {"boiled_beef": {"black_sesame_seed": 1}}, {"mountain_papaya": {"black_sesame_seed": 2}}, {"uncured_smoked_pork": {"black_sesame_seed": 1}}, {"spearmint": {"black_sesame_seed": 1}}, {"raw_beef": {"black_sesame_seed": 1}}, {"chinese_star_anise": {"black_sesame_seed": 1}}, {"boiled_crab": {"black_sesame_seed": 1}}, {"pawpaw": {"black_sesame_seed": 1}}, {"italian_lime": {"black_sesame_seed": 5}}, {"wheat_bread": {"black_sesame_seed": 1}}, {"calabash_nutmeg": {"black_sesame_seed": 3}}, {"yeast": {"black_sesame_seed": 1}}, {"choke_cherry": {"black_sesame_seed": 2}}, {"chokeberry": {"black_sesame_seed": 5}}, {"rice_husk": {"black_sesame_seed": 1}}, {"goat_cheese": {"black_sesame_seed": 1}}, {"finocchoi_fennel_oil": {"black_sesame_seed": 1}}, {"thai_pepper": {"black_sesame_seed": 5}}, {"sauvignon_grape": {"black_sesame_seed": 3}}, {"rose_apple": {"black_sesame_seed": 3}}, {"sour_cherry": {"black_sesame_seed": 2}}, {"crisp_bread": {"black_sesame_seed": 3}}, {"pepper": {"black_sesame_seed": 5}}, {"corn_mint": {"black_sesame_seed": 5}}, {"dried_kidney_bean": {"black_sesame_seed": 4}}, {"origanum_floribundum": {"black_sesame_seed": 1}}, {"hinoki_oil": {"black_sesame_seed": 1}}, {"prickly_pear": {"black_sesame_seed": 2}}, {"porcini": {"black_sesame_seed": 1}}, {"palm": {"black_sesame_seed": 1}}, {"cumin_fruit_oil": {"black_sesame_seed": 1}}, {"raspberry": {"black_sesame_seed": 7}}, {"pimento": {"black_sesame_seed": 2}}, {"fermented_russian_black_tea": {"black_sesame_seed": 7}}, {"ceylon_citronella": {"black_sesame_seed": 1}}, {"hinoki": {"black_sesame_seed": 1}}, {"eucalyptus": {"black_sesame_seed": 6}}, {"tarragon": {"black_sesame_seed": 1}}, {"mantis_shrimp": {"black_sesame_seed": 1}}, {"citrus_peel": {"black_sesame_seed": 7}}, {"grilled_pork": {"black_sesame_seed": 1}}, {"green_bell_pepper": {"black_sesame_seed": 5}}, {"peru_balsam": {"black_sesame_seed": 2}}, {"elderberry_fruit": {"black_sesame_seed": 2}}, {"cranberry": {"black_sesame_seed": 7}}, {"red_currant": {"black_sesame_seed": 5}}, {"orange_peel": {"black_sesame_seed": 2}}, {"raw_bean": {"black_sesame_seed": 4}}, {"corn": {"black_sesame_seed": 3}}, {"galanga": {"black_sesame_seed": 1}}, {"lima_bean": {"black_sesame_seed": 4}}, {"brewed_tea": {"black_sesame_seed": 7}}, {"feta_cheese": {"black_sesame_seed": 1}}, {"butter": {"black_sesame_seed": 3}}, {"oregano_oil": {"black_sesame_seed": 1}}, {"orthodon_citraliferum": {"black_sesame_seed": 1}}, {"satureia_thymera": {"black_sesame_seed": 1}}, {"sweetfish": {"black_sesame_seed": 3}}, {"prunus": {"black_sesame_seed": 2}}, {"turmeric": {"black_sesame_seed": 4}}, {"perovski_abrotanoides_oil": {"black_sesame_seed": 1}}, {"clove_oil": {"black_sesame_seed": 2}}, {"litchi": {"black_sesame_seed": 3}}, {"kelp": {"black_sesame_seed": 1}}, {"tahiti_vanilla": {"black_sesame_seed": 2}}, {"feijoa": {"black_sesame_seed": 4}}, {"globefish": {"black_sesame_seed": 3}}, {"caraway": {"black_sesame_seed": 2}}, {"japanese_peppermint_oil": {"black_sesame_seed": 1}}, {"lovage": {"black_sesame_seed": 7}}, {"dill": {"black_sesame_seed": 9}}, {"mackerel": {"black_sesame_seed": 3}}, {"mexican_lime": {"black_sesame_seed": 5}}, {"pecan": {"black_sesame_seed": 3}}, {"mushroom": {"black_sesame_seed": 1}}, {"lovage_leaf": {"black_sesame_seed": 1}}, {"eel": {"black_sesame_seed": 3}}, {"cognac": {"black_sesame_seed": 5}}, {"fried_beef": {"black_sesame_seed": 2}}, {"red_bean": {"black_sesame_seed": 4}}, {"star_anise": {"black_sesame_seed": 1}}, {"citrus_juice": {"black_sesame_seed": 2}}, {"neroli_bigarade": {"black_sesame_seed": 1}}, {"hop_oil": {"black_sesame_seed": 3}}, {"roasted_chicken": {"black_sesame_seed": 1}}, {"blue_cheese": {"black_sesame_seed": 1}}, {"pouching_tea": {"black_sesame_seed": 7}}, {"domiati_cheese": {"black_sesame_seed": 1}}, {"callitris": {"black_sesame_seed": 1}}, {"roasted_green_tea": {"black_sesame_seed": 7}}, {"cherimoya": {"black_sesame_seed": 1}}, {"elder_flower": {"black_sesame_seed": 1}}, {"guava": {"black_sesame_seed": 6}}, {"lime_peel_oil": {"black_sesame_seed": 3}}, {"matsutake": {"black_sesame_seed": 1}}, {"olive": {"black_sesame_seed": 2}}, {"clove": {"black_sesame_seed": 5}}, {"ceylon_tea_cinnamon_leaf": {"black_sesame_seed": 1}}, {"sperm_whale_oil": {"black_sesame_seed": 1}}, {"california_orange_peel": {"black_sesame_seed": 2}}, {"rye_bread": {"black_sesame_seed": 3}}, {"citrus": {"black_sesame_seed": 4}}, {"mozzarella_cheese": {"black_sesame_seed": 1}}, {"petitgrain": {"black_sesame_seed": 1}}, {"boiled_chicken": {"black_sesame_seed": 1}}, {"roasted_turkey": {"black_sesame_seed": 1}}, {"dill_seed": {"black_sesame_seed": 17}}, {"mandarin": {"black_sesame_seed": 4}}, {"scallop": {"black_sesame_seed": 1}}, {"corn_oil": {"black_sesame_seed": 2}}, {"carrot": {"black_sesame_seed": 4}}, {"eucalyptus_globulus_oil": {"black_sesame_seed": 1}}, {"white_bread": {"black_sesame_seed": 1}}, {"java_citronella_oil": {"black_sesame_seed": 2}}, {"rosemary": {"black_sesame_seed": 5}}, {"tamarind": {"black_sesame_seed": 4}}, {"scotch_spearmint": {"black_sesame_seed": 4}}, {"rabbiteye_blueberry": {"black_sesame_seed": 5}}, {"fennel_oil": {"black_sesame_seed": 1}}, {"tilsit_cheese": {"black_sesame_seed": 1}}, {"squid": {"black_sesame_seed": 1}}, {"cardamom": {"black_sesame_seed": 7}}, {"tea": {"black_sesame_seed": 7}}, {"pilchard": {"black_sesame_seed": 3}}, {"starfruit": {"black_sesame_seed": 4}}, {"wild_strawberry": {"black_sesame_seed": 5}}, {"malt": {"black_sesame_seed": 3}}, {"tomato": {"black_sesame_seed": 3}}, {"lemon": {"black_sesame_seed": 5}}, {"loquat": {"black_sesame_seed": 3}}, {"roquefort_cheese": {"black_sesame_seed": 1}}, {"mentha_silvestris_oil": {"black_sesame_seed": 1}}, {"palm_fruit": {"black_sesame_seed": 2}}, {"mandarin_peel_oil": {"black_sesame_seed": 3}}, {"fig": {"black_sesame_seed": 2}}, {"kola_tea": {"black_sesame_seed": 7}}, {"japanese_peppermint": {"black_sesame_seed": 7}}, {"caja_fruit": {"black_sesame_seed": 2}}, {"watercress": {"black_sesame_seed": 1}}, {"hog_plum": {"black_sesame_seed": 2}}, {"buckwheat": {"black_sesame_seed": 2}}, {"red_wine": {"black_sesame_seed": 3}}, {"botrytized_wine": {"black_sesame_seed": 3}}, {"ethiopian_pepper": {"black_sesame_seed": 5}}, {"smoked_salmon": {"black_sesame_seed": 3}}, {"lamb": {"black_sesame_seed": 2}}, {"mace": {"black_sesame_seed": 6}}, {"echinacea": {"black_sesame_seed": 1}}, {"cottage_cheese": {"black_sesame_seed": 1}}]} -"#; - let mut parser = ondemand::Parser::default(); - let padded_string = data.to_padded_string(); - let mut doc = parser.iterate(&padded_string)?; +fn main() -> Result<(), Box> { + let data = r#"{"ingredients": [{"rose_wine": {"black_sesame_seed": 3}}, {"myrtleberry": {"black_sesame_seed": 5}}, {"grilled_beef": {"black_sesame_seed": 1}}, {"parmesan_cheese": {"black_sesame_seed": 2}}, {"strawberry_jam": {"black_sesame_seed": 1}}, {"tuna": {"black_sesame_seed": 3}}, {"black_tea": {"black_sesame_seed": 8}}, {"japanese_mint": {"black_sesame_seed": 5}}, {"pork": {"black_sesame_seed": 1}}, {"french_lavender": {"black_sesame_seed": 2}}, {"nutmeg": {"black_sesame_seed": 12}}, {"carob_fruit": {"black_sesame_seed": 2}}, {"mastic_gum_leaf_oil": {"black_sesame_seed": 2}}, {"california_pepper": {"black_sesame_seed": 5}}, {"orange_peel_oil": {"black_sesame_seed": 2}}, {"eucalyptus_oil": {"black_sesame_seed": 4}}, {"monarda_punctata": {"black_sesame_seed": 1}}, {"dried_green_tea": {"black_sesame_seed": 7}}, {"sake": {"black_sesame_seed": 2}}, {"currant": {"black_sesame_seed": 2}}, {"sweet_grass_oil": {"black_sesame_seed": 2}}, {"safflower_seed": {"black_sesame_seed": 17}}, {"sparkling_wine": {"black_sesame_seed": 3}}, {"enokidake": {"black_sesame_seed": 1}}, {"raw_peanut": {"black_sesame_seed": 1}}, {"cucumber": {"black_sesame_seed": 1}}, {"russian_cheese": {"black_sesame_seed": 1}}, {"catfish": {"black_sesame_seed": 3}}, {"origanum": {"black_sesame_seed": 2}}, {"roasted_beef": {"black_sesame_seed": 1}}, {"limburger_cheese": {"black_sesame_seed": 1}}, {"cantaloupe": {"black_sesame_seed": 3}}, {"corn_mint_oil": {"black_sesame_seed": 5}}, {"marjoram": {"black_sesame_seed": 4}}, {"herring": {"black_sesame_seed": 3}}, {"kumquat_peel_oil": {"black_sesame_seed": 1}}, {"cheese": {"black_sesame_seed": 1}}, {"lavender": {"black_sesame_seed": 1}}, {"cayenne": {"black_sesame_seed": 5}}, {"red_kidney_bean": {"black_sesame_seed": 4}}, {"mangosteen": {"black_sesame_seed": 2}}, {"lovage_root": {"black_sesame_seed": 1}}, {"mastic_gum": {"black_sesame_seed": 2}}, {"liver": {"black_sesame_seed": 1}}, {"thyme": {"black_sesame_seed": 3}}, {"oregano": {"black_sesame_seed": 1}}, {"lamb_liver": {"black_sesame_seed": 1}}, {"fruit": {"black_sesame_seed": 2}}, {"crab": {"black_sesame_seed": 1}}, {"french_peppermint": {"black_sesame_seed": 7}}, {"jamaican_rum": {"black_sesame_seed": 4}}, {"orange": {"black_sesame_seed": 7}}, {"romano_cheese": {"black_sesame_seed": 2}}, {"keta_salmon": {"black_sesame_seed": 3}}, {"juniper_berry": {"black_sesame_seed": 5}}, {"cheddar_cheese": {"black_sesame_seed": 1}}, {"calytrix_tetragona_oil": {"black_sesame_seed": 1}}, {"sea_bass": {"black_sesame_seed": 3}}, {"raw_pork": {"black_sesame_seed": 1}}, {"pear": {"black_sesame_seed": 2}}, {"pike": {"black_sesame_seed": 3}}, {"peanut": {"black_sesame_seed": 1}}, {"kumquat": {"black_sesame_seed": 2}}, {"european_cranberry": {"black_sesame_seed": 7}}, {"lemongrass": {"black_sesame_seed": 1}}, {"gin": {"black_sesame_seed": 1}}, {"orange_juice": {"black_sesame_seed": 1}}, {"dried_fig": {"black_sesame_seed": 2}}, {"sassafras": {"black_sesame_seed": 1}}, {"geranium": {"black_sesame_seed": 1}}, {"java_citronella": {"black_sesame_seed": 2}}, {"bourbon_whiskey": {"black_sesame_seed": 1}}, {"peanut_butter": {"black_sesame_seed": 1}}, {"smoked_fish": {"black_sesame_seed": 3}}, {"pennyroyal": {"black_sesame_seed": 1}}, {"roasted_shrimp": {"black_sesame_seed": 1}}, {"concord_grape": {"black_sesame_seed": 3}}, {"munster_cheese": {"black_sesame_seed": 1}}, {"sage": {"black_sesame_seed": 3}}, {"cocoa": {"black_sesame_seed": 3}}, {"champagne_wine": {"black_sesame_seed": 3}}, {"chicory_root": {"black_sesame_seed": 2}}, {"chamaecyparis_formosensis_oil": {"black_sesame_seed": 1}}, {"crowberry": {"black_sesame_seed": 5}}, {"water_apple": {"black_sesame_seed": 2}}, {"mint_oil": {"black_sesame_seed": 4}}, {"wheaten_bread": {"black_sesame_seed": 1}}, {"durian": {"black_sesame_seed": 2}}, {"cloudberry": {"black_sesame_seed": 7}}, {"japanese_star_anise": {"black_sesame_seed": 1}}, {"fried_cured_pork": {"black_sesame_seed": 1}}, {"satsuma": {"black_sesame_seed": 4}}, {"winter_savory": {"black_sesame_seed": 1}}, {"cowberry": {"black_sesame_seed": 5}}, {"snap_bean": {"black_sesame_seed": 4}}, {"cabernet_sauvignon_grape": {"black_sesame_seed": 3}}, {"port_wine": {"black_sesame_seed": 3}}, {"blackberry": {"black_sesame_seed": 7}}, {"tea_tree_oil": {"black_sesame_seed": 2}}, {"shiitake": {"black_sesame_seed": 1}}, {"cumin": {"black_sesame_seed": 6}}, {"prune": {"black_sesame_seed": 2}}, {"bog_blueberry": {"black_sesame_seed": 5}}, {"celery": {"black_sesame_seed": 4}}, {"boiled_pork": {"black_sesame_seed": 1}}, {"rooibus_tea": {"black_sesame_seed": 7}}, {"eucalyptus_globulus": {"black_sesame_seed": 1}}, {"tabasco_pepper": {"black_sesame_seed": 5}}, {"fermented_shrimp": {"black_sesame_seed": 1}}, {"jackfruit": {"black_sesame_seed": 2}}, {"gruyere_cheese": {"black_sesame_seed": 3}}, {"bread": {"black_sesame_seed": 1}}, {"pineapple": {"black_sesame_seed": 1}}, {"peppermint": {"black_sesame_seed": 7}}, {"cruciferae_seed": {"black_sesame_seed": 17}}, {"israeli_orange": {"black_sesame_seed": 7}}, {"monarda_punctata_oil": {"black_sesame_seed": 1}}, {"california_orange": {"black_sesame_seed": 7}}, {"peppermint_oil": {"black_sesame_seed": 5}}, {"ceylon_tea": {"black_sesame_seed": 7}}, {"cod": {"black_sesame_seed": 3}}, {"roasted_peanut": {"black_sesame_seed": 1}}, {"fish": {"black_sesame_seed": 3}}, {"plum": {"black_sesame_seed": 3}}, {"fennel": {"black_sesame_seed": 3}}, {"toasted_oat": {"black_sesame_seed": 1}}, {"cured_pork": {"black_sesame_seed": 4}}, {"pork_liver": {"black_sesame_seed": 1}}, {"mastic_gum_fruit_oil": {"black_sesame_seed": 1}}, {"passion_fruit": {"black_sesame_seed": 7}}, {"parsley": {"black_sesame_seed": 7}}, {"soybean": {"black_sesame_seed": 4}}, {"strawberry": {"black_sesame_seed": 6}}, {"scotch_spearmint_oil": {"black_sesame_seed": 4}}, {"pork_sausage": {"black_sesame_seed": 1}}, {"hernandia_peltata_oil": {"black_sesame_seed": 1}}, {"lemongrass_oil": {"black_sesame_seed": 1}}, {"roasted_filbert": {"black_sesame_seed": 3}}, {"wild_berry": {"black_sesame_seed": 5}}, {"wine": {"black_sesame_seed": 3}}, {"raw_lean_fish": {"black_sesame_seed": 3}}, {"roasted_lamb": {"black_sesame_seed": 2}}, {"pinto_bean": {"black_sesame_seed": 4}}, {"spearmint_oil": {"black_sesame_seed": 1}}, {"chicken": {"black_sesame_seed": 1}}, {"guarana": {"black_sesame_seed": 2}}, {"ocimum_viride": {"black_sesame_seed": 1}}, {"lantana_camara_oil": {"black_sesame_seed": 1}}, {"roman_chamomile": {"black_sesame_seed": 2}}, {"mate": {"black_sesame_seed": 1}}, {"roasted_mate": {"black_sesame_seed": 1}}, {"palmarosa": {"black_sesame_seed": 1}}, {"clary_sage": {"black_sesame_seed": 1}}, {"mint": {"black_sesame_seed": 5}}, {"myrtle": {"black_sesame_seed": 4}}, {"cabernet_sauvignon_wine": {"black_sesame_seed": 3}}, {"fermented_tea": {"black_sesame_seed": 7}}, {"brown_rice": {"black_sesame_seed": 1}}, {"mastic_gum_oil": {"black_sesame_seed": 2}}, {"vanilla": {"black_sesame_seed": 3}}, {"popcorn": {"black_sesame_seed": 2}}, {"eucalyptus_macarthurii": {"black_sesame_seed": 1}}, {"shrimp": {"black_sesame_seed": 1}}, {"malagueta_pepper": {"black_sesame_seed": 5}}, {"milk": {"black_sesame_seed": 4}}, {"muscadine_grape": {"black_sesame_seed": 3}}, {"mung_bean": {"black_sesame_seed": 4}}, {"kaffir_lime": {"black_sesame_seed": 5}}, {"huckleberry": {"black_sesame_seed": 5}}, {"tangerine_juice": {"black_sesame_seed": 1}}, {"muscat_grape": {"black_sesame_seed": 3}}, {"eucalyptus_bakeries_oil": {"black_sesame_seed": 1}}, {"smoked_pork_belly": {"black_sesame_seed": 1}}, {"camembert_cheese": {"black_sesame_seed": 1}}, {"haddock": {"black_sesame_seed": 3}}, {"cymbopogon_sennaarensis": {"black_sesame_seed": 1}}, {"whitefish": {"black_sesame_seed": 3}}, {"yellow_passion_fruit": {"black_sesame_seed": 1}}, {"calytrix_tetragona": {"black_sesame_seed": 1}}, {"roasted_malt": {"black_sesame_seed": 3}}, {"thymus": {"black_sesame_seed": 6}}, {"mandarin_peel": {"black_sesame_seed": 3}}, {"loganberry": {"black_sesame_seed": 5}}, {"tangerine_peel_oil": {"black_sesame_seed": 1}}, {"mango": {"black_sesame_seed": 6}}, {"roman_chamomile_oil": {"black_sesame_seed": 1}}, {"muskmelon": {"black_sesame_seed": 3}}, {"roasted_chicory_root": {"black_sesame_seed": 2}}, {"sherry": {"black_sesame_seed": 3}}, {"fatty_fish": {"black_sesame_seed": 3}}, {"lime_juice": {"black_sesame_seed": 1}}, {"dried_black_tea": {"black_sesame_seed": 7}}, {"malay_apple": {"black_sesame_seed": 2}}, {"navy_bean": {"black_sesame_seed": 4}}, {"smoked_pork": {"black_sesame_seed": 1}}, {"mutton_liver": {"black_sesame_seed": 1}}, {"seychelles_tea": {"black_sesame_seed": 7}}, {"lime": {"black_sesame_seed": 5}}, {"raw_fish": {"black_sesame_seed": 3}}, {"papaya": {"black_sesame_seed": 5}}, {"green_tea": {"black_sesame_seed": 7}}, {"citrus_peel_oil": {"black_sesame_seed": 7}}, {"seed": {"black_sesame_seed": 17}}, {"raw_fatty_fish": {"black_sesame_seed": 3}}, {"parsnip_fruit": {"black_sesame_seed": 2}}, {"parsnip": {"black_sesame_seed": 1}}, {"blenheim_apricot": {"black_sesame_seed": 2}}, {"buchu": {"black_sesame_seed": 4}}, {"blueberry": {"black_sesame_seed": 6}}, {"sauvignon_blanc_grape": {"black_sesame_seed": 3}}, {"kiwi": {"black_sesame_seed": 2}}, {"white_wine": {"black_sesame_seed": 4}}, {"long_pepper": {"black_sesame_seed": 5}}, {"fried_pork": {"black_sesame_seed": 1}}, {"kidney_bean": {"black_sesame_seed": 4}}, {"wild_raspberry": {"black_sesame_seed": 5}}, {"licorice": {"black_sesame_seed": 3}}, {"grapefruit": {"black_sesame_seed": 4}}, {"roasted_coconut": {"black_sesame_seed": 1}}, {"buchu_oil": {"black_sesame_seed": 4}}, {"guinea_pepper": {"black_sesame_seed": 5}}, {"burley_tobacco": {"black_sesame_seed": 1}}, {"monkey_orange": {"black_sesame_seed": 4}}, {"cooked_apple": {"black_sesame_seed": 2}}, {"roasted_cocoa": {"black_sesame_seed": 3}}, {"cream_cheese": {"black_sesame_seed": 1}}, {"smoked_fatty_fish": {"black_sesame_seed": 3}}, {"oatmeal": {"black_sesame_seed": 2}}, {"ocimum_gratissimum": {"black_sesame_seed": 1}}, {"coconut": {"black_sesame_seed": 1}}, {"roasted_pecan": {"black_sesame_seed": 3}}, {"horse_mackerel": {"black_sesame_seed": 3}}, {"peach": {"black_sesame_seed": 2}}, {"dwarf_quince": {"black_sesame_seed": 2}}, {"seed_oil": {"black_sesame_seed": 3}}, {"lingonberry": {"black_sesame_seed": 2}}, {"capsicum": {"black_sesame_seed": 5}}, {"leaf": {"black_sesame_seed": 6}}, {"jasmine_tea": {"black_sesame_seed": 7}}, {"elderberry": {"black_sesame_seed": 7}}, {"cape_gooseberry": {"black_sesame_seed": 6}}, {"roasted_spanish_peanut": {"black_sesame_seed": 1}}, {"lean_fish": {"black_sesame_seed": 3}}, {"comte_cheese": {"black_sesame_seed": 1}}, {"root": {"black_sesame_seed": 1}}, {"ginger": {"black_sesame_seed": 10}}, {"cherry": {"black_sesame_seed": 2}}, {"eucalyptus_dives": {"black_sesame_seed": 1}}, {"uncured_boiled_pork": {"black_sesame_seed": 1}}, {"raw_lamb": {"black_sesame_seed": 2}}, {"salmon": {"black_sesame_seed": 3}}, {"crownberry": {"black_sesame_seed": 5}}, {"rapeseed": {"black_sesame_seed": 17}}, {"dried_parsley": {"black_sesame_seed": 7}}, {"lemon_balm": {"black_sesame_seed": 3}}, {"roasted_barley": {"black_sesame_seed": 2}}, {"chicken_liver": {"black_sesame_seed": 1}}, {"tangerine": {"black_sesame_seed": 5}}, {"cilantro": {"black_sesame_seed": 1}}, {"fenugreek": {"black_sesame_seed": 1}}, {"swiss_cheese": {"black_sesame_seed": 1}}, {"raw_chicken": {"black_sesame_seed": 1}}, {"sheep_cheese": {"black_sesame_seed": 1}}, {"celery_seed": {"black_sesame_seed": 3}}, {"french_bean": {"black_sesame_seed": 4}}, {"whiskey": {"black_sesame_seed": 3}}, {"tuber": {"black_sesame_seed": 1}}, {"grape": {"black_sesame_seed": 3}}, {"coffee": {"black_sesame_seed": 3}}, {"filbert": {"black_sesame_seed": 2}}, {"peanut_oil": {"black_sesame_seed": 1}}, {"quince": {"black_sesame_seed": 2}}, {"spanish_sage": {"black_sesame_seed": 1}}, {"lemon_peel_oil": {"black_sesame_seed": 2}}, {"smoked_herring": {"black_sesame_seed": 3}}, {"coriander": {"black_sesame_seed": 6}}, {"rice": {"black_sesame_seed": 1}}, {"cinnamon": {"black_sesame_seed": 7}}, {"roasted_pork": {"black_sesame_seed": 1}}, {"chinese_quince": {"black_sesame_seed": 3}}, {"chive": {"black_sesame_seed": 3}}, {"grapefruit_juice": {"black_sesame_seed": 4}}, {"fried_chicken": {"black_sesame_seed": 2}}, {"emmental_cheese": {"black_sesame_seed": 1}}, {"melon": {"black_sesame_seed": 3}}, {"laurel": {"black_sesame_seed": 7}}, {" nectarine": {"black_sesame_seed": 4}}, {"wort": {"black_sesame_seed": 3}}, {"rum": {"black_sesame_seed": 4}}, {"caraway_seed": {"black_sesame_seed": 1}}, {"calamus": {"black_sesame_seed": 2}}, {"lemon_peel": {"black_sesame_seed": 2}}, {"watermelon": {"black_sesame_seed": 2}}, {"capsicum_annuum": {"black_sesame_seed": 5}}, {"hop": {"black_sesame_seed": 3}}, {"uncured_pork": {"black_sesame_seed": 1}}, {"provolone_cheese": {"black_sesame_seed": 1}}, {"boiled_beef": {"black_sesame_seed": 1}}, {"mountain_papaya": {"black_sesame_seed": 2}}, {"uncured_smoked_pork": {"black_sesame_seed": 1}}, {"spearmint": {"black_sesame_seed": 1}}, {"raw_beef": {"black_sesame_seed": 1}}, {"chinese_star_anise": {"black_sesame_seed": 1}}, {"boiled_crab": {"black_sesame_seed": 1}}, {"pawpaw": {"black_sesame_seed": 1}}, {"italian_lime": {"black_sesame_seed": 5}}, {"wheat_bread": {"black_sesame_seed": 1}}, {"calabash_nutmeg": {"black_sesame_seed": 3}}, {"yeast": {"black_sesame_seed": 1}}, {"choke_cherry": {"black_sesame_seed": 2}}, {"chokeberry": {"black_sesame_seed": 5}}, {"rice_husk": {"black_sesame_seed": 1}}, {"goat_cheese": {"black_sesame_seed": 1}}, {"finocchoi_fennel_oil": {"black_sesame_seed": 1}}, {"thai_pepper": {"black_sesame_seed": 5}}, {"raw_pork": {"black_sesame_seed": 1}}, {"pear": {"black_sesame_seed": 2}}, {"pike": {"black_sesame_seed": 3}}, {"peanut": {"black_sesame_seed": 1}}, {"kumquat": {"black_sesame_seed": 2}}, {"european_cranberry": {"black_sesame_seed": 7}}, {"lemongrass": {"black_sesame_seed": 1}}, {"gin": {"black_sesame_seed": 1}}, {"orange_juice": {"black_sesame_seed": 1}}, {"dried_fig": {"black_sesame_seed": 2}}, {"sassafras": {"black_sesame_seed": 1}}, {"geranium": {"black_sesame_seed": 1}}, {"java_citronella": {"black_sesame_seed": 2}}, {"bourbon_whiskey": {"black_sesame_seed": 1}}, {"peanut_butter": {"black_sesame_seed": 1}}, {"smoked_fish": {"black_sesame_seed": 3}}, {"pennyroyal": {"black_sesame_seed": 1}}, {"roasted_shrimp": {"black_sesame_seed": 1}}, {"concord_grape": {"black_sesame_seed": 3}}, {"munster_cheese": {"black_sesame_seed": 1}}, {"sage": {"black_sesame_seed": 3}}, {"cocoa": {"black_sesame_seed": 3}}, {"champagne_wine": {"black_sesame_seed": 3}}, {"chicory_root": {"black_sesame_seed": 2}}, {"chamaecyparis_formosensis_oil": {"black_sesame_seed": 1}}, {"crowberry": {"black_sesame_seed": 5}}, {"water_apple": {"black_sesame_seed": 2}}, {"mint_oil": {"black_sesame_seed": 4}}, {"wheaten_bread": {"black_sesame_seed": 1}}, {"durian": {"black_sesame_seed": 2}}, {"cloudberry": {"black_sesame_seed": 7}}, {"japanese_star_anise": {"black_sesame_seed": 1}}, {"fried_cured_pork": {"black_sesame_seed": 1}}, {"satsuma": {"black_sesame_seed": 4}}, {"winter_savory": {"black_sesame_seed": 1}}, {"cowberry": {"black_sesame_seed": 5}}, {"snap_bean": {"black_sesame_seed": 4}}, {"cabernet_sauvignon_grape": {"black_sesame_seed": 3}}, {"port_wine": {"black_sesame_seed": 3}}, {"blackberry": {"black_sesame_seed": 7}}, {"tea_tree_oil": {"black_sesame_seed": 2}}, {"shiitake": {"black_sesame_seed": 1}}, {"cumin": {"black_sesame_seed": 6}}, {"prune": {"black_sesame_seed": 2}}, {"bog_blueberry": {"black_sesame_seed": 5}}, {"celery": {"black_sesame_seed": 4}}, {"boiled_pork": {"black_sesame_seed": 1}}, {"rooibus_tea": {"black_sesame_seed": 7}}, {"eucalyptus_globulus": {"black_sesame_seed": 1}}, {"tabasco_pepper": {"black_sesame_seed": 5}}, {"fermented_shrimp": {"black_sesame_seed": 1}}, {"jackfruit": {"black_sesame_seed": 2}}, {"gruyere_cheese": {"black_sesame_seed": 3}}, {"bread": {"black_sesame_seed": 1}}, {"pineapple": {"black_sesame_seed": 1}}, {"peppermint": {"black_sesame_seed": 7}}, {"cruciferae_seed": {"black_sesame_seed": 17}}, {"israeli_orange": {"black_sesame_seed": 7}}, {"monarda_punctata_oil": {"black_sesame_seed": 1}}, {"california_orange": {"black_sesame_seed": 7}}, {"peppermint_oil": {"black_sesame_seed": 5}}, {"ceylon_tea": {"black_sesame_seed": 7}}, {"cod": {"black_sesame_seed": 3}}, {"roasted_peanut": {"black_sesame_seed": 1}}, {"fish": {"black_sesame_seed": 3}}, {"plum": {"black_sesame_seed": 3}}, {"fennel": {"black_sesame_seed": 3}}, {"toasted_oat": {"black_sesame_seed": 1}}, {"cured_pork": {"black_sesame_seed": 4}}, {"pork_liver": {"black_sesame_seed": 1}}, {"mastic_gum_fruit_oil": {"black_sesame_seed": 1}}, {"passion_fruit": {"black_sesame_seed": 7}}, {"parsley": {"black_sesame_seed": 7}}, {"soybean": {"black_sesame_seed": 4}}, {"strawberry": {"black_sesame_seed": 6}}, {"scotch_spearmint_oil": {"black_sesame_seed": 4}}, {"pork_sausage": {"black_sesame_seed": 1}}, {"hernandia_peltata_oil": {"black_sesame_seed": 1}}, {"lemongrass_oil": {"black_sesame_seed": 1}}, {"roasted_filbert": {"black_sesame_seed": 3}}, {"wild_berry": {"black_sesame_seed": 5}}, {"wine": {"black_sesame_seed": 3}}, {"raw_lean_fish": {"black_sesame_seed": 3}}, {"roasted_lamb": {"black_sesame_seed": 2}}, {"pinto_bean": {"black_sesame_seed": 4}}, {"spearmint_oil": {"black_sesame_seed": 1}}, {"chicken": {"black_sesame_seed": 1}}, {"guarana": {"black_sesame_seed": 2}}, {"ocimum_viride": {"black_sesame_seed": 1}}, {"lantana_camara_oil": {"black_sesame_seed": 1}}, {"roman_chamomile": {"black_sesame_seed": 2}}, {"mate": {"black_sesame_seed": 1}}, {"roasted_mate": {"black_sesame_seed": 1}}, {"palmarosa": {"black_sesame_seed": 1}}, {"clary_sage": {"black_sesame_seed": 1}}, {"mint": {"black_sesame_seed": 5}}, {"myrtle": {"black_sesame_seed": 4}}, {"cabernet_sauvignon_wine": {"black_sesame_seed": 3}}, {"fermented_tea": {"black_sesame_seed": 7}}, {"brown_rice": {"black_sesame_seed": 1}}, {"mastic_gum_oil": {"black_sesame_seed": 2}}, {"vanilla": {"black_sesame_seed": 3}}, {"popcorn": {"black_sesame_seed": 2}}, {"eucalyptus_macarthurii": {"black_sesame_seed": 1}}, {"shrimp": {"black_sesame_seed": 1}}, {"malagueta_pepper": {"black_sesame_seed": 5}}, {"milk": {"black_sesame_seed": 4}}, {"muscadine_grape": {"black_sesame_seed": 3}}, {"mung_bean": {"black_sesame_seed": 4}}, {"kaffir_lime": {"black_sesame_seed": 5}}, {"huckleberry": {"black_sesame_seed": 5}}, {"tangerine_juice": {"black_sesame_seed": 1}}, {"muscat_grape": {"black_sesame_seed": 3}}, {"eucalyptus_bakeries_oil": {"black_sesame_seed": 1}}, {"smoked_pork_belly": {"black_sesame_seed": 1}}, {"camembert_cheese": {"black_sesame_seed": 1}}, {"haddock": {"black_sesame_seed": 3}}, {"cymbopogon_sennaarensis": {"black_sesame_seed": 1}}, {"whitefish": {"black_sesame_seed": 3}}, {"yellow_passion_fruit": {"black_sesame_seed": 1}}, {"calytrix_tetragona": {"black_sesame_seed": 1}}, {"roasted_malt": {"black_sesame_seed": 3}}, {"thymus": {"black_sesame_seed": 6}}, {"mandarin_peel": {"black_sesame_seed": 3}}, {"loganberry": {"black_sesame_seed": 5}}, {"tangerine_peel_oil": {"black_sesame_seed": 1}}, {"mango": {"black_sesame_seed": 6}}, {"roman_chamomile_oil": {"black_sesame_seed": 1}}, {"muskmelon": {"black_sesame_seed": 3}}, {"roasted_chicory_root": {"black_sesame_seed": 2}}, {"sherry": {"black_sesame_seed": 3}}, {"fatty_fish": {"black_sesame_seed": 3}}, {"lime_juice": {"black_sesame_seed": 1}}, {"black_sesame_seed": 3}}, {"cottage_cheese": {"black_sesame_seed": 1}}]}"#; + let mut parser = Parser::default(); + let v = parser.parse_to_value(data)?; + let ingredients = v.get("ingredients").unwrap().as_array().unwrap(); - let mut arr = doc.get_object()?.at_pointer("/ingredients")?.get_array()?; - - for value in arr.iter()? { - let mut object = value?.get_object()?; - - for field in object.iter()? { - let mut field = field?; - - let key = field.unescaped_key(false)?; - let mut value = field.take_value(); - println!("key: {} | value: {}", key, value.get_object()?.raw_json()?); + println!("Ingredients list:"); + for item in ingredients { + let obj = item.as_object().unwrap(); + for (key, val) in obj { + let inner_obj = val.as_object().unwrap(); + println!("ingredient: {} | value: {:?}", key, inner_obj); } } diff --git a/examples/perf_sample.rs b/examples/perf_sample.rs new file mode 100644 index 0000000..0fb933e --- /dev/null +++ b/examples/perf_sample.rs @@ -0,0 +1,63 @@ +//! Throwaway binary for perf sampling of the Large struct deserialization path. +//! Run under `perf record` to locate CPU hotspots. +use serde::{Deserialize, Serialize}; +use simdjson_rust::{dom::Parser, serde::from_tape}; + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] +struct UserProfile { + id: u64, + name: String, + email: String, + active: bool, + friends: Vec, + score: f64, +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] +struct LargeData { + users: Vec, + version: String, + timestamp: u64, +} + +fn make_large_data() -> String { + let users = (0..500) + .map(|id| UserProfile { + id, + name: format!("User Name {}", id), + email: format!("user.{}@example.com", id), + active: id % 2 == 0, + friends: (0..(id % 20)).map(|i| 10000 + i).collect(), + score: (id as f64) * 1.5, + }) + .collect(); + let data = LargeData { + users, + version: "v1.0.0-benchmark".to_string(), + timestamp: 1719999999, + }; + serde_json::to_string(&data).unwrap() +} + +fn main() { + let json = make_large_data(); + let mut parser = Parser::default(); + // Warm up the parser's internal buffers. + for _ in 0..100 { + let tape = parser.parse_str(&json).unwrap(); + let _res: LargeData = from_tape(tape).unwrap(); + } + // Hot loop: this is what perf samples. + let iters = std::env::args() + .nth(1) + .map(|s| s.parse::().unwrap()) + .unwrap_or(20000); + let mut acc = 0u64; + for _ in 0..iters { + let tape = parser.parse_str(&json).unwrap(); + let res: LargeData = from_tape(tape).unwrap(); + acc = acc.wrapping_add(res.users.len() as u64); + } + // Prevent dead-code elimination. + std::hint::black_box(acc); +} diff --git a/examples/quickstart.rs b/examples/quickstart.rs index d82c443..3a0ab93 100644 --- a/examples/quickstart.rs +++ b/examples/quickstart.rs @@ -1,13 +1,21 @@ -use simdjson_rust::{ondemand, prelude::*, Result}; +use simdjson_rust::dom::Parser; -fn main() -> Result<()> { - let ps = load_padded_string("simdjson-sys/simdjson/jsonexamples/twitter.json")?; - let mut parser = ondemand::Parser::default(); - let mut tweets = parser.iterate(&ps)?; - println!( - "{} results.", - tweets.at_pointer("/search_metadata/count")?.get_uint64()? - ); +fn main() -> Result<(), Box> { + let json = r#"{ + "search_metadata": { + "count": 100 + } + }"#; + let mut parser = Parser::default(); + let v = parser.parse_to_value(json)?; + let count = v + .get("search_metadata") + .unwrap() + .get("count") + .unwrap() + .as_u64() + .unwrap(); + println!("{} results.", count); Ok(()) } diff --git a/examples/simple.rs b/examples/simple.rs index 61160ef..95f0b1b 100644 --- a/examples/simple.rs +++ b/examples/simple.rs @@ -1,26 +1,36 @@ -use simdjson_rust::{dom, ondemand, prelude::*}; +#[cfg(feature = "serde")] +use serde::Deserialize; +#[cfg(feature = "serde")] +use simdjson_rust::serde::from_str; -fn main() -> simdjson_rust::Result<()> { - let ps = make_padded_string("[0,1,2,3]"); +#[cfg(feature = "serde")] +#[derive(Debug, Deserialize, PartialEq)] +struct Point { + x: i64, + y: i64, +} - // ondemand api. - { - let mut parser = ondemand::Parser::default(); - let mut doc = parser.iterate(&ps)?; - let mut array = doc.get_array()?; - for (index, value) in array.iter()?.enumerate() { - assert_eq!(index as u64, value?.get_uint64()?); - } - } +fn main() -> Result<(), Box> { + use simdjson_rust::dom::Parser; + + // 1. Parsing directly to our zero-copy DOM Value enum (Always available) + let mut parser = Parser::default(); + let val = parser.parse_to_value(r#"{"points": [10, 20], "name": "rust"}"#)?; + + let name = val.get("name").unwrap().as_str().unwrap(); + let points = val.get("points").unwrap().as_array().unwrap(); + println!("Name: {}", name); + println!( + "Points: [{}, {}]", + points[0].as_i64().unwrap(), + points[1].as_i64().unwrap() + ); - // dom api. + // 2. Direct zero-copy deserialization into custom structs via Serde + #[cfg(feature = "serde")] { - let mut parser = dom::Parser::default(); - let elem = parser.parse(&ps)?; - let arr = elem.get_array()?; - for (index, value) in arr.iter().enumerate() { - assert_eq!(index as u64, value.get_uint64()?); - } + let point: Point = from_str(r#"{"x": 100, "y": -200}"#)?; + println!("Deserialized point: {:?}", point); } Ok(()) diff --git a/justfile b/justfile new file mode 100644 index 0000000..9efde82 --- /dev/null +++ b/justfile @@ -0,0 +1,9 @@ + + +copy-compile-commands: + fd -IH compile_commands target/ -x cp {} simdjson-sys/ + +# Run the parser bench tuned for the local CPU. +# Requires `.cargo/config.toml` (copy from .cargo/config.toml.example). +bench: + cargo bench --bench parser_bench --features native diff --git a/mise.toml b/mise.toml new file mode 100644 index 0000000..b43e489 --- /dev/null +++ b/mise.toml @@ -0,0 +1,4 @@ +[tools] +cargo-release = "latest" +fd = "latest" +just = "latest" diff --git a/rustfmt.toml b/rustfmt.toml index 96d691f..c45153d 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -1,6 +1,6 @@ unstable_features = true -version = "Two" +style_edition = "2024" group_imports = "StdExternalCrate" imports_granularity = "Crate" diff --git a/simdjson-sys/CMakeLists.txt b/simdjson-sys/CMakeLists.txt new file mode 100644 index 0000000..df3dd70 --- /dev/null +++ b/simdjson-sys/CMakeLists.txt @@ -0,0 +1,19 @@ +cmake_minimum_required(VERSION 3.15) +project(simdjson_sys LANGUAGES CXX) + +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +include(FetchContent) + +fetchcontent_declare( + simdjson + GIT_REPOSITORY + https://github.com/simdjson/simdjson.git + GIT_TAG + v4.6.4 + GIT_SHALLOW + TRUE) +fetchcontent_makeavailable(simdjson) + +install(TARGETS simdjson ARCHIVE DESTINATION lib) +install(DIRECTORY ${simdjson_SOURCE_DIR}/include/ DESTINATION include) diff --git a/simdjson-sys/Cargo.toml b/simdjson-sys/Cargo.toml index 63efe30..e604f71 100644 --- a/simdjson-sys/Cargo.toml +++ b/simdjson-sys/Cargo.toml @@ -1,20 +1,27 @@ [package] name = "simdjson-sys" -version = "0.1.0-alpha.2" +version = "0.2.0-alpha.1" edition = "2021" -authors = ["SunDoge <384813529@qq.com>"] -license = "Apache-2.0" description = "Low level Rust bindings for simdjson." -homepage = "https://crates.io/crates/simdjson-sys" documentation = "https://docs.rs/simdjson-sys" -repository = "https://github.com/SunDoge/simdjson-rust/tree/master/simdjson-sys" readme = "README.md" +homepage = "https://crates.io/crates/simdjson-sys" +repository = "https://github.com/SunDoge/simdjson-rust/tree/master/simdjson-sys" +license = "Apache-2.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -exclude = ["simdjson/", "!simdjson/singleheader/simdjson.*"] +exclude = ["!simdjson/singleheader/simdjson.*", "simdjson/"] [dependencies] +cxx = "1.0.197" [build-dependencies] -bindgen = "0.66.1" -cc = { version = "1.0.83", features = ["parallel"] } +cmake = "0.1" +cxx-build = "1.0.197" + +[features] +# Compile the bundled simdjson (and C++ bridge) with `-march=native` for the +# building machine. Off by default so published builds stay portable; enable +# locally for maximum speed on your own CPU. +default = [] +native = [] diff --git a/simdjson-sys/build.rs b/simdjson-sys/build.rs index a9c00cb..04875f5 100644 --- a/simdjson-sys/build.rs +++ b/simdjson-sys/build.rs @@ -1,40 +1,27 @@ -use std::{env, path::PathBuf}; - fn main() { - cc::Build::new() - .cpp(true) - .flag_if_supported("-std=c++17") - .flag_if_supported("/std:c++20") // error C7555: use of designated initializers requires at least '/std:c++20' - .flag_if_supported("-pthread") - .flag_if_supported("-O3") - .flag_if_supported("/O2") - .flag_if_supported("-DNDEBUG") - .flag_if_supported("/DNDEBUG") - .include("simdjson/singleheader") - .file("src/simdjson_c_api.cpp") - .file("simdjson/singleheader/simdjson.cpp") - .cargo_metadata(true) - .compile("simdjson_c_api"); - - let bindings = bindgen::Builder::default() - // The input header we would like to generate - // bindings for. - .header("src/simdjson_c_api.h") - // Tell cargo to invalidate the built crate whenever any of the - // included header files changed. - .parse_callbacks(Box::new(bindgen::CargoCallbacks)) - // Finish the builder and generate the bindings. - .generate() - // Unwrap the Result and panic on failure. - .expect("Unable to generate bindings"); + let native = std::env::var_os("CARGO_FEATURE_NATIVE").is_some(); - println!("cargo:rerun-if-changed=src/simdjson_c_api.h"); - println!("cargo:rerun-if-changed=src/simdjson_c_api.cpp"); + let mut cmake_cfg = cmake::Config::new("."); + if native { + // Tune the C++ kernels for the building CPU. Off by default so that + // published builds remain portable across machines. + cmake_cfg.cxxflag("-march=native"); + } + let dst = cmake_cfg.build(); + let include_dir = dst.join("include"); + let lib_dir = dst.join("lib"); - // Write the bindings to the $OUT_DIR/bindings.rs file. - let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); + cxx_build::bridge("src/lib.rs") + .include("src") + .include(&include_dir) + .file("src/simdjson_dom_bridge.cpp") + .std("c++17") + .flag_if_supported("/std:c++20") // simdjson uses designated initializers on MSVC. + .compile("simdjson_sys_bridge"); - bindings - .write_to_file(out_path.join("bindings.rs")) - .expect("Couldn't write bindings!"); + println!("cargo:rustc-link-search=native={}", lib_dir.display()); + println!("cargo:rustc-link-lib=static=simdjson"); + println!("cargo:rerun-if-changed=src/simdjson_dom_bridge.h"); + println!("cargo:rerun-if-changed=src/simdjson_dom_bridge.cpp"); + println!("cargo:rerun-if-changed=CMakeLists.txt"); } diff --git a/simdjson-sys/src/lib.rs b/simdjson-sys/src/lib.rs index 705b3b1..ed66e0f 100644 --- a/simdjson-sys/src/lib.rs +++ b/simdjson-sys/src/lib.rs @@ -1,9 +1,71 @@ -#![allow(non_upper_case_globals)] -#![allow(non_camel_case_types)] -#![allow(non_snake_case)] +#[cxx::bridge(namespace = "simdjson_sys::dom")] +pub mod dom_ffi { + struct TapeView<'a> { + tape: &'a [u64], + string_buf: &'a [u8], + } -include!(concat!(env!("OUT_DIR"), "/bindings.rs")); + unsafe extern "C++" { + include!("simdjson.h"); + include!("simdjson_dom_bridge.h"); + + #[namespace = "simdjson::dom"] + type parser; + + fn parser_new(max_capacity: usize) -> UniquePtr; + fn parser_parse(parser: Pin<&mut parser>, json: &[u8], realloc_if_needed: bool) -> i32; + fn parser_get_tape_view(parser: &parser) -> TapeView<'_>; + } +} pub const SIMDJSON_PADDING: usize = 64; pub const SIMDJSON_MAXSIZE_BYTES: usize = 0xFFFFFFFF; pub const DEFAULT_BATCH_SIZE: usize = 1000000; + +#[cfg(test)] +mod tests { + use super::*; + + pub struct Parser { + inner: cxx::UniquePtr, + } + + impl Default for Parser { + fn default() -> Self { + Self::new(SIMDJSON_MAXSIZE_BYTES) + } + } + + impl Parser { + pub fn new(max_capacity: usize) -> Self { + Self { + inner: dom_ffi::parser_new(max_capacity), + } + } + + pub fn parse_string(&mut self, json: &mut String) -> i32 { + if json.capacity() < json.len() + SIMDJSON_PADDING { + json.reserve(SIMDJSON_PADDING); + } + + dom_ffi::parser_parse(self.inner.pin_mut(), json.as_bytes(), false) + } + + pub fn get_tape_view(&self) -> dom_ffi::TapeView<'_> { + dom_ffi::parser_get_tape_view(self.inner.as_ref().expect("parser must not be null")) + } + } + + #[test] + fn parser_parses_string_and_exposes_tape_view() { + let mut parser = Parser::default(); + let mut json = String::from(r#"{"answer":42,"message":"ok"}"#); + + assert_eq!(parser.parse_string(&mut json), 0); + + let view = parser.get_tape_view(); + assert!(!view.tape.is_empty()); + assert_eq!(view.tape[0] >> 56, u64::from(b'r')); + assert!(!view.string_buf.is_empty()); + } +} diff --git a/simdjson-sys/src/simdjson_dom_bridge.cpp b/simdjson-sys/src/simdjson_dom_bridge.cpp new file mode 100644 index 0000000..32a43f5 --- /dev/null +++ b/simdjson-sys/src/simdjson_dom_bridge.cpp @@ -0,0 +1,57 @@ +#include "simdjson_dom_bridge.h" + +#include "simdjson/dom/element.h" +#include "simdjson/dom/parser.h" + +namespace simdjson_sys::dom { + +namespace { + +constexpr uint64_t JSON_VALUE_MASK = 0x00FFFFFFFFFFFFFF; + +size_t roundup_n(size_t value, size_t n) { return (value + n - 1) & ~(n - 1); } + +size_t string_buf_capacity(size_t document_capacity) { + return roundup_n(5 * document_capacity / 3 + simdjson::SIMDJSON_PADDING, 64); +} + +} // namespace + +std::unique_ptr parser_new(std::size_t max_capacity) { + return std::make_unique(max_capacity); +} + +int32_t parser_parse(simdjson::dom::parser &parser, + rust::Slice json, + bool realloc_if_needed) { + simdjson::dom::element element; + const auto error = parser.parse(json.data(), json.size(), realloc_if_needed) + .get(element); + return static_cast(error); +} + +TapeView parser_get_tape_view(const simdjson::dom::parser &parser) { + const auto &doc = parser.doc; + if (!doc.tape || !doc.string_buf) { + return TapeView{ + rust::Slice(), + rust::Slice(), + }; + } + + const size_t tape_len = static_cast(doc.tape[0] & JSON_VALUE_MASK); + if (tape_len == 0) { + return TapeView{ + rust::Slice(), + rust::Slice(), + }; + } + + const size_t string_len = string_buf_capacity(doc.capacity()); + return TapeView{ + rust::Slice(doc.tape.get(), tape_len), + rust::Slice(doc.string_buf.get(), string_len), + }; +} + +} // namespace simdjson_sys::dom diff --git a/simdjson-sys/src/simdjson_dom_bridge.h b/simdjson-sys/src/simdjson_dom_bridge.h new file mode 100644 index 0000000..2f735c1 --- /dev/null +++ b/simdjson-sys/src/simdjson_dom_bridge.h @@ -0,0 +1,18 @@ +#pragma once + +#include "rust/cxx.h" + +#include +#include +#include + +#include "simdjson-sys/src/lib.rs.h" + +namespace simdjson_sys::dom { + +std::unique_ptr parser_new(std::size_t max_capacity); +int32_t parser_parse(simdjson::dom::parser &parser, + rust::Slice json, bool realloc_if_needed); +TapeView parser_get_tape_view(const simdjson::dom::parser &parser); + +} // namespace simdjson_sys::dom diff --git a/src/dom.rs b/src/dom.rs new file mode 100644 index 0000000..36cca19 --- /dev/null +++ b/src/dom.rs @@ -0,0 +1,188 @@ +//! High-level DOM parser wrapping `simdjson_sys`. +//! +//! This module provides a safe, ergonomic Rust API over the simdjson DOM +//! parser. After parsing, the result is available as a zero-copy +//! [`Value`](crate::tape::Value), or as a raw +//! [`TapeRef`](crate::tape::TapeRef) for custom traversal. + +use simdjson_sys::{SIMDJSON_PADDING, dom_ffi}; +use snafu::prelude::*; + +use crate::{error::SimdJsonError, tape::TapeRef}; + +/// The error type returned by the DOM parser. +#[derive(Debug, Snafu)] +#[snafu(visibility(pub(crate)))] +pub enum Error { + #[snafu(display("simdjson error: {source}"))] + SimdJson { source: SimdJsonError }, + + #[snafu(display( + "the JSON input buffer must have at least {} bytes of extra capacity", + SIMDJSON_PADDING + ))] + InsufficientPadding, + + #[snafu(display("tape error: {source}"))] + Tape { + #[snafu(backtrace)] + source: crate::tape::Error, + }, +} + +pub type Result = std::result::Result; + +impl From for Error { + fn from(source: crate::tape::Error) -> Self { + Error::Tape { source } + } +} + +/// A simdjson DOM parser. +/// +/// The parser owns an internal buffer. After calling one of the `parse*` +/// methods the tape and string buffer remain valid until the next parse call. +/// +/// # Example +/// +/// ```rust +/// use simdjson_rust::dom::Parser; +/// +/// let mut parser = Parser::default(); +/// let value = parser.parse_str(r#"{"hello": "world"}"#).unwrap(); +/// println!("{value:?}"); +/// ``` +pub struct Parser { + inner: cxx::UniquePtr, + /// Reuse the padded buffer to avoid reallocation on every parse. + padded: Vec, +} + +impl Default for Parser { + fn default() -> Self { + Self::new(simdjson_sys::SIMDJSON_MAXSIZE_BYTES) + } +} + +impl Parser { + /// Create a parser that can handle documents up to `max_capacity` bytes. + pub fn new(max_capacity: usize) -> Self { + Self { + inner: dom_ffi::parser_new(max_capacity), + padded: Vec::new(), + } + } + + /// Parse a JSON byte slice. + /// + /// The slice must have at least [`SIMDJSON_PADDING`] bytes of extra + /// capacity beyond its length. Use [`parse_str`](Self::parse_str) or + /// [`parse_padded`](Self::parse_padded) for convenience variants that + /// handle padding automatically. + /// + /// # Errors + /// + /// Returns [`Error::SimdJson`] if the parser rejects the input. + pub fn parse_bytes_with_padding(&mut self, json: &[u8]) -> Result> { + let code = dom_ffi::parser_parse(self.inner.pin_mut(), json, false); + SimdJsonError::check_code(code).context(SimdJsonSnafu)?; + Ok(self.tape_ref()) + } + + /// Parse a JSON string, automatically adding the required padding. + /// + /// The padding is added on a private copy of the string, so the original + /// `json` argument does not need to be mutable. + pub fn parse_str(&mut self, json: &str) -> Result> { + self.parse_bytes(json.as_bytes()) + } + + /// Parse a JSON byte slice, automatically adding the required padding. + pub fn parse_bytes(&mut self, json: &[u8]) -> Result> { + // simdjson reads up to SIMDJSON_PADDING bytes past the end of the + // slice pointer when realloc_if_needed=false. We allocate a buffer + // with that extra space, copy the JSON into it, zero the tail, then + // pass a slice of the original length. simdjson uses the slice length + // as the document length but reads the zero-padded tail unsafely. + let json_len = json.len(); + self.padded.clear(); + self.padded.reserve(json_len + SIMDJSON_PADDING); + self.padded.extend_from_slice(json); + // Zero-fill the SIMDJSON_PADDING extra bytes that simdjson will read. + self.padded + .extend(std::iter::repeat_n(0u8, SIMDJSON_PADDING)); + + // Pass only the JSON length; the zero tail is read past the slice end. + let code = dom_ffi::parser_parse(self.inner.pin_mut(), &self.padded[..json_len], false); + SimdJsonError::check_code(code).context(SimdJsonSnafu)?; + Ok(self.tape_ref()) + } + + /// Parse a JSON string that is already padded (has `>= SIMDJSON_PADDING` + /// bytes of extra space in its allocation). + /// + /// The string must have `s.capacity() >= s.len() + SIMDJSON_PADDING`. + pub fn parse_padded(&mut self, json: &mut String) -> Result> { + ensure!( + json.capacity() >= json.len() + SIMDJSON_PADDING, + InsufficientPaddingSnafu + ); + let code = dom_ffi::parser_parse(self.inner.pin_mut(), json.as_bytes(), false); + SimdJsonError::check_code(code).context(SimdJsonSnafu)?; + Ok(self.tape_ref()) + } + + /// Returns a reference to the current tape. + /// + /// This is only meaningful after a successful `parse*` call. + pub fn tape_ref(&self) -> TapeRef<'_> { + let view = + dom_ffi::parser_get_tape_view(self.inner.as_ref().expect("parser must not be null")); + TapeRef::new(view.tape, view.string_buf) + } + + /// Parse a JSON string and return a zero-copy `Value`. + pub fn parse_to_value(&mut self, json: &str) -> Result> { + let mut tape = self.parse_str(json)?; + Ok(tape.parse_value()?) + } + + /// Parse a JSON byte slice and return a zero-copy `Value`. + pub fn parse_bytes_to_value(&mut self, json: &[u8]) -> Result> { + let mut tape = self.parse_bytes(json)?; + Ok(tape.parse_value()?) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_null_tape() -> Result<()> { + let mut p = Parser::default(); + let tape = p.parse_str("null")?; + // First word is always the root 'r' node + assert_eq!(tape.tape_type(), Some(crate::tape::TapeType::Root)); + Ok(()) + } + + #[test] + fn parse_object_tape() -> Result<()> { + let mut p = Parser::default(); + let tape = p.parse_str(r#"{"answer":42}"#)?; + // First word is root 'r' + assert_eq!(tape.tape_type(), Some(crate::tape::TapeType::Root)); + Ok(()) + } + + #[test] + fn parse_to_value_works() -> Result<()> { + let mut p = Parser::default(); + let v = p.parse_to_value(r#"{"answer":42,"ok":true,"name":"rust"}"#)?; + assert_eq!(v.get("answer").unwrap().as_i64(), Some(42)); + assert_eq!(v.get("ok").unwrap().as_bool(), Some(true)); + assert_eq!(v.get("name").unwrap().as_str(), Some("rust")); + Ok(()) + } +} diff --git a/src/dom/array.rs b/src/dom/array.rs deleted file mode 100644 index 4cc50b3..0000000 --- a/src/dom/array.rs +++ /dev/null @@ -1,112 +0,0 @@ -use std::{marker::PhantomData, ptr::NonNull}; - -use simdjson_sys as ffi; - -use super::{document::Document, element::Element}; -use crate::{ - macros::{impl_drop, map_ptr_result}, - Result, -}; - -pub struct Array<'a> { - ptr: NonNull, - _doc: PhantomData<&'a Document>, -} - -impl<'a> Array<'a> { - pub fn new(ptr: NonNull) -> Self { - Self { - ptr, - _doc: PhantomData, - } - } - - pub fn iter(&self) -> ArrayIter { - let begin = unsafe { NonNull::new_unchecked(ffi::SJ_DOM_array_begin(self.ptr.as_ptr())) }; - let end = unsafe { NonNull::new_unchecked(ffi::SJ_DOM_array_end(self.ptr.as_ptr())) }; - ArrayIter::new(begin, end) - } - - pub fn size(&self) -> usize { - unsafe { ffi::SJ_DOM_array_size(self.ptr.as_ptr()) } - } - - pub fn number_of_slots(&self) -> usize { - unsafe { ffi::SJ_DOM_array_number_of_slots(self.ptr.as_ptr()) } - } - - pub fn at_pointer(&self, json_pointer: &str) -> Result> { - map_ptr_result!(ffi::SJ_DOM_array_at_pointer( - self.ptr.as_ptr(), - json_pointer.as_ptr().cast(), - json_pointer.len() - )) - .map(Element::new) - } - - pub fn at(&self, index: usize) -> Result> { - map_ptr_result!(ffi::SJ_DOM_array_at(self.ptr.as_ptr(), index)).map(Element::new) - } -} - -impl_drop!(Array<'a>, ffi::SJ_DOM_array_free); - -pub struct ArrayIter<'a> { - begin: NonNull, - end: NonNull, - running: bool, - _doc: PhantomData<&'a Document>, -} - -impl<'a> ArrayIter<'a> { - pub fn new( - begin: NonNull, - end: NonNull, - ) -> Self { - Self { - begin, - end, - running: false, - _doc: PhantomData, - } - } - - pub fn get(&self) -> Element<'a> { - let ptr = unsafe { ffi::SJ_DOM_array_iterator_get(self.begin.as_ptr()) }; - Element::new(unsafe { NonNull::new_unchecked(ptr) }) - } - - pub fn step(&mut self) { - unsafe { ffi::SJ_DOM_array_iterator_step(self.begin.as_ptr()) } - } - - pub fn not_equal(&self) -> bool { - unsafe { ffi::SJ_DOM_array_iterator_not_equal(self.begin.as_ptr(), self.end.as_ptr()) } - } -} - -impl<'a> Drop for ArrayIter<'a> { - fn drop(&mut self) { - unsafe { - ffi::SJ_DOM_array_iterator_free(self.begin.as_ptr()); - ffi::SJ_DOM_array_iterator_free(self.end.as_ptr()); - } - } -} - -impl<'a> Iterator for ArrayIter<'a> { - type Item = Element<'a>; - - fn next(&mut self) -> Option { - if self.running { - self.step(); - } - - if self.not_equal() { - self.running = true; - Some(self.get()) - } else { - None - } - } -} diff --git a/src/dom/document.rs b/src/dom/document.rs deleted file mode 100644 index 44f190c..0000000 --- a/src/dom/document.rs +++ /dev/null @@ -1,36 +0,0 @@ -use std::ptr::NonNull; - -use simdjson_sys as ffi; - -use super::Element; -use crate::macros::impl_drop; - -pub struct Document { - ptr: NonNull, -} - -impl Default for Document { - fn default() -> Self { - Self { - ptr: unsafe { NonNull::new_unchecked(ffi::SJ_DOM_document_new()) }, - } - } -} - -impl Document { - pub fn new(ptr: NonNull) -> Self { - Self { ptr } - } - - pub fn root(&self) -> Element<'_> { - Element::new(unsafe { - NonNull::new_unchecked(ffi::SJ_DOM_document_root(self.ptr.as_ptr())) - }) - } - - pub fn as_ptr(&self) -> *mut ffi::SJ_DOM_document { - self.ptr.as_ptr() - } -} - -impl_drop!(Document, ffi::SJ_DOM_document_free); diff --git a/src/dom/document_stream.rs b/src/dom/document_stream.rs deleted file mode 100644 index b326847..0000000 --- a/src/dom/document_stream.rs +++ /dev/null @@ -1,84 +0,0 @@ -use std::{marker::PhantomData, ptr::NonNull}; - -use simdjson_sys as ffi; - -use super::Element; -use crate::{ - macros::{impl_drop, map_ptr_result}, - Result, -}; - -pub struct DocumentStream { - ptr: NonNull, -} - -impl DocumentStream { - pub fn new(ptr: NonNull) -> Self { - Self { ptr } - } - - pub fn iter(&self) -> DocumentStreamIter { - let begin = - unsafe { NonNull::new_unchecked(ffi::SJ_DOM_document_stream_begin(self.ptr.as_ptr())) }; - let end = - unsafe { NonNull::new_unchecked(ffi::SJ_DOM_document_stream_end(self.ptr.as_ptr())) }; - DocumentStreamIter::new(begin, end) - } -} - -impl_drop!(DocumentStream, ffi::SJ_DOM_document_stream_free); - -pub struct DocumentStreamIter<'a> { - begin: NonNull, - end: NonNull, - running: bool, - _parser: PhantomData<&'a DocumentStream>, -} - -impl<'a> DocumentStreamIter<'a> { - pub fn new( - begin: NonNull, - end: NonNull, - ) -> Self { - Self { - begin, - end, - running: false, - _parser: PhantomData, - } - } - - pub fn get(&self) -> Result> { - map_ptr_result!(ffi::SJ_DOM_document_stream_iterator_get( - self.begin.as_ptr() - )) - .map(Element::new) - } - - pub fn step(&mut self) { - unsafe { ffi::SJ_DOM_document_stream_iterator_step(self.begin.as_ptr()) } - } - - pub fn not_equal(&self) -> bool { - unsafe { - ffi::SJ_DOM_document_stream_iterator_not_equal(self.begin.as_ptr(), self.end.as_ptr()) - } - } -} - -impl<'a> Iterator for DocumentStreamIter<'a> { - type Item = Result>; - - fn next(&mut self) -> Option { - if self.running { - self.step(); - } - - if self.not_equal() { - self.running = true; - Some(self.get()) - } else { - None - } - } -} diff --git a/src/dom/element.rs b/src/dom/element.rs deleted file mode 100644 index 15beade..0000000 --- a/src/dom/element.rs +++ /dev/null @@ -1,96 +0,0 @@ -use std::{marker::PhantomData, ptr::NonNull}; - -use simdjson_sys as ffi; - -use super::{array::Array, document::Document, object::Object}; -use crate::{ - macros::{impl_drop, map_primitive_result, map_ptr_result}, - utils::string_view_struct_to_str, - Result, -}; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum ElementType { - Array = '[' as _, - Object = '{' as _, - Int64 = 'l' as _, - UInt64 = 'u' as _, - Double = 'd' as _, - String = '"' as _, - Bool = 't' as _, - NullValue = 'n' as _, -} - -impl From for ElementType { - fn from(value: i32) -> Self { - match value as u8 as char { - '[' => Self::Array, - '{' => Self::Object, - 'l' => Self::Int64, - 'u' => Self::UInt64, - 'd' => Self::Double, - '"' => Self::String, - 't' => Self::Bool, - 'n' => Self::NullValue, - _ => unreachable!(), - } - } -} - -pub struct Element<'a> { - ptr: NonNull, - _doc: PhantomData<&'a Document>, -} - -impl<'a> Element<'a> { - pub fn new(ptr: NonNull) -> Self { - Self { - ptr, - _doc: PhantomData, - } - } - - pub fn get_type(&self) -> ElementType { - unsafe { ElementType::from(ffi::SJ_DOM_element_type(self.ptr.as_ptr())) } - } - - pub fn get_array(&self) -> Result { - map_ptr_result!(ffi::SJ_DOM_element_get_array(self.ptr.as_ptr())).map(Array::new) - } - - pub fn get_object(&self) -> Result { - map_ptr_result!(ffi::SJ_DOM_element_get_object(self.ptr.as_ptr())).map(Object::new) - } - - pub fn get_string(&self) -> Result<&'a str> { - map_primitive_result!(ffi::SJ_DOM_element_get_string(self.ptr.as_ptr())) - .map(string_view_struct_to_str) - } - - pub fn get_int64(&self) -> Result { - map_primitive_result!(ffi::SJ_DOM_element_get_int64(self.ptr.as_ptr())) - } - - pub fn get_uint64(&self) -> Result { - map_primitive_result!(ffi::SJ_DOM_element_get_uint64(self.ptr.as_ptr())) - } - - pub fn get_double(&self) -> Result { - map_primitive_result!(ffi::SJ_DOM_element_get_double(self.ptr.as_ptr())) - } - - pub fn get_bool(&self) -> Result { - map_primitive_result!(ffi::SJ_DOM_element_get_bool(self.ptr.as_ptr())) - } - - pub fn at_pointer(&self, json_pointer: &str) -> Result { - map_ptr_result!(ffi::SJ_DOM_element_at_pointer( - self.ptr.as_ptr(), - json_pointer.as_ptr().cast(), - json_pointer.len() - )) - .map(Element::new) - } -} - -impl_drop!(Element<'a>, ffi::SJ_DOM_element_free); diff --git a/src/dom/mod.rs b/src/dom/mod.rs deleted file mode 100644 index 0b8da53..0000000 --- a/src/dom/mod.rs +++ /dev/null @@ -1,13 +0,0 @@ -mod array; -mod document; -mod document_stream; -mod element; -mod object; -mod parser; - -pub use array::{Array, ArrayIter}; -pub use document::Document; -pub use document_stream::{DocumentStream, DocumentStreamIter}; -pub use element::{Element, ElementType}; -pub use object::{Object, ObjectIter}; -pub use parser::Parser; diff --git a/src/dom/object.rs b/src/dom/object.rs deleted file mode 100644 index aa25591..0000000 --- a/src/dom/object.rs +++ /dev/null @@ -1,90 +0,0 @@ -use std::{marker::PhantomData, ptr::NonNull}; - -use simdjson_sys as ffi; - -use super::{document::Document, Element}; -use crate::{macros::impl_drop, utils::string_view_struct_to_str}; - -pub struct Object<'a> { - ptr: NonNull, - _doc: PhantomData<&'a Document>, -} - -impl<'a> Object<'a> { - pub fn new(ptr: NonNull) -> Self { - Self { - ptr, - _doc: PhantomData, - } - } - - pub fn iter(&self) -> ObjectIter { - let begin = unsafe { NonNull::new_unchecked(ffi::SJ_DOM_object_begin(self.ptr.as_ptr())) }; - let end = unsafe { NonNull::new_unchecked(ffi::SJ_DOM_object_end(self.ptr.as_ptr())) }; - ObjectIter::new(begin, end) - } -} - -pub struct ObjectIter<'a> { - begin: NonNull, - end: NonNull, - running: bool, - _doc: PhantomData<&'a Document>, -} - -impl<'a> ObjectIter<'a> { - pub fn new( - begin: NonNull, - end: NonNull, - ) -> Self { - Self { - begin, - end, - running: false, - _doc: PhantomData, - } - } - - pub fn get(&self) -> (&'a str, Element<'a>) { - let kv = unsafe { ffi::SJ_DOM_object_iterator_get(self.begin.as_ptr()) }; - let key = string_view_struct_to_str(kv.key); - let value = Element::new(unsafe { NonNull::new_unchecked(kv.value) }); - (key, value) - } - - pub fn step(&mut self) { - unsafe { ffi::SJ_DOM_object_iterator_step(self.begin.as_ptr()) } - } - - pub fn not_equal(&self) -> bool { - unsafe { ffi::SJ_DOM_object_iterator_not_equal(self.begin.as_ptr(), self.end.as_ptr()) } - } -} - -impl<'a> Drop for ObjectIter<'a> { - fn drop(&mut self) { - unsafe { - ffi::SJ_DOM_object_iterator_free(self.begin.as_ptr()); - ffi::SJ_DOM_object_iterator_free(self.end.as_ptr()); - } - } -} - -impl<'a> Iterator for ObjectIter<'a> { - type Item = (&'a str, Element<'a>); - - fn next(&mut self) -> Option { - if self.running { - self.step(); - } - - if self.not_equal() { - self.running = true; - Some(self.get()) - } else { - None - } - } -} - -impl_drop!(Object<'a>, ffi::SJ_DOM_object_free); diff --git a/src/dom/parser.rs b/src/dom/parser.rs deleted file mode 100644 index 8b8baa0..0000000 --- a/src/dom/parser.rs +++ /dev/null @@ -1,101 +0,0 @@ -use std::ptr::NonNull; - -use ffi::DEFAULT_BATCH_SIZE; -use simdjson_sys as ffi; - -use super::{document::Document, document_stream::DocumentStream, element::Element}; -use crate::{ - macros::{impl_drop, map_ptr_result}, - Result, -}; - -pub struct Parser { - ptr: NonNull, -} - -impl Default for Parser { - fn default() -> Self { - Self::new(ffi::SIMDJSON_MAXSIZE_BYTES) - } -} - -impl Parser { - pub fn new(max_capacity: usize) -> Self { - let ptr = unsafe { NonNull::new_unchecked(ffi::SJ_DOM_parser_new(max_capacity)) }; - Self { ptr } - } - - pub fn parse(&mut self, padded_string: &String) -> Result { - map_ptr_result!(ffi::SJ_DOM_parser_parse( - self.ptr.as_ptr(), - padded_string.as_ptr().cast(), - padded_string.len() - )) - .map(Element::new) - } - - pub fn parse_into_document<'d>( - &self, - doc: &'d mut Document, - padded_string: &String, - ) -> Result> { - map_ptr_result!(ffi::SJ_DOM_parser_parse_into_document( - self.ptr.as_ptr(), - doc.as_ptr(), - padded_string.as_ptr().cast(), - padded_string.len() - )) - .map(Element::new) - } - - pub fn parse_many(&mut self, padded_string: &String) -> Result { - self.parse_batch(padded_string, DEFAULT_BATCH_SIZE) - } - - pub fn parse_batch( - &mut self, - padded_string: &String, - batch_size: usize, - ) -> Result { - map_ptr_result!(ffi::SJ_DOM_parser_parse_many( - self.ptr.as_ptr(), - padded_string.as_ptr().cast(), - padded_string.len(), - batch_size - )) - .map(DocumentStream::new) - } -} - -impl_drop!(Parser, ffi::SJ_DOM_parser_free); - -#[cfg(test)] -mod tests { - use super::*; - use crate::prelude::*; - - #[test] - fn parse() { - let ps = "1".to_padded_string(); - let mut parser = Parser::default(); - let elem = parser.parse(&ps).unwrap(); - assert_eq!(elem.get_uint64().unwrap(), 1); - } - - #[test] - fn parse_into_document() { - let ps = "[1,2,3]".to_padded_string(); - let parser = Parser::default(); - let mut doc = Document::default(); - let elem = parser.parse_into_document(&mut doc, &ps).unwrap(); - assert_eq!( - elem.get_array() - .unwrap() - .at(0) - .unwrap() - .get_uint64() - .unwrap(), - 1 - ); - } -} diff --git a/src/error.rs b/src/error.rs index 3fdba0e..3fb0e2c 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,144 +1,123 @@ -use thiserror::Error; +//! Simdjson error code translations and strong type definitions. -pub type Result = std::result::Result; +use snafu::prelude::*; -#[derive(Debug, Error)] +/// Strongly typed error enum representing C++ simdjson error codes. +#[derive(Debug, Snafu, Clone, Copy, PartialEq, Eq)] pub enum SimdJsonError { - #[error("This parser can't support a document that big")] + #[snafu(display("CAPACITY: This parser can't support a document that big"))] Capacity, - #[error("Error allocating memory, we're most likely out of memory")] + #[snafu(display("MEMALLOC: Memory allocation failure"))] MemAlloc, - #[error("Something went wrong while writing to the tape")] + #[snafu(display("TAPE_ERROR: The JSON is so badly formed we couldn't build a tape"))] TapeError, - #[error("The JSON document was too deep (too many nested objects and arrays)")] + #[snafu(display("DEPTH_ERROR: Your document exceeds the maximum depth"))] DepthError, - #[error("Problem while parsing a string")] + #[snafu(display("STRING_ERROR: Problem while parsing a string"))] StringError, - #[error("Problem while parsing an atom starting with the letter 't'")] + #[snafu(display("T_ATOM_ERROR: Problem while parsing an atom starting with 't'"))] TAtomError, - #[error("Problem while parsing an atom starting with the letter 'f'")] + #[snafu(display("F_ATOM_ERROR: Problem while parsing an atom starting with 'f'"))] FAtomError, - #[error("Problem while parsing an atom starting with the letter 'n'")] + #[snafu(display("N_ATOM_ERROR: Problem while parsing an atom starting with 'n'"))] NAtomError, - #[error("Problem while parsing a number")] + #[snafu(display("NUMBER_ERROR: Problem while parsing a number"))] NumberError, - #[error("The input is not valid UTF-8")] + #[snafu(display("UTF8_ERROR: The input is not valid UTF-8"))] Utf8Error, - #[error("Uninitialized")] + #[snafu(display("UNINITIALIZED: Uninitialized or an empty parser"))] Uninitialized, - #[error("Empty: no JSON found")] + #[snafu(display("EMPTY: No structural element found"))] Empty, - #[error("Within strings, some characters must be escaped, we found unescaped characters")] + #[snafu(display("UNESCAPED_CHARS: Found unescaped characters in a string"))] UnescapedChars, - #[error("A string is opened, but never closed.")] + #[snafu(display("UNCLOSED_STRING: Unclosed string"))] UnclosedString, - #[error( - "simdjson does not have an implementation supported by this CPU architecture (perhaps \ - it's a non-SIMD CPU?)." - )] + #[snafu(display("UNSUPPORTED_ARCHITECTURE: Unsupported architecture"))] UnsupportedArchitecture, - #[error("The JSON element does not have the requested type.")] + #[snafu(display("INCORRECT_TYPE: Element has the wrong type for this operation"))] IncorrectType, - #[error("The JSON number is too large or too small to fit within the requested type.")] + #[snafu(display("NUMBER_OUT_OF_RANGE: Number is too large or too small"))] NumberOutOfRange, - #[error("Attempted to access an element of a JSON array that is beyond its length.")] + #[snafu(display("INDEX_OUT_OF_BOUNDS: Array index is too large"))] IndexOutOfBounds, - #[error("The JSON field referenced does not exist in this object.")] + #[snafu(display("NO_SUCH_FIELD: The key was not found in the object"))] NoSuchField, - #[error("Error reading the file.")] + #[snafu(display("IO_ERROR: Error reading file"))] IoError, - #[error("Invalid JSON pointer syntax.")] + #[snafu(display("INVALID_JSON_POINTER: Invalid JSON pointer syntax"))] InvalidJsonPointer, - #[error("Invalid URI fragment syntax.")] + #[snafu(display("INVALID_URI_FRAGMENT: Fragment is not valid"))] InvalidUriFragment, - #[error("todo")] + #[snafu(display("UNEXPECTED_ERROR: Something went wrong, this is a bug in simdjson"))] UnexpectedError, - #[error("todo")] - ParserInUse, - - #[error("todo")] - OutOfOrderIteration, - - #[error("todo")] - InsufficientPadding, - - #[error("todo")] - IncompleteArrayOrObject, - - #[error("todo")] - ScalarDocumentAsValue, - - #[error("todo")] - OutOfBounds, - - #[error("todo")] - TailingContent, - - #[error("todo")] - NumErrorCodes, - - #[error("todo")] - StdIoError(#[from] std::io::Error), + #[snafu(display("UNKNOWN_ERROR: Unknown error code {code}"))] + Unknown { code: i32 }, } -impl From for SimdJsonError { - fn from(error_code: i32) -> Self { - match error_code { - 1 => SimdJsonError::Capacity, - 2 => SimdJsonError::MemAlloc, - 3 => SimdJsonError::TapeError, - 4 => SimdJsonError::DepthError, - 5 => SimdJsonError::StringError, - 6 => SimdJsonError::TAtomError, - 7 => SimdJsonError::FAtomError, - 8 => SimdJsonError::NAtomError, - 9 => SimdJsonError::NumberError, - 10 => SimdJsonError::Utf8Error, - 11 => SimdJsonError::Uninitialized, - 12 => SimdJsonError::Empty, - 13 => SimdJsonError::UnescapedChars, - 14 => SimdJsonError::UnclosedString, - 15 => SimdJsonError::UnsupportedArchitecture, - 16 => SimdJsonError::IncorrectType, - 17 => SimdJsonError::NumberOutOfRange, - 18 => SimdJsonError::IndexOutOfBounds, - 19 => SimdJsonError::NoSuchField, - 20 => SimdJsonError::IoError, - 21 => SimdJsonError::InvalidJsonPointer, - 22 => SimdJsonError::InvalidUriFragment, - 23 => SimdJsonError::UnexpectedError, - 24 => SimdJsonError::ParserInUse, - 25 => SimdJsonError::OutOfOrderIteration, - 26 => SimdJsonError::InsufficientPadding, - 27 => SimdJsonError::IncompleteArrayOrObject, - 28 => SimdJsonError::ScalarDocumentAsValue, - 29 => SimdJsonError::OutOfBounds, - 30 => SimdJsonError::TailingContent, - 31 => SimdJsonError::NumErrorCodes, - x => panic!("Unknown error code: {}", x), +impl SimdJsonError { + /// Convert a raw C++ simdjson error code into a strongly typed + /// `SimdJsonError`. + /// + /// If the code is `0` (SUCCESS), this returns `None`. For all other codes, + /// this returns `Some(SimdJsonError)`. + pub fn from_code(code: i32) -> Option { + match code { + 0 => None, + 1 => Some(Self::Capacity), + 2 => Some(Self::MemAlloc), + 3 => Some(Self::TapeError), + 4 => Some(Self::DepthError), + 5 => Some(Self::StringError), + 6 => Some(Self::TAtomError), + 7 => Some(Self::FAtomError), + 8 => Some(Self::NAtomError), + 9 => Some(Self::NumberError), + 10 => Some(Self::Utf8Error), + 11 => Some(Self::Uninitialized), + 12 => Some(Self::Empty), + 13 => Some(Self::UnescapedChars), + 14 => Some(Self::UnclosedString), + 15 => Some(Self::UnsupportedArchitecture), + 16 => Some(Self::IncorrectType), + 17 => Some(Self::NumberOutOfRange), + 18 => Some(Self::IndexOutOfBounds), + 19 => Some(Self::NoSuchField), + 20 => Some(Self::IoError), + 21 => Some(Self::InvalidJsonPointer), + 22 => Some(Self::InvalidUriFragment), + 23 => Some(Self::UnexpectedError), + other => Some(Self::Unknown { code: other }), + } + } + + pub fn check_code(code: i32) -> Result<(), Self> { + match Self::from_code(code) { + Some(err) => Err(err), + None => Ok(()), } } } diff --git a/src/lib.rs b/src/lib.rs index bdf072e..46c0fed 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,16 +1,45 @@ -mod macros; +//! # simdjson-rust +//! +//! High-performance JSON parsing for Rust, backed by the +//! [simdjson](https://github.com/simdjson/simdjson) C++ library. +//! +//! ## Quick Start +//! +//! ```rust +//! use simdjson_rust::dom::Parser; +//! +//! let mut parser = Parser::default(); +//! let value = parser +//! .parse_to_value(r#"{"hello": "world", "n": 42}"#) +//! .unwrap(); +//! println!("{value:?}"); +//! ``` +//! +//! ## Serde Support +//! +//! With the default `serde` feature you can deserialize directly into any +//! `serde::Deserialize` type: +//! +//! ```rust,ignore +//! use serde::Deserialize; +//! use simdjson_rust::serde::from_str; +//! +//! #[derive(Deserialize, Debug)] +//! struct Config { +//! name: String, +//! value: u64, +//! } +//! +//! let cfg: Config = from_str(r#"{"name": "example", "value": 99}"#).unwrap(); +//! println!("{cfg:?}"); +//! ``` pub mod dom; -mod error; -pub mod ondemand; -pub mod padded_string; -pub mod prelude; -mod utils; +pub mod error; +pub mod tape; -pub use error::{Result, SimdJsonError}; -pub use simdjson_sys::{SIMDJSON_MAXSIZE_BYTES, SIMDJSON_PADDING}; +#[cfg(feature = "serde")] +pub mod serde; -// pub mod serde; - -#[cfg(test)] -mod tests {} +/// Re-export the simdjson-sys padding constant. +pub use simdjson_sys::SIMDJSON_PADDING; diff --git a/src/macros.rs b/src/macros.rs deleted file mode 100644 index 9090b5f..0000000 --- a/src/macros.rs +++ /dev/null @@ -1,76 +0,0 @@ -macro_rules! impl_drop { - ($name:ident < $($lt:lifetime),+ > , $free_fn:expr) => { - impl<$($lt),+> Drop for $name<$($lt),+> { - fn drop(&mut self) { - unsafe { - $free_fn(self.ptr.as_ptr()); - } - } - } - }; - ($name:ty, $free_fn:expr) => { - impl Drop for $name { - fn drop(&mut self) { - unsafe { - $free_fn(self.ptr.as_ptr()); - } - } - } - }; -} - -macro_rules! map_result { - (primitive, $func_call:expr, $get_code:expr, $get_value:expr) => { - unsafe { - let ptr = $func_call; - let code = $get_code(ptr); - if code == 0 { - Ok($get_value(ptr)) - } else { - Err(crate::error::SimdJsonError::from(code)) - } - } - }; - ($func_call:expr, $get_code:expr, $get_value:expr) => { - unsafe { - let ptr = $func_call; - let code = $get_code(ptr); - if code == 0 { - Ok(std::ptr::NonNull::new_unchecked($get_value(ptr))) - } else { - Err(crate::error::SimdJsonError::from(code)) - } - } - }; -} - -macro_rules! map_ptr_result { - ($call:expr) => { - unsafe { - let res = $call; - if res.error == 0 { - Ok(std::ptr::NonNull::new_unchecked(res.value)) - } else { - Err(crate::error::SimdJsonError::from(res.error)) - } - } - }; -} - -macro_rules! map_primitive_result { - ($call:expr) => { - unsafe { - let res = $call; - if res.error == 0 { - Ok(res.value) - } else { - Err(crate::error::SimdJsonError::from(res.error)) - } - } - }; -} - -pub(crate) use impl_drop; -pub(crate) use map_primitive_result; -pub(crate) use map_ptr_result; -pub(crate) use map_result; diff --git a/src/ondemand/array.rs b/src/ondemand/array.rs deleted file mode 100644 index 6afafc8..0000000 --- a/src/ondemand/array.rs +++ /dev/null @@ -1,94 +0,0 @@ -use std::{marker::PhantomData, ptr::NonNull}; - -use simdjson_sys as ffi; - -use super::{array_iterator::ArrayIterator, document::Document, value::Value}; -use crate::{ - error::Result, - macros::{impl_drop, map_result}, - utils::string_view_to_str, -}; - -pub struct Array<'a> { - ptr: NonNull, - _doc: PhantomData<&'a mut Document<'a, 'a>>, -} - -impl<'a> Array<'a> { - pub fn new(ptr: NonNull) -> Self { - Self { - ptr, - _doc: PhantomData, - } - } - - pub fn count_elements(&mut self) -> Result { - map_result!( - primitive, - ffi::SJ_OD_array_count_elements(self.ptr.as_mut()), - ffi::size_t_result_error, - ffi::size_t_result_value_unsafe - ) - } - - pub fn is_empty(&mut self) -> Result { - map_result!( - primitive, - ffi::SJ_OD_array_is_empty(self.ptr.as_mut()), - ffi::bool_result_error, - ffi::bool_result_value_unsafe - ) - } - - pub fn at(&mut self, index: usize) -> Result> { - map_result!( - ffi::SJ_OD_array_at(self.ptr.as_mut(), index), - ffi::SJ_OD_value_result_error, - ffi::SJ_OD_value_result_value_unsafe - ) - .map(Value::new) - } - - pub fn at_pointer(&mut self, key: &str) -> Result> { - map_result!( - ffi::SJ_OD_array_at_pointer(self.ptr.as_mut(), key.as_ptr().cast(), key.len()), - ffi::SJ_OD_value_result_error, - ffi::SJ_OD_value_result_value_unsafe - ) - .map(Value::new) - } - - pub fn reset(&mut self) -> Result { - map_result!( - primitive, - ffi::SJ_OD_array_reset(self.ptr.as_mut()), - ffi::bool_result_error, - ffi::bool_result_value_unsafe - ) - } - - pub fn iter(&mut self) -> Result { - let begin = map_result!( - ffi::SJ_OD_array_begin(self.ptr.as_mut()), - ffi::SJ_OD_array_iterator_result_error, - ffi::SJ_OD_array_iterator_result_value_unsafe - )?; - let end = map_result!( - ffi::SJ_OD_array_end(self.ptr.as_mut()), - ffi::SJ_OD_array_iterator_result_error, - ffi::SJ_OD_array_iterator_result_value_unsafe - )?; - Ok(ArrayIterator::new(begin, end)) - } - - pub fn raw_json(&mut self) -> Result<&'a str> { - let sv = map_result!( - ffi::SJ_OD_array_raw_json(self.ptr.as_mut()), - ffi::STD_string_view_result_error, - ffi::STD_string_view_result_value_unsafe - )?; - Ok(string_view_to_str(sv)) - } -} - -impl_drop!(Array<'a>, ffi::SJ_OD_array_free); diff --git a/src/ondemand/array_iterator.rs b/src/ondemand/array_iterator.rs deleted file mode 100644 index 06095ce..0000000 --- a/src/ondemand/array_iterator.rs +++ /dev/null @@ -1,89 +0,0 @@ -use std::{marker::PhantomData, ptr::NonNull}; - -use simdjson_sys as ffi; - -use super::{document::Document, value::Value}; -use crate::{error::Result, macros::map_result}; - -pub struct ArrayIterator<'a> { - begin: NonNull, - end: NonNull, - running: bool, - _doc: PhantomData<&'a mut Document<'a, 'a>>, -} - -impl<'a> ArrayIterator<'a> { - pub fn new( - begin: NonNull, - end: NonNull, - ) -> Self { - Self { - begin, - end, - running: false, - _doc: PhantomData, - } - } - - pub fn get(&mut self) -> Result> { - map_result!( - ffi::SJ_OD_array_iterator_get(self.begin.as_mut()), - ffi::SJ_OD_value_result_error, - ffi::SJ_OD_value_result_value_unsafe - ) - .map(Value::new) - } - - pub fn not_equal(&self) -> bool { - unsafe { ffi::SJ_OD_array_iterator_not_equal(self.begin.as_ref(), self.end.as_ref()) } - } - - pub fn step(&mut self) { - unsafe { ffi::SJ_OD_array_iterator_step(self.begin.as_mut()) } - } -} - -impl<'a> Drop for ArrayIterator<'a> { - fn drop(&mut self) { - unsafe { - ffi::SJ_OD_array_iterator_free(self.begin.as_mut()); - ffi::SJ_OD_array_iterator_free(self.end.as_mut()); - } - } -} - -impl<'a> Iterator for ArrayIterator<'a> { - type Item = Result>; - - fn next(&mut self) -> Option { - if self.running { - self.step(); - } - - if self.not_equal() { - self.running = true; - Some(self.get()) - } else { - self.running = false; - None - } - } -} - -#[cfg(test)] -mod tests { - use crate::{ondemand::parser::Parser, padded_string::make_padded_string}; - - #[test] - fn test_iter() { - let mut parser = Parser::default(); - let ps = make_padded_string("[1,2,3]"); - let mut doc = parser.iterate(&ps).unwrap(); - // drop(ps); - let mut arr = doc.get_array().unwrap(); - for (v, num) in arr.iter().unwrap().zip([1u64, 2, 3]) { - let res = v.unwrap().get_uint64().unwrap(); - assert_eq!(res, num); - } - } -} diff --git a/src/ondemand/document.rs b/src/ondemand/document.rs deleted file mode 100644 index 9953d87..0000000 --- a/src/ondemand/document.rs +++ /dev/null @@ -1,175 +0,0 @@ -use std::{marker::PhantomData, ptr::NonNull}; - -use simdjson_sys as ffi; - -use super::{array::Array, number::Number, object::Object, parser::Parser, value::Value, JsonType}; -use crate::{ - error::Result, - macros::{impl_drop, map_result}, - utils::string_view_to_str, -}; - -pub struct Document<'p, 's> { - ptr: NonNull, - _parser: PhantomData<&'p mut Parser>, - _padded_string: PhantomData<&'s String>, -} -impl<'p, 's> Document<'p, 's> { - pub fn new(ptr: NonNull) -> Self { - Self { - ptr, - _parser: PhantomData, - _padded_string: PhantomData, - } - } - - pub fn get_uint64(&mut self) -> Result { - map_result!( - primitive, - ffi::SJ_OD_document_get_uint64(self.ptr.as_mut()), - ffi::uint64_t_result_error, - ffi::uint64_t_result_value_unsafe - ) - } - - pub fn get_int64(&mut self) -> Result { - map_result!( - primitive, - ffi::SJ_OD_document_get_int64(self.ptr.as_mut()), - ffi::int64_t_result_error, - ffi::int64_t_result_value_unsafe - ) - } - - pub fn get_bool(&mut self) -> Result { - map_result!( - primitive, - ffi::SJ_OD_document_get_bool(self.ptr.as_mut()), - ffi::bool_result_error, - ffi::bool_result_value_unsafe - ) - } - - pub fn get_double(&mut self) -> Result { - map_result!( - primitive, - ffi::SJ_OD_document_get_double(self.ptr.as_mut()), - ffi::double_result_error, - ffi::double_result_value_unsafe - ) - } - - pub fn get_value<'a>(&mut self) -> Result> { - map_result!( - ffi::SJ_OD_document_get_value(self.ptr.as_mut()), - ffi::SJ_OD_value_result_error, - ffi::SJ_OD_value_result_value_unsafe - ) - .map(Value::new) - } - - pub fn get_array<'a>(&mut self) -> Result> { - map_result!( - ffi::SJ_OD_document_get_array(self.ptr.as_mut()), - ffi::SJ_OD_array_result_error, - ffi::SJ_OD_array_result_value_unsafe - ) - .map(Array::new) - } - - pub fn get_object<'a>(&mut self) -> Result> { - map_result!( - ffi::SJ_OD_document_get_object(self.ptr.as_mut()), - ffi::SJ_OD_object_result_error, - ffi::SJ_OD_object_result_value_unsafe - ) - .map(Object::new) - } - - pub fn get_wobbly_string<'a>(&mut self) -> Result<&'a str> { - let sv = map_result!( - ffi::SJ_OD_document_get_wobbly_string(self.ptr.as_mut()), - ffi::STD_string_view_result_error, - ffi::STD_string_view_result_value_unsafe - )?; - Ok(string_view_to_str(sv)) - } - - pub fn get_string<'a>(&mut self) -> Result<&'a str> { - let sv = map_result!( - ffi::SJ_OD_document_get_string(self.ptr.as_mut()), - ffi::STD_string_view_result_error, - ffi::STD_string_view_result_value_unsafe - )?; - Ok(string_view_to_str(sv)) - } - - pub fn at_pointer<'a>(&mut self, json_pointer: &str) -> Result> { - map_result!( - ffi::SJ_OD_document_at_pointer( - self.ptr.as_mut(), - json_pointer.as_ptr().cast(), - json_pointer.len() - ), - ffi::SJ_OD_value_result_error, - ffi::SJ_OD_value_result_value_unsafe - ) - .map(Value::new) - } - - pub fn get_number<'a>(&mut self) -> Result> { - map_result!( - ffi::SJ_OD_document_get_number(self.ptr.as_mut()), - ffi::SJ_OD_number_result_error, - ffi::SJ_OD_number_result_value_unsafe - ) - .map(Number::new) - } - - pub fn is_null(&mut self) -> Result { - map_result!( - primitive, - ffi::SJ_OD_document_is_null(self.ptr.as_mut()), - ffi::bool_result_error, - ffi::bool_result_value_unsafe - ) - } - - pub fn json_type(&mut self) -> Result { - let json_type = map_result!( - primitive, - ffi::SJ_OD_document_type(self.ptr.as_mut()), - ffi::int_result_error, - ffi::int_result_value_unsafe - )?; - Ok(JsonType::from(json_type)) - } -} - -impl_drop!(Document<'p, 's>, ffi::SJ_OD_document_free); - -#[cfg(test)] -mod tests { - use crate::{ondemand, prelude::*}; - - #[test] - fn get_bool() { - let mut parser = ondemand::Parser::default(); - - { - let json = "true".to_padded_string(); - let mut doc = parser.iterate(&json).unwrap(); - assert!(doc.get_bool().unwrap()); - } - { - let json = "false".to_padded_string(); - let mut doc = parser.iterate(&json).unwrap(); - assert!(!doc.get_bool().unwrap()); - } - { - let json = "1".to_padded_string(); - let mut doc = parser.iterate(&json).unwrap(); - assert!(doc.get_bool().is_err()); - } - } -} diff --git a/src/ondemand/field.rs b/src/ondemand/field.rs deleted file mode 100644 index cf65733..0000000 --- a/src/ondemand/field.rs +++ /dev/null @@ -1,53 +0,0 @@ -use std::{marker::PhantomData, ptr::NonNull}; - -use simdjson_sys as ffi; - -use super::{document::Document, value::Value}; -use crate::{ - error::Result, - macros::{impl_drop, map_result}, - utils::string_view_to_str, -}; - -pub struct Field<'a> { - ptr: NonNull, - _doc: PhantomData<&'a mut Document<'a, 'a>>, -} - -impl<'a> Field<'a> { - pub fn new(ptr: NonNull) -> Self { - Self { - ptr, - _doc: PhantomData, - } - } - - pub fn unescaped_key(&mut self, allow_replacement: bool) -> Result<&'a str> { - let sv = map_result!( - ffi::SJ_OD_field_unescaped_key(self.ptr.as_mut(), allow_replacement), - ffi::STD_string_view_result_error, - ffi::STD_string_view_result_value_unsafe - )?; - Ok(string_view_to_str(sv)) - } - - // Double free error. - // pub fn value(&mut self) -> Value { - // let ptr = unsafe { - // let ptr = ffi::SJ_OD_field_value(self.ptr.as_mut()); - // NonNull::new_unchecked(ptr) - // }; - // Value::new(ptr) - // } - - pub fn take_value(self) -> Value<'a> { - let ptr = unsafe { - let ptr = ffi::SJ_OD_field_take_value(self.ptr.as_ptr()); - NonNull::new_unchecked(ptr) - }; - - Value::new(ptr) - } -} - -impl_drop!(Field<'a>, ffi::SJ_OD_field_free); diff --git a/src/ondemand/json_type.rs b/src/ondemand/json_type.rs deleted file mode 100644 index 52a24f6..0000000 --- a/src/ondemand/json_type.rs +++ /dev/null @@ -1,41 +0,0 @@ -#[derive(Debug, PartialEq)] -pub enum JsonType { - Array = 1, - Object, - Number, - String, - Boolean, - Null, -} - -impl From for JsonType { - fn from(value: i32) -> Self { - match value { - 1 => JsonType::Array, - 2 => JsonType::Object, - 3 => JsonType::Number, - 4 => JsonType::String, - 5 => JsonType::Boolean, - 6 => JsonType::Null, - _ => panic!("Invalid JsonType value: {}", value), - } - } -} - -#[derive(Debug, PartialEq)] -pub enum NumberType { - FloatingPointNumber = 1, - SignedInteger, - UnsignedInteger, -} - -impl From for NumberType { - fn from(value: i32) -> Self { - match value { - 1 => NumberType::FloatingPointNumber, - 2 => NumberType::SignedInteger, - 3 => NumberType::UnsignedInteger, - _ => panic!("Invalid NumberType value: {}", value), - } - } -} diff --git a/src/ondemand/mod.rs b/src/ondemand/mod.rs deleted file mode 100644 index 8f646a6..0000000 --- a/src/ondemand/mod.rs +++ /dev/null @@ -1,21 +0,0 @@ -mod array; -mod array_iterator; -mod document; -mod field; -mod json_type; -mod number; -mod object; -mod object_iterator; -mod parser; -mod value; - -pub use array::Array; -pub use array_iterator::ArrayIterator; -pub use document::Document; -pub use field::Field; -pub use json_type::{JsonType, NumberType}; -pub use number::Number; -pub use object::Object; -pub use object_iterator::ObjectIterator; -pub use parser::Parser; -pub use value::Value; diff --git a/src/ondemand/number.rs b/src/ondemand/number.rs deleted file mode 100644 index b9814cd..0000000 --- a/src/ondemand/number.rs +++ /dev/null @@ -1,38 +0,0 @@ -use std::{marker::PhantomData, ptr::NonNull}; - -use simdjson_sys as ffi; - -use super::{Document, NumberType}; -use crate::macros::impl_drop; - -pub struct Number<'a> { - ptr: NonNull, - _doc: PhantomData<&'a mut Document<'a, 'a>>, -} - -impl<'a> Number<'a> { - pub fn new(ptr: NonNull) -> Self { - Self { - ptr, - _doc: PhantomData, - } - } - - pub fn get_uint64(&mut self) -> u64 { - unsafe { ffi::SJ_OD_number_get_uint64(self.ptr.as_mut()) } - } - - pub fn get_int64(&mut self) -> i64 { - unsafe { ffi::SJ_OD_number_get_int64(self.ptr.as_mut()) } - } - - pub fn get_double(&mut self) -> f64 { - unsafe { ffi::SJ_OD_number_get_double(self.ptr.as_mut()) } - } - - pub fn get_number_type(&mut self) -> NumberType { - unsafe { ffi::SJ_OD_number_get_number_type(self.ptr.as_mut()) }.into() - } -} - -impl_drop!(Number<'a>, ffi::SJ_OD_number_free); diff --git a/src/ondemand/object.rs b/src/ondemand/object.rs deleted file mode 100644 index db552ae..0000000 --- a/src/ondemand/object.rs +++ /dev/null @@ -1,111 +0,0 @@ -use std::{marker::PhantomData, ptr::NonNull}; - -use simdjson_sys as ffi; - -use super::{document::Document, object_iterator::ObjectIterator, value::Value}; -use crate::{ - error::Result, - macros::{impl_drop, map_result}, - utils::string_view_to_str, -}; - -pub struct Object<'a> { - ptr: NonNull, - _doc: PhantomData<&'a mut Document<'a, 'a>>, -} - -impl<'a> Object<'a> { - pub fn new(ptr: NonNull) -> Self { - Self { - ptr, - _doc: PhantomData, - } - } - - pub fn at_pointer(&mut self, json_pointer: &str) -> Result> { - map_result!( - ffi::SJ_OD_object_at_pointer( - self.ptr.as_mut(), - json_pointer.as_ptr().cast(), - json_pointer.len() - ), - ffi::SJ_OD_value_result_error, - ffi::SJ_OD_value_result_value_unsafe - ) - .map(Value::new) - } - - pub fn iter(&mut self) -> Result> { - let begin = map_result!( - ffi::SJ_OD_object_begin(self.ptr.as_mut()), - ffi::SJ_OD_object_iterator_result_error, - ffi::SJ_OD_object_iterator_result_value_unsafe - )?; - let end = map_result!( - ffi::SJ_OD_object_end(self.ptr.as_mut()), - ffi::SJ_OD_object_iterator_result_error, - ffi::SJ_OD_object_iterator_result_value_unsafe - )?; - Ok(ObjectIterator::new(begin, end)) - } - - pub fn raw_json(&mut self) -> Result<&'a str> { - let sv = map_result!( - ffi::SJ_OD_object_raw_json(self.ptr.as_mut()), - ffi::STD_string_view_result_error, - ffi::STD_string_view_result_value_unsafe - )?; - Ok(string_view_to_str(sv)) - } - - pub fn find_field(&mut self, key: &str) -> Result> { - map_result!( - ffi::SJ_OD_object_find_field(self.ptr.as_mut(), key.as_ptr().cast(), key.len()), - ffi::SJ_OD_value_result_error, - ffi::SJ_OD_value_result_value_unsafe - ) - .map(Value::new) - } - - pub fn find_field_unordered(&mut self, key: &str) -> Result> { - map_result!( - ffi::SJ_OD_object_find_field_unordered( - self.ptr.as_mut(), - key.as_ptr().cast(), - key.len() - ), - ffi::SJ_OD_value_result_error, - ffi::SJ_OD_value_result_value_unsafe - ) - .map(Value::new) - } - - pub fn count_fields(&mut self) -> Result { - map_result!( - primitive, - ffi::SJ_OD_object_count_fields(self.ptr.as_mut()), - ffi::size_t_result_error, - ffi::size_t_result_value_unsafe - ) - } - - pub fn is_empty(&mut self) -> Result { - map_result!( - primitive, - ffi::SJ_OD_object_is_empty(self.ptr.as_mut()), - ffi::bool_result_error, - ffi::bool_result_value_unsafe - ) - } - - pub fn reset(&mut self) -> Result { - map_result!( - primitive, - ffi::SJ_OD_object_reset(self.ptr.as_mut()), - ffi::bool_result_error, - ffi::bool_result_value_unsafe - ) - } -} - -impl_drop!(Object<'a>, ffi::SJ_OD_object_free); diff --git a/src/ondemand/object_iterator.rs b/src/ondemand/object_iterator.rs deleted file mode 100644 index 4ad88a9..0000000 --- a/src/ondemand/object_iterator.rs +++ /dev/null @@ -1,71 +0,0 @@ -use std::{marker::PhantomData, ptr::NonNull}; - -use simdjson_sys as ffi; - -use super::{document::Document, field::Field}; -use crate::{error::Result, macros::map_result}; - -pub struct ObjectIterator<'a> { - begin: NonNull, - end: NonNull, - running: bool, - _doc: PhantomData<&'a mut Document<'a, 'a>>, -} - -impl<'a> ObjectIterator<'a> { - pub fn new( - begin: NonNull, - end: NonNull, - ) -> Self { - Self { - begin, - end, - running: false, - _doc: PhantomData, - } - } - - pub fn get(&mut self) -> Result> { - map_result!( - ffi::SJ_OD_object_iterator_get(self.begin.as_mut()), - ffi::SJ_OD_field_result_error, - ffi::SJ_OD_field_result_value_unsafe - ) - .map(Field::new) - } - - pub fn not_equal(&self) -> bool { - unsafe { ffi::SJ_OD_object_iterator_not_equal(self.begin.as_ref(), self.end.as_ref()) } - } - - pub fn step(&mut self) { - unsafe { ffi::SJ_OD_object_iterator_step(self.begin.as_mut()) } - } -} - -impl<'a> Drop for ObjectIterator<'a> { - fn drop(&mut self) { - unsafe { - ffi::SJ_OD_object_iterator_free(self.begin.as_mut()); - ffi::SJ_OD_object_iterator_free(self.end.as_mut()); - } - } -} - -impl<'a> Iterator for ObjectIterator<'a> { - type Item = Result>; - - fn next(&mut self) -> Option { - if self.running { - self.step(); - } - - if self.not_equal() { - self.running = true; - Some(self.get()) - } else { - self.running = false; - None - } - } -} diff --git a/src/ondemand/parser.rs b/src/ondemand/parser.rs deleted file mode 100644 index c0ad10e..0000000 --- a/src/ondemand/parser.rs +++ /dev/null @@ -1,73 +0,0 @@ -use std::ptr::NonNull; - -use simdjson_sys as ffi; - -use super::document::Document; -use crate::{ - error::Result, - macros::{impl_drop, map_result}, -}; - -pub struct Parser { - ptr: NonNull, -} - -impl Default for Parser { - fn default() -> Self { - Parser::new(ffi::SIMDJSON_MAXSIZE_BYTES) - } -} - -impl Parser { - pub fn new(max_capacity: usize) -> Self { - let ptr = unsafe { NonNull::new_unchecked(ffi::SJ_OD_parser_new(max_capacity)) }; - Self { ptr } - } - - pub fn iterate<'p, 's>(&'p mut self, padded_string: &'s String) -> Result> { - map_result!( - ffi::SJ_OD_parser_iterate_padded_string_view( - self.ptr.as_mut(), - padded_string.as_ptr().cast(), - padded_string.len(), - padded_string.capacity() - ), - ffi::SJ_OD_document_result_error, - ffi::SJ_OD_document_result_value_unsafe - ) - .map(Document::new) - } -} - -impl_drop!(Parser, ffi::SJ_OD_parser_free); - -#[cfg(test)] -mod tests { - use super::*; - use crate::padded_string::make_padded_string; - - #[test] - fn test_new() { - let mut parser = Parser::default(); - let ps = make_padded_string("[1,2,3]"); - let mut doc = parser.iterate(&ps).unwrap(); - // drop(ps); - // doc.get_array().unwrap(); - let arr = doc.get_array().unwrap(); - // drop(doc); - // for v in arr.iter().unwrap() { - // let _ = v.unwrap().get_uint64().unwrap(); - // } - // doc.get_value().unwrap(); - - drop(arr); - drop(doc); - - let ps2 = make_padded_string("1"); - let mut doc2 = parser.iterate(&ps2).unwrap(); - let v = doc2.get_int64().unwrap(); - assert_eq!(v, 1); - let v = doc2.get_uint64().unwrap(); - assert_eq!(v, 1); - } -} diff --git a/src/ondemand/value.rs b/src/ondemand/value.rs deleted file mode 100644 index f9959df..0000000 --- a/src/ondemand/value.rs +++ /dev/null @@ -1,130 +0,0 @@ -use std::{marker::PhantomData, ptr::NonNull}; - -use simdjson_sys as ffi; - -use super::{array::Array, document::Document, number::Number, object::Object, JsonType}; -use crate::{ - error::Result, - macros::{impl_drop, map_result}, - utils::string_view_to_str, -}; - -pub struct Value<'a> { - ptr: NonNull, - _doc: PhantomData<&'a mut Document<'a, 'a>>, -} - -impl<'a> Value<'a> { - pub fn new(ptr: NonNull) -> Self { - Self { - ptr, - _doc: PhantomData, - } - } - - pub fn get_uint64(&mut self) -> Result { - map_result!( - primitive, - ffi::SJ_OD_value_get_uint64(self.ptr.as_mut()), - ffi::uint64_t_result_error, - ffi::uint64_t_result_value_unsafe - ) - } - - pub fn get_int64(&mut self) -> Result { - map_result!( - primitive, - ffi::SJ_OD_value_get_int64(self.ptr.as_mut()), - ffi::int64_t_result_error, - ffi::int64_t_result_value_unsafe - ) - } - - pub fn get_bool(&mut self) -> Result { - map_result!( - primitive, - ffi::SJ_OD_value_get_bool(self.ptr.as_mut()), - ffi::bool_result_error, - ffi::bool_result_value_unsafe - ) - } - - pub fn get_double(&mut self) -> Result { - map_result!( - primitive, - ffi::SJ_OD_value_get_double(self.ptr.as_mut()), - ffi::double_result_error, - ffi::double_result_value_unsafe - ) - } - - pub fn get_array(&mut self) -> Result> { - map_result!( - ffi::SJ_OD_value_get_array(self.ptr.as_mut()), - ffi::SJ_OD_array_result_error, - ffi::SJ_OD_array_result_value_unsafe - ) - .map(Array::new) - } - - pub fn get_object(&mut self) -> Result> { - map_result!( - ffi::SJ_OD_value_get_object(self.ptr.as_mut()), - ffi::SJ_OD_object_result_error, - ffi::SJ_OD_object_result_value_unsafe - ) - .map(Object::new) - } - - pub fn get_number(&mut self) -> Result> { - map_result!( - ffi::SJ_OD_value_get_number(self.ptr.as_mut()), - ffi::SJ_OD_number_result_error, - ffi::SJ_OD_number_result_value_unsafe - ) - .map(Number::new) - } - - pub fn get_string(&mut self, allow_replacement: bool) -> Result<&'a str> { - let sv = map_result!( - ffi::SJ_OD_value_get_string(self.ptr.as_mut(), allow_replacement), - ffi::STD_string_view_result_error, - ffi::STD_string_view_result_value_unsafe - )?; - Ok(string_view_to_str(sv)) - } - - pub fn at_pointer(&mut self, json_pointer: &str) -> Result> { - map_result!( - ffi::SJ_OD_value_at_pointer( - self.ptr.as_mut(), - json_pointer.as_ptr().cast(), - json_pointer.len() - ), - ffi::SJ_OD_value_result_error, - ffi::SJ_OD_value_result_value_unsafe - ) - .map(Value::new) - } - - pub fn is_null(&mut self) -> Result { - map_result!( - primitive, - ffi::SJ_OD_value_is_null(self.ptr.as_mut()), - ffi::bool_result_error, - ffi::bool_result_value_unsafe - ) - } - - pub fn json_type(&mut self) -> Result { - let json_type = map_result!( - primitive, - ffi::SJ_OD_value_type(self.ptr.as_mut()), - ffi::int_result_error, - ffi::int_result_value_unsafe - )?; - Ok(JsonType::from(json_type)) - } -} - -impl_drop!(Value<'a>, ffi::SJ_OD_value_free); diff --git a/src/padded_string.rs b/src/padded_string.rs deleted file mode 100644 index d4800c9..0000000 --- a/src/padded_string.rs +++ /dev/null @@ -1,49 +0,0 @@ -use std::{io::Read, path::Path}; - -use simdjson_sys as ffi; - -pub fn make_padded_string(s: &str) -> String { - let mut ps = String::with_capacity(s.len() + ffi::SIMDJSON_PADDING); - ps.push_str(s); - ps -} - -pub fn load_padded_string>(path: P) -> std::io::Result { - let mut file = std::fs::File::open(path)?; - let len = file.metadata()?.len() as usize; - let mut buf = String::with_capacity(len + ffi::SIMDJSON_PADDING); - file.read_to_string(&mut buf)?; - Ok(buf) -} - -pub trait ToPaddedString { - fn to_padded_string(&self) -> String; -} - -pub trait IntoPaddedString { - fn into_padded_string(self) -> String; -} - -impl ToPaddedString for &str { - fn to_padded_string(&self) -> String { - make_padded_string(self) - } -} - -impl ToPaddedString for &mut str { - fn to_padded_string(&self) -> String { - make_padded_string(self) - } -} - -impl IntoPaddedString for String { - fn into_padded_string(mut self) -> String { - if self.capacity() < self.len() + ffi::SIMDJSON_PADDING { - self.reserve(ffi::SIMDJSON_PADDING); - } - self - } -} - -#[cfg(test)] -mod tests {} diff --git a/src/prelude.rs b/src/prelude.rs deleted file mode 100644 index f519ca8..0000000 --- a/src/prelude.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub use crate::padded_string::{ - load_padded_string, make_padded_string, IntoPaddedString, ToPaddedString, -}; diff --git a/src/serde.rs b/src/serde.rs new file mode 100644 index 0000000..d4377ce --- /dev/null +++ b/src/serde.rs @@ -0,0 +1,823 @@ +//! `serde::Deserializer` implementation backed by the simdjson tape. +//! +//! This allows you to deserialize any `serde::Deserialize` type directly from +//! simdjson's DOM tape without first converting to an intermediate AST. +//! +//! # Example +//! +//! ```rust +//! use serde::Deserialize; +//! use simdjson_rust::serde::from_str; +//! +//! #[derive(Debug, Deserialize, PartialEq)] +//! struct Point { +//! x: f64, +//! y: f64, +//! } +//! +//! let p: Point = from_str(r#"{"x": 1.0, "y": 2.0}"#).unwrap(); +//! assert_eq!(p, Point { x: 1.0, y: 2.0 }); +//! ``` + +use serde::{ + Deserialize, + de::{self, DeserializeSeed, EnumAccess, MapAccess, SeqAccess, VariantAccess, Visitor}, +}; +use snafu::prelude::*; + +use crate::{ + dom::Parser, + tape::{self, TapeRef, TapeType}, +}; + +/// The error type returned by the Serde deserializer. +#[derive(Debug, Snafu)] +#[snafu(visibility(pub(crate)))] +pub enum Error { + #[snafu(display("serde error: {message}"))] + Custom { message: String }, + + /// A malformed or unsupported tape state was encountered while + /// deserializing. Carries the structured [`tape::Error`] so callers can + /// match on the concrete failure. + #[snafu(transparent)] + Tape { + #[snafu(backtrace)] + source: tape::Error, + }, + + #[snafu(display("dom error: {source}"))] + Dom { + #[snafu(backtrace)] + source: crate::dom::Error, + }, +} + +pub type Result = std::result::Result; + +impl From for Error { + fn from(source: crate::dom::Error) -> Self { + Error::Dom { source } + } +} + +impl de::Error for Error { + fn custom(msg: T) -> Self { + Error::Custom { + message: msg.to_string(), + } + } +} + +/// Deserialize `T` from a JSON string using the simdjson DOM tape. +pub fn from_str Deserialize<'de>>(json: &str) -> Result { + let mut parser = Parser::default(); + let tape = parser.parse_str(json)?; + let mut de = TapeDeserializer { tape }; + T::deserialize(&mut de) +} + +/// Deserialize `T` from a JSON byte slice using the simdjson DOM tape. +pub fn from_bytes Deserialize<'de>>(json: &[u8]) -> Result { + let mut parser = Parser::default(); + let tape = parser.parse_bytes(json)?; + let mut de = TapeDeserializer { tape }; + T::deserialize(&mut de) +} + +/// Deserialize `T` directly from a parsed `TapeRef` cursor. +/// +/// This allows you to reuse the `Parser` instance across multiple documents, +/// which is highly recommended for performance. +pub fn from_tape<'de, T: Deserialize<'de>>(tape: TapeRef<'de>) -> Result { + let mut de = TapeDeserializer { tape }; + T::deserialize(&mut de) +} + +/// A `serde::Deserializer` that reads directly from the simdjson tape. +/// +/// The lifetime `'de` is tied to the parser that owns the tape, enabling +/// zero-copy `&str` deserialization. +pub struct TapeDeserializer<'de> { + tape: TapeRef<'de>, +} + +/// Internal helper: deserialize a single JSON value from a mutable tape cursor. +/// +/// This is the workhorse used by all `SeqAccess`/`MapAccess`/`VariantAccess` +/// impls so they can share the same tape cursor. +/// +/// Dispatches on the raw tag byte of the current tape word rather than going +/// through `Option`, avoiding an enum round-trip and a second match +/// on every node. +#[inline] +fn deserialize_value<'de, V: Visitor<'de>>( + tape: &mut TapeRef<'de>, + visitor: V, +) -> Result { + if tape.pos >= tape.tape.len() { + return Err(tape::OutOfBoundsSnafu { + pos: tape.pos, + len: tape.tape.len(), + } + .build() + .into()); + } + let word = tape.current_word_public(); + let tag = (word >> 56) as u8; + match tag { + b'r' => { + tape.skip(1); + deserialize_value(tape, visitor) + } + b'n' => { + tape.skip(1); + visitor.visit_unit() + } + b't' => { + tape.skip(1); + visitor.visit_bool(true) + } + b'f' => { + tape.skip(1); + visitor.visit_bool(false) + } + b'l' => { + let v = tape.get_int64()?; + tape.skip(2); + visitor.visit_i64(v) + } + b'u' => { + let v = tape.get_uint64()?; + tape.skip(2); + visitor.visit_u64(v) + } + b'd' => { + let v = tape.get_double()?; + tape.skip(2); + visitor.visit_f64(v) + } + b'"' => { + let s: &'de str = tape.get_string()?; + tape.skip(1); + visitor.visit_borrowed_str(s) + } + b'[' => { + let end_idx = tape.scope_close_idx(); + let count = tape.scope_count(); + tape.skip(1); // past `[` + let result = visitor.visit_seq(TapeSeqAccess { + tape, + end_idx, + count, + })?; + tape.skip(1); // past `]` + Ok(result) + } + b'{' => { + let end_idx = tape.scope_close_idx(); + tape.skip(1); // past `{` + let result = visitor.visit_map(TapeMapAccess { tape, end_idx })?; + tape.skip(1); // past `}` + Ok(result) + } + _ => match TapeType::from_tape_word(word) { + // Recognised tag that isn't a valid value position (e.g. a stray + // `}` / `]`). `tag` is known, so this is unexpected, not unknown. + Some(_) => Err(tape::UnexpectedTapeTypeSnafu { + found: tag, + pos: tape.pos(), + } + .build() + .into()), + None => Err(tape::UnknownTapeTypeSnafu { + tag, + pos: tape.pos(), + } + .build() + .into()), + }, + } +} + +/// Skip a complete JSON value on the tape without deserializing it. +fn skip_value(tape: &mut TapeRef<'_>) -> Result<()> { + match tape.tape_type() { + Some(TapeType::Null) | Some(TapeType::True) | Some(TapeType::False) => { + tape.skip(1); + } + Some(TapeType::Int64) | Some(TapeType::Uint64) | Some(TapeType::Double) => { + tape.skip(2); + } + Some(TapeType::String) => { + tape.skip(1); + } + Some(TapeType::StartArray) | Some(TapeType::StartObject) => { + // scope_close_idx = position of the closing `]` / `}` entry + let end_idx = tape.scope_close_idx(); + tape.seek_public(end_idx); + tape.skip(1); // past the closing bracket + } + Some(t) => { + return Err(tape::UnexpectedTapeTypeSnafu { + found: t as u8, + pos: tape.pos(), + } + .build() + .into()); + } + None => { + return Err(tape::UnknownTapeTypeSnafu { + tag: (tape.current_word_public() >> 56) as u8, + pos: tape.pos(), + } + .build() + .into()); + } + } + Ok(()) +} + +// --------------------------------------------------------------------------- +// SeqAccess for arrays +// --------------------------------------------------------------------------- + +struct TapeSeqAccess<'a, 'de: 'a> { + tape: &'a mut TapeRef<'de>, + end_idx: usize, + /// Element count read from the `[` tape word, used as a `size_hint` so + /// serde can pre-allocate the destination `Vec` and avoid a grow storm. + count: usize, +} + +impl<'a, 'de> SeqAccess<'de> for TapeSeqAccess<'a, 'de> { + type Error = Error; + + fn next_element_seed>(&mut self, seed: T) -> Result> { + if self.tape.pos() >= self.end_idx { + return Ok(None); + } + seed.deserialize(&mut TapeRefDeserializer { tape: self.tape }) + .map(Some) + } + + fn size_hint(&self) -> Option { + Some(self.count) + } +} + +// --------------------------------------------------------------------------- +// MapAccess for objects +// --------------------------------------------------------------------------- + +struct TapeMapAccess<'a, 'de: 'a> { + tape: &'a mut TapeRef<'de>, + end_idx: usize, +} + +impl<'a, 'de> MapAccess<'de> for TapeMapAccess<'a, 'de> { + type Error = Error; + + fn next_key_seed>(&mut self, seed: K) -> Result> { + if self.tape.pos() >= self.end_idx { + return Ok(None); + } + // Keys in object tape are always String nodes. + let word = self.tape.current_word_public(); + let tag = (word >> 56) as u8; + if tag != b'"' { + return Err(tape::UnexpectedTapeTypeSnafu { + found: tag, + pos: self.tape.pos(), + } + .build() + .into()); + } + let key: &'de str = self.tape.get_string()?; + self.tape.skip(1); + let key_deserializer = de::value::BorrowedStrDeserializer::::new(key); + seed.deserialize(key_deserializer).map(Some) + } + + fn next_value_seed>(&mut self, seed: V) -> Result { + seed.deserialize(&mut TapeRefDeserializer { tape: self.tape }) + } +} + +// --------------------------------------------------------------------------- +// EnumAccess / VariantAccess +// --------------------------------------------------------------------------- + +struct TapeEnumAccess<'a, 'de: 'a> { + tape: &'a mut TapeRef<'de>, + end_idx: usize, +} + +impl<'a, 'de> EnumAccess<'de> for TapeEnumAccess<'a, 'de> { + type Error = Error; + type Variant = TapeVariantAccess<'a, 'de>; + + fn variant_seed>(self, seed: V) -> Result<(V::Value, Self::Variant)> { + let word = self.tape.current_word_public(); + let tag = (word >> 56) as u8; + if tag != b'"' { + return Err(tape::UnexpectedTapeTypeSnafu { + found: tag, + pos: self.tape.pos(), + } + .build() + .into()); + } + let name: &'de str = self.tape.get_string()?; + self.tape.skip(1); + let name_deserializer = de::value::BorrowedStrDeserializer::::new(name); + let val = seed.deserialize(name_deserializer)?; + Ok(( + val, + TapeVariantAccess { + tape: self.tape, + end_idx: self.end_idx, + }, + )) + } +} + +struct TapeVariantAccess<'a, 'de: 'a> { + tape: &'a mut TapeRef<'de>, + #[allow(dead_code)] + end_idx: usize, +} + +impl<'a, 'de> VariantAccess<'de> for TapeVariantAccess<'a, 'de> { + type Error = Error; + + fn unit_variant(self) -> Result<()> { + Ok(()) + } + + fn newtype_variant_seed>(self, seed: T) -> Result { + seed.deserialize(&mut TapeRefDeserializer { tape: self.tape }) + } + + fn tuple_variant>(self, _len: usize, visitor: V) -> Result { + deserialize_value(self.tape, visitor) + } + + fn struct_variant>( + self, + _fields: &'static [&'static str], + visitor: V, + ) -> Result { + deserialize_value(self.tape, visitor) + } +} + +// --------------------------------------------------------------------------- +// TapeRefDeserializer - thin wrapper that delegates to `deserialize_value` +// --------------------------------------------------------------------------- + +/// A thin `serde::Deserializer` that mutates a shared `TapeRef` cursor. +/// +/// This is used by `SeqAccess`, `MapAccess`, and `VariantAccess` to +/// delegate element deserialization. +struct TapeRefDeserializer<'a, 'de: 'a> { + tape: &'a mut TapeRef<'de>, +} + +macro_rules! forward_to_deserialize_value { + ($($fn:ident)*) => { + $( + fn $fn>(self, visitor: V) -> Result { + deserialize_value(self.tape, visitor) + } + )* + }; +} + +impl<'a, 'de> de::Deserializer<'de> for &mut TapeRefDeserializer<'a, 'de> { + type Error = Error; + + // All other methods can forward to `deserialize_any`. + forward_to_deserialize_value! { + deserialize_bool + deserialize_i8 deserialize_i16 deserialize_i32 deserialize_i64 + deserialize_u8 deserialize_u16 deserialize_u32 deserialize_u64 + deserialize_f32 deserialize_f64 + deserialize_char deserialize_str deserialize_string + deserialize_bytes deserialize_byte_buf + deserialize_unit + deserialize_seq deserialize_map + deserialize_identifier + } + + fn deserialize_any>(self, visitor: V) -> Result { + deserialize_value(self.tape, visitor) + } + + fn deserialize_option>(self, visitor: V) -> Result { + if self.tape.tape_type() == Some(TapeType::Null) { + self.tape.skip(1); + visitor.visit_none() + } else { + visitor.visit_some(self) + } + } + + fn deserialize_newtype_struct>( + self, + _name: &'static str, + visitor: V, + ) -> Result { + visitor.visit_newtype_struct(self) + } + + fn deserialize_enum>( + self, + _name: &'static str, + _variants: &'static [&'static str], + visitor: V, + ) -> Result { + match self.tape.tape_type() { + Some(TapeType::String) => { + let s: &'de str = self.tape.get_string()?; + self.tape.skip(1); + let enum_deserializer = de::value::StrDeserializer::::new(s); + visitor.visit_enum(enum_deserializer) + } + Some(TapeType::StartObject) => { + let end_idx = self.tape.scope_close_idx(); + self.tape.skip(1); // past `{` + let result = visitor.visit_enum(TapeEnumAccess { + tape: self.tape, + end_idx, + })?; + self.tape.skip(1); // past `}` + Ok(result) + } + _ => deserialize_value(self.tape, visitor), + } + } + + fn deserialize_ignored_any>(self, visitor: V) -> Result { + skip_value(self.tape)?; + visitor.visit_unit() + } + + fn deserialize_unit_struct>( + self, + _name: &'static str, + visitor: V, + ) -> Result { + deserialize_value(self.tape, visitor) + } + + fn deserialize_tuple>(self, _len: usize, visitor: V) -> Result { + deserialize_value(self.tape, visitor) + } + + fn deserialize_tuple_struct>( + self, + _name: &'static str, + _len: usize, + visitor: V, + ) -> Result { + deserialize_value(self.tape, visitor) + } + + fn deserialize_struct>( + self, + _name: &'static str, + _fields: &'static [&'static str], + visitor: V, + ) -> Result { + deserialize_value(self.tape, visitor) + } +} + +// --------------------------------------------------------------------------- +// TapeDeserializer (top-level entry point) +// --------------------------------------------------------------------------- + +impl<'de> de::Deserializer<'de> for &mut TapeDeserializer<'de> { + type Error = Error; + + fn deserialize_any>(self, visitor: V) -> Result { + // Skip root if present. + if self.tape.tape_type() == Some(TapeType::Root) { + self.tape.skip(1); + } + deserialize_value(&mut self.tape, visitor) + } + + fn deserialize_option>(self, visitor: V) -> Result { + if self.tape.tape_type() == Some(TapeType::Root) { + self.tape.skip(1); + } + if self.tape.tape_type() == Some(TapeType::Null) { + self.tape.skip(1); + visitor.visit_none() + } else { + visitor.visit_some(self) + } + } + + fn deserialize_newtype_struct>( + self, + _name: &'static str, + visitor: V, + ) -> Result { + visitor.visit_newtype_struct(self) + } + + fn deserialize_enum>( + self, + _name: &'static str, + _variants: &'static [&'static str], + visitor: V, + ) -> Result { + if self.tape.tape_type() == Some(TapeType::Root) { + self.tape.skip(1); + } + match self.tape.tape_type() { + Some(TapeType::String) => { + let s: &'de str = self.tape.get_string()?; + self.tape.skip(1); + let enum_deserializer = de::value::StrDeserializer::::new(s); + visitor.visit_enum(enum_deserializer) + } + Some(TapeType::StartObject) => { + let end_idx = self.tape.scope_close_idx(); + self.tape.skip(1); + let result = visitor.visit_enum(TapeEnumAccess { + tape: &mut self.tape, + end_idx, + })?; + self.tape.skip(1); // past `}` + Ok(result) + } + _ => { + let t = &mut self.tape; + deserialize_value(t, visitor) + } + } + } + + fn deserialize_ignored_any>(self, visitor: V) -> Result { + if self.tape.tape_type() == Some(TapeType::Root) { + self.tape.skip(1); + } + skip_value(&mut self.tape)?; + visitor.visit_unit() + } + + fn deserialize_bool>(self, visitor: V) -> Result { + self.deserialize_any(visitor) + } + + fn deserialize_i8>(self, visitor: V) -> Result { + self.deserialize_any(visitor) + } + + fn deserialize_i16>(self, visitor: V) -> Result { + self.deserialize_any(visitor) + } + + fn deserialize_i32>(self, visitor: V) -> Result { + self.deserialize_any(visitor) + } + + fn deserialize_i64>(self, visitor: V) -> Result { + self.deserialize_any(visitor) + } + + fn deserialize_u8>(self, visitor: V) -> Result { + self.deserialize_any(visitor) + } + + fn deserialize_u16>(self, visitor: V) -> Result { + self.deserialize_any(visitor) + } + + fn deserialize_u32>(self, visitor: V) -> Result { + self.deserialize_any(visitor) + } + + fn deserialize_u64>(self, visitor: V) -> Result { + self.deserialize_any(visitor) + } + + fn deserialize_f32>(self, visitor: V) -> Result { + self.deserialize_any(visitor) + } + + fn deserialize_f64>(self, visitor: V) -> Result { + self.deserialize_any(visitor) + } + + fn deserialize_char>(self, visitor: V) -> Result { + self.deserialize_any(visitor) + } + + fn deserialize_str>(self, visitor: V) -> Result { + self.deserialize_any(visitor) + } + + fn deserialize_string>(self, visitor: V) -> Result { + self.deserialize_any(visitor) + } + + fn deserialize_bytes>(self, visitor: V) -> Result { + self.deserialize_any(visitor) + } + + fn deserialize_byte_buf>(self, visitor: V) -> Result { + self.deserialize_any(visitor) + } + + fn deserialize_unit>(self, visitor: V) -> Result { + self.deserialize_any(visitor) + } + + fn deserialize_unit_struct>( + self, + _name: &'static str, + visitor: V, + ) -> Result { + self.deserialize_any(visitor) + } + + fn deserialize_seq>(self, visitor: V) -> Result { + self.deserialize_any(visitor) + } + + fn deserialize_tuple>(self, _len: usize, visitor: V) -> Result { + self.deserialize_any(visitor) + } + + fn deserialize_tuple_struct>( + self, + _name: &'static str, + _len: usize, + visitor: V, + ) -> Result { + self.deserialize_any(visitor) + } + + fn deserialize_map>(self, visitor: V) -> Result { + self.deserialize_any(visitor) + } + + fn deserialize_struct>( + self, + _name: &'static str, + _fields: &'static [&'static str], + visitor: V, + ) -> Result { + self.deserialize_any(visitor) + } + + fn deserialize_identifier>(self, visitor: V) -> Result { + self.deserialize_any(visitor) + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use serde::Deserialize; + + use super::*; + + #[derive(Debug, Deserialize, PartialEq)] + struct Point { + x: f64, + y: f64, + } + + #[derive(Debug, Deserialize, PartialEq)] + struct Nested { + name: String, + value: i64, + tags: Vec, + } + + #[derive(Debug, Deserialize, PartialEq)] + #[serde(rename_all = "snake_case")] + enum Color { + Red, + Green, + Blue, + Custom { r: u8, g: u8, b: u8 }, + } + + #[test] + fn deserialize_struct() { + let p: Point = from_str(r#"{"x": 1.5, "y": -2.0}"#).unwrap(); + assert_eq!(p, Point { x: 1.5, y: -2.0 }); + } + + #[test] + fn deserialize_nested() { + let n: Nested = from_str(r#"{"name":"hello","value":42,"tags":["a","b","c"]}"#).unwrap(); + assert_eq!(n.name, "hello"); + assert_eq!(n.value, 42); + assert_eq!(n.tags, vec!["a", "b", "c"]); + } + + #[test] + fn deserialize_vec_of_ints() { + let v: Vec = from_str("[1, 2, 3, -4]").unwrap(); + assert_eq!(v, vec![1, 2, 3, -4]); + } + + #[test] + fn deserialize_option_some() { + let v: Option = from_str("42").unwrap(); + assert_eq!(v, Some(42)); + } + + #[test] + fn deserialize_option_none() { + let v: Option = from_str("null").unwrap(); + assert_eq!(v, None); + } + + #[test] + fn deserialize_bool_true() { + assert!(from_str::("true").unwrap()); + } + + #[test] + fn deserialize_bool_false() { + assert!(!from_str::("false").unwrap()); + } + + #[test] + fn deserialize_unit_enum() { + let c: Color = from_str(r#""red""#).unwrap(); + assert_eq!(c, Color::Red); + } + + #[test] + fn deserialize_struct_enum() { + let c: Color = from_str(r#"{"custom": {"r": 10, "g": 20, "b": 30}}"#).unwrap(); + assert_eq!( + c, + Color::Custom { + r: 10, + g: 20, + b: 30 + } + ); + } + + #[test] + fn deserialize_f64() { + let v: f64 = from_str("1.5").unwrap(); + assert_eq!(v, 1.5_f64); + } + + #[test] + fn deserialize_string() { + let s: String = from_str(r#""hello world""#).unwrap(); + assert_eq!(s, "hello world"); + } + + #[test] + fn deserialize_nested_arrays() { + let v: Vec> = from_str("[[1,2],[3,4],[5]]").unwrap(); + assert_eq!(v, vec![vec![1, 2], vec![3, 4], vec![5]]); + } + + #[derive(Debug, Deserialize, PartialEq)] + struct LargeObject { + id: u64, + name: String, + active: bool, + score: f64, + tags: Vec, + meta: Option, + } + + #[test] + fn deserialize_large_object() { + let json = r#"{ + "id": 12345, + "name": "simdjson", + "active": true, + "score": 9.99, + "tags": ["fast", "safe", "rust"], + "meta": null + }"#; + let v: LargeObject = from_str(json).unwrap(); + assert_eq!(v.id, 12345); + assert_eq!(v.name, "simdjson"); + assert!(v.active); + assert!((v.score - 9.99).abs() < 1e-6); + assert_eq!(v.tags[0], "fast"); + assert!(v.meta.is_none()); + } +} diff --git a/src/serde/de.rs b/src/serde/de.rs deleted file mode 100644 index 3f4a130..0000000 --- a/src/serde/de.rs +++ /dev/null @@ -1,366 +0,0 @@ -use crate::dom::array::ArrayIter; -use crate::dom::element::{Element, ElementType}; -use crate::dom::object::ObjectIter; - -use crate::error::SimdJsonError; -use crate::libsimdjson::ffi; -use serde::de::{ - Deserialize, DeserializeSeed, Deserializer, IntoDeserializer, MapAccess, SeqAccess, Visitor, -}; - -// pub struct ElementVisitor; - -// impl<'de> Visitor for ElementVisitor { - -// } - -pub fn from_element<'a, T>(element: &'a Element<'a>) -> Result -where - T: Deserialize<'a>, -{ - // let mut parser = Parser::default(); - // let mut doc = parser.parse_str(s)?; - let t = T::deserialize(element)?; - Ok(t) -} - -impl<'de, 'a> Deserializer<'de> for &'a Element<'a> { - type Error = SimdJsonError; - - fn deserialize_any(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - match self.get_type() { - ElementType::NullValue => self.deserialize_unit(visitor), - ElementType::Bool => self.deserialize_bool(visitor), - ElementType::String => self.deserialize_str(visitor), - ElementType::Uint64 => self.deserialize_u64(visitor), - ElementType::Int64 => self.deserialize_i64(visitor), - ElementType::Array => self.deserialize_seq(visitor), - ElementType::Object => self.deserialize_map(visitor), - ElementType::Double => self.deserialize_f64(visitor), - } - } - - fn deserialize_bool(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - visitor.visit_bool(self.get_bool()?) - } - - fn deserialize_unit(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - visitor.visit_unit() - } - - fn deserialize_i8(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - visitor.visit_i8(self.get_i64()? as i8) - } - - fn deserialize_i16(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - visitor.visit_i16(self.get_i64()? as i16) - } - - fn deserialize_i32(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - visitor.visit_i32(self.get_i64()? as i32) - } - - fn deserialize_i64(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - visitor.visit_i64(self.get_i64()?) - } - - fn deserialize_u8(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - visitor.visit_u8(self.get_u64()? as u8) - } - - fn deserialize_u16(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - visitor.visit_u16(self.get_u64()? as u16) - } - - fn deserialize_u32(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - visitor.visit_u32(self.get_u64()? as u32) - } - - fn deserialize_u64(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - visitor.visit_u64(self.get_u64()?) - } - - // Float parsing is stupidly hard. - fn deserialize_f32(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - visitor.visit_f32(self.get_f64()? as f32) - } - - // Float parsing is stupidly hard. - fn deserialize_f64(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - visitor.visit_f64(self.get_f64()?) - } - - // The `Serializer` implementation on the previous page serialized chars as - // single-character strings so handle that representation here. - fn deserialize_char(self, _visitor: V) -> Result - where - V: Visitor<'de>, - { - // Parse a string, check that it is one character, call `visit_char`. - unimplemented!() - } - - // Refer to the "Understanding deserializer lifetimes" page for information - // about the three deserialization flavors of strings in Serde. - fn deserialize_str(self, _visitor: V) -> Result - where - V: Visitor<'de>, - { - unimplemented!() - } - - fn deserialize_string(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - visitor.visit_string(self.get_string()?) - } - - // The `Serializer` implementation on the previous page serialized byte - // arrays as JSON arrays of bytes. Handle that representation here. - fn deserialize_bytes(self, _visitor: V) -> Result - where - V: Visitor<'de>, - { - unimplemented!() - } - - fn deserialize_byte_buf(self, _visitor: V) -> Result - where - V: Visitor<'de>, - { - unimplemented!() - } - - // An absent optional is represented as the JSON `null` and a present - // optional is represented as just the contained value. - // - // As commented in `Serializer` implementation, this is a lossy - // representation. For example the values `Some(())` and `None` both - // serialize as just `null`. Unfortunately this is typically what people - // expect when working with JSON. Other formats are encouraged to behave - // more intelligently if possible. - fn deserialize_option(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - if self.is_null() { - visitor.visit_none() - } else { - visitor.visit_some(self) - } - } - - // Unit struct means a named value containing no data. - fn deserialize_unit_struct( - self, - _name: &'static str, - visitor: V, - ) -> Result - where - V: Visitor<'de>, - { - self.deserialize_unit(visitor) - } - - // As is done here, serializers are encouraged to treat newtype structs as - // insignificant wrappers around the data they contain. That means not - // parsing anything other than the contained value. - fn deserialize_newtype_struct( - self, - _name: &'static str, - visitor: V, - ) -> Result - where - V: Visitor<'de>, - { - visitor.visit_newtype_struct(self) - } - - // Deserialization of compound types like sequences and maps happens by - // passing the visitor an "Access" object that gives it the ability to - // iterate through the data contained in the sequence. - fn deserialize_seq(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - // Parse the opening bracket of the sequence. - // de::Deserializer::deserialize_seq(self.into_iter(), visitor) - // unimplemented!() - let value = visitor.visit_seq(ArrayIter::new(&self.get_array()?))?; - Ok(value) - } - - // Tuples look just like sequences in JSON. Some formats may be able to - // represent tuples more efficiently. - // - // As indicated by the length parameter, the `Deserialize` implementation - // for a tuple in the Serde data model is required to know the length of the - // tuple before even looking at the input data. - fn deserialize_tuple(self, _len: usize, visitor: V) -> Result - where - V: Visitor<'de>, - { - self.deserialize_seq(visitor) - } - - // Tuple structs look just like sequences in JSON. - fn deserialize_tuple_struct( - self, - _name: &'static str, - _len: usize, - visitor: V, - ) -> Result - where - V: Visitor<'de>, - { - self.deserialize_seq(visitor) - } - - // Much like `deserialize_seq` but calls the visitors `visit_map` method - // with a `MapAccess` implementation, rather than the visitor's `visit_seq` - // method with a `SeqAccess` implementation. - fn deserialize_map(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - visitor.visit_map(ObjectIter::new(&self.get_object()?)) - } - - // Structs look just like maps in JSON. - // - // Notice the `fields` parameter - a "struct" in the Serde data model means - // that the `Deserialize` implementation is required to know what the fields - // are before even looking at the input data. Any key-value pairing in which - // the fields cannot be known ahead of time is probably a map. - fn deserialize_struct( - self, - _name: &'static str, - _fields: &'static [&'static str], - visitor: V, - ) -> Result - where - V: Visitor<'de>, - { - self.deserialize_map(visitor) - } - - fn deserialize_enum( - self, - _name: &'static str, - _variants: &'static [&'static str], - _visitor: V, - ) -> Result - where - V: Visitor<'de>, - { - unimplemented!() - } - - // An identifier in Serde is the type that identifies a field of a struct or - // the variant of an enum. In JSON, struct fields and enum variants are - // represented as strings. In other formats they may be represented as - // numeric indices. - fn deserialize_identifier(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - self.deserialize_str(visitor) - } - - // Like `deserialize_any` but indicates to the `Deserializer` that it makes - // no difference which `Visitor` method is called because the data is - // ignored. - // - // Some deserializers are able to implement this more efficiently than - // `deserialize_any`, for example by rapidly skipping over matched - // delimiters without paying close attention to the data in between. - // - // Some formats are not able to implement this at all. Formats that can - // implement `deserialize_any` and `deserialize_ignored_any` are known as - // self-describing. - fn deserialize_ignored_any(self, visitor: V) -> Result - where - V: Visitor<'de>, - { - self.deserialize_any(visitor) - } -} - -impl<'de, 'a> SeqAccess<'de> for ArrayIter<'a> { - type Error = SimdJsonError; - - fn next_element_seed(&mut self, seed: T) -> Result, Self::Error> - where - T: DeserializeSeed<'de>, - { - if let Some(element) = self.next() { - seed.deserialize(&element).map(Some) - } else { - Ok(None) - } - } -} - -impl<'de, 'a> MapAccess<'de> for ObjectIter<'a> { - type Error = SimdJsonError; - - fn next_key_seed(&mut self, seed: K) -> Result, Self::Error> - where - K: DeserializeSeed<'de>, - { - if self.has_next() { - seed.deserialize(self.key().into_deserializer()).map(Some) - } else { - Ok(None) - } - } - - fn next_value_seed(&mut self, seed: V) -> Result - where - V: DeserializeSeed<'de>, - { - let result = seed.deserialize(&self.value()); - ffi::object_iterator_next(self.ptr.pin_mut()); - result - } -} diff --git a/src/serde/mod.rs b/src/serde/mod.rs deleted file mode 100644 index dacf8af..0000000 --- a/src/serde/mod.rs +++ /dev/null @@ -1,29 +0,0 @@ -pub mod de; - -#[cfg(test)] -mod tests { - use super::*; - // use super::element::GetValue; - use crate::dom::parser::Parser; - use serde::Deserialize; - - #[test] - fn test_element() -> Result<(), Box> { - let mut parser = Parser::default(); - let elm = parser.parse(r#"[true, false]"#)?; - println!("{}", elm); - let a: Vec = de::from_element(&elm)?; - assert_eq!(vec![true, false], a); - - #[derive(Debug, Deserialize)] - struct A { - field1: bool, - } - let elm = parser.parse(r#"{"field1": false}"#)?; - println!("{}", elm); - let a: A = de::from_element(&elm)?; - assert!(!a.field1); - - Ok(()) - } -} diff --git a/src/tape.rs b/src/tape.rs new file mode 100644 index 0000000..3037acb --- /dev/null +++ b/src/tape.rs @@ -0,0 +1,780 @@ +//! Tape parser for simdjson DOM tape format. +//! +//! The simdjson DOM parser serializes parsed JSON into a "tape" - an array of +//! 64-bit words. Each word encodes its node type in the top 8 bits and a +//! payload (e.g., string offset, number of elements, or sibling index) in the +//! lower 56 bits. +//! +//! ## Tape type byte values (top 8 bits of each tape word) +//! +//! | Char | Hex | Meaning | +//! |------|------|-------------------------------------------------| +//! | `r` | 0x72 | Root node (always first and last entries) | +//! | `{` | 0x7B | Start of object (payload = end-of-scope index) | +//! | `}` | 0x7D | End of object (payload = start index) | +//! | `[` | 0x5B | Start of array (payload = end-of-scope index) | +//! | `]` | 0x5D | End of array (payload = start index) | +//! | `"` | 0x22 | String (payload = byte offset into string_buf) | +//! | `l` | 0x6C | Signed integer (next tape word = i64 bits) | +//! | `u` | 0x75 | Unsigned integer(next tape word = u64) | +//! | `d` | 0x64 | Double (next tape word = f64 bits) | +//! | `t` | 0x74 | Boolean true | +//! | `f` | 0x66 | Boolean false | +//! | `n` | 0x6E | Null | + +use std::str; + +use snafu::prelude::*; + +/// The error type returned by Tape operations. +/// +/// Each variant names the concrete failure and carries the positions and +/// lengths relevant to it, so callers can inspect *what* went wrong and +/// *where*, rather than parsing a formatted message. +#[derive(Debug, Snafu)] +#[snafu(visibility(pub(crate)))] +pub enum Error { + /// The tape cursor advanced past the end of the tape. + #[snafu(display("tape position {pos} out of bounds (len={len})"))] + OutOfBounds { pos: usize, len: usize }, + + /// The tape word's tag byte is not a recognised `TapeType`. + #[snafu(display("unknown tape type byte 0x{tag:02X} at position {pos}"))] + UnknownTapeType { tag: u8, pos: usize }, + + /// The tape word's type is recognised but not valid at this position + /// (e.g. a closing `}` where a value is expected, or a non-string node + /// where an object key is expected). + #[snafu(display("unexpected tape type 0x{found:02X} at position {pos}"))] + UnexpectedTapeType { found: u8, pos: usize }, + + /// A string node's offset or length runs past the end of `string_buf`. + #[snafu(display("string at offset {offset} out of bounds (string_buf len={len})"))] + StringOutOfBounds { offset: usize, len: usize }, + + /// A number node is missing its second (value) tape word. + #[snafu(display("value word missing for tape node at position {pos}"))] + MissingValueWord { pos: usize }, + + /// The string bytes at the node's offset are not valid UTF-8. + #[snafu(display("invalid UTF-8 in string at offset {pos}"))] + InvalidUtf8 { + source: std::str::Utf8Error, + pos: usize, + }, +} + +pub type Result = std::result::Result; + +/// Mask to extract the lower 56 bits (payload) from a tape word. +const PAYLOAD_MASK: u64 = 0x00FF_FFFF_FFFF_FFFF; + +/// Tape type byte values encoded in the top 8 bits of each tape word. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub enum TapeType { + Root = b'r', + StartObject = b'{', + EndObject = b'}', + StartArray = b'[', + EndArray = b']', + String = b'"', + Int64 = b'l', + Uint64 = b'u', + Double = b'd', + True = b't', + False = b'f', + Null = b'n', +} + +impl TapeType { + /// Decode the tape type from the top byte of a tape word. + #[inline] + pub fn from_tape_word(word: u64) -> Option { + let tag = (word >> 56) as u8; + match tag { + b'r' => Some(Self::Root), + b'{' => Some(Self::StartObject), + b'}' => Some(Self::EndObject), + b'[' => Some(Self::StartArray), + b']' => Some(Self::EndArray), + b'"' => Some(Self::String), + b'l' => Some(Self::Int64), + b'u' => Some(Self::Uint64), + b'd' => Some(Self::Double), + b't' => Some(Self::True), + b'f' => Some(Self::False), + b'n' => Some(Self::Null), + _ => None, + } + } +} +/// A zero-copy representation of a JSON value parsed from the simdjson tape. +#[derive(Debug, Clone, PartialEq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize))] +pub enum Value<'a> { + Null, + Bool(bool), + Int64(i64), + Uint64(u64), + Double(f64), + String(&'a str), + Array(Vec>), + Object(Vec<(&'a str, Value<'a>)>), +} + +impl<'a> Value<'a> { + /// Look up a field in a JSON object by key. + pub fn get(&self, key: &str) -> Option<&Value<'a>> { + match self { + Value::Object(fields) => fields.iter().find(|(k, _)| *k == key).map(|(_, v)| v), + _ => None, + } + } + + /// Look up an element in a JSON array by index. + pub fn get_index(&self, index: usize) -> Option<&Value<'a>> { + match self { + Value::Array(arr) => arr.get(index), + _ => None, + } + } + + /// Returns true if the value is Null. + pub fn is_null(&self) -> bool { + matches!(self, Value::Null) + } + + /// Returns the boolean value if this is a Bool. + pub fn as_bool(&self) -> Option { + match self { + Value::Bool(b) => Some(*b), + _ => None, + } + } + + /// Returns the i64 value if this is an Int64 or a compatible Uint64. + pub fn as_i64(&self) -> Option { + match self { + Value::Int64(i) => Some(*i), + Value::Uint64(u) => (*u).try_into().ok(), + _ => None, + } + } + + /// Returns the u64 value if this is a Uint64 or a compatible Int64. + pub fn as_u64(&self) -> Option { + match self { + Value::Uint64(u) => Some(*u), + Value::Int64(i) => (*i).try_into().ok(), + _ => None, + } + } + + /// Returns the f64 value if this is a Double. + pub fn as_f64(&self) -> Option { + match self { + Value::Double(f) => Some(*f), + _ => None, + } + } + + /// Returns the borrowed string slice if this is a String. + pub fn as_str(&self) -> Option<&'a str> { + match self { + Value::String(s) => Some(*s), + _ => None, + } + } + + /// Returns the borrowed array slice if this is an Array. + pub fn as_array(&self) -> Option<&[Value<'a>]> { + match self { + Value::Array(arr) => Some(arr), + _ => None, + } + } + + /// Returns the borrowed object slice if this is an Object. + pub fn as_object(&self) -> Option<&[(&'a str, Value<'a>)]> { + match self { + Value::Object(obj) => Some(obj), + _ => None, + } + } +} + +/// A cursor over the simdjson DOM tape. +/// +/// `TapeRef` borrows both the tape and the string buffer slices from a +/// [`simdjson_sys::dom_ffi::TapeView`]. It maintains a `pos` cursor that +/// advances as values are decoded. +/// +/// The lifetime `'a` is tied to the [`crate::dom::Parser`] that owns the +/// underlying memory, ensuring the tape remains valid. +#[derive(Debug, Clone, Copy)] +pub struct TapeRef<'a> { + pub(crate) tape: &'a [u64], + pub(crate) string_buf: &'a [u8], + /// Current index into `tape`. + pub(crate) pos: usize, +} + +impl<'a> TapeRef<'a> { + /// Create a new tape cursor positioned at index 0 (the root `r` node). + pub fn new(tape: &'a [u64], string_buf: &'a [u8]) -> Self { + Self { + tape, + string_buf, + pos: 0, + } + } + + /// Returns the current position in the tape. + #[inline] + pub fn pos(&self) -> usize { + self.pos + } + + /// Returns the raw tape word at the current position. + #[inline] + pub(crate) fn current_word(&self) -> u64 { + self.tape[self.pos] + } + + /// Public accessor for the raw tape word (needed by serde module). + #[inline] + pub fn current_word_public(&self) -> u64 { + self.current_word() + } + + /// Returns the type of the current tape node. + #[inline] + pub fn tape_type(&self) -> Option { + if self.pos >= self.tape.len() { + return None; + } + TapeType::from_tape_word(self.current_word()) + } + + /// Returns the lower-56-bit payload of the current tape word. + #[inline] + pub(crate) fn payload(&self) -> u64 { + self.current_word() & PAYLOAD_MASK + } + + /// Public accessor for payload (needed by serde module). + #[inline] + pub fn payload_public(&self) -> u64 { + self.payload() + } + + /// Returns the tape index of the closing `}` or `]` for the current + /// `StartObject` or `StartArray` node. + /// + /// In simdjson's tape format, the opening bracket (`{` or `[`) stores a + /// packed payload: the lower 32 bits are the **next-sibling index** (the + /// tape position of the first element *after* the closing bracket), and + /// the upper 24 bits encode the element count. The closing bracket is + /// therefore at `next_sibling - 1`. + #[inline] + pub fn scope_close_idx(&self) -> usize { + // Extract lower 32 bits = next sibling index. + let next_sibling = (self.payload() & 0xFFFF_FFFF) as usize; + // The closing `}` or `]` is one position before the next sibling. + next_sibling.saturating_sub(1) + } + + /// Returns the number of direct children in the current `[` / `{` scope. + /// + /// simdjson packs the element count into bits 32–55 of the opening + /// bracket's tape word (mask `0xFFFFFF` after a `>> 32`). This matches + /// `simdjson::internal::tape_ref::scope_count`. + #[inline] + pub fn scope_count(&self) -> usize { + ((self.current_word() >> 32) & 0xFFFF_FF) as usize + } + + /// Advance the cursor by `n` tape words. + #[inline] + pub fn skip(&mut self, n: usize) { + self.pos += n; + } + + /// Move the cursor to an absolute tape index. + #[inline] + pub fn seek_public(&mut self, pos: usize) { + self.pos = pos; + } + + /// Decode the string at the current position. + /// + /// The string buffer layout is: 4-byte little-endian length prefix followed + /// by raw UTF-8 bytes. The tape word payload is a byte offset into + /// `string_buf`. + /// + /// The returned `&str` has lifetime `'a` (tied to the parser), so it can + /// be used for zero-copy deserialization. + #[inline] + pub fn get_string(&self) -> Result<&'a str> { + debug_assert_eq!( + self.tape_type(), + Some(TapeType::String), + "get_string called on non-string node" + ); + let offset = self.payload() as usize; + ensure!( + offset + 4 <= self.string_buf.len(), + StringOutOfBoundsSnafu { + offset, + len: self.string_buf.len(), + } + ); + // First 4 bytes: little-endian u32 length. + let len_bytes = &self.string_buf[offset..offset + 4]; + let len = u32::from_le_bytes(len_bytes.try_into().unwrap()) as usize; + let start = offset + 4; + let end = start + len; + ensure!( + end <= self.string_buf.len(), + StringOutOfBoundsSnafu { + offset, + len: self.string_buf.len(), + } + ); + // SAFETY: simdjson validates UTF-8 while building the string buffer + // during stage2; a parse that reaches this point has already passed + // simdjson's UTF-8 check (a failure would have surfaced as a + // `SimdJsonError::Utf8Error`/`StringError` from `parser_parse`). + // Re-validating here showed up as ~11% of CPU in profiling, all of it + // redundant. + Ok(unsafe { str::from_utf8_unchecked(&self.string_buf[start..end]) }) + } + + /// Decode the `i64` value at the current position. + /// + /// The actual integer is stored in the *next* tape word (reinterpreted as + /// `i64` bits). + #[inline] + pub fn get_int64(&self) -> Result { + debug_assert_eq!( + self.tape_type(), + Some(TapeType::Int64), + "get_int64 called on non-int64 node" + ); + let next = self.pos + 1; + ensure!( + next < self.tape.len(), + MissingValueWordSnafu { pos: self.pos } + ); + Ok(self.tape[next] as i64) + } + + /// Decode the `u64` value at the current position. + #[inline] + pub fn get_uint64(&self) -> Result { + debug_assert_eq!( + self.tape_type(), + Some(TapeType::Uint64), + "get_uint64 called on non-uint64 node" + ); + let next = self.pos + 1; + ensure!( + next < self.tape.len(), + MissingValueWordSnafu { pos: self.pos } + ); + Ok(self.tape[next]) + } + + /// Decode the `f64` value at the current position. + #[inline] + pub fn get_double(&self) -> Result { + debug_assert_eq!( + self.tape_type(), + Some(TapeType::Double), + "get_double called on non-double node" + ); + let next = self.pos + 1; + ensure!( + next < self.tape.len(), + MissingValueWordSnafu { pos: self.pos } + ); + Ok(f64::from_bits(self.tape[next])) + } + + /// Parse the value at the current tape position into a zero-copy `Value`. + /// + /// After a successful call the cursor is advanced past the parsed value, + /// ready to decode the next element. + pub fn parse_value(&mut self) -> Result> { + ensure!( + self.pos < self.tape.len(), + OutOfBoundsSnafu { + pos: self.pos, + len: self.tape.len(), + } + ); + + match self.tape_type() { + Some(TapeType::Root) => { + // Skip the root node and parse the document value. + self.skip(1); + self.parse_value() + } + Some(TapeType::Null) => { + self.skip(1); + Ok(Value::Null) + } + Some(TapeType::True) => { + self.skip(1); + Ok(Value::Bool(true)) + } + Some(TapeType::False) => { + self.skip(1); + Ok(Value::Bool(false)) + } + Some(TapeType::Int64) => { + let v = self.get_int64()?; + self.skip(2); // type word + value word + Ok(Value::Int64(v)) + } + Some(TapeType::Uint64) => { + let v = self.get_uint64()?; + self.skip(2); + Ok(Value::Uint64(v)) + } + Some(TapeType::Double) => { + let v = self.get_double()?; + self.skip(2); + Ok(Value::Double(v)) + } + Some(TapeType::String) => { + let s = self.get_string()?; + self.skip(1); + Ok(Value::String(s)) + } + Some(TapeType::StartArray) => { + // scope_close_idx = position of the matching `]` node + let end_idx = self.scope_close_idx(); + self.skip(1); // move past `[` + let mut arr = Vec::new(); + while self.pos < end_idx { + let v = self.parse_value()?; + arr.push(v); + } + // Now at the `]` node; skip it. + self.skip(1); + Ok(Value::Array(arr)) + } + Some(TapeType::StartObject) => { + // scope_close_idx = position of the matching `}` node + let end_idx = self.scope_close_idx(); + self.skip(1); // move past `{` + let mut obj = Vec::new(); + while self.pos < end_idx { + // Each field is: key (String node) then value + let key = self.get_string()?; + self.skip(1); // past key string word + let value = self.parse_value()?; + obj.push((key, value)); + } + // Now at the `}` node; skip it. + self.skip(1); + Ok(Value::Object(obj)) + } + Some(t) => UnexpectedTapeTypeSnafu { + found: t as u8, + pos: self.pos, + } + .fail(), + None => UnknownTapeTypeSnafu { + tag: (self.current_word() >> 56) as u8, + pos: self.pos, + } + .fail(), + } + } + + /// Visit the tape using a [`TapeVisitor`]. + /// + /// This is the low-level API for custom traversal. The visitor receives + /// structured callbacks as the tape is walked. + pub fn visit>(&mut self, visitor: &mut V) -> Result { + ensure!( + self.pos < self.tape.len(), + OutOfBoundsSnafu { + pos: self.pos, + len: self.tape.len(), + } + ); + + match self.tape_type() { + Some(TapeType::Root) => { + self.skip(1); + self.visit(visitor) + } + Some(TapeType::Null) => { + self.skip(1); + visitor.visit_null() + } + Some(TapeType::True) => { + self.skip(1); + visitor.visit_bool(true) + } + Some(TapeType::False) => { + self.skip(1); + visitor.visit_bool(false) + } + Some(TapeType::Int64) => { + let v = self.get_int64()?; + self.skip(2); + visitor.visit_i64(v) + } + Some(TapeType::Uint64) => { + let v = self.get_uint64()?; + self.skip(2); + visitor.visit_u64(v) + } + Some(TapeType::Double) => { + let v = self.get_double()?; + self.skip(2); + visitor.visit_f64(v) + } + Some(TapeType::String) => { + let s = self.get_string()?; + self.skip(1); + visitor.visit_str(s) + } + Some(TapeType::StartArray) => { + let end_idx = self.scope_close_idx(); + self.skip(1); + let result = visitor.visit_array(self, end_idx)?; + self.skip(1); // past `]` + Ok(result) + } + Some(TapeType::StartObject) => { + let end_idx = self.scope_close_idx(); + self.skip(1); + let result = visitor.visit_object(self, end_idx)?; + self.skip(1); // past `}` + Ok(result) + } + Some(t) => UnexpectedTapeTypeSnafu { + found: t as u8, + pos: self.pos, + } + .fail(), + None => UnknownTapeTypeSnafu { + tag: (self.current_word() >> 56) as u8, + pos: self.pos, + } + .fail(), + } + } +} + +/// Callback-based visitor interface for the simdjson tape. +/// +/// Implement this trait to build custom data structures from the tape without +/// allocating intermediate `Value` nodes. +pub trait TapeVisitor<'a> { + /// The value produced after visiting a complete JSON value. + type Output; + + fn visit_null(&mut self) -> Result; + fn visit_bool(&mut self, v: bool) -> Result; + fn visit_i64(&mut self, v: i64) -> Result; + fn visit_u64(&mut self, v: u64) -> Result; + fn visit_f64(&mut self, v: f64) -> Result; + + /// Visit a string value. The `s` borrow is tied to the tape lifetime `'a`. + fn visit_str(&mut self, s: &'a str) -> Result; + + /// Visit an array. The implementor should call `tape.visit(self)` for each + /// element while `tape.pos() < end_idx`. + fn visit_array(&mut self, tape: &mut TapeRef<'a>, end_idx: usize) -> Result; + + /// Visit an object. Fields are key–value pairs; the implementor should + /// read key strings and call `tape.visit(self)` for values while + /// `tape.pos() < end_idx`. + fn visit_object(&mut self, tape: &mut TapeRef<'a>, end_idx: usize) -> Result; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tape_type_decoding() { + let words: &[u64] = &[ + (b'r' as u64) << 56 | 3, + (b'n' as u64) << 56, + (b'r' as u64) << 56, + ]; + let mut r = TapeRef::new(words, &[]); + assert_eq!(r.tape_type(), Some(TapeType::Root)); + r.skip(1); + assert_eq!(r.tape_type(), Some(TapeType::Null)); + } + + #[test] + fn tape_bool_types() { + let words: &[u64] = &[(b't' as u64) << 56, (b'f' as u64) << 56]; + let r = TapeRef::new(words, &[]); + assert_eq!(r.tape_type(), Some(TapeType::True)); + let mut r2 = r; + r2.skip(1); + assert_eq!(r2.tape_type(), Some(TapeType::False)); + } + + #[test] + fn tape_type_from_tape_word() { + assert_eq!( + TapeType::from_tape_word((b'r' as u64) << 56), + Some(TapeType::Root) + ); + assert_eq!( + TapeType::from_tape_word((b'{' as u64) << 56), + Some(TapeType::StartObject) + ); + assert_eq!( + TapeType::from_tape_word((b'}' as u64) << 56), + Some(TapeType::EndObject) + ); + assert_eq!( + TapeType::from_tape_word((b'[' as u64) << 56), + Some(TapeType::StartArray) + ); + assert_eq!( + TapeType::from_tape_word((b']' as u64) << 56), + Some(TapeType::EndArray) + ); + assert_eq!( + TapeType::from_tape_word((b'"' as u64) << 56), + Some(TapeType::String) + ); + assert_eq!( + TapeType::from_tape_word((b'l' as u64) << 56), + Some(TapeType::Int64) + ); + assert_eq!( + TapeType::from_tape_word((b'u' as u64) << 56), + Some(TapeType::Uint64) + ); + assert_eq!( + TapeType::from_tape_word((b'd' as u64) << 56), + Some(TapeType::Double) + ); + assert_eq!( + TapeType::from_tape_word((b't' as u64) << 56), + Some(TapeType::True) + ); + assert_eq!( + TapeType::from_tape_word((b'f' as u64) << 56), + Some(TapeType::False) + ); + assert_eq!( + TapeType::from_tape_word((b'n' as u64) << 56), + Some(TapeType::Null) + ); + assert_eq!(TapeType::from_tape_word(0xAA00_0000_0000_0000), None); + } + + /// A simple visitor that counts nodes. + #[cfg(feature = "serde")] + struct NodeCounter { + count: usize, + } + + #[cfg(feature = "serde")] + impl<'a> TapeVisitor<'a> for NodeCounter { + type Output = usize; + + fn visit_null(&mut self) -> Result { + self.count += 1; + Ok(self.count) + } + + fn visit_bool(&mut self, _v: bool) -> Result { + self.count += 1; + Ok(self.count) + } + + fn visit_i64(&mut self, _v: i64) -> Result { + self.count += 1; + Ok(self.count) + } + + fn visit_u64(&mut self, _v: u64) -> Result { + self.count += 1; + Ok(self.count) + } + + fn visit_f64(&mut self, _v: f64) -> Result { + self.count += 1; + Ok(self.count) + } + + fn visit_str(&mut self, _s: &'a str) -> Result { + self.count += 1; + Ok(self.count) + } + + fn visit_array(&mut self, tape: &mut TapeRef<'a>, end_idx: usize) -> Result { + while tape.pos() < end_idx { + tape.visit(self)?; + } + Ok(self.count) + } + + fn visit_object(&mut self, tape: &mut TapeRef<'a>, end_idx: usize) -> Result { + while tape.pos() < end_idx { + // Key + tape.visit(self)?; + // Value + tape.visit(self)?; + } + Ok(self.count) + } + } + + #[cfg(feature = "serde")] + #[test] + fn tape_visitor_counts_nodes() { + use crate::dom::Parser; + let mut p = Parser::default(); + // {"a": 1, "b": [true, false]} → nodes: "a", 1, "b", true, false = 5 + let mut tape = p.parse_str(r#"{"a":1,"b":[true,false]}"#).unwrap(); + let mut counter = NodeCounter { count: 0 }; + tape.visit(&mut counter).unwrap(); + assert_eq!(counter.count, 5); + } + + #[test] + fn parse_value_empty_array() { + use crate::dom::Parser; + let mut p = Parser::default(); + let mut tape = p.parse_str("[]").unwrap(); + let v = tape.parse_value().unwrap(); + assert_eq!(v, Value::Array(vec![])); + } + + #[test] + fn parse_value_empty_object() { + use crate::dom::Parser; + let mut p = Parser::default(); + let mut tape = p.parse_str("{}").unwrap(); + let v = tape.parse_value().unwrap(); + assert_eq!(v, Value::Object(vec![])); + } + + #[test] + fn parse_value_uint64() { + use crate::dom::Parser; + let mut p = Parser::default(); + // Large positive integer that simdjson stores as u64 + let mut tape = p.parse_str("18446744073709551615").unwrap(); // u64::MAX + let v = tape.parse_value().unwrap(); + assert_eq!(v.as_u64(), Some(u64::MAX)); + } +} diff --git a/src/utils.rs b/src/utils.rs deleted file mode 100644 index 6e667d1..0000000 --- a/src/utils.rs +++ /dev/null @@ -1,24 +0,0 @@ -use std::ptr::NonNull; - -use simdjson_sys as ffi; - -#[inline] -pub fn string_view_to_str<'a>(sv: NonNull) -> &'a str { - let s = unsafe { - let s = std::slice::from_raw_parts( - ffi::STD_string_view_data(sv.as_ptr()).cast(), - ffi::STD_string_view_size(sv.as_ptr()), - ); - std::str::from_utf8_unchecked(s) - }; - unsafe { ffi::STD_string_view_free(sv.as_ptr()) }; - s -} - -#[inline] -pub fn string_view_struct_to_str<'a>(sv: ffi::SJ_string_view) -> &'a str { - unsafe { - let s = std::slice::from_raw_parts(sv.data.cast(), sv.len); - std::str::from_utf8_unchecked(s) - } -} diff --git a/tests/basic_tests.rs b/tests/basic_tests.rs index 0d729ef..dc0c08f 100644 --- a/tests/basic_tests.rs +++ b/tests/basic_tests.rs @@ -1,43 +1,187 @@ -// #[cfg(test)] -// mod number_tests { - -// use simdjson_rust::dom; -// use simdjson_rust::error::SimdJsonError; - -// // ulp distance -// // Marc B. Reynolds, 2016-2019 -// // Public Domain under http://unlicense.org, see link for details. -// // adapted by D. Lemire -// #[allow(dead_code)] -// fn f64_ulp_dist(a: f64, b: f64) -> u64 { -// // let ua: u64 = transmute(a); -// // let ub: u64 = transmute(b); -// let (ua, ub): (u64, u64) = (a.to_bits(), b.to_bits()); - -// if (ub ^ ua) as i64 >= 0 { -// if (ua - ub) as i64 >= 0 { -// ua - ub -// } else { -// ub - ua -// } -// } else { -// ua + ub + 0x80000000 -// } -// } - -// #[test] -// fn small_integers() -> Result<(), SimdJsonError> { -// let mut parser = dom::parser::Parser::default(); - -// for _m in 10..20 { -// for i in -1024..1024 { -// let s = i.to_string(); -// let actual = parser.parse(&s)?.get_i64()?; - -// assert_eq!(actual, i); -// } -// } - -// Ok(()) -// } -// } +//! Integration tests for simdjson-rust tape parsing. + +#[cfg(test)] +mod tape_integration { + use simdjson_rust::{dom::Parser, tape::TapeType}; + + #[test] + fn parse_null_returns_root_node() { + let mut p = Parser::default(); + let tape = p.parse_str("null").unwrap(); + assert_eq!(tape.tape_type(), Some(TapeType::Root)); + } + + #[test] + fn parse_object_tape_structure() { + let mut p = Parser::default(); + let tape = p.parse_str(r#"{"answer":42}"#).unwrap(); + assert_eq!(tape.tape_type(), Some(TapeType::Root)); + } + + #[test] + fn parse_to_value_null() { + let mut p = Parser::default(); + let v = p.parse_to_value("null").unwrap(); + assert!(v.is_null()); + } + + #[test] + fn parse_to_value_bool() { + let mut p = Parser::default(); + assert_eq!(p.parse_to_value("true").unwrap().as_bool(), Some(true)); + assert_eq!(p.parse_to_value("false").unwrap().as_bool(), Some(false)); + } + + #[test] + fn parse_to_value_integers() { + let mut p = Parser::default(); + assert_eq!(p.parse_to_value("42").unwrap().as_u64(), Some(42)); + assert_eq!(p.parse_to_value("-100").unwrap().as_i64(), Some(-100)); + } + + #[test] + fn parse_to_value_float() { + let mut p = Parser::default(); + let v = p.parse_to_value("1.5").unwrap(); + assert_eq!(v.as_f64().unwrap(), 1.5_f64); + } + + #[test] + fn parse_to_value_string() { + let mut p = Parser::default(); + let v = p.parse_to_value(r#""hello world""#).unwrap(); + assert_eq!(v.as_str().unwrap(), "hello world"); + } + + #[test] + fn parse_to_value_array() { + let mut p = Parser::default(); + let v = p.parse_to_value("[1, 2, 3]").unwrap(); + assert!(v.as_array().is_some()); + let arr = v.as_array().unwrap(); + assert_eq!(arr.len(), 3); + assert_eq!(arr[0].as_u64(), Some(1)); + assert_eq!(arr[1].as_u64(), Some(2)); + assert_eq!(arr[2].as_u64(), Some(3)); + } + + #[test] + fn parse_to_value_object() { + let mut p = Parser::default(); + let v = p + .parse_to_value(r#"{"answer": 42, "ratio": 1.5, "ok": true}"#) + .unwrap(); + assert_eq!(v.get("answer").unwrap().as_u64(), Some(42)); + assert_eq!(v.get("ratio").unwrap().as_f64(), Some(1.5)); + assert_eq!(v.get("ok").unwrap().as_bool(), Some(true)); + } + + #[test] + fn parse_to_value_nested() { + let mut p = Parser::default(); + let v = p + .parse_to_value(r#"{"outer": {"inner": [1, 2, {"deep": true}]}}"#) + .unwrap(); + let outer = v.get("outer").unwrap(); + let inner = outer.get("inner").unwrap(); + let arr = inner.as_array().unwrap(); + assert_eq!(arr[0].as_u64(), Some(1)); + assert_eq!(arr[1].as_u64(), Some(2)); + assert_eq!(arr[2].get("deep").unwrap().as_bool(), Some(true)); + } + + #[test] + fn parse_to_value_unicode() { + let mut p = Parser::default(); + let v = p + .parse_to_value(r#"{"emoji": "🦀", "chinese": "你好"}"#) + .unwrap(); + assert_eq!(v.get("emoji").unwrap().as_str().unwrap(), "🦀"); + assert_eq!(v.get("chinese").unwrap().as_str().unwrap(), "你好"); + } + + #[cfg(feature = "serde")] + mod with_serde { + use serde::Deserialize; + use simdjson_rust::serde::{from_bytes, from_str}; + + #[derive(Debug, Deserialize, PartialEq)] + struct Person { + name: String, + age: u64, + active: bool, + } + + #[test] + fn from_str_struct() { + let p: Person = from_str(r#"{"name": "Alice", "age": 30, "active": true}"#).unwrap(); + assert_eq!( + p, + Person { + name: "Alice".to_string(), + age: 30, + active: true + } + ); + } + + #[test] + fn from_bytes_struct() { + let json = br#"{"name": "Bob", "age": 25, "active": false}"#; + let p: Person = from_bytes(json).unwrap(); + assert_eq!( + p, + Person { + name: "Bob".to_string(), + age: 25, + active: false + } + ); + } + + #[derive(Debug, Deserialize, PartialEq)] + struct WithOptional { + required: String, + optional: Option, + } + + #[test] + fn from_str_with_null_optional() { + let v: WithOptional = from_str(r#"{"required": "test", "optional": null}"#).unwrap(); + assert_eq!(v.required, "test"); + assert_eq!(v.optional, None); + } + + #[test] + fn from_str_with_some_optional() { + let v: WithOptional = from_str(r#"{"required": "test", "optional": 99}"#).unwrap(); + assert_eq!(v.optional, Some(99)); + } + + #[derive(Debug, Deserialize, PartialEq)] + #[serde(rename_all = "lowercase")] + enum Status { + Ok, + Error, + } + + #[test] + fn from_str_unit_enum() { + let s: Status = from_str(r#""ok""#).unwrap(); + assert_eq!(s, Status::Ok); + let e: Status = from_str(r#""error""#).unwrap(); + assert_eq!(e, Status::Error); + } + + #[test] + fn from_str_vec_of_structs() { + let v: Vec = from_str( + r#"[{"name":"A","age":1,"active":true},{"name":"B","age":2,"active":false}]"#, + ) + .unwrap(); + assert_eq!(v.len(), 2); + assert_eq!(v[0].name, "A"); + assert_eq!(v[1].age, 2); + } + } +}