From 5d6cc8207dbb08746d58f7e07a83645235ddf3e5 Mon Sep 17 00:00:00 2001 From: Tyler Beason Date: Wed, 17 Jun 2026 11:58:30 -0500 Subject: [PATCH 1/2] Add symbol joining to to_dataframe (symbols kwarg) Records carry only the opaque numeric instrument_id; the human-readable symbol lives in metadata.mappings. to_dataframe(store; symbols=true) now joins the raw_symbol onto the frame as a :symbol column, resolved per record from store.metadata.mappings via DatabentoBinaryEncoding's new symbol helpers. The join keys on instrument_id/ts_event, follows instrument-id rolls across the query range, and yields missing for unmatched rows. It applies only when the data was fetched with stype_out = SType.INSTRUMENT_ID (the get_range default). Default is symbols=false, so existing to_dataframe calls are unaffected. Re-exports symbol_map / symbol_for / add_symbol_column! from DatabentoBinaryEncoding so streaming consumers (foreach_record, which returns the metadata) can build the lookup once and resolve symbols per record. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 10 ++++++++++ src/DatabentoAPI.jl | 4 ++++ src/conversion.jl | 19 ++++++++++++++++--- test/test_conversion.jl | 26 ++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef85d19..7226145 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- **Symbol joining on `to_dataframe`.** `to_dataframe(store; symbols=true)` joins + the human-readable `raw_symbol` onto the frame as a `:symbol` column, resolved + per record from `store.metadata.mappings` (records carry only the opaque + numeric `instrument_id`). The join follows instrument-id rolls across the query + range and yields `missing` for unmatched rows; it applies only when the data + was fetched with `stype_out = SType.INSTRUMENT_ID` (the `get_range` default). + The underlying helpers `symbol_map`, `symbol_for`, and `add_symbol_column!` are + re-exported from DatabentoBinaryEncoding for streaming consumers + (`foreach_record`, which returns the metadata). Requires the corresponding + DatabentoBinaryEncoding release. - **Chunked + concurrent `get_range`** (#33). New `chunk` kwarg (a `Dates.Period`, e.g. `Year(1)`) splits `[start_dt, end_dt)` into half-open chunks fetched concurrently under a semaphore (`concurrency` kwarg, default diff --git a/src/DatabentoAPI.jl b/src/DatabentoAPI.jl index d9b4119..a3253d9 100644 --- a/src/DatabentoAPI.jl +++ b/src/DatabentoAPI.jl @@ -52,6 +52,7 @@ using URIs import DatabentoBinaryEncoding as DBN using DatabentoBinaryEncoding: Schema, SType, Compression, Encoding, Action, Side, InstrumentClass +using DatabentoBinaryEncoding: symbol_map, symbol_for, add_symbol_column! include("errors.jl") include("enums.jl") @@ -94,6 +95,9 @@ export load_api_key, default_config_path # Store + conversion export DBNStore, to_dataframe, to_csv, to_json, to_parquet, to_file +# Symbol resolution (re-exported from DatabentoBinaryEncoding) +export symbol_map, symbol_for, add_symbol_column! + # Clients export Historical, Live diff --git a/src/conversion.jl b/src/conversion.jl index d59191f..4776e0f 100644 --- a/src/conversion.jl +++ b/src/conversion.jl @@ -2,11 +2,24 @@ # does not depend on DataFrames/CSV/Parquet directly — DBN.jl already pulls them in. """ - to_dataframe(store) + to_dataframe(store; symbols=false) -Convert a `DBNStore` to a `DataFrame` via DBN.jl's `records_to_dataframe`. +Convert a `DBNStore` to a `DataFrame` via DBN's `records_to_dataframe`. + +Pass `symbols=true` to join the human-readable `raw_symbol` onto the frame as a +`:symbol` column, resolved per record from `store.metadata.mappings` (records +themselves carry only the opaque numeric `instrument_id`). The join keys on +`instrument_id`/`ts_event`, follows instrument-id rolls across the query range, +and yields `missing` for unmatched rows. It only applies when the data was +fetched with `stype_out = SType.INSTRUMENT_ID` (the [`get_range`](@ref) default); +any other `stype_out` leaves the column `missing` with a single warning. + +For streaming consumers ([`foreach_record`](@ref), which returns the +`DBN.Metadata`), build a [`symbol_map`](@ref) once from that metadata and call +[`symbol_for`](@ref) per record instead. """ -to_dataframe(s::DBNStore) = DBN.records_to_dataframe(s.records) +to_dataframe(s::DBNStore; symbols::Bool = false) = + DBN.records_to_dataframe(s.records, s.metadata; symbols = symbols) """ to_file(store, path) diff --git a/test/test_conversion.jl b/test/test_conversion.jl index 8bd1241..84e9889 100644 --- a/test/test_conversion.jl +++ b/test/test_conversion.jl @@ -40,4 +40,30 @@ end @test length(recs) == 1 end end + + @testset "to_dataframe symbol join" begin + # _tiny_store's record: instrument_id=101, ts_event = 2023-11-14 (UTC), + # so YYYYMMDD = 20231114. Add a mapping covering that date. + s = _tiny_store() + md = DBN.Metadata( + s.metadata.version, s.metadata.dataset, s.metadata.schema, + s.metadata.start_ts, s.metadata.end_ts, s.metadata.limit, + s.metadata.stype_in, s.metadata.stype_out, s.metadata.ts_out, + s.metadata.symbols, s.metadata.partial, s.metadata.not_found, + [("AAPL", "101", 20231101, 20231201)], + ) + store = DBNStore(md, s.records) + + # off by default: no :symbol column, same frame as before + @test !hasproperty(to_dataframe(store), :symbol) + + # on: joins the raw symbol + df = to_dataframe(store; symbols = true) + @test df.symbol == ["AAPL"] + + # re-exported helpers are usable directly (e.g. for foreach_record) + smap = symbol_map(md) + @test symbol_for(smap, 101, store.records[1].hd.ts_event) == "AAPL" + @test symbol_for(smap, 999, store.records[1].hd.ts_event) === nothing + end end From 6e76a7f57291f515ba7ab3a32e6379f38c2dfc2e Mon Sep 17 00:00:00 2001 From: Tyler Beason Date: Wed, 17 Jun 2026 12:18:26 -0500 Subject: [PATCH 2/2] Require DatabentoBinaryEncoding 0.1.4; fix docstring @refs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses Codex P1: this package now imports symbol_map / symbol_for / add_symbol_column! from DatabentoBinaryEncoding, which first exist in its 0.1.4 release. Bump the [compat] lower bound from 0.1.2 to 0.1.4 so a fresh install can't resolve an older DBN that lacks these bindings (which would make `using DatabentoAPI` fail during module load). Also drop the [`symbol_map`](@ref) / [`symbol_for`](@ref) cross-reference links from the to_dataframe docstring — those helpers are documented in the DatabentoBinaryEncoding docs, not spliced into this package's API reference, so the @refs would fail Documenter's cross_references check. Use plain code spans. Co-Authored-By: Claude Opus 4.8 (1M context) --- Project.toml | 2 +- src/conversion.jl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Project.toml b/Project.toml index c33aad1..bef5bad 100644 --- a/Project.toml +++ b/Project.toml @@ -22,7 +22,7 @@ URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" [compat] CodecZstd = "0.8" DataFrames = "1.7" -DatabentoBinaryEncoding = "0.1.2" +DatabentoBinaryEncoding = "0.1.4" EnumX = "1.0" HTTP = "1.10" JSON3 = "1.14" diff --git a/src/conversion.jl b/src/conversion.jl index 4776e0f..a7f6dba 100644 --- a/src/conversion.jl +++ b/src/conversion.jl @@ -15,8 +15,8 @@ fetched with `stype_out = SType.INSTRUMENT_ID` (the [`get_range`](@ref) default) any other `stype_out` leaves the column `missing` with a single warning. For streaming consumers ([`foreach_record`](@ref), which returns the -`DBN.Metadata`), build a [`symbol_map`](@ref) once from that metadata and call -[`symbol_for`](@ref) per record instead. +`DBN.Metadata`), build a `symbol_map` once from that metadata and call +`symbol_for` per record instead (both re-exported from DatabentoBinaryEncoding). """ to_dataframe(s::DBNStore; symbols::Bool = false) = DBN.records_to_dataframe(s.records, s.metadata; symbols = symbols)