Symbol joining on to_dataframe (instrument_id → raw_symbol)

CHANGELOG.md

@@ -8,6 +8,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Added
+- **Symbol joining on `to_dataframe`.** `to_dataframe(store; symbols=true)` joins
+  the human-readable `raw_symbol` onto the frame as a `:symbol` column, resolved
+  per record from `store.metadata.mappings` (records carry only the opaque
+  numeric `instrument_id`). The join follows instrument-id rolls across the query
+  range and yields `missing` for unmatched rows; it applies only when the data
+  was fetched with `stype_out = SType.INSTRUMENT_ID` (the `get_range` default).
+  The underlying helpers `symbol_map`, `symbol_for`, and `add_symbol_column!` are
+  re-exported from DatabentoBinaryEncoding for streaming consumers
+  (`foreach_record`, which returns the metadata). Requires the corresponding
+  DatabentoBinaryEncoding release.
 - **Chunked + concurrent `get_range`** (#33). New `chunk` kwarg (a
   `Dates.Period`, e.g. `Year(1)`) splits `[start_dt, end_dt)` into half-open
   chunks fetched concurrently under a semaphore (`concurrency` kwarg, default

Project.toml

@@ -22,7 +22,7 @@ URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"
 [compat]
 CodecZstd = "0.8"
 DataFrames = "1.7"
-DatabentoBinaryEncoding = "0.1.2"
+DatabentoBinaryEncoding = "0.1.4"
 EnumX = "1.0"
 HTTP = "1.10"
 JSON3 = "1.14"

src/DatabentoAPI.jl

@@ -52,6 +52,7 @@ using URIs
 
 import DatabentoBinaryEncoding as DBN
 using DatabentoBinaryEncoding: Schema, SType, Compression, Encoding, Action, Side, InstrumentClass
+using DatabentoBinaryEncoding: symbol_map, symbol_for, add_symbol_column!
 
 include("errors.jl")
 include("enums.jl")
@@ -94,6 +95,9 @@ export load_api_key, default_config_path
 # Store + conversion
 export DBNStore, to_dataframe, to_csv, to_json, to_parquet, to_file
 
+# Symbol resolution (re-exported from DatabentoBinaryEncoding)
+export symbol_map, symbol_for, add_symbol_column!
+
 # Clients
 export Historical, Live
 

src/conversion.jl

@@ -2,11 +2,24 @@
 # does not depend on DataFrames/CSV/Parquet directly — DBN.jl already pulls them in.
 
 """
-    to_dataframe(store)
+    to_dataframe(store; symbols=false)
 
-Convert a `DBNStore` to a `DataFrame` via DBN.jl's `records_to_dataframe`.
+Convert a `DBNStore` to a `DataFrame` via DBN's `records_to_dataframe`.
+
+Pass `symbols=true` to join the human-readable `raw_symbol` onto the frame as a
+`:symbol` column, resolved per record from `store.metadata.mappings` (records
+themselves carry only the opaque numeric `instrument_id`). The join keys on
+`instrument_id`/`ts_event`, follows instrument-id rolls across the query range,
+and yields `missing` for unmatched rows. It only applies when the data was
+fetched with `stype_out = SType.INSTRUMENT_ID` (the [`get_range`](@ref) default);
+any other `stype_out` leaves the column `missing` with a single warning.
+
+For streaming consumers ([`foreach_record`](@ref), which returns the
+`DBN.Metadata`), build a `symbol_map` once from that metadata and call
+`symbol_for` per record instead (both re-exported from DatabentoBinaryEncoding).
 """
-to_dataframe(s::DBNStore) = DBN.records_to_dataframe(s.records)
+to_dataframe(s::DBNStore; symbols::Bool = false) =
+    DBN.records_to_dataframe(s.records, s.metadata; symbols = symbols)
 
 """
     to_file(store, path)

test/test_conversion.jl

@@ -40,4 +40,30 @@ end
             @test length(recs) == 1
         end
     end
+
+    @testset "to_dataframe symbol join" begin
+        # _tiny_store's record: instrument_id=101, ts_event = 2023-11-14 (UTC),
+        # so YYYYMMDD = 20231114. Add a mapping covering that date.
+        s = _tiny_store()
+        md = DBN.Metadata(
+            s.metadata.version, s.metadata.dataset, s.metadata.schema,
+            s.metadata.start_ts, s.metadata.end_ts, s.metadata.limit,
+            s.metadata.stype_in, s.metadata.stype_out, s.metadata.ts_out,
+            s.metadata.symbols, s.metadata.partial, s.metadata.not_found,
+            [("AAPL", "101", 20231101, 20231201)],
+        )
+        store = DBNStore(md, s.records)
+
+        # off by default: no :symbol column, same frame as before
+        @test !hasproperty(to_dataframe(store), :symbol)
+
+        # on: joins the raw symbol
+        df = to_dataframe(store; symbols = true)
+        @test df.symbol == ["AAPL"]
+
+        # re-exported helpers are usable directly (e.g. for foreach_record)
+        smap = symbol_map(md)
+        @test symbol_for(smap, 101, store.records[1].hd.ts_event) == "AAPL"
+        @test symbol_for(smap, 999, store.records[1].hd.ts_event) === nothing
+    end
 end