From beef0a29160b17aa343cb6b9e9b39fc7858cc4fe Mon Sep 17 00:00:00 2001 From: Tyler Beason Date: Thu, 25 Jun 2026 09:27:55 -0500 Subject: [PATCH 1/2] Upgrade to HTTP.jl 2.4 and release v0.3.1 HTTP.jl 2.x is a ground-up rewrite. All real-HTTP usage is isolated in src/http.jl (plus one direct dispatcher call in batch.jl): - readtimeout -> read_idle_timeout (2.x rename; same idle-timeout semantics) - retry=false -> retries=0 (2.x has no `retry` boolean) - HTTP.Exceptions.{HTTPError,TimeoutError,ConnectError} -> top-level HTTP.* (the Exceptions submodule is gone) - streaming uses startread's returned Response instead of http.message - JSON3.read(resp.body) -> JSON3.read(Vector{UInt8}(resp.body)); in 2.x resp.body is an HTTP.BytesBody wrapper JSON3 doesn't recognize - _is_retryable_exception keeps connect timeouts retryable: 2.x surfaces them as TimeoutError(operation="connect") rather than ConnectError Tests: HTTP.Exceptions.X -> HTTP.X and TimeoutError(n) -> TimeoutError("read", n) (the 1-arg ctor is gone). HTTP.Response mock ctors are backward compatible. Compat bumped to "2.4" in Project.toml and benchmark/Project.toml. Bumps the package to v0.3.1. Supersedes #42. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 11 +++++++ Project.toml | 4 +-- benchmark/Project.toml | 3 ++ src/historical/batch.jl | 10 +++--- src/http.jl | 51 ++++++++++++++++++------------ test/test_historical_timeseries.jl | 12 +++---- test/test_http.jl | 6 ++-- 7 files changed, 61 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 637a64a..815bc6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.1] - 2026-06-25 + ### Added - **Live symbol resolution.** A `Live` client now keeps a running `instrument_id → symbol` map, built by the reader from the gateway's @@ -18,6 +20,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 typed mode the mapping is captured before any `control_channel` overflow drop, so it's reliable even without draining the control channel. +### Changed +- **Upgraded to HTTP.jl 2.4** (from 1.10). HTTP.jl 2.x is a ground-up rewrite + with renamed keywords and top-level exception types; the changes here are + internal (`readtimeout` → `read_idle_timeout`, `retry=false` → `retries=0`, + `HTTP.Exceptions.X` → `HTTP.X`, and using `startread`'s returned response) and + do not affect the public API. Connect timeouts (now surfaced as + `HTTP.TimeoutError(operation="connect")` rather than `HTTP.ConnectError`) + remain retryable, as before. + ## [0.3.0] - 2026-06-22 ### Added diff --git a/Project.toml b/Project.toml index c97b560..cd3cb35 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "DatabentoAPI" uuid = "feb0aed8-3291-460b-9688-521dcb17a4bf" -version = "0.3.0" +version = "0.3.1" authors = ["Tyler Beason "] [deps] @@ -24,7 +24,7 @@ CodecZstd = "0.8" DataFrames = "1.7" DatabentoBinaryEncoding = "0.1.4" EnumX = "1.0" -HTTP = "1.10" +HTTP = "2.4" JSON3 = "1.14" SHA = "0.7.0" TOML = "1.0.3" diff --git a/benchmark/Project.toml b/benchmark/Project.toml index db47b0f..488ded5 100644 --- a/benchmark/Project.toml +++ b/benchmark/Project.toml @@ -11,6 +11,9 @@ Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" TranscodingStreams = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" +[compat] +HTTP = "2.4" + [sources] DatabentoAPI = {path = ".."} DatabentoBinaryEncoding = {path = "../../DatabentoBinaryEncoding.jl"} diff --git a/src/historical/batch.jl b/src/historical/batch.jl index 0d51134..0e2e288 100644 --- a/src/historical/batch.jl +++ b/src/historical/batch.jl @@ -156,11 +156,11 @@ function batch_download(c::Historical; "Accept" => "application/octet-stream", "User-Agent" => c.http.user_agent], UInt8[]; - status_exception = false, - readtimeout = c.http.timeout, - connect_timeout = c.http.connect_timeout, - retry = false, - decompress = false) + status_exception = false, + read_idle_timeout = c.http.timeout, # HTTP 2.x rename of `readtimeout` + connect_timeout = c.http.connect_timeout, + retries = 0, # HTTP 2.x: disable built-in retry (was `retry=false`) + decompress = false) if resp.status >= 400 T = resp.status < 500 ? BentoClientError : BentoServerError throw(http_error_from_response(T, resp.status, diff --git a/src/http.jl b/src/http.jl index 9cf2ac3..ddd2f03 100644 --- a/src/http.jl +++ b/src/http.jl @@ -42,14 +42,16 @@ _default_http_request(method, url, headers, body; kwargs...) = # (`stream_opener(c, method, url, headers, qpairs) do status, headers, io ...`). function _default_http_stream(consume, c, method, url, headers, qpairs) HTTP.open(method, url, headers; - query = qpairs, - status_exception = false, - decompress = false, - retry = false, - readtimeout = c.timeout, - connect_timeout = c.connect_timeout) do http - HTTP.startread(http) - return consume(http.message.status, http.message.headers, http) + query = qpairs, + status_exception = false, + decompress = false, + retries = 0, # HTTP 2.x: disable built-in retry (was `retry=false`) + read_idle_timeout = c.timeout, # HTTP 2.x rename of `readtimeout` + connect_timeout = c.connect_timeout) do http + # In HTTP 2.x `startread` returns the response head directly, so we read + # status/headers off it rather than reaching into `http.message`. + resp = HTTP.startread(http) + return consume(resp.status, resp.headers, http) end end @@ -85,10 +87,13 @@ _is_retryable_status(status::Integer) = status == 429 || (500 <= status < 600 && # Read timeouts are excluded: for long-range queries the server-side assembly # time is deterministic, so a request that exceeded the read timeout once will # exceed it on every retry — retrying just multiplies the wall-clock cost by -# (1 + max_retries) with zero success probability. Connect timeouts surface as -# the distinct `HTTP.ConnectError` and remain retryable. +# (1 + max_retries) with zero success probability. Connect timeouts, by +# contrast, are transient and stay retryable: under HTTP 2.x a connect timeout +# surfaces as a `HTTP.TimeoutError` tagged `operation == "connect"` (no longer a +# distinct `HTTP.ConnectError`), so we special-case it back in. _is_retryable_exception(e) = - e isa HTTP.Exceptions.HTTPError && !(e isa HTTP.Exceptions.TimeoutError) + (e isa HTTP.HTTPError && !(e isa HTTP.TimeoutError)) || + (e isa HTTP.TimeoutError && e.operation == "connect") # Parse a `Retry-After` header. The delta-seconds form is honored (capped); # the rarer HTTP-date form falls back to `nothing` so the caller uses backoff. @@ -193,9 +198,9 @@ function request(c::HTTPClient, method::Symbol, path::AbstractString; c.dispatcher(method_str, url, headers, body_bytes; query = qpairs, status_exception = false, - readtimeout = c.timeout, + read_idle_timeout = c.timeout, # HTTP 2.x rename of `readtimeout` connect_timeout = c.connect_timeout, - retry = false, + retries = 0, # HTTP 2.x: disable built-in retry (was `retry=false`) decompress = false) catch e # Transient transport failure: back off and retry until the budget @@ -204,10 +209,13 @@ function request(c::HTTPClient, method::Symbol, path::AbstractString; _retry_wait(c, attempt, nothing) continue end - # Read timeout: not retryable (see _is_retryable_exception). Map to - # a BentoError naming the actual remedy — HTTP.jl's raw TimeoutError - # gives no hint that it's the *client* giving up. - e isa HTTP.Exceptions.TimeoutError && throw(BentoTimeoutError(c.timeout)) + # Read/request timeout: not retryable (see _is_retryable_exception). + # Map to a BentoError naming the actual remedy — HTTP.jl's raw + # TimeoutError gives no hint that it's the *client* giving up. A + # connect timeout (operation == "connect") is excluded: it's + # retryable above and not a sign the read budget was too small. + e isa HTTP.TimeoutError && e.operation != "connect" && + throw(BentoTimeoutError(c.timeout)) rethrow() end @@ -238,7 +246,9 @@ Issue a GET request and parse the JSON response body via JSON3. """ function get_json(c::HTTPClient, path::AbstractString; query = nothing) resp = request(c, :GET, path; query = query, accept = "application/json") - return JSON3.read(resp.body) + # HTTP 2.x: `resp.body` is an `HTTP.BytesBody`, not a `Vector{UInt8}`; JSON3 + # doesn't recognize it, so materialize the bytes first. + return JSON3.read(Vector{UInt8}(resp.body)) end """ @@ -248,7 +258,8 @@ Issue a POST request with a form-encoded body and parse the JSON response. """ function post_json(c::HTTPClient, path::AbstractString; body = nothing) resp = request(c, :POST, path; body = body, accept = "application/json") - return JSON3.read(resp.body) + # HTTP 2.x: materialize the BytesBody before handing it to JSON3 (see get_json). + return JSON3.read(Vector{UInt8}(resp.body)) end """ @@ -357,7 +368,7 @@ function open_stream(f, c::HTTPClient, path::AbstractString; # itself, so it is a *different* object than f_err[]. An identical # object means the consumer's own code threw it: propagate that # untouched rather than blaming the Databento stream.) - e isa HTTP.Exceptions.TimeoutError && e !== f_err[] && + e isa HTTP.TimeoutError && e.operation != "connect" && e !== f_err[] && throw(BentoTimeoutError(c.timeout)) rethrow() end diff --git a/test/test_historical_timeseries.jl b/test/test_historical_timeseries.jl index a29b8c5..f2b2f63 100644 --- a/test/test_historical_timeseries.jl +++ b/test/test_historical_timeseries.jl @@ -554,7 +554,7 @@ end calls = Ref(0) opener = (consume, c, method, url, headers, qpairs) -> begin calls[] += 1 - throw(HTTP.Exceptions.TimeoutError(100)) + throw(HTTP.TimeoutError("read", 100)) end c = Historical("test-key"; gateway = "https://hist.test", stream_opener = opener, retry_sleep = _NOSLEEP_TS, timeout = 100) @@ -566,17 +566,17 @@ end end @testset "consumer-raised HTTP TimeoutError is not blamed on the stream" begin - # If the callback's own code throws HTTP.Exceptions.TimeoutError (e.g. it - # makes its own HTTP request), that must propagate untouched — mapping it - # to BentoTimeoutError would misreport the Databento stream as the culprit. + # If the callback's own code throws HTTP.TimeoutError (e.g. it makes its own + # HTTP request), that must propagate untouched — mapping it to + # BentoTimeoutError would misreport the Databento stream as the culprit. bytes, _ = _build_sample_dbn_zstd() opener = _seq_opener([(200, Pair{String,String}[], bytes)]) c = Historical("test-key"; gateway = "https://hist.test", stream_opener = opener, retry_sleep = _NOSLEEP_TS, timeout = 100) - @test_throws HTTP.Exceptions.TimeoutError DatabentoAPI.foreach_record(c; + @test_throws HTTP.TimeoutError DatabentoAPI.foreach_record(c; dataset = "XNAS.ITCH", schema = Schema.TRADES, symbols = ["AAPL"], start_dt = "2024-01-02T14:30:00", end_dt = "2024-01-02T14:31:00") do rec - throw(HTTP.Exceptions.TimeoutError(42)) # consumer's own timeout + throw(HTTP.TimeoutError("read", 42)) # consumer's own timeout end end diff --git a/test/test_http.jl b/test/test_http.jl index d254512..75d7b67 100644 --- a/test/test_http.jl +++ b/test/test_http.jl @@ -124,7 +124,7 @@ const _NOSLEEP = _ -> nothing calls = Ref(0) function mock(method, url, headers, body; kwargs...) calls[] += 1 - calls[] < 2 && throw(HTTP.Exceptions.ConnectError(url, ErrorException("refused"))) + calls[] < 2 && throw(HTTP.ConnectError(url, ErrorException("refused"))) HTTP.Response(200; body = "{}") end c = HTTPClient("k", "https://example.test"; @@ -138,7 +138,7 @@ const _NOSLEEP = _ -> nothing calls = Ref(0) function mock(method, url, headers, body; kwargs...) calls[] += 1 - throw(HTTP.Exceptions.TimeoutError(100)) + throw(HTTP.TimeoutError("read", 100)) end c = HTTPClient("k", "https://example.test"; dispatcher = mock, retry_sleep = _NOSLEEP, timeout = 100) @@ -162,7 +162,7 @@ const _NOSLEEP = _ -> nothing calls = Ref(0) function mock(method, url, headers, body; kwargs...) calls[] += 1 - calls[] < 3 && throw(HTTP.Exceptions.ConnectError(url, ErrorException("refused"))) + calls[] < 3 && throw(HTTP.ConnectError(url, ErrorException("refused"))) HTTP.Response(200; body = "{}") end c = HTTPClient("k", "https://example.test"; From 32ef7664fb8b177319c78f7755cdf1c7261e1913 Mon Sep 17 00:00:00 2001 From: Tyler Beason Date: Thu, 25 Jun 2026 09:59:39 -0500 Subject: [PATCH 2/2] Treat TLS-handshake timeouts as retryable connection failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HTTP 2.x's connect_timeout budget covers DNS/dial *and* the TLS handshake. A handshake deadline surfaces as TimeoutError(operation="tls_handshake"), not "connect", so the previous predicate skipped the retry loop and then mislabeled it as a read timeout (BentoTimeoutError → "raise the read timeout"), when it's really a transient connection failure. Generalize the connect-timeout special-case to a _CONNECT_PHASE_TIMEOUT_OPS set {"connect", "tls_handshake"} via _is_connect_timeout, used by both the retry predicate and the two BentoTimeoutError mapping sites. Adds a regression test. Addresses Codex review feedback on #45. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/http.jl | 27 ++++++++++++++++++--------- test/test_http.jl | 20 ++++++++++++++++++++ 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/src/http.jl b/src/http.jl index ddd2f03..c17cad5 100644 --- a/src/http.jl +++ b/src/http.jl @@ -82,18 +82,27 @@ HTTPClient(api_key::AbstractString, base_url::AbstractString; # 501 Not Implemented (a permanent "this server can't do that"). _is_retryable_status(status::Integer) = status == 429 || (500 <= status < 600 && status != 501) +# Connection-phase timeout operations (HTTP 2.x tags `TimeoutError` with the +# phase that stalled). The `connect_timeout` budget covers DNS/dial *and* the +# TLS handshake, so a connection failure surfaces as either `"connect"` or +# `"tls_handshake"` — both are transient and should be treated like the old +# `HTTP.ConnectError`, not like a read timeout. +const _CONNECT_PHASE_TIMEOUT_OPS = ("connect", "tls_handshake") +_is_connect_timeout(e) = + e isa HTTP.TimeoutError && e.operation in _CONNECT_PHASE_TIMEOUT_OPS + # Transient transport-layer failures from HTTP.jl (connect/request errors). # Status errors don't appear here because we pass status_exception=false. # Read timeouts are excluded: for long-range queries the server-side assembly # time is deterministic, so a request that exceeded the read timeout once will # exceed it on every retry — retrying just multiplies the wall-clock cost by -# (1 + max_retries) with zero success probability. Connect timeouts, by -# contrast, are transient and stay retryable: under HTTP 2.x a connect timeout -# surfaces as a `HTTP.TimeoutError` tagged `operation == "connect"` (no longer a -# distinct `HTTP.ConnectError`), so we special-case it back in. +# (1 + max_retries) with zero success probability. Connection-phase timeouts, by +# contrast, are transient and stay retryable: under HTTP 2.x they surface as a +# `HTTP.TimeoutError` (no longer a distinct `HTTP.ConnectError`), so we +# special-case them back in via `_is_connect_timeout`. _is_retryable_exception(e) = (e isa HTTP.HTTPError && !(e isa HTTP.TimeoutError)) || - (e isa HTTP.TimeoutError && e.operation == "connect") + _is_connect_timeout(e) # Parse a `Retry-After` header. The delta-seconds form is honored (capped); # the rarer HTTP-date form falls back to `nothing` so the caller uses backoff. @@ -212,9 +221,9 @@ function request(c::HTTPClient, method::Symbol, path::AbstractString; # Read/request timeout: not retryable (see _is_retryable_exception). # Map to a BentoError naming the actual remedy — HTTP.jl's raw # TimeoutError gives no hint that it's the *client* giving up. A - # connect timeout (operation == "connect") is excluded: it's - # retryable above and not a sign the read budget was too small. - e isa HTTP.TimeoutError && e.operation != "connect" && + # connection-phase timeout is excluded: it's retryable above and not + # a sign the read budget was too small. + e isa HTTP.TimeoutError && !_is_connect_timeout(e) && throw(BentoTimeoutError(c.timeout)) rethrow() end @@ -368,7 +377,7 @@ function open_stream(f, c::HTTPClient, path::AbstractString; # itself, so it is a *different* object than f_err[]. An identical # object means the consumer's own code threw it: propagate that # untouched rather than blaming the Databento stream.) - e isa HTTP.TimeoutError && e.operation != "connect" && e !== f_err[] && + e isa HTTP.TimeoutError && !_is_connect_timeout(e) && e !== f_err[] && throw(BentoTimeoutError(c.timeout)) rethrow() end diff --git a/test/test_http.jl b/test/test_http.jl index 75d7b67..a4e0a59 100644 --- a/test/test_http.jl +++ b/test/test_http.jl @@ -172,6 +172,26 @@ const _NOSLEEP = _ -> nothing @test calls[] == 3 end + @testset "connection-phase timeouts (connect/tls_handshake) remain retryable" begin + # HTTP 2.x reports connection-phase deadline failures as TimeoutError + # tagged with the phase. The connect_timeout budget covers both the + # dial ("connect") and the TLS handshake ("tls_handshake"); both are + # transient and must be retried, not mapped to BentoTimeoutError. + for op in ("connect", "tls_handshake") + calls = Ref(0) + function mock(method, url, headers, body; kwargs...) + calls[] += 1 + calls[] < 3 && throw(HTTP.TimeoutError(op, 30)) + HTTP.Response(200; body = "{}") + end + c = HTTPClient("k", "https://example.test"; + dispatcher = mock, retry_sleep = _NOSLEEP) + resp = request(c, :GET, "/v0/foo") + @test resp.status == 200 + @test calls[] == 3 + end + end + @testset "DEFAULT_TIMEOUT raised for long-range queries" begin @test DEFAULT_TIMEOUT == 600 end