Upgrade to HTTP.jl 2.4 (v0.3.1)

CHANGELOG.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.3.1] - 2026-06-25
+
 ### Added
 - **Live symbol resolution.** A `Live` client now keeps a running
   `instrument_id → symbol` map, built by the reader from the gateway's
@@ -18,6 +20,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   typed mode the mapping is captured before any `control_channel` overflow drop,
   so it's reliable even without draining the control channel.
 
+### Changed
+- **Upgraded to HTTP.jl 2.4** (from 1.10). HTTP.jl 2.x is a ground-up rewrite
+  with renamed keywords and top-level exception types; the changes here are
+  internal (`readtimeout` → `read_idle_timeout`, `retry=false` → `retries=0`,
+  `HTTP.Exceptions.X` → `HTTP.X`, and using `startread`'s returned response) and
+  do not affect the public API. Connect timeouts (now surfaced as
+  `HTTP.TimeoutError(operation="connect")` rather than `HTTP.ConnectError`)
+  remain retryable, as before.
+
 ## [0.3.0] - 2026-06-22
 
 ### Added

Project.toml

@@ -1,6 +1,6 @@
 name = "DatabentoAPI"
 uuid = "feb0aed8-3291-460b-9688-521dcb17a4bf"
-version = "0.3.0"
+version = "0.3.1"
 authors = ["Tyler Beason <tbeas12@gmail.com>"]
 
 [deps]
@@ -24,7 +24,7 @@ CodecZstd = "0.8"
 DataFrames = "1.7"
 DatabentoBinaryEncoding = "0.1.4"
 EnumX = "1.0"
-HTTP = "1.10"
+HTTP = "2.4"
 JSON3 = "1.14"
 SHA = "0.7.0"
 TOML = "1.0.3"

benchmark/Project.toml

@@ -11,6 +11,9 @@ Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 TranscodingStreams = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"
 
+[compat]
+HTTP = "2.4"
+
 [sources]
 DatabentoAPI = {path = ".."}
 DatabentoBinaryEncoding = {path = "../../DatabentoBinaryEncoding.jl"}

src/historical/batch.jl

@@ -156,11 +156,11 @@ function batch_download(c::Historical;
              "Accept"        => "application/octet-stream",
              "User-Agent"    => c.http.user_agent],
             UInt8[];
-            status_exception = false,
-            readtimeout      = c.http.timeout,
-            connect_timeout  = c.http.connect_timeout,
-            retry            = false,
-            decompress       = false)
+            status_exception  = false,
+            read_idle_timeout = c.http.timeout,        # HTTP 2.x rename of `readtimeout`
+            connect_timeout   = c.http.connect_timeout,
+            retries           = 0,                       # HTTP 2.x: disable built-in retry (was `retry=false`)
+            decompress        = false)
         if resp.status >= 400
             T = resp.status < 500 ? BentoClientError : BentoServerError
             throw(http_error_from_response(T, resp.status,

src/http.jl

@@ -42,14 +42,16 @@ _default_http_request(method, url, headers, body; kwargs...) =
 # (`stream_opener(c, method, url, headers, qpairs) do status, headers, io ...`).
 function _default_http_stream(consume, c, method, url, headers, qpairs)
     HTTP.open(method, url, headers;
-              query = qpairs,
-              status_exception = false,
-              decompress = false,
-              retry = false,
-              readtimeout = c.timeout,
-              connect_timeout = c.connect_timeout) do http
-        HTTP.startread(http)
-        return consume(http.message.status, http.message.headers, http)
+              query             = qpairs,
+              status_exception  = false,
+              decompress        = false,
+              retries           = 0,           # HTTP 2.x: disable built-in retry (was `retry=false`)
+              read_idle_timeout = c.timeout,   # HTTP 2.x rename of `readtimeout`
+              connect_timeout   = c.connect_timeout) do http
+        # In HTTP 2.x `startread` returns the response head directly, so we read
+        # status/headers off it rather than reaching into `http.message`.
+        resp = HTTP.startread(http)
+        return consume(resp.status, resp.headers, http)
     end
 end
 
@@ -80,15 +82,27 @@ HTTPClient(api_key::AbstractString, base_url::AbstractString;
 # 501 Not Implemented (a permanent "this server can't do that").
 _is_retryable_status(status::Integer) = status == 429 || (500 <= status < 600 && status != 501)
 
+# Connection-phase timeout operations (HTTP 2.x tags `TimeoutError` with the
+# phase that stalled). The `connect_timeout` budget covers DNS/dial *and* the
+# TLS handshake, so a connection failure surfaces as either `"connect"` or
+# `"tls_handshake"` — both are transient and should be treated like the old
+# `HTTP.ConnectError`, not like a read timeout.
+const _CONNECT_PHASE_TIMEOUT_OPS = ("connect", "tls_handshake")
+_is_connect_timeout(e) =
+    e isa HTTP.TimeoutError && e.operation in _CONNECT_PHASE_TIMEOUT_OPS
+
 # Transient transport-layer failures from HTTP.jl (connect/request errors).
 # Status errors don't appear here because we pass status_exception=false.
 # Read timeouts are excluded: for long-range queries the server-side assembly
 # time is deterministic, so a request that exceeded the read timeout once will
 # exceed it on every retry — retrying just multiplies the wall-clock cost by
-# (1 + max_retries) with zero success probability. Connect timeouts surface as
-# the distinct `HTTP.ConnectError` and remain retryable.
+# (1 + max_retries) with zero success probability. Connection-phase timeouts, by
+# contrast, are transient and stay retryable: under HTTP 2.x they surface as a
+# `HTTP.TimeoutError` (no longer a distinct `HTTP.ConnectError`), so we
+# special-case them back in via `_is_connect_timeout`.
 _is_retryable_exception(e) =
-    e isa HTTP.Exceptions.HTTPError && !(e isa HTTP.Exceptions.TimeoutError)
+    (e isa HTTP.HTTPError && !(e isa HTTP.TimeoutError)) ||
+    _is_connect_timeout(e)
 
 # Parse a `Retry-After` header. The delta-seconds form is honored (capped);
 # the rarer HTTP-date form falls back to `nothing` so the caller uses backoff.
@@ -193,9 +207,9 @@ function request(c::HTTPClient, method::Symbol, path::AbstractString;
             c.dispatcher(method_str, url, headers, body_bytes;
                          query              = qpairs,
                          status_exception   = false,
-                         readtimeout        = c.timeout,
+                         read_idle_timeout  = c.timeout,        # HTTP 2.x rename of `readtimeout`
                          connect_timeout    = c.connect_timeout,
-                         retry              = false,
+                         retries            = 0,                 # HTTP 2.x: disable built-in retry (was `retry=false`)
                          decompress         = false)
         catch e
             # Transient transport failure: back off and retry until the budget
@@ -204,10 +218,13 @@ function request(c::HTTPClient, method::Symbol, path::AbstractString;
                 _retry_wait(c, attempt, nothing)
                 continue
             end
-            # Read timeout: not retryable (see _is_retryable_exception). Map to
-            # a BentoError naming the actual remedy — HTTP.jl's raw TimeoutError
-            # gives no hint that it's the *client* giving up.
-            e isa HTTP.Exceptions.TimeoutError && throw(BentoTimeoutError(c.timeout))
+            # Read/request timeout: not retryable (see _is_retryable_exception).
+            # Map to a BentoError naming the actual remedy — HTTP.jl's raw
+            # TimeoutError gives no hint that it's the *client* giving up. A
+            # connection-phase timeout is excluded: it's retryable above and not
+            # a sign the read budget was too small.
+            e isa HTTP.TimeoutError && !_is_connect_timeout(e) &&
+                throw(BentoTimeoutError(c.timeout))
             rethrow()
         end
 
@@ -238,7 +255,9 @@ Issue a GET request and parse the JSON response body via JSON3.
 """
 function get_json(c::HTTPClient, path::AbstractString; query = nothing)
     resp = request(c, :GET, path; query = query, accept = "application/json")
-    return JSON3.read(resp.body)
+    # HTTP 2.x: `resp.body` is an `HTTP.BytesBody`, not a `Vector{UInt8}`; JSON3
+    # doesn't recognize it, so materialize the bytes first.
+    return JSON3.read(Vector{UInt8}(resp.body))
 end
 
 """
@@ -248,7 +267,8 @@ Issue a POST request with a form-encoded body and parse the JSON response.
 """
 function post_json(c::HTTPClient, path::AbstractString; body = nothing)
     resp = request(c, :POST, path; body = body, accept = "application/json")
-    return JSON3.read(resp.body)
+    # HTTP 2.x: materialize the BytesBody before handing it to JSON3 (see get_json).
+    return JSON3.read(Vector{UInt8}(resp.body))
 end
 
 """
@@ -357,7 +377,7 @@ function open_stream(f, c::HTTPClient, path::AbstractString;
             # itself, so it is a *different* object than f_err[]. An identical
             # object means the consumer's own code threw it: propagate that
             # untouched rather than blaming the Databento stream.)
-            e isa HTTP.Exceptions.TimeoutError && e !== f_err[] &&
+            e isa HTTP.TimeoutError && !_is_connect_timeout(e) && e !== f_err[] &&
                 throw(BentoTimeoutError(c.timeout))
             rethrow()
         end

test/test_historical_timeseries.jl

@@ -554,7 +554,7 @@ end
     calls = Ref(0)
     opener = (consume, c, method, url, headers, qpairs) -> begin
         calls[] += 1
-        throw(HTTP.Exceptions.TimeoutError(100))
+        throw(HTTP.TimeoutError("read", 100))
     end
     c = Historical("test-key"; gateway = "https://hist.test",
                    stream_opener = opener, retry_sleep = _NOSLEEP_TS, timeout = 100)
@@ -566,17 +566,17 @@ end
 end
 
 @testset "consumer-raised HTTP TimeoutError is not blamed on the stream" begin
-    # If the callback's own code throws HTTP.Exceptions.TimeoutError (e.g. it
-    # makes its own HTTP request), that must propagate untouched — mapping it
-    # to BentoTimeoutError would misreport the Databento stream as the culprit.
+    # If the callback's own code throws HTTP.TimeoutError (e.g. it makes its own
+    # HTTP request), that must propagate untouched — mapping it to
+    # BentoTimeoutError would misreport the Databento stream as the culprit.
     bytes, _ = _build_sample_dbn_zstd()
     opener = _seq_opener([(200, Pair{String,String}[], bytes)])
     c = Historical("test-key"; gateway = "https://hist.test",
                    stream_opener = opener, retry_sleep = _NOSLEEP_TS, timeout = 100)
-    @test_throws HTTP.Exceptions.TimeoutError DatabentoAPI.foreach_record(c;
+    @test_throws HTTP.TimeoutError DatabentoAPI.foreach_record(c;
         dataset = "XNAS.ITCH", schema = Schema.TRADES, symbols = ["AAPL"],
         start_dt = "2024-01-02T14:30:00", end_dt = "2024-01-02T14:31:00") do rec
-        throw(HTTP.Exceptions.TimeoutError(42))   # consumer's own timeout
+        throw(HTTP.TimeoutError("read", 42))   # consumer's own timeout
     end
 end
 

test/test_http.jl

@@ -124,7 +124,7 @@ const _NOSLEEP = _ -> nothing
         calls = Ref(0)
         function mock(method, url, headers, body; kwargs...)
             calls[] += 1
-            calls[] < 2 && throw(HTTP.Exceptions.ConnectError(url, ErrorException("refused")))
+            calls[] < 2 && throw(HTTP.ConnectError(url, ErrorException("refused")))
             HTTP.Response(200; body = "{}")
         end
         c = HTTPClient("k", "https://example.test";
@@ -138,7 +138,7 @@ const _NOSLEEP = _ -> nothing
         calls = Ref(0)
         function mock(method, url, headers, body; kwargs...)
             calls[] += 1
-            throw(HTTP.Exceptions.TimeoutError(100))
+            throw(HTTP.TimeoutError("read", 100))
         end
         c = HTTPClient("k", "https://example.test";
                        dispatcher = mock, retry_sleep = _NOSLEEP, timeout = 100)
@@ -162,7 +162,7 @@ const _NOSLEEP = _ -> nothing
         calls = Ref(0)
         function mock(method, url, headers, body; kwargs...)
             calls[] += 1
-            calls[] < 3 && throw(HTTP.Exceptions.ConnectError(url, ErrorException("refused")))
+            calls[] < 3 && throw(HTTP.ConnectError(url, ErrorException("refused")))
             HTTP.Response(200; body = "{}")
         end
         c = HTTPClient("k", "https://example.test";
@@ -172,6 +172,26 @@ const _NOSLEEP = _ -> nothing
         @test calls[] == 3
     end
 
+    @testset "connection-phase timeouts (connect/tls_handshake) remain retryable" begin
+        # HTTP 2.x reports connection-phase deadline failures as TimeoutError
+        # tagged with the phase. The connect_timeout budget covers both the
+        # dial ("connect") and the TLS handshake ("tls_handshake"); both are
+        # transient and must be retried, not mapped to BentoTimeoutError.
+        for op in ("connect", "tls_handshake")
+            calls = Ref(0)
+            function mock(method, url, headers, body; kwargs...)
+                calls[] += 1
+                calls[] < 3 && throw(HTTP.TimeoutError(op, 30))
+                HTTP.Response(200; body = "{}")
+            end
+            c = HTTPClient("k", "https://example.test";
+                           dispatcher = mock, retry_sleep = _NOSLEEP)
+            resp = request(c, :GET, "/v0/foo")
+            @test resp.status == 200
+            @test calls[] == 3
+        end
+    end
+
     @testset "DEFAULT_TIMEOUT raised for long-range queries" begin
         @test DEFAULT_TIMEOUT == 600
     end