From 62ff22fd955ab6e1beaed4d5cd0d7546f5742dc2 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Wed, 5 Aug 2026 23:38:10 -0600 Subject: [PATCH 01/23] 1.0 release readiness: protocol fixes, org-transfer cleanup, API polish Protocol/correctness fixes: - copy_from no longer hangs forever when the COPY statement errors before CopyInResponse (e.g. missing table): ReadyForQuery now terminates the wait loop and the server error is surfaced with the connection usable - copy_from/copy_to now reject non-COPY and wrong-direction statements cleanly (CopyFail abort) instead of deadlocking or silently succeeding - COPY statements via DBInterface.execute or Postgres.cursor now abort the server-side copy (CopyFail + fresh Sync, since the pre-copy Sync is ignored in copy-in mode) and throw a clear error pointing at copy_from/copy_to, keeping the connection usable - IPv6 host literals are bracketed when forming the transport address ([::1]:5432); unix socket paths get a clear unsupported error - GC.@preserve added around unsafe_string(pointer(buf)) parsing loops (error/notice/notification/parameter-status/describe/command-tag) - password material (cleartext and md5 hash) is redacted from debug logs - PostgresInterfaceError now subtypes Exception - parse_numeric throws a clear error for NaN/Infinity numeric values - cursor(conn, sql) rolls back its own transaction if cursor setup fails API surface polish: - ConnectionParams.debug/reconnect are honored by connect (kwargs still override); sslservername plumbed through ConnectionParams, keyword DSNs, URIs, and all ConnectionPool constructors; style kwarg available on the DSN/params connect and pool paths - removed the accepted-but-ignored `binary` kwarg from execute - public declarations for the supported API surface (Julia 1.11+) - docstrings for the public API (connection, pool, transactions, COPY, LISTEN/NOTIFY, type registry, statement cache, cursor, errors, types) Org-transfer/metadata cleanup: - deploydocs points at JuliaDatabases/Postgres.jl - LICENSE names Postgres.jl (was template text naming Example.jl) - README: CI/docs/codecov badges, raw"..." on $1 examples so they are copy-pasteable, style-based query logging (set_query_logger! was removed pre-1.0 but docs still showed it), sslservername documented - docs: same fixes, plus reference blocks for the newly documented types - Parsers compat tightened to "2" (0.3/1 predate APIs in use); unused Logging dependency dropped; dead code removed (ERROR_CODE, DATETIME_OPTIONS, parse_to_julia_array, commented-out escape/lastrowid) Tests: COPY misuse/error-path coverage, ConnectionParams debug/reconnect, sslservername parsing, IPv6/unix-socket address formation, numeric special values, updated export-surface test for public names. Co-Authored-By: Claude Fable 5 --- LICENSE.md | 4 +- Project.toml | 3 +- README.md | 28 ++-- docs/make.jl | 2 +- docs/src/index.md | 29 +++- docs/src/manual.md | 20 ++- src/Postgres.jl | 300 ++++++++++++++++++++++++++++++++++++--- src/api/API.jl | 135 ++++++++++++------ src/api/types.jl | 24 +++- src/array_parsing.jl | 10 +- src/connection_string.jl | 32 ++++- src/execute.jl | 55 ++++++- test/runtests.jl | 97 ++++++++++++- 13 files changed, 626 insertions(+), 113 deletions(-) diff --git a/LICENSE.md b/LICENSE.md index 295bef4..2639a23 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,6 @@ -The Example.jl package is licensed under the MIT "Expat" License: +The Postgres.jl package is licensed under the MIT "Expat" License: -> Copyright (c) 2022: Jacob Quinn +> Copyright (c) 2022: Jacob Quinn and contributors > > Permission is hereby granted, free of charge, to any person obtaining > a copy of this software and associated documentation files (the diff --git a/Project.toml b/Project.toml index 165d80d..210cc27 100644 --- a/Project.toml +++ b/Project.toml @@ -8,7 +8,6 @@ ConcurrentUtilities = "f0e56b4a-5159-44fe-b623-3e5288b988bb" DBInterface = "a10d1c49-ce27-4219-8d33-6db1a4562965" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" MD5 = "6ac74813-4b46-53a4-afec-0b5dc9d7885c" Parsers = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" @@ -24,7 +23,7 @@ DBInterface = "2.5" Harbor = "1" JSON = "1" MD5 = "0.2" -Parsers = "0.3, 1, 2" +Parsers = "2" Reseau = "1.1" SASLAuth = "1" StructUtils = "2" diff --git a/README.md b/README.md index 91f0f23..9ec0bab 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,10 @@ # Postgres.jl -Postgres.jl is a PostgreSQL client that implements the v3 wire protocol with `DBInterface` and `Tables` integration. +[![CI](https://github.com/JuliaDatabases/Postgres.jl/actions/workflows/CI.yml/badge.svg)](https://github.com/JuliaDatabases/Postgres.jl/actions/workflows/CI.yml) +[![docs](https://img.shields.io/badge/docs-dev-blue.svg)](https://JuliaDatabases.github.io/Postgres.jl/dev/) +[![codecov](https://codecov.io/gh/JuliaDatabases/Postgres.jl/branch/main/graph/badge.svg)](https://codecov.io/gh/JuliaDatabases/Postgres.jl) + +Postgres.jl is a pure-Julia PostgreSQL client that implements the v3 wire protocol with `DBInterface` and `Tables` integration. ## Installation @@ -33,7 +37,7 @@ Connection options support: - PostgreSQL URIs such as `postgresql://postgres:postgres@127.0.0.1:5432/postgres`. - Environment defaults: `PGHOST`, `PGPORT`, `PGUSER`, `PGPASSWORD`, `PGDATABASE`, `PGAPPNAME`, `PGCONNECT_TIMEOUT`, and TLS-related `PGSSL*` variables. - `sslmode` values: `disable`, `prefer`, `require`, `verify-full` (only `verify-full` enforces certificate verification). -- TLS files: `sslrootcert`, `sslcert`, `sslkey`, `sslcapath`. +- TLS files: `sslrootcert`, `sslcert`, `sslkey`, `sslcapath`; `sslservername` overrides the TLS SNI hostname when connecting to a pre-resolved address. - `connect_timeout` (seconds) and `statement_timeout` (milliseconds). - `application_name` and `statement_cache_maxsize`. @@ -51,9 +55,9 @@ DBInterface.close!(conn) ```julia using Postgres, DBInterface, Tables conn = DBInterface.connect(Postgres.Connection, "host=127.0.0.1;user=postgres;password=postgres;dbname=postgres") -rows = Tables.rowtable(DBInterface.execute(conn, "SELECT $1::int AS val", (42,))) +rows = Tables.rowtable(DBInterface.execute(conn, raw"SELECT $1::int AS val", (42,))) @show rows[1].val -stmt = DBInterface.prepare(conn, "SELECT $1::int AS val") +stmt = DBInterface.prepare(conn, raw"SELECT $1::int AS val") rows = Tables.rowtable(DBInterface.execute(stmt, (7,))) DBInterface.close!(stmt) DBInterface.close!(conn) @@ -87,7 +91,7 @@ StructUtils.@tags struct ProfileSummary createdAt::DateTime &(postgres=(name=:created_at,),) end -profile = DBInterface.execute(conn, """ +profile = DBInterface.execute(conn, raw""" SELECT profile_id, first_name, last_name, created_at FROM profiles WHERE profile_id = $1 @@ -184,14 +188,18 @@ DBInterface.close!(conn) `Numeric` values are returned as `Postgres.Numeric`, `interval` values as `Dates.Period` or `Dates.CompoundPeriod`, and range types as `Postgres.PostgresRange{T}`. -## Query logging +## Query logging and driver styles + +Driver behavior — query logging, server notices, asynchronous notifications — is customized by defining a driver "style": subtype `Postgres.AbstractPostgresStyle`, overload the behavior hooks for it, and pass an instance via the `style` connection keyword. ```julia using Postgres, DBInterface -conn = DBInterface.connect(Postgres.Connection, "host=127.0.0.1;user=postgres;password=postgres;dbname=postgres") -Postgres.set_query_logger!(conn) do event, info - @show event info.success info.duration_ns -end + +struct LoggingStyle <: Postgres.AbstractPostgresStyle end +Postgres.query_logging_enabled(::LoggingStyle) = true +Postgres.query_logger(::LoggingStyle, event::Symbol, info::NamedTuple) = @info "query" event info.success info.duration_ns + +conn = DBInterface.connect(Postgres.Connection, "host=127.0.0.1;user=postgres;password=postgres;dbname=postgres"; style=LoggingStyle()) DBInterface.execute(conn, "SELECT 1") DBInterface.close!(conn) ``` diff --git a/docs/make.jl b/docs/make.jl index a70d638..83c2d65 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -10,5 +10,5 @@ makedocs( ) if get(ENV, "POSTGRES_DOCS_DEPLOY", "false") == "true" - deploydocs(repo = "github.com/quinnj/Postgres.jl.git", push_preview = true) + deploydocs(repo = "github.com/JuliaDatabases/Postgres.jl.git", push_preview = true) end diff --git a/docs/src/index.md b/docs/src/index.md index de09dc5..4916e6c 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -18,7 +18,7 @@ Postgres.jl accepts DSN strings or PostgreSQL URIs and supports: - libpq-style keyword strings such as `host=127.0.0.1 port=5432 user=postgres dbname=postgres`. - Environment defaults from `PGHOST`, `PGPORT`, `PGUSER`, `PGPASSWORD`, `PGDATABASE`, `PGAPPNAME`, `PGCONNECT_TIMEOUT`, and TLS-related `PGSSL*` variables. - `sslmode` values: `disable`, `prefer`, `require`, `verify-full` (only `verify-full` verifies certificates). -- TLS files: `sslrootcert`, `sslcert`, `sslkey`, `sslcapath`. +- TLS files: `sslrootcert`, `sslcert`, `sslkey`, `sslcapath`; `sslservername` overrides the TLS SNI hostname when connecting to a pre-resolved address. - `connect_timeout` (seconds), `statement_timeout` (milliseconds). - `application_name` and `statement_cache_maxsize`. @@ -33,7 +33,7 @@ DBInterface.close!(conn) ```julia using Postgres, DBInterface, Tables conn = DBInterface.connect(Postgres.Connection, "host=127.0.0.1;user=postgres;password=postgres;dbname=postgres") -rows = Tables.rowtable(DBInterface.execute(conn, "SELECT $1::int AS val", (42,))) +rows = Tables.rowtable(DBInterface.execute(conn, raw"SELECT $1::int AS val", (42,))) @show rows[1].val DBInterface.close!(conn) ``` @@ -65,7 +65,7 @@ StructUtils.@tags struct ProfileSummary createdAt::DateTime &(postgres=(name=:created_at,),) end -profile = DBInterface.execute(conn, """ +profile = DBInterface.execute(conn, raw""" SELECT profile_id, first_name, last_name, created_at FROM profiles WHERE profile_id = $1 @@ -162,12 +162,16 @@ DBInterface.close!(conn) ## Query logging +Query logging (and other driver behavior) is customized with a driver style; see the [Manual](@ref) for details. + ```julia using Postgres, DBInterface -conn = DBInterface.connect(Postgres.Connection, "host=127.0.0.1;user=postgres;password=postgres;dbname=postgres") -Postgres.set_query_logger!(conn) do event, info - @show event info.success info.duration_ns -end + +struct LoggingStyle <: Postgres.AbstractPostgresStyle end +Postgres.query_logging_enabled(::LoggingStyle) = true +Postgres.query_logger(::LoggingStyle, event::Symbol, info::NamedTuple) = @info "query" event info.success info.duration_ns + +conn = DBInterface.connect(Postgres.Connection, "host=127.0.0.1;user=postgres;password=postgres;dbname=postgres"; style=LoggingStyle()) DBInterface.execute(conn, "SELECT 1") DBInterface.close!(conn) ``` @@ -187,6 +191,17 @@ DBInterface.close!(pool) `Postgres.Error` includes SQLSTATE information. Use `Postgres.cancel_query!(conn)` to cancel a running query. +## Reference + ```@autodocs Modules = [Postgres] ``` + +```@docs +Postgres.Error +Postgres.Notification +Postgres.Numeric +Postgres.PostgresRange +Postgres.ConnectionParams +Postgres.parse_dsn +``` diff --git a/docs/src/manual.md b/docs/src/manual.md index a0d010f..f7d2662 100644 --- a/docs/src/manual.md +++ b/docs/src/manual.md @@ -92,7 +92,7 @@ StructUtils.@tags struct ProfileSummary createdAt::DateTime &(postgres=(name=:created_at,),) end -profile = DBInterface.execute(conn, """ +profile = DBInterface.execute(conn, raw""" SELECT profile_id, first_name, last_name, created_at FROM profiles WHERE profile_id = $1 @@ -117,6 +117,16 @@ that style, and pass an instance with the `style` connection keyword. The default `Postgres.PostgresStyle` keeps query logging disabled and reports server notices through Julia's logger. +```julia +struct LoggingStyle <: Postgres.AbstractPostgresStyle end +Postgres.query_logging_enabled(::LoggingStyle) = true +Postgres.query_logger(::LoggingStyle, event::Symbol, info::NamedTuple) = @info "query" event info.success info.duration_ns +Postgres.notice_callback(::LoggingStyle, notice) = @info "notice" notice +Postgres.notification_callback(::LoggingStyle, notification) = @info "notification" notification + +conn = DBInterface.connect(Postgres.Connection, "host=127.0.0.1 user=postgres dbname=postgres"; style=LoggingStyle()) +``` + ```@docs Postgres.AbstractPostgresStyle Postgres.PostgresStyle @@ -124,17 +134,17 @@ Postgres.PostgresStyle ## Parameters And Prepared Statements -Use PostgreSQL placeholders (`$1`, `$2`, ...) and pass a tuple or other iterable of parameter values. +Use PostgreSQL placeholders (`$1`, `$2`, ...) and pass a tuple or other iterable of parameter values. Note the use of `raw"..."` strings so that `$1` is not treated as Julia string interpolation. ```julia -rows = Tables.rowtable(DBInterface.execute(conn, "SELECT $1::int + $2::int AS total", (20, 22))) +rows = Tables.rowtable(DBInterface.execute(conn, raw"SELECT $1::int + $2::int AS total", (20, 22))) @show only(rows).total ``` Prepared statements can be created explicitly. Postgres.jl also caches prepared statements internally with LRU eviction; set `statement_cache_maxsize=0` to disable caching. ```julia -stmt = DBInterface.prepare(conn, "SELECT $1::text AS value") +stmt = DBInterface.prepare(conn, raw"SELECT $1::text AS value") rows = Tables.rowtable(DBInterface.execute(stmt, ("prepared",))) DBInterface.close!(stmt) ``` @@ -143,7 +153,7 @@ Bulk inserts can use `DBInterface.executemany`. ```julia DBInterface.execute(conn, "CREATE TEMP TABLE demo_many (id int, name text)") -stmt = DBInterface.prepare(conn, "INSERT INTO demo_many VALUES ($1, $2)") +stmt = DBInterface.prepare(conn, raw"INSERT INTO demo_many VALUES ($1, $2)") DBInterface.executemany(stmt, ([1, 2, 3], ["a", "b", "c"])) DBInterface.close!(stmt) ``` diff --git a/src/Postgres.jl b/src/Postgres.jl index 0c1944d..1d39e55 100644 --- a/src/Postgres.jl +++ b/src/Postgres.jl @@ -4,8 +4,15 @@ using DBInterface, Dates, UUIDs, Parsers, Tables, StructUtils, JSON, ConcurrentU export DBInterface -# For non-api errors that happen in Postgres.jl -struct PostgresInterfaceError +""" + Postgres.PostgresInterfaceError <: Exception + +A client-side error raised by Postgres.jl itself (closed connections, +parameter-count mismatches, unsupported features, ...), as opposed to +[`Postgres.Error`](@ref Postgres.API.Error), which represents an error +reported by the server. +""" +struct PostgresInterfaceError <: Exception msg::String end Base.showerror(io::IO, e::PostgresInterfaceError) = print(io, e.msg) @@ -18,6 +25,39 @@ using .ConnectionString const Pools = ConcurrentUtilities.Pools const ReseauConn = Union{Reseau.TCP.Conn, Reseau.TLS.Conn} +""" + Postgres.Connection + +A single connection to a PostgreSQL server, created via +`DBInterface.connect(Postgres.Connection, ...)`: + + DBInterface.connect(Postgres.Connection, host, user, password; dbname, port=5432, kwargs...) + DBInterface.connect(Postgres.Connection, dsn::String; kwargs...) + DBInterface.connect(Postgres.Connection, params::ConnectionParams; kwargs...) + +`dsn` may be a libpq-style keyword string (`"host=127.0.0.1 user=postgres dbname=postgres"`) +or a PostgreSQL URI (`"postgresql://user:pass@host:5432/dbname?sslmode=require"`). + +Supported keyword arguments (all but the last three also available as DSN/URI +options): + +- `dbname`, `port`, `application_name` +- `connect_timeout` (seconds), `statement_timeout` (milliseconds) +- `sslmode` (`"disable"`, `"prefer"` (default), `"require"`, `"verify-full"`), + `sslrootcert`, `sslcert`, `sslkey`, `sslcapath`, and `sslservername` + (TLS SNI override for pre-resolved hosts) +- `statement_cache_maxsize`: LRU prepared-statement cache size (default 100; `0` disables) +- `reconnect`: automatically reconnect and re-prepare statements if the + connection is found dead (default `false`; never reconnects mid-transaction) +- `style`: a custom [`AbstractPostgresStyle`](@ref Postgres.API.AbstractPostgresStyle) + for query logging / notice / notification behavior +- `debug`: log wire protocol messages + +Connections are safe for concurrent use from multiple tasks: operations are +serialized on an internal lock. Close with `DBInterface.close!(conn)` or +`close(conn)`; the do-block form `DBInterface.connect(f, Postgres.Connection, ...)` +closes automatically. +""" mutable struct Connection{T, S <: API.AbstractPostgresStyle} <: DBInterface.Connection const lock::ReentrantLock socket::ReseauConn @@ -70,8 +110,6 @@ mutable struct Connection{T, S <: API.AbstractPostgresStyle} <: DBInterface.Conn statement_timeout_val = statement_timeout === nothing ? nothing : Int(statement_timeout) sslservername_val = sslservername === nothing ? nothing : String(sslservername) maxsize = max(0, Int(statement_cache_maxsize)) - #TODO: if values have spaces, need to single-quote them - # also need to escape single quotes/backslahes then with backslashes socket, pid, skey, server_params = API.connect(host, port, dbname, user, password, debug, app_name, timeout, sslmode_val, sslrootcert_val, sslcert_val, sslkey_val, sslcapath_val, sslservername_val, statement_timeout_val) registry = Dict(API.DEFAULT_TYPE_REGISTRY) return new{Statement{typeof(style)}, typeof(style)}(ReentrantLock(), socket, host, user, password, dbname, port, app_name, timeout, sslmode_val, sslrootcert_val, sslcert_val, sslkey_val, sslcapath_val, sslservername_val, statement_timeout_val, pid, skey, Dict{String, Statement{typeof(style)}}(), maxsize, 0, server_params, registry, false, reconnect, debug, style, false, 0, 1) @@ -121,10 +159,20 @@ function evict_lru_statement!(conn::Connection) return end +""" + Postgres.get_cached_statements(conn) -> Dict{String, Statement} + +Return a copy of the connection's prepared-statement cache, keyed by SQL text. +""" function get_cached_statements(conn::Connection) @lock conn.lock copy(conn.statements) end +""" + Postgres.clear_statement_cache!(conn) + +Close all server-side prepared statements in the connection's cache and empty it. +""" function clear_statement_cache!(conn::Connection) @lock conn.lock begin for (sql, stmt) in conn.statements @@ -135,6 +183,12 @@ function clear_statement_cache!(conn::Connection) return conn end +""" + Postgres.set_statement_cache_maxsize!(conn, maxsize) + +Set the maximum number of prepared statements the connection caches (LRU +eviction). `0` disables caching and closes all currently cached statements. +""" function set_statement_cache_maxsize!(conn::Connection, maxsize::Integer) @lock conn.lock begin conn.statement_cache_maxsize = max(0, Int(maxsize)) @@ -153,18 +207,41 @@ function set_statement_cache_maxsize!(conn::Connection, maxsize::Integer) return conn end +""" + Postgres.get_server_parameter(conn, name) -> Union{String, Nothing} + +Return the server-reported value of runtime parameter `name` (e.g. +`"server_version"`, `"TimeZone"`), or `nothing` if the server has not reported it. +""" get_server_parameter(conn::Connection, param::String) = @lock conn.lock get(conn.server_parameters, param, nothing) +""" + Postgres.get_server_parameters(conn) -> Dict{String, String} + +Return a copy of all runtime parameters the server has reported on this connection. +""" get_server_parameters(conn::Connection) = @lock conn.lock copy(conn.server_parameters) # NOTE: runtime callback setters are gone — customize behavior by passing a custom # AbstractPostgresStyle to Connection(; style=...) and overloading the style-first # interface methods (query_logger / notice_callback / notification_callback). +""" + Postgres.get_statement_timeout(conn) -> Union{Int, Nothing} + +Return the statement timeout (milliseconds) configured on the connection, or +`nothing` if none was set. +""" function get_statement_timeout(conn::Connection) return @lock conn.lock conn.statement_timeout end +""" + Postgres.set_statement_timeout!(conn, timeout) + +Set the server `statement_timeout` for the connection, in milliseconds. +`nothing` or `0` disables the timeout. +""" function set_statement_timeout!(conn::Connection, timeout::Union{Integer, Nothing}) timeout_val = timeout === nothing ? 0 : max(0, Int(timeout)) DBInterface.execute(conn, "SET statement_timeout = $timeout_val") @@ -172,24 +249,55 @@ function set_statement_timeout!(conn::Connection, timeout::Union{Integer, Nothin return conn end +""" + Postgres.escape_identifier(name) -> String + +Quote a string for use as a SQL identifier (double-quoted, embedded quotes doubled). +""" function escape_identifier(name::AbstractString) return string("\"", replace(name, "\"" => "\"\""), "\"") end +""" + Postgres.escape_literal(val) -> String + +Quote a string for use as a SQL literal (single-quoted, embedded quotes +doubled). Prefer query parameters (`\$1`, `\$2`, ...) over literal interpolation +whenever possible. +""" function escape_literal(val::AbstractString) return string("'", replace(val, "'" => "''"), "'") end +""" + Postgres.listen!(conn, channel) + +Execute `LISTEN channel` so the connection receives notifications for +`channel`. Use [`wait_for_notification`](@ref Postgres.wait_for_notification) +to block until one arrives. +""" function listen!(conn::Connection, channel::AbstractString) DBInterface.execute(conn, "LISTEN $(escape_identifier(channel))") return conn end +""" + Postgres.unlisten!(conn, channel) + +Execute `UNLISTEN channel` to stop receiving notifications for `channel`. +""" function unlisten!(conn::Connection, channel::AbstractString) DBInterface.execute(conn, "UNLISTEN $(escape_identifier(channel))") return conn end +""" + Postgres.notify!(conn, channel, payload=nothing) + +Execute `NOTIFY channel` (with optional `payload`), delivering a +[`Notification`](@ref Postgres.API.Notification) to all connections listening +on `channel`. +""" function notify!(conn::Connection, channel::AbstractString, payload::Union{AbstractString, Nothing}=nothing) channel_ident = escape_identifier(channel) sql = payload === nothing ? "NOTIFY $channel_ident" : "NOTIFY $channel_ident, $(escape_literal(payload))" @@ -229,6 +337,15 @@ end const NOTIFICATION_POLL_INTERVAL_NS = Int64(100_000_000) +""" + Postgres.wait_for_notification(conn; timeout=nothing) -> Union{Notification, Nothing} + +Block until a `NOTIFY` message arrives on the connection (see +[`listen!`](@ref Postgres.listen!)) and return it as a +[`Notification`](@ref Postgres.API.Notification). With a `timeout` (seconds), +return `nothing` if no notification arrives in time. The connection lock is +held while waiting, so use a dedicated connection for listening. +""" function wait_for_notification(conn::Connection; timeout::Union{Real, Nothing}=nothing) start_time = time() @lock conn.lock begin @@ -269,6 +386,13 @@ function wait_for_notification(conn::Connection; timeout::Union{Real, Nothing}=n end end +""" + Postgres.copy_from(conn, sql, data) + +Execute a `COPY ... FROM STDIN` statement, streaming `data` (an `IO`, string, +or byte vector) to the server. Supports all COPY formats, including +`(FORMAT BINARY)`. Returns `conn`. +""" function copy_from(conn::Connection, sql::AbstractString, data::IO; debug::Bool=false) log_enabled = API.query_logging_enabled(conn.style) start_ns = log_enabled ? time_ns() : 0 @@ -291,6 +415,13 @@ function copy_from(conn::Connection, sql::AbstractString, data; debug::Bool=fals return copy_from(conn, sql, buffer; debug=debug) end +""" + Postgres.copy_to(conn, sql, [dest::IO]) + +Execute a `COPY ... TO STDOUT` statement. With a `dest` IO, the copy stream is +written to it and `dest` is returned; without one, the raw bytes are returned +as a `Vector{UInt8}`. +""" function copy_to(conn::Connection, sql::AbstractString, dest::IO; debug::Bool=false) log_enabled = API.query_logging_enabled(conn.style) start_ns = log_enabled ? time_ns() : 0 @@ -314,6 +445,16 @@ function copy_to(conn::Connection, sql::AbstractString; debug::Bool=false) return take!(buffer) end +""" + Postgres.register_type!(conn, oid, julia_type; parser=nothing) + +Register a mapping from PostgreSQL type `oid` to `julia_type` in the +connection's type registry. `parser` is a `(val::String, registry) -> value` +function that converts the wire text; without one, values are returned as +`String`. See also [`register_enum!`](@ref Postgres.register_enum!), +[`register_composite!`](@ref Postgres.register_composite!), and +[`register_range!`](@ref Postgres.register_range!). +""" function register_type!(conn::Connection, oid::Integer, julia_type::Type; parser::Union{Function, Nothing}=nothing) @lock conn.lock API.register_type!(conn.type_registry, oid, julia_type; parser=parser) return conn @@ -330,6 +471,12 @@ function lookup_type_oid(conn::Connection, name::AbstractString, schema::Abstrac return Int(rows[1].oid) end +""" + Postgres.register_enum!(conn, name; schema="public", julia_type=Symbol) + +Look up the enum type `schema.name` on the server and register it so values +are returned as `julia_type` (by default `Symbol`). +""" function register_enum!(conn::Connection, name::AbstractString; schema::AbstractString="public", julia_type::Type=Symbol) oid = lookup_type_oid(conn, name, schema) parser = julia_type === Symbol ? (val, registry) -> Symbol(val) : nothing @@ -337,6 +484,12 @@ function register_enum!(conn::Connection, name::AbstractString; schema::Abstract return conn end +""" + Postgres.register_composite!(conn, name; schema="public") + +Look up the composite type `schema.name` on the server and register it so +values are returned as `NamedTuple`s with the composite's field names. +""" function register_composite!(conn::Connection, name::AbstractString; schema::AbstractString="public") rows = Tables.rowtable(DBInterface.execute(conn, """ SELECT t.oid, a.attname, a.atttypid @@ -367,6 +520,13 @@ function register_composite!(conn::Connection, name::AbstractString; schema::Abs return conn end +""" + Postgres.register_range!(conn, name; schema="public") + +Look up the range type `schema.name` on the server and register it so values +are returned as [`PostgresRange`](@ref Postgres.API.PostgresRange) of the +range's element type. +""" function register_range!(conn::Connection, name::AbstractString; schema::AbstractString="public") rows = Tables.rowtable(DBInterface.execute(conn, """ SELECT t.oid, r.rngsubtype @@ -384,6 +544,14 @@ function register_range!(conn::Connection, name::AbstractString; schema::Abstrac return conn end +""" + Postgres.cancel_query!(conn) + +Send a PostgreSQL CancelRequest for the query currently running on `conn` +(over a separate, short-lived connection, so it works while `conn` is busy). +The cancelled query fails with a [`Postgres.Error`](@ref Postgres.API.Error) +with SQLSTATE `57014`. +""" function cancel_query!(conn::Connection) host = conn.host port = conn.port @@ -427,15 +595,13 @@ function DBInterface.connect(::Type{Connection}, host::AbstractString, user::Abs Connection(host=host, user=user, password=passwd, dbname=dbname, port=port, debug=debug, reconnect=reconnect, application_name=application_name, connect_timeout=connect_timeout, sslmode=sslmode, sslrootcert=sslrootcert, sslcert=sslcert, sslkey=sslkey, sslcapath=sslcapath, sslservername=sslservername, statement_timeout=statement_timeout, statement_cache_maxsize=statement_cache_maxsize, style=style) end -function DBInterface.connect(::Type{Connection}, dsn::String; debug::Bool=false, reconnect::Bool=false, statement_cache_maxsize::Union{Integer, Nothing}=nothing) - params = parse_dsn(dsn) - actual_maxsize = isnothing(statement_cache_maxsize) ? params.statement_cache_maxsize : statement_cache_maxsize - Connection(host=params.host, user=params.user, password=params.password, dbname=params.dbname, port=params.port, debug=debug, reconnect=reconnect, application_name=params.application_name, connect_timeout=params.connect_timeout, sslmode=params.sslmode, sslrootcert=params.sslrootcert, sslcert=params.sslcert, sslkey=params.sslkey, sslcapath=params.sslcapath, statement_timeout=params.statement_timeout, statement_cache_maxsize=actual_maxsize) +function DBInterface.connect(::Type{Connection}, dsn::String; debug::Union{Bool, Nothing}=nothing, reconnect::Union{Bool, Nothing}=nothing, statement_cache_maxsize::Union{Integer, Nothing}=nothing, style::API.AbstractPostgresStyle=PostgresStyle()) + return DBInterface.connect(Connection, parse_dsn(dsn); debug=debug, reconnect=reconnect, statement_cache_maxsize=statement_cache_maxsize, style=style) end -function DBInterface.connect(::Type{Connection}, params::ConnectionParams; debug::Bool=false, reconnect::Bool=false, statement_cache_maxsize::Union{Integer, Nothing}=nothing) +function DBInterface.connect(::Type{Connection}, params::ConnectionParams; debug::Union{Bool, Nothing}=nothing, reconnect::Union{Bool, Nothing}=nothing, statement_cache_maxsize::Union{Integer, Nothing}=nothing, style::API.AbstractPostgresStyle=PostgresStyle()) actual_maxsize = isnothing(statement_cache_maxsize) ? params.statement_cache_maxsize : statement_cache_maxsize - Connection(host=params.host, user=params.user, password=params.password, dbname=params.dbname, port=params.port, debug=debug, reconnect=reconnect, application_name=params.application_name, connect_timeout=params.connect_timeout, sslmode=params.sslmode, sslrootcert=params.sslrootcert, sslcert=params.sslcert, sslkey=params.sslkey, sslcapath=params.sslcapath, statement_timeout=params.statement_timeout, statement_cache_maxsize=actual_maxsize) + Connection(host=params.host, user=params.user, password=params.password, dbname=params.dbname, port=params.port, debug=something(debug, params.debug), reconnect=something(reconnect, params.reconnect), application_name=params.application_name, connect_timeout=params.connect_timeout, sslmode=params.sslmode, sslrootcert=params.sslrootcert, sslcert=params.sslcert, sslkey=params.sslkey, sslcapath=params.sslcapath, sslservername=params.sslservername, statement_timeout=params.statement_timeout, statement_cache_maxsize=actual_maxsize, style=style) end function DBInterface.connect(f::Function, ::Type{Connection}, args...; kwargs...) @@ -458,6 +624,21 @@ function DBInterface.close!(conn::Connection) end Base.close(conn::Connection) = DBInterface.close!(conn) +""" + Postgres.ConnectionPool + +A pool of [`Connection`](@ref Postgres.Connection)s, created lazily up to +`limit` and reused across [`acquire`](@ref Postgres.acquire)/[`release`](@ref +Postgres.release) cycles (dead connections are replaced transparently): + + ConnectionPool(Postgres.Connection, host, user, password; limit=10, kwargs...) + ConnectionPool(dsn::String; limit=10, kwargs...) + ConnectionPool(params::ConnectionParams; limit=10, kwargs...) + ConnectionPool(connector::Function; limit=10) + +Prefer [`with_connection`](@ref Postgres.with_connection) over manual +acquire/release. Close all pooled connections with `DBInterface.close!(pool)`. +""" struct ConnectionPool pool::Pools.Pool connector::Function @@ -468,21 +649,17 @@ function ConnectionPool(connector::Function; limit::Integer=10) return ConnectionPool(pool, connector) end -function ConnectionPool(::Type{Connection}, host::AbstractString, user::AbstractString, passwd::Union{AbstractString, Nothing}; dbname::AbstractString="", port::Integer=5432, debug::Bool=false, reconnect::Bool=false, application_name::Union{AbstractString, Nothing}=nothing, connect_timeout::Union{Integer, Nothing}=nothing, sslmode::Union{AbstractString, Nothing}=nothing, sslrootcert::Union{AbstractString, Nothing}=nothing, sslcert::Union{AbstractString, Nothing}=nothing, sslkey::Union{AbstractString, Nothing}=nothing, sslcapath::Union{AbstractString, Nothing}=nothing, statement_timeout::Union{Integer, Nothing}=nothing, statement_cache_maxsize::Integer=100, limit::Integer=10, style::API.AbstractPostgresStyle=PostgresStyle()) - connector = () -> DBInterface.connect(Connection, host, user, passwd; dbname=dbname, port=port, debug=debug, reconnect=reconnect, application_name=application_name, connect_timeout=connect_timeout, sslmode=sslmode, sslrootcert=sslrootcert, sslcert=sslcert, sslkey=sslkey, sslcapath=sslcapath, statement_timeout=statement_timeout, statement_cache_maxsize=statement_cache_maxsize, style=style) +function ConnectionPool(::Type{Connection}, host::AbstractString, user::AbstractString, passwd::Union{AbstractString, Nothing}; dbname::AbstractString="", port::Integer=5432, debug::Bool=false, reconnect::Bool=false, application_name::Union{AbstractString, Nothing}=nothing, connect_timeout::Union{Integer, Nothing}=nothing, sslmode::Union{AbstractString, Nothing}=nothing, sslrootcert::Union{AbstractString, Nothing}=nothing, sslcert::Union{AbstractString, Nothing}=nothing, sslkey::Union{AbstractString, Nothing}=nothing, sslcapath::Union{AbstractString, Nothing}=nothing, sslservername::Union{AbstractString, Nothing}=nothing, statement_timeout::Union{Integer, Nothing}=nothing, statement_cache_maxsize::Integer=100, limit::Integer=10, style::API.AbstractPostgresStyle=PostgresStyle()) + connector = () -> DBInterface.connect(Connection, host, user, passwd; dbname=dbname, port=port, debug=debug, reconnect=reconnect, application_name=application_name, connect_timeout=connect_timeout, sslmode=sslmode, sslrootcert=sslrootcert, sslcert=sslcert, sslkey=sslkey, sslcapath=sslcapath, sslservername=sslservername, statement_timeout=statement_timeout, statement_cache_maxsize=statement_cache_maxsize, style=style) return ConnectionPool(connector; limit=limit) end -function ConnectionPool(dsn::String; debug::Bool=false, reconnect::Bool=false, statement_cache_maxsize::Union{Integer, Nothing}=nothing, limit::Integer=10) - params = parse_dsn(dsn) - actual_maxsize = isnothing(statement_cache_maxsize) ? params.statement_cache_maxsize : statement_cache_maxsize - connector = () -> DBInterface.connect(Connection, params; debug=debug, reconnect=reconnect, statement_cache_maxsize=actual_maxsize) - return ConnectionPool(connector; limit=limit) +function ConnectionPool(dsn::String; debug::Union{Bool, Nothing}=nothing, reconnect::Union{Bool, Nothing}=nothing, statement_cache_maxsize::Union{Integer, Nothing}=nothing, limit::Integer=10, style::API.AbstractPostgresStyle=PostgresStyle()) + return ConnectionPool(parse_dsn(dsn); debug=debug, reconnect=reconnect, statement_cache_maxsize=statement_cache_maxsize, limit=limit, style=style) end -function ConnectionPool(params::ConnectionParams; debug::Bool=false, reconnect::Bool=false, statement_cache_maxsize::Union{Integer, Nothing}=nothing, limit::Integer=10) - actual_maxsize = isnothing(statement_cache_maxsize) ? params.statement_cache_maxsize : statement_cache_maxsize - connector = () -> DBInterface.connect(Connection, params; debug=debug, reconnect=reconnect, statement_cache_maxsize=actual_maxsize) +function ConnectionPool(params::ConnectionParams; debug::Union{Bool, Nothing}=nothing, reconnect::Union{Bool, Nothing}=nothing, statement_cache_maxsize::Union{Integer, Nothing}=nothing, limit::Integer=10, style::API.AbstractPostgresStyle=PostgresStyle()) + connector = () -> DBInterface.connect(Connection, params; debug=debug, reconnect=reconnect, statement_cache_maxsize=statement_cache_maxsize, style=style) return ConnectionPool(connector; limit=limit) end @@ -491,11 +668,23 @@ function pool_isvalid(conn::Connection) return valid end +""" + Postgres.acquire(pool; forcenew=false) -> Connection + +Take a connection from the pool, creating one if none is available (blocking +if the pool is at its limit). Return it with [`release`](@ref Postgres.release). +""" function acquire(pool::ConnectionPool; forcenew::Bool=false) conn = Pools.acquire(pool.connector, pool.pool; forcenew=forcenew, isvalid=pool_isvalid) return conn end +""" + Postgres.release(pool, conn) + +Return a connection previously taken with [`acquire`](@ref Postgres.acquire) +to the pool. +""" function release(pool::ConnectionPool, conn::Connection) if pool_isvalid(conn) Pools.release(pool.pool, conn) @@ -505,6 +694,12 @@ function release(pool::ConnectionPool, conn::Connection) return pool end +""" + Postgres.with_connection(f, pool; forcenew=false) + +Acquire a connection from the pool, call `f(conn)`, and release the connection +back to the pool afterwards. Returns `f`'s result. +""" function with_connection(f::Function, pool::ConnectionPool; forcenew::Bool=false) conn = acquire(pool; forcenew=forcenew) try @@ -543,6 +738,15 @@ function execute_simple(conn::Connection, sql::String) return conn end +""" + Postgres.start_transaction(conn) + +Begin a transaction (`BEGIN`). If a transaction is already open, create a +savepoint instead, so transactions nest. Pair with [`commit`](@ref +Postgres.commit) or [`rollback`](@ref Postgres.rollback); prefer +[`transaction`](@ref Postgres.transaction) or +[`@transaction`](@ref Postgres.@transaction) for automatic handling. +""" function start_transaction(conn::Connection) @lock conn.lock begin checkconn(conn) @@ -560,8 +764,18 @@ function start_transaction(conn::Connection) return conn end +""" + Postgres.in_transaction(conn) -> Bool + +Whether the connection currently has an open transaction. +""" in_transaction(conn::Connection) = @lock conn.lock conn.in_transaction +""" + Postgres.commit(conn) + +Commit the current transaction (or release one level of transaction nesting). +""" function commit(conn::Connection) @lock conn.lock begin checkconn(conn) @@ -579,6 +793,12 @@ function commit(conn::Connection) return conn end +""" + Postgres.rollback(conn) + +Roll back the current transaction (or, in a nested transaction, roll back to +the enclosing savepoint). +""" function rollback(conn::Connection) @lock conn.lock begin checkconn(conn) @@ -597,6 +817,16 @@ function rollback(conn::Connection) return conn end +""" + Postgres.transaction(f, conn) + +Run `f(conn)` inside a transaction: committed if `f` returns normally, rolled +back if it throws. Nested calls use savepoints. Returns `f`'s result. + + Postgres.transaction(conn) do conn + DBInterface.execute(conn, "INSERT INTO t VALUES (1)") + end +""" function transaction(f::F, conn::Connection) where {F} start_transaction(conn) try @@ -621,6 +851,12 @@ function DBInterface.transaction(f::F, conn::Connection) where {F} end end +""" + Postgres.@transaction conn expr + +Run `expr` inside a transaction: committed if it completes, rolled back if it +throws. Evaluates to `expr`'s value. +""" macro transaction(conn, expr) quote local success = false @@ -637,8 +873,6 @@ macro transaction(conn, expr) end end -# escape(conn::Connection, s::AbstractString) = API.escape(conn.pg, s) - struct Describe resultset::Any end @@ -671,6 +905,12 @@ function Base.show(io::IO, desc::Describe) end end +""" + Postgres.describe(conn, table; schema="public") + +Return a printable summary of a table's columns: name, type, nullability, +default, primary-key flag, and foreign-key reference. +""" function describe(conn::Connection, table::AbstractString; schema::String="public") Describe(DBInterface.execute(conn, """ WITH column_info AS ( @@ -721,4 +961,20 @@ function describe(conn::Connection, table::AbstractString; schema::String="publi """, (table, schema))) end +# the supported API surface (`public` requires Julia 1.11+; the names are +# parsed from a string so the file still loads on 1.10) +@static if VERSION >= v"1.11" + eval(Meta.parse( + "public Connection, ConnectionPool, ConnectionParams, PostgresInterfaceError, " * + "Error, Notification, Numeric, PostgresRange, AbstractPostgresStyle, PostgresStyle, " * + "query_logging_enabled, query_logger, notice_callback, notification_callback, parse_dsn, " * + "transaction, @transaction, start_transaction, commit, rollback, in_transaction, " * + "cursor, copy_from, copy_to, listen!, unlisten!, notify!, wait_for_notification, " * + "register_type!, register_enum!, register_composite!, register_range!, " * + "command_tag, rows_affected, cancel_query!, escape_identifier, escape_literal, " * + "get_cached_statements, clear_statement_cache!, set_statement_cache_maxsize!, " * + "get_server_parameter, get_server_parameters, get_statement_timeout, set_statement_timeout!, " * + "acquire, release, with_connection, describe")) +end + end diff --git a/src/api/API.jl b/src/api/API.jl index 9a07b5d..d79e350 100644 --- a/src/api/API.jl +++ b/src/api/API.jl @@ -1,13 +1,21 @@ module API -using UUIDs, Dates, Reseau, SASLAuth, MD5, Parsers, StructUtils, Logging, JSON, Random +using UUIDs, Dates, Reseau, SASLAuth, MD5, Parsers, StructUtils, JSON, Random import ..PostgresInterfaceError -export PostgresStyle, AbstractPostgresStyle, query_logging_enabled, query_logger, notice_callback, notification_callback, Error, Notification, Numeric, PostgresRange +export PostgresStyle, AbstractPostgresStyle, query_logging_enabled, query_logger, notice_callback, notification_callback, Error, Notification, Numeric, PostgresRange, cancel_request const ReseauConn = Union{Reseau.TCP.Conn, Reseau.TLS.Conn} const SKIP_BUFFER_SIZE = 8192 +""" + Postgres.Error <: Exception + +A PostgreSQL server error (an `ErrorResponse` message). Carries the fields the +server reported: `severity`, `code` (the SQLSTATE, e.g. `"23505"`), `message`, +and optional context such as `detail`, `hint`, `position`, `schema`, `table`, +`column`, and `constraint`. +""" struct Error <: Exception severity::String code::String @@ -28,6 +36,14 @@ struct Error <: Exception routine::Union{String, Nothing} end +""" + Postgres.Notification + +An asynchronous `NOTIFY` message received from the server, with the notifying +backend's `pid`, the `channel` name, and the `payload` string (empty when the +notification had no payload). See `Postgres.listen!` and +`Postgres.wait_for_notification`. +""" struct Notification pid::Int32 channel::String @@ -51,28 +67,6 @@ function Base.showerror(io::IO, e::Error) return end -# error code => (name, should_be_shown) -const ERROR_CODE = Dict{Char, Tuple{String, Bool}}( - 'S' => ("Severity", true), - 'V' => ("Severity", false), - 'C' => ("Code", false), - 'M' => ("Message", true), - 'D' => ("Detail", true), - 'H' => ("Hint", true), - 'P' => ("Position", false), - 'p' => ("Internal Position", false), - 'q' => ("Internal Query", false), - 'W' => ("Where", true), - 's' => ("Schema Name", true), - 't' => ("Table Name", true), - 'c' => ("Column Name", true), - 'd' => ("Data Type Name", true), - 'n' => ("Constraint Name", true), - 'F' => ("File", false), - 'L' => ("Line", false), - 'R' => ("Routine", false), -) - function errorResponse(len, socket, debug) buf = read(socket, len) # parse error fields @@ -94,7 +88,7 @@ function errorResponse(len, socket, debug) file = nothing line = nothing routine = nothing - while i < len + GC.@preserve buf while i < len ccode = Char(buf[i]) i += 1 val = unsafe_string(pointer(buf, i)) @@ -144,7 +138,7 @@ function noticeResponse(len, socket) buf = read(socket, len) i = 1 notice = Dict{String, String}() - while i < length(buf) + GC.@preserve buf while i < length(buf) ccode = Char(buf[i]) i += 1 val = unsafe_string(pointer(buf, i)) @@ -161,9 +155,11 @@ function notificationResponse(len, socket) channel = "" payload = "" if !isempty(buf) - channel = unsafe_string(pointer(buf, i)) - i += sizeof(channel) + 1 - i <= length(buf) && (payload = unsafe_string(pointer(buf, i))) + GC.@preserve buf begin + channel = unsafe_string(pointer(buf, i)) + i += sizeof(channel) + 1 + i <= length(buf) && (payload = unsafe_string(pointer(buf, i))) + end end return Notification(pid, channel, payload) end @@ -390,7 +386,7 @@ function waitfor(socket, debug::Bool, codes::Vararg{Char, N}) where {N} # parameter status buf = read(socket, len) i = 1 - while i < len + GC.@preserve buf while i < len j = findnext(isequal(UInt8(0)), buf, i) j === nothing && break key = unsafe_string(pointer(buf, i), j - i) @@ -428,6 +424,14 @@ function waitfor(socket, debug::Bool, codes::Vararg{Char, N}) where {N} return pid, skey, server_params end +# password material must never reach the debug log: password-bearing messages +# are written with debug=false and a redacted line is logged instead +function write_password_message(socket, debug::Bool, password::String) + debug && @info "sending message: p, (password redacted)" + writemessage(socket, false, 'p', password) + return +end + function authRequest(debug, len, socket, user, password, client::Union{Nothing, SASLAuth.SCRAMSHA256Client}=nothing) auth_code = ntoh(read(socket, Int32)) debug && @info "auth code: $auth_code" @@ -440,7 +444,7 @@ function authRequest(debug, len, socket, user, password, client::Union{Nothing, throw(Error("kerberos v5 authentication not supported")) elseif auth_code == 3 # send cleartext password message - writemessage(socket, debug, 'p', password) + write_password_message(socket, debug, password) mt, len = readheader(socket, debug) if mt == UInt8('E') # error @@ -464,7 +468,7 @@ function authRequest(debug, len, socket, user, password, client::Union{Nothing, # Calculate the MD5 password pass = string("md5", bytes2hex(md5(vcat(Vector{UInt8}(bytes2hex(md5(string(password, user)))), salt)))) # Send password message - writemessage(socket, debug, 'p', pass) + write_password_message(socket, debug, pass) mt, len = readheader(socket, debug) if mt == UInt8('E') # error @@ -553,8 +557,17 @@ end connectsocket(host::AbstractString, port::Integer; connect_timeout::Union{Int, Nothing}=nothing) = connectsocket(host, port, connect_timeout) +# `host:port` for Reseau, with IPv6 literals bracketed (`[::1]:5432`) so the +# address parser doesn't reject the extra colons +function hostport_address(host::AbstractString, port::Integer) + startswith(host, '/') && throw(PostgresInterfaceError("unix socket connections are not supported; provide a TCP host")) + h = String(host) + (startswith(h, '[') && endswith(h, ']')) && return string(h, ":", Int(port)) + return occursin(':', h) ? string("[", h, "]:", Int(port)) : string(h, ":", Int(port)) +end + function connectsocket(host::AbstractString, port::Integer, @nospecialize(connect_timeout::Union{Int, Nothing})) - address = string(host, ":", Int(port)) + address = hostport_address(host, port) return if connect_timeout === nothing Reseau.TCP.connect(address) else @@ -696,7 +709,7 @@ function describeprepared(socket, name::String, debug::Bool) ncols = Int(ntoh(read(socket, Int16))) buf = read(socket, len - 2) i = 1 - while i < len - 2 + GC.@preserve buf while i < len - 2 ptr = pointer(buf, i) plen = Int(@ccall strlen(ptr::Ptr{Cvoid})::Csize_t) name = _symbol(ptr, plen) @@ -772,7 +785,7 @@ end function commandComplete(len, socket) buf = read(socket, len) isempty(buf) && return "" - return unsafe_string(pointer(buf)) + return GC.@preserve buf unsafe_string(pointer(buf)) end function rows_affected_from_command_tag(tag::String) @@ -789,6 +802,8 @@ function StructUtils.applyeach(::AbstractPostgresStyle, f, e::Exec) nrows = 0 server_error = nothing consumer_error = nothing + copy_in_statement = false + copy_out_statement = false try while true mt, len = readheader(e.socket, e.debug) @@ -821,6 +836,22 @@ function StructUtils.applyeach(::AbstractPostgresStyle, f, e::Exec) elseif mt == UInt8('T') || mt == UInt8('n') || mt == UInt8('I') || mt == UInt8('S') # row description / no data / empty query response / parameter status skipbytes!(e.socket, len) + elseif mt == UInt8('G') + # CopyInResponse: the statement was a COPY ... FROM STDIN, which + # execute doesn't support. Abort the copy with CopyFail so the + # server returns to ready and the connection stays usable; a + # clear client error is thrown below. A fresh Sync must follow: + # the one sent with Bind/Execute was ignored during copy-in + # mode, and the aborted extended-query sequence only reaches + # ReadyForQuery once a Sync arrives after the error. + skipbytes!(e.socket, len) + copy_in_statement = true + writemessages(e.socket, e.debug, ('f', "COPY FROM STDIN is not supported via execute"), ('S',)) + elseif mt == UInt8('H') || mt == UInt8('d') || mt == UInt8('c') + # CopyOutResponse/CopyData/CopyDone: drain the copy-out stream + # through ReadyForQuery; a clear client error is thrown below + mt == UInt8('H') && (copy_out_statement = true) + skipbytes!(e.socket, len) elseif mt == UInt8('A') # notification response notification = notificationResponse(len, e.socket) @@ -843,6 +874,11 @@ function StructUtils.applyeach(::AbstractPostgresStyle, f, e::Exec) server_error === nothing || throw(server_error) rethrow() end + # COPY misuse throws a clear client error (the server error from the + # aborted copy would be confusing); the stream was drained above, so the + # connection stays usable + copy_in_statement && throw(Error("COPY ... FROM STDIN is not supported via execute; use Postgres.copy_from")) + copy_out_statement && throw(Error("COPY ... TO STDOUT is not supported via execute; use Postgres.copy_to")) # server errors take precedence; otherwise surface a consumer error that # aborted materialization (the stream was still drained above) server_error === nothing || throw(server_error) @@ -899,13 +935,20 @@ exec(socket::ReseauConn, query::String, debug::Bool) = exec(PostgresStyle(), soc function copy_in(style::S, socket, query::String, source::IO, debug::Bool) where {S <: AbstractPostgresStyle} writemessage(socket, debug, 'Q', query) error_msg = nothing + copy_started = false while true mt, len = readheader(socket, debug) if mt == UInt8('G') skipbytes!(socket, len) + copy_started = true break elseif mt == UInt8('E') error_msg = errorResponse(len, socket, debug) + elseif mt == UInt8('Z') + # ReadyForQuery without CopyInResponse: the statement errored or + # wasn't a COPY ... FROM STDIN; the stream is back at ready + skipbytes!(socket, len) + break elseif mt == UInt8('N') notice = noticeResponse(len, socket) notice_callback(style, notice) @@ -917,6 +960,7 @@ function copy_in(style::S, socket, query::String, source::IO, debug::Bool) where end end error_msg === nothing || throw(error_msg) + copy_started || throw(Error("statement did not initiate COPY ... FROM STDIN")) buf = Vector{UInt8}(undef, 16384) while !eof(source) n = readbytes!(source, buf, length(buf)) @@ -951,16 +995,26 @@ end function copy_out(style::S, socket, query::String, dest::IO, debug::Bool) where {S <: AbstractPostgresStyle} writemessage(socket, debug, 'Q', query) error_msg = nothing + copy_started = false + wrong_direction = false while true mt, len = readheader(socket, debug) if mt == UInt8('H') skipbytes!(socket, len) + copy_started = true elseif mt == UInt8('d') write(dest, read(socket, len)) elseif mt == UInt8('c') skipbytes!(socket, len) elseif mt == UInt8('C') skipbytes!(socket, len) + elseif mt == UInt8('G') + # CopyInResponse: the statement was COPY ... FROM STDIN. The server + # is now waiting on us for data, so abort the copy with CopyFail to + # return the stream to ready instead of deadlocking. + skipbytes!(socket, len) + wrong_direction = true + writemessage(socket, debug, 'f', "COPY FROM STDIN is not supported via copy_to") elseif mt == UInt8('E') error_msg = errorResponse(len, socket, debug) elseif mt == UInt8('N') @@ -976,7 +1030,9 @@ function copy_out(style::S, socket, query::String, dest::IO, debug::Bool) where skipbytes!(socket, len) end end + wrong_direction && throw(Error("statement initiated COPY ... FROM STDIN; use Postgres.copy_from")) error_msg === nothing || throw(error_msg) + copy_started || throw(Error("statement did not initiate COPY ... TO STDOUT")) return dest end @@ -1004,15 +1060,6 @@ function cancel_request(host::String, port::Int, pid::Int32, skey::Int32, debug: end end -export cancel_request - -# function escape(conn::PGconn, s::AbstractString) -# str = C.PQescapeLiteral(ptr, s, sizeof(s)) -# escaped = unsafe_string(str) -# C.PQfreemem(str) -# return escaped -# end - include("../array_parsing.jl") using .ArrayParsing diff --git a/src/api/types.jl b/src/api/types.jl index a2cc1c1..6616495 100644 --- a/src/api/types.jl +++ b/src/api/types.jl @@ -40,6 +40,17 @@ StructUtils.lift(::AbstractPostgresStyle, ::Type{T}, x::T) where {T<:JSON.LazyVa StructUtils.lift(::AbstractPostgresStyle, ::Type{T}, x::T, tags) where {T<:JSON.LazyValue} = x, nothing +""" + Postgres.Numeric + +Exact decimal representation of a PostgreSQL `numeric`/`decimal` value: +`coeff * 10^-scale`, where `coeff` is a `BigInt` and `scale` the number of +digits after the decimal point. Preserves the value and scale exactly (no +floating-point rounding). `print`/`string` produce the decimal text form. + +The PostgreSQL special values `NaN`, `Infinity`, and `-Infinity` cannot be +represented and throw an error when encountered. +""" struct Numeric coeff::BigInt scale::Int @@ -48,6 +59,14 @@ StructUtils.structlike(::AbstractPostgresStyle, ::Type{Numeric}) = false Base.:(==)(a::Numeric, b::Numeric) = a.coeff == b.coeff && a.scale == b.scale +""" + Postgres.PostgresRange{T} + +A PostgreSQL range value (`int4range`, `numrange`, `tstzrange`, ...). `lower` +and `upper` are the bounds (`missing` when unbounded), `lower_inclusive` and +`upper_inclusive` indicate whether each bound is inclusive, and `empty` is +`true` for the empty range. +""" struct PostgresRange{T} lower::Union{T, Missing} upper::Union{T, Missing} @@ -190,8 +209,6 @@ function register_type!(registry::Dict{Int, TypeInfo}, oid::Integer, julia_type: return registry end -const DATETIME_OPTIONS = Parsers.Options(dateformat=dateformat"yyyy-mm-dd HH:MM:SS.s") - @inline function tzoffset_seconds(offset::AbstractString) isempty(offset) && return 0 sign = offset[1] == '-' ? -1 : 1 @@ -315,6 +332,9 @@ Base.show(io::IO, num::Numeric) = print(io, numeric_string(num)) function parse_numeric(val::String) stripped = strip(val) stripped == "" && return Numeric(BigInt(0), 0) + lowered = lowercase(stripped) + (lowered == "nan" || lowered == "infinity" || lowered == "-infinity" || lowered == "+infinity") && + throw(PostgresInterfaceError("postgres numeric special value \"$stripped\" cannot be represented as Postgres.Numeric")) sign = 1 if stripped[1] == '-' sign = -1 diff --git a/src/array_parsing.jl b/src/array_parsing.jl index d7b5d67..0ab1608 100644 --- a/src/array_parsing.jl +++ b/src/array_parsing.jl @@ -174,14 +174,6 @@ function parse_array(str::String, inner_type::Type{T}) where {T} return coerce_array(Any[value], inner_type) end -function parse_to_julia_array(str::String, julia_type::Type) - try - return parse_array(str, julia_type) - catch - return str - end -end - -export parse_array, parse_to_julia_array +export parse_array end diff --git a/src/connection_string.jl b/src/connection_string.jl index f636e23..ef94b9d 100644 --- a/src/connection_string.jl +++ b/src/connection_string.jl @@ -2,6 +2,21 @@ module ConnectionString using URIs +""" + Postgres.ConnectionParams(; host="localhost", port=5432, user="", password=nothing, + dbname="", kwargs...) + +Structured connection options, an alternative to DSN strings: + + params = Postgres.ConnectionParams(host="127.0.0.1", user="postgres", dbname="postgres") + conn = DBInterface.connect(Postgres.Connection, params) + +Also produced by `Postgres.parse_dsn`. Supported keyword +arguments mirror the connection keywords: `application_name`, +`connect_timeout`, `sslmode`, `sslrootcert`, `sslcert`, `sslkey`, `sslcapath`, +`sslservername`, `statement_timeout`, `statement_cache_maxsize`, `debug`, and +`reconnect`. +""" struct ConnectionParams host::String port::Int @@ -15,14 +30,15 @@ struct ConnectionParams sslcert::Union{String, Nothing} sslkey::Union{String, Nothing} sslcapath::Union{String, Nothing} + sslservername::Union{String, Nothing} statement_timeout::Union{Int, Nothing} statement_cache_maxsize::Int debug::Bool reconnect::Bool end -function ConnectionParams(; host::String="localhost", port::Int=5432, user::String="", password::Union{String, Nothing}=nothing, dbname::String="", application_name::Union{String, Nothing}=nothing, connect_timeout::Union{Int, Nothing}=nothing, sslmode::Union{String, Nothing}=nothing, sslrootcert::Union{String, Nothing}=nothing, sslcert::Union{String, Nothing}=nothing, sslkey::Union{String, Nothing}=nothing, sslcapath::Union{String, Nothing}=nothing, statement_timeout::Union{Int, Nothing}=nothing, statement_cache_maxsize::Int=100, debug::Bool=false, reconnect::Bool=false) - return ConnectionParams(host, port, user, password, dbname, application_name, connect_timeout, sslmode, sslrootcert, sslcert, sslkey, sslcapath, statement_timeout, statement_cache_maxsize, debug, reconnect) +function ConnectionParams(; host::String="localhost", port::Int=5432, user::String="", password::Union{String, Nothing}=nothing, dbname::String="", application_name::Union{String, Nothing}=nothing, connect_timeout::Union{Int, Nothing}=nothing, sslmode::Union{String, Nothing}=nothing, sslrootcert::Union{String, Nothing}=nothing, sslcert::Union{String, Nothing}=nothing, sslkey::Union{String, Nothing}=nothing, sslcapath::Union{String, Nothing}=nothing, sslservername::Union{String, Nothing}=nothing, statement_timeout::Union{Int, Nothing}=nothing, statement_cache_maxsize::Int=100, debug::Bool=false, reconnect::Bool=false) + return ConnectionParams(host, port, user, password, dbname, application_name, connect_timeout, sslmode, sslrootcert, sslcert, sslkey, sslcapath, sslservername, statement_timeout, statement_cache_maxsize, debug, reconnect) end default_user() = get(ENV, "PGUSER", get(ENV, "USER", get(ENV, "USERNAME", ""))) @@ -81,6 +97,7 @@ function params_from_values(values::Dict{String, String}) sslcert=get(merged, "sslcert", nothing), sslkey=get(merged, "sslkey", nothing), sslcapath=get(merged, "sslcapath", nothing), + sslservername=get(merged, "sslservername", nothing), statement_timeout=parse_optional_int(get(merged, "statement_timeout", nothing)), statement_cache_maxsize=parse(Int, get(merged, "statement_cache_maxsize", "100")), ) @@ -143,6 +160,15 @@ function parse_keyword_dsn(dsn::String) return values end +""" + Postgres.parse_dsn(dsn) -> ConnectionParams + +Parse a libpq-style keyword string (`"host=127.0.0.1 user=postgres"`) or a +PostgreSQL URI (`"postgresql://user:pass@host:5432/dbname"`) into +`ConnectionParams`. Unset options fall back +to the `PGHOST`, `PGPORT`, `PGUSER`, `PGPASSWORD`, `PGDATABASE`, `PGAPPNAME`, +`PGCONNECT_TIMEOUT`, and `PGSSL*` environment variables, then to defaults. +""" function parse_dsn(dsn::String) lowered = lowercase(dsn) (startswith(lowered, "postgres://") || startswith(lowered, "postgresql://")) && return parse_uri(dsn) @@ -178,7 +204,7 @@ function parse_uri(uri::String) query = String(parsed.query) if !isempty(query) params = URIs.queryparams(query) - for key in ("host", "port", "user", "password", "dbname", "application_name", "connect_timeout", "sslmode", "sslrootcert", "sslcert", "sslkey", "sslcapath", "statement_timeout", "statement_cache_maxsize") + for key in ("host", "port", "user", "password", "dbname", "application_name", "connect_timeout", "sslmode", "sslrootcert", "sslcert", "sslkey", "sslcapath", "sslservername", "statement_timeout", "statement_cache_maxsize") haskey(params, key) && (values[key] = params[key]) end end diff --git a/src/execute.jl b/src/execute.jl index 93f9715..7484062 100644 --- a/src/execute.jl +++ b/src/execute.jl @@ -16,7 +16,21 @@ end Base.size(r::Result) = (length(r.rows),) Base.getindex(r::Result, i::Integer) = r.rows[i] + +""" + Postgres.command_tag(result) -> Union{String, Nothing} + +The PostgreSQL command completion tag for the executed statement, e.g. +`"SELECT 5"`, `"INSERT 0 2"`, or `"UPDATE 3"`. +""" command_tag(r::Result) = r.command_tag + +""" + Postgres.rows_affected(result) -> Union{Int, Nothing} + +The number of rows the statement affected (parsed from the command tag), or +`nothing` when the statement doesn't report one. +""" rows_affected(r::Result) = r.rows_affected getdata(r::ResultRow) = getfield(r, :data) @@ -36,8 +50,6 @@ Tables.getcolumn(r::ResultRow, nm::Symbol) = Tables.getcolumn(r, getlookup(r)[nm Tables.schema(r::Result) = Tables.Schema(r.names, r.types) -# DBInterface.lastrowid(result::Result) = API.lastrowid(result.result) - function DBInterface.close!(::Result) return end @@ -273,6 +285,7 @@ function read_portal_batch!(cursor::Cursor) rows = ResultRow[] error_msg = nothing consumer_error = nothing + copy_statement = false done = false try while true @@ -299,6 +312,19 @@ function read_portal_batch!(cursor::Cursor) elseif mt == UInt8('C') API.skipbytes!(conn.socket, len) done = true + elseif mt == UInt8('G') + # CopyInResponse: a COPY ... FROM STDIN statement was used with + # a cursor. Abort the copy with CopyFail so the stream returns + # to ready instead of deadlocking; a clear error is thrown + # below. A fresh Sync must follow: the one sent with + # Bind/Execute was ignored during copy-in mode. + API.skipbytes!(conn.socket, len) + copy_statement = true + API.writemessages(conn.socket, conn.debug, ('f', "COPY FROM STDIN is not supported via cursor"), ('S',)) + elseif mt == UInt8('H') || mt == UInt8('d') || mt == UInt8('c') + # CopyOutResponse/CopyData/CopyDone: drain the copy-out stream + mt == UInt8('H') && (copy_statement = true) + API.skipbytes!(conn.socket, len) elseif mt == UInt8('N') notice = API.noticeResponse(len, conn.socket) API.notice_callback(conn.style, notice) @@ -320,6 +346,10 @@ function read_portal_batch!(cursor::Cursor) error_msg === nothing || throw(error_msg) rethrow() end + if copy_statement + cursor.done = true + throw(PostgresInterfaceError("COPY statements are not supported via cursor; use Postgres.copy_from or Postgres.copy_to")) + end error_msg === nothing || throw(error_msg) consumer_error === nothing || throw(consumer_error) cursor.buffer = rows @@ -349,7 +379,7 @@ function Base.iterate(cursor::Cursor, state=nothing) return row, nothing end -function DBInterface.execute(stmt::Statement, params=nothing, ::Type{T}=Any; debug::Bool=false, binary::Bool=false) where {T} +function DBInterface.execute(stmt::Statement, params=nothing, ::Type{T}=Any; debug::Bool=false) where {T} style = stmt.conn.style log_enabled = API.query_logging_enabled(style) start_ns = log_enabled ? time_ns() : 0 @@ -422,9 +452,24 @@ function cursor(stmt::Statement, params=nothing; fetchsize::Integer=1000, owns_t end end +""" + Postgres.cursor(conn, sql, params=nothing; fetchsize=1000) -> Cursor + Postgres.cursor(stmt, params=nothing; fetchsize=1000) -> Cursor + +Execute a query and stream its result rows in batches of `fetchsize` instead +of materializing them all at once. The returned cursor iterates rows; close it +with `DBInterface.close!(cursor)`. A cursor requires a transaction: one is +started (and committed on close) if the connection isn't already in one. +""" function cursor(conn::Connection, sql::AbstractString, params=nothing; fetchsize::Integer=1000, debug::Bool=false) owns_transaction = false in_transaction(conn) || (start_transaction(conn); owns_transaction = true) - stmt = DBInterface.prepare(conn, sql; debug=debug) - return cursor(stmt, params; fetchsize=fetchsize, owns_transaction=owns_transaction) + try + stmt = DBInterface.prepare(conn, sql; debug=debug) + return cursor(stmt, params; fetchsize=fetchsize, owns_transaction=owns_transaction) + catch + # don't leave the transaction we started dangling on a failed cursor + owns_transaction && isopen(conn) && in_transaction(conn) && rollback(conn) + rethrow() + end end diff --git a/test/runtests.jl b/test/runtests.jl index 76609c5..016185b 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -418,7 +418,27 @@ end @testset "Postgres" begin @testset "Export Surface" begin - @test Set(names(Postgres)) == Set([:DBInterface, :Postgres]) + exported = Set([:DBInterface, :Postgres]) + @static if VERSION >= v"1.11" + # names() includes `public` declarations on Julia 1.11+ + public_names = Set([ + :Connection, :ConnectionPool, :ConnectionParams, :PostgresInterfaceError, + :Error, :Notification, :Numeric, :PostgresRange, :AbstractPostgresStyle, :PostgresStyle, + :query_logging_enabled, :query_logger, :notice_callback, :notification_callback, :parse_dsn, + :transaction, Symbol("@transaction"), :start_transaction, :commit, :rollback, :in_transaction, + :cursor, :copy_from, :copy_to, :listen!, :unlisten!, :notify!, :wait_for_notification, + :register_type!, :register_enum!, :register_composite!, :register_range!, + :command_tag, :rows_affected, :cancel_query!, :escape_identifier, :escape_literal, + :get_cached_statements, :clear_statement_cache!, :set_statement_cache_maxsize!, + :get_server_parameter, :get_server_parameters, :get_statement_timeout, :set_statement_timeout!, + :acquire, :release, :with_connection, :describe, + ]) + @test Set(names(Postgres)) == union(exported, public_names) + else + @test Set(names(Postgres)) == exported + end + @test Postgres.PostgresInterfaceError <: Exception + @test Postgres.Error <: Exception end @testset "Connection String Parsing" begin @@ -470,6 +490,18 @@ end @test query_host_params.port == 5436 @test query_host_params.dbname == "postgres" + sni_params = Postgres.parse_dsn("host=203.0.113.7 sslmode=require sslservername=db.example.com") + @test sni_params.sslservername == "db.example.com" + sni_uri_params = Postgres.parse_dsn("postgresql://postgres@203.0.113.7/postgres?sslservername=db.example.com") + @test sni_uri_params.sslservername == "db.example.com" + + # IPv6 hosts must be bracketed for the transport address parser; + # unix socket paths are rejected with a clear error + @test Postgres.API.hostport_address("::1", 5432) == "[::1]:5432" + @test Postgres.API.hostport_address("127.0.0.1", 5432) == "127.0.0.1:5432" + @test Postgres.API.hostport_address("db.example.com", 6432) == "db.example.com:6432" + @test_throws Postgres.PostgresInterfaceError Postgres.API.hostport_address("/var/run/postgresql", 5432) + withenv( "PGHOST" => "envhost", "PGPORT" => "5544", @@ -502,6 +534,10 @@ end @test string(Postgres.API.parse_numeric("-0.00120")) == "-0.00120" @test string(Postgres.API.parse_numeric("1.23e3")) == "1230" @test Postgres.API.parse_numeric("+42") == Postgres.Numeric(BigInt(42), 0) + # numeric special values can't be represented and must fail clearly + @test_throws Postgres.PostgresInterfaceError Postgres.API.parse_numeric("NaN") + @test_throws Postgres.PostgresInterfaceError Postgres.API.parse_numeric("Infinity") + @test_throws Postgres.PostgresInterfaceError Postgres.API.parse_numeric("-Infinity") @test Postgres.API.parse_value(1184, "2024-02-13 05:28:17+02", registry) == DateTime(2024, 2, 13, 3, 28, 17) @test Postgres.API.parse_value(1184, "2024-02-13 05:28:17+02:30", registry) == DateTime(2024, 2, 13, 2, 58, 17) @@ -877,6 +913,24 @@ end conn3 = DBInterface.connect(Postgres.Connection, params_with_app) @test isopen(conn3) DBInterface.close!(conn3) + + # ConnectionParams debug/reconnect fields are honored (and + # overridable via keyword arguments) + params_reconnect = Postgres.ConnectionParams( + host=cfg.host, + port=cfg.port, + user=cfg.user, + password=cfg.password, + dbname=cfg.dbname, + reconnect=true + ) + conn4 = DBInterface.connect(Postgres.Connection, params_reconnect) + @test conn4.reconnect + @test !conn4.debug + DBInterface.close!(conn4) + conn5 = DBInterface.connect(Postgres.Connection, params_reconnect; reconnect=false) + @test !conn5.reconnect + DBInterface.close!(conn5) end @testset "SSL Modes" begin @@ -1048,6 +1102,47 @@ end Postgres.copy_from(conn, "COPY copy_test FROM STDIN (FORMAT BINARY)", binary_copy) rows2 = Tables.rowtable(DBInterface.execute(conn, "SELECT count(*) AS count FROM copy_test")) @test rows2[1].count == 2 + + # an invalid COPY statement errors (instead of hanging) and + # leaves the connection usable + @test_throws Postgres.API.Error Postgres.copy_from(conn, "COPY nonexistent_copy_tbl FROM STDIN", "1\n") + @test Tables.rowtable(DBInterface.execute(conn, "SELECT 1 AS a"))[1].a == 1 + + # non-COPY and wrong-direction statements are rejected + # cleanly, without desyncing or deadlocking the connection + @test_throws Postgres.API.Error Postgres.copy_from(conn, "SELECT 1", "1\n") + @test_throws Postgres.API.Error Postgres.copy_to(conn, "SELECT 1") + @test_throws Postgres.API.Error Postgres.copy_from(conn, "COPY copy_test TO STDOUT", "1\talpha\n") + @test_throws Postgres.API.Error Postgres.copy_to(conn, "COPY copy_test FROM STDIN") + @test Tables.rowtable(DBInterface.execute(conn, "SELECT 2 AS a"))[1].a == 2 + + # COPY via execute throws a clear client error pointing at + # copy_from/copy_to and keeps the connection usable + err = try + DBInterface.execute(conn, "COPY copy_test FROM STDIN") + nothing + catch e + e + end + @test err isa Postgres.API.Error + @test occursin("copy_from", err.message) + err = try + DBInterface.execute(conn, "COPY copy_test TO STDOUT") + nothing + catch e + e + end + @test err isa Postgres.API.Error + @test occursin("copy_to", err.message) + @test Tables.rowtable(DBInterface.execute(conn, "SELECT 3 AS a"))[1].a == 3 + + # COPY via cursor errors cleanly, keeps the connection + # usable, and doesn't leave its transaction open + @test_throws Postgres.PostgresInterfaceError Postgres.cursor(conn, "COPY copy_test TO STDOUT") + @test !Postgres.in_transaction(conn) + @test_throws Postgres.PostgresInterfaceError Postgres.cursor(conn, "COPY copy_test FROM STDIN") + @test !Postgres.in_transaction(conn) + @test Tables.rowtable(DBInterface.execute(conn, "SELECT 4 AS a"))[1].a == 4 end @testset "Cursor Streaming" begin From ac85a2f83a7feb8427e2c9b749aa48ca71628abb Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 6 Aug 2026 00:02:05 -0600 Subject: [PATCH 02/23] Address independent review findings - add the GC.@preserve missed in update_server_parameters! (same unsafe_string(pointer) pattern fixed elsewhere in this PR) - consistent exception taxonomy: COPY misuse (non-COPY, wrong-direction, via execute/cursor) now throws PostgresInterfaceError everywhere instead of Error in some paths and PostgresInterfaceError in others; the Error docstring now documents the residual protocol-level client uses - copy-out error precedence: a genuine mid-stream server error (e.g. COPY (SELECT 1/0) TO STDOUT) is surfaced instead of being masked by the misuse error in the execute and cursor paths - copy_in's post-data drain aborts a second CopyInResponse (multi-statement query strings) with CopyFail instead of deadlocking - README: drop inaccurate "pure-Julia" (TLS uses OpenSSL_jll via Reseau) - tests for all of the above Co-Authored-By: Claude Fable 5 --- README.md | 2 +- src/Postgres.jl | 2 +- src/api/API.jl | 35 ++++++++++++++++++++++++++--------- src/execute.jl | 19 ++++++++++++++----- test/runtests.jl | 25 +++++++++++++++---------- 5 files changed, 57 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 9ec0bab..4bd6704 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![docs](https://img.shields.io/badge/docs-dev-blue.svg)](https://JuliaDatabases.github.io/Postgres.jl/dev/) [![codecov](https://codecov.io/gh/JuliaDatabases/Postgres.jl/branch/main/graph/badge.svg)](https://codecov.io/gh/JuliaDatabases/Postgres.jl) -Postgres.jl is a pure-Julia PostgreSQL client that implements the v3 wire protocol with `DBInterface` and `Tables` integration. +Postgres.jl is a PostgreSQL client written in Julia that implements the v3 wire protocol with `DBInterface` and `Tables` integration. ## Installation diff --git a/src/Postgres.jl b/src/Postgres.jl index 1d39e55..d9c1ed6 100644 --- a/src/Postgres.jl +++ b/src/Postgres.jl @@ -307,7 +307,7 @@ end function update_server_parameters!(conn::Connection, buf::Vector{UInt8}) i = 1 - while i < length(buf) + GC.@preserve buf while i < length(buf) j = findnext(isequal(UInt8(0)), buf, i) j === nothing && break key = unsafe_string(pointer(buf, i), j - i) diff --git a/src/api/API.jl b/src/api/API.jl index d79e350..ebf765e 100644 --- a/src/api/API.jl +++ b/src/api/API.jl @@ -14,7 +14,10 @@ const SKIP_BUFFER_SIZE = 8192 A PostgreSQL server error (an `ErrorResponse` message). Carries the fields the server reported: `severity`, `code` (the SQLSTATE, e.g. `"23505"`), `message`, and optional context such as `detail`, `hint`, `position`, `schema`, `table`, -`column`, and `constraint`. +`column`, and `constraint`. A small number of protocol-level failures detected +client-side (unsupported authentication methods, protocol desync) also use +this type, with an empty `code`; other client-side failures throw +`Postgres.PostgresInterfaceError`. """ struct Error <: Exception severity::String @@ -874,11 +877,16 @@ function StructUtils.applyeach(::AbstractPostgresStyle, f, e::Exec) server_error === nothing || throw(server_error) rethrow() end - # COPY misuse throws a clear client error (the server error from the - # aborted copy would be confusing); the stream was drained above, so the - # connection stays usable - copy_in_statement && throw(Error("COPY ... FROM STDIN is not supported via execute; use Postgres.copy_from")) - copy_out_statement && throw(Error("COPY ... TO STDOUT is not supported via execute; use Postgres.copy_to")) + # COPY misuse throws a clear client error; the stream was drained above, + # so the connection stays usable. For copy-in the server error is just the + # CopyFail artifact, so the client error wins; for copy-out a server error + # is a genuine mid-stream failure (e.g. inside COPY (SELECT ...) TO + # STDOUT) and is more informative than the misuse error. + copy_in_statement && throw(PostgresInterfaceError("COPY ... FROM STDIN is not supported via execute; use Postgres.copy_from")) + if copy_out_statement + server_error === nothing || throw(server_error) + throw(PostgresInterfaceError("COPY ... TO STDOUT is not supported via execute; use Postgres.copy_to")) + end # server errors take precedence; otherwise surface a consumer error that # aborted materialization (the stream was still drained above) server_error === nothing || throw(server_error) @@ -960,7 +968,7 @@ function copy_in(style::S, socket, query::String, source::IO, debug::Bool) where end end error_msg === nothing || throw(error_msg) - copy_started || throw(Error("statement did not initiate COPY ... FROM STDIN")) + copy_started || throw(PostgresInterfaceError("statement did not initiate COPY ... FROM STDIN")) buf = Vector{UInt8}(undef, 16384) while !eof(source) n = readbytes!(source, buf, length(buf)) @@ -969,12 +977,20 @@ function copy_in(style::S, socket, query::String, source::IO, debug::Bool) where end writemessage(socket, debug, 'c') error_msg = nothing + second_copy = false while true mt, len = readheader(socket, debug) if mt == UInt8('E') error_msg = errorResponse(len, socket, debug) elseif mt == UInt8('C') skipbytes!(socket, len) + elseif mt == UInt8('G') + # a second CopyInResponse (multi-statement query string): the + # server is waiting for more copy data, so abort with CopyFail + # instead of deadlocking; a clear error is thrown below + skipbytes!(socket, len) + second_copy = true + writemessage(socket, debug, 'f', "copy_from supports a single COPY FROM STDIN statement") elseif mt == UInt8('N') notice = noticeResponse(len, socket) notice_callback(style, notice) @@ -988,6 +1004,7 @@ function copy_in(style::S, socket, query::String, source::IO, debug::Bool) where skipbytes!(socket, len) end end + second_copy && throw(PostgresInterfaceError("copy_from supports a single COPY ... FROM STDIN statement per call")) error_msg === nothing || throw(error_msg) return end @@ -1030,9 +1047,9 @@ function copy_out(style::S, socket, query::String, dest::IO, debug::Bool) where skipbytes!(socket, len) end end - wrong_direction && throw(Error("statement initiated COPY ... FROM STDIN; use Postgres.copy_from")) + wrong_direction && throw(PostgresInterfaceError("statement initiated COPY ... FROM STDIN; use Postgres.copy_from")) error_msg === nothing || throw(error_msg) - copy_started || throw(Error("statement did not initiate COPY ... TO STDOUT")) + copy_started || throw(PostgresInterfaceError("statement did not initiate COPY ... TO STDOUT")) return dest end diff --git a/src/execute.jl b/src/execute.jl index 7484062..b8ffd72 100644 --- a/src/execute.jl +++ b/src/execute.jl @@ -285,7 +285,8 @@ function read_portal_batch!(cursor::Cursor) rows = ResultRow[] error_msg = nothing consumer_error = nothing - copy_statement = false + copy_in_statement = false + copy_out_statement = false done = false try while true @@ -319,11 +320,11 @@ function read_portal_batch!(cursor::Cursor) # below. A fresh Sync must follow: the one sent with # Bind/Execute was ignored during copy-in mode. API.skipbytes!(conn.socket, len) - copy_statement = true + copy_in_statement = true API.writemessages(conn.socket, conn.debug, ('f', "COPY FROM STDIN is not supported via cursor"), ('S',)) elseif mt == UInt8('H') || mt == UInt8('d') || mt == UInt8('c') # CopyOutResponse/CopyData/CopyDone: drain the copy-out stream - mt == UInt8('H') && (copy_statement = true) + mt == UInt8('H') && (copy_out_statement = true) API.skipbytes!(conn.socket, len) elseif mt == UInt8('N') notice = API.noticeResponse(len, conn.socket) @@ -346,9 +347,17 @@ function read_portal_batch!(cursor::Cursor) error_msg === nothing || throw(error_msg) rethrow() end - if copy_statement + # same precedence as the execute path: for copy-in the server error is + # just the CopyFail artifact; for copy-out a server error is a genuine + # mid-stream failure and wins over the misuse error + if copy_in_statement cursor.done = true - throw(PostgresInterfaceError("COPY statements are not supported via cursor; use Postgres.copy_from or Postgres.copy_to")) + throw(PostgresInterfaceError("COPY ... FROM STDIN is not supported via cursor; use Postgres.copy_from")) + end + if copy_out_statement + cursor.done = true + error_msg === nothing || throw(error_msg) + throw(PostgresInterfaceError("COPY ... TO STDOUT is not supported via cursor; use Postgres.copy_to")) end error_msg === nothing || throw(error_msg) consumer_error === nothing || throw(consumer_error) diff --git a/test/runtests.jl b/test/runtests.jl index 016185b..50a7c70 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1108,14 +1108,19 @@ end @test_throws Postgres.API.Error Postgres.copy_from(conn, "COPY nonexistent_copy_tbl FROM STDIN", "1\n") @test Tables.rowtable(DBInterface.execute(conn, "SELECT 1 AS a"))[1].a == 1 - # non-COPY and wrong-direction statements are rejected - # cleanly, without desyncing or deadlocking the connection - @test_throws Postgres.API.Error Postgres.copy_from(conn, "SELECT 1", "1\n") - @test_throws Postgres.API.Error Postgres.copy_to(conn, "SELECT 1") - @test_throws Postgres.API.Error Postgres.copy_from(conn, "COPY copy_test TO STDOUT", "1\talpha\n") - @test_throws Postgres.API.Error Postgres.copy_to(conn, "COPY copy_test FROM STDIN") + # non-COPY, wrong-direction, and multi-statement COPY calls + # are rejected cleanly, without desyncing or deadlocking + @test_throws Postgres.PostgresInterfaceError Postgres.copy_from(conn, "SELECT 1", "1\n") + @test_throws Postgres.PostgresInterfaceError Postgres.copy_to(conn, "SELECT 1") + @test_throws Postgres.PostgresInterfaceError Postgres.copy_from(conn, "COPY copy_test TO STDOUT", "1\talpha\n") + @test_throws Postgres.PostgresInterfaceError Postgres.copy_to(conn, "COPY copy_test FROM STDIN") + @test_throws Postgres.PostgresInterfaceError Postgres.copy_from(conn, "COPY copy_test (id, name) FROM STDIN; COPY copy_test (id, name) FROM STDIN", "9\tomega\n") @test Tables.rowtable(DBInterface.execute(conn, "SELECT 2 AS a"))[1].a == 2 + # a genuine mid-stream server error during copy-out wins + # over the misuse error and the connection stays usable + @test_throws Postgres.API.Error Postgres.copy_to(conn, "COPY (SELECT 1/0) TO STDOUT") + # COPY via execute throws a clear client error pointing at # copy_from/copy_to and keeps the connection usable err = try @@ -1124,16 +1129,16 @@ end catch e e end - @test err isa Postgres.API.Error - @test occursin("copy_from", err.message) + @test err isa Postgres.PostgresInterfaceError + @test occursin("copy_from", err.msg) err = try DBInterface.execute(conn, "COPY copy_test TO STDOUT") nothing catch e e end - @test err isa Postgres.API.Error - @test occursin("copy_to", err.message) + @test err isa Postgres.PostgresInterfaceError + @test occursin("copy_to", err.msg) @test Tables.rowtable(DBInterface.execute(conn, "SELECT 3 AS a"))[1].a == 3 # COPY via cursor errors cleanly, keeps the connection From 9b02386b47b593e29374722f1f7885470d364d47 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 6 Aug 2026 00:14:50 -0600 Subject: [PATCH 03/23] Harden copy_in/copy_out against user-IO failures mid-copy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A failing data source in copy_from now aborts the copy with CopyFail and drains to ReadyForQuery, so the connection stays usable and the source's error propagates. A failing dest IO (or socket failure) mid copy-out closes the socket instead of leaving a desynced connection that still looks valid to pool_isvalid — matching the mid-stream bail behavior of the execute and cursor paths. Co-Authored-By: Claude Fable 5 --- src/api/API.jl | 195 +++++++++++++++++++++++++++-------------------- test/runtests.jl | 22 ++++++ 2 files changed, 135 insertions(+), 82 deletions(-) diff --git a/src/api/API.jl b/src/api/API.jl index ebf765e..c2d68a3 100644 --- a/src/api/API.jl +++ b/src/api/API.jl @@ -944,65 +944,89 @@ function copy_in(style::S, socket, query::String, source::IO, debug::Bool) where writemessage(socket, debug, 'Q', query) error_msg = nothing copy_started = false - while true - mt, len = readheader(socket, debug) - if mt == UInt8('G') - skipbytes!(socket, len) - copy_started = true - break - elseif mt == UInt8('E') - error_msg = errorResponse(len, socket, debug) - elseif mt == UInt8('Z') - # ReadyForQuery without CopyInResponse: the statement errored or - # wasn't a COPY ... FROM STDIN; the stream is back at ready - skipbytes!(socket, len) - break - elseif mt == UInt8('N') - notice = noticeResponse(len, socket) - notice_callback(style, notice) - elseif mt == UInt8('A') - notification = notificationResponse(len, socket) - notification_callback(style, notification) - else - skipbytes!(socket, len) + try + while true + mt, len = readheader(socket, debug) + if mt == UInt8('G') + skipbytes!(socket, len) + copy_started = true + break + elseif mt == UInt8('E') + error_msg = errorResponse(len, socket, debug) + elseif mt == UInt8('Z') + # ReadyForQuery without CopyInResponse: the statement errored or + # wasn't a COPY ... FROM STDIN; the stream is back at ready + skipbytes!(socket, len) + break + elseif mt == UInt8('N') + notice = noticeResponse(len, socket) + notice_callback(style, notice) + elseif mt == UInt8('A') + notification = notificationResponse(len, socket) + notification_callback(style, notification) + else + skipbytes!(socket, len) + end end + catch + # bailed mid-stream: the position is unknowable, never reuse the socket + close(socket) + rethrow() end error_msg === nothing || throw(error_msg) copy_started || throw(PostgresInterfaceError("statement did not initiate COPY ... FROM STDIN")) - buf = Vector{UInt8}(undef, 16384) - while !eof(source) - n = readbytes!(source, buf, length(buf)) - n == 0 && break - writemessage(socket, debug, 'd', view(buf, 1:n)) + try + buf = Vector{UInt8}(undef, 16384) + while !eof(source) + n = readbytes!(source, buf, length(buf)) + n == 0 && break + writemessage(socket, debug, 'd', view(buf, 1:n)) + end + writemessage(socket, debug, 'c') + catch + # the user's data source failed mid-copy: abort the copy so the + # connection returns to ready, then rethrow the source error + try + writemessage(socket, debug, 'f', "client-side data source failed") + drain_to_ready!(socket, debug) + catch + close(socket) + end + rethrow() end - writemessage(socket, debug, 'c') error_msg = nothing second_copy = false - while true - mt, len = readheader(socket, debug) - if mt == UInt8('E') - error_msg = errorResponse(len, socket, debug) - elseif mt == UInt8('C') - skipbytes!(socket, len) - elseif mt == UInt8('G') - # a second CopyInResponse (multi-statement query string): the - # server is waiting for more copy data, so abort with CopyFail - # instead of deadlocking; a clear error is thrown below - skipbytes!(socket, len) - second_copy = true - writemessage(socket, debug, 'f', "copy_from supports a single COPY FROM STDIN statement") - elseif mt == UInt8('N') - notice = noticeResponse(len, socket) - notice_callback(style, notice) - elseif mt == UInt8('A') - notification = notificationResponse(len, socket) - notification_callback(style, notification) - elseif mt == UInt8('Z') - skipbytes!(socket, len) - break - else - skipbytes!(socket, len) + try + while true + mt, len = readheader(socket, debug) + if mt == UInt8('E') + error_msg = errorResponse(len, socket, debug) + elseif mt == UInt8('C') + skipbytes!(socket, len) + elseif mt == UInt8('G') + # a second CopyInResponse (multi-statement query string): the + # server is waiting for more copy data, so abort with CopyFail + # instead of deadlocking; a clear error is thrown below + skipbytes!(socket, len) + second_copy = true + writemessage(socket, debug, 'f', "copy_from supports a single COPY FROM STDIN statement") + elseif mt == UInt8('N') + notice = noticeResponse(len, socket) + notice_callback(style, notice) + elseif mt == UInt8('A') + notification = notificationResponse(len, socket) + notification_callback(style, notification) + elseif mt == UInt8('Z') + skipbytes!(socket, len) + break + else + skipbytes!(socket, len) + end end + catch + # bailed mid-stream: the position is unknowable, never reuse the socket + close(socket) + rethrow() end second_copy && throw(PostgresInterfaceError("copy_from supports a single COPY ... FROM STDIN statement per call")) error_msg === nothing || throw(error_msg) @@ -1014,38 +1038,45 @@ function copy_out(style::S, socket, query::String, dest::IO, debug::Bool) where error_msg = nothing copy_started = false wrong_direction = false - while true - mt, len = readheader(socket, debug) - if mt == UInt8('H') - skipbytes!(socket, len) - copy_started = true - elseif mt == UInt8('d') - write(dest, read(socket, len)) - elseif mt == UInt8('c') - skipbytes!(socket, len) - elseif mt == UInt8('C') - skipbytes!(socket, len) - elseif mt == UInt8('G') - # CopyInResponse: the statement was COPY ... FROM STDIN. The server - # is now waiting on us for data, so abort the copy with CopyFail to - # return the stream to ready instead of deadlocking. - skipbytes!(socket, len) - wrong_direction = true - writemessage(socket, debug, 'f', "COPY FROM STDIN is not supported via copy_to") - elseif mt == UInt8('E') - error_msg = errorResponse(len, socket, debug) - elseif mt == UInt8('N') - notice = noticeResponse(len, socket) - notice_callback(style, notice) - elseif mt == UInt8('A') - notification = notificationResponse(len, socket) - notification_callback(style, notification) - elseif mt == UInt8('Z') - skipbytes!(socket, len) - break - else - skipbytes!(socket, len) + try + while true + mt, len = readheader(socket, debug) + if mt == UInt8('H') + skipbytes!(socket, len) + copy_started = true + elseif mt == UInt8('d') + write(dest, read(socket, len)) + elseif mt == UInt8('c') + skipbytes!(socket, len) + elseif mt == UInt8('C') + skipbytes!(socket, len) + elseif mt == UInt8('G') + # CopyInResponse: the statement was COPY ... FROM STDIN. The server + # is now waiting on us for data, so abort the copy with CopyFail to + # return the stream to ready instead of deadlocking. + skipbytes!(socket, len) + wrong_direction = true + writemessage(socket, debug, 'f', "COPY FROM STDIN is not supported via copy_to") + elseif mt == UInt8('E') + error_msg = errorResponse(len, socket, debug) + elseif mt == UInt8('N') + notice = noticeResponse(len, socket) + notice_callback(style, notice) + elseif mt == UInt8('A') + notification = notificationResponse(len, socket) + notification_callback(style, notification) + elseif mt == UInt8('Z') + skipbytes!(socket, len) + break + else + skipbytes!(socket, len) + end end + catch + # bailed mid-stream (socket failure, or the user's dest IO threw): + # the position is unknowable, never reuse the socket + close(socket) + rethrow() end wrong_direction && throw(PostgresInterfaceError("statement initiated COPY ... FROM STDIN; use Postgres.copy_from")) error_msg === nothing || throw(error_msg) diff --git a/test/runtests.jl b/test/runtests.jl index 50a7c70..f08706b 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -61,6 +61,14 @@ struct Int8Row s::String end +# user-IO failure injection for the COPY hardening tests +struct ThrowingSource <: IO end +Base.eof(::ThrowingSource) = false +Base.readbytes!(::ThrowingSource, ::Vector{UInt8}, n) = error("source failed") + +struct FailingDest <: IO end +Base.write(::FailingDest, ::Vector{UInt8}) = error("dest failed") + struct PgConfig host::String port::Int @@ -1121,6 +1129,20 @@ end # over the misuse error and the connection stays usable @test_throws Postgres.API.Error Postgres.copy_to(conn, "COPY (SELECT 1/0) TO STDOUT") + # a failing user data source aborts the copy with CopyFail + # and the connection stays usable + @test_throws ErrorException Postgres.copy_from(conn, "COPY copy_test (id, name) FROM STDIN", ThrowingSource()) + @test Tables.rowtable(DBInterface.execute(conn, "SELECT 5 AS a"))[1].a == 5 + + # a failing dest IO mid copy-out closes the connection + # instead of leaving a desynced socket that looks usable + conn_copyfail = DBInterface.connect(Postgres.Connection, cfg.host, cfg.user, cfg.password; dbname=cfg.dbname, port=cfg.port) + DBInterface.execute(conn_copyfail, "CREATE TEMP TABLE copy_out_fail (id int)") + DBInterface.execute(conn_copyfail, "INSERT INTO copy_out_fail VALUES (1), (2)") + @test_throws ErrorException Postgres.copy_to(conn_copyfail, "COPY copy_out_fail TO STDOUT", FailingDest()) + @test !isopen(conn_copyfail.socket) + DBInterface.close!(conn_copyfail) + # COPY via execute throws a clear client error pointing at # copy_from/copy_to and keeps the connection usable err = try From 8bc5a42b750c324a4ae5f37702b3e02d1de7d91a Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 6 Aug 2026 00:24:00 -0600 Subject: [PATCH 04/23] Address security review findings - redact SASL/SCRAM messages from the debug log: the earlier redaction covered only cleartext and md5, leaving the client proof (and thus an offline brute-force oracle for the password) in the log for SCRAM-SHA-256, the default auth method on modern PostgreSQL - bound all message-field parsing by the buffer actually received. Reseau's read returns a short buffer at EOF without throwing, so loops bounded on the server-declared length, and the unbounded unsafe_string(pointer(buf)) NUL scans, could read past the allocation and surface heap bytes as column names, command tags, or error text. Adds cstring_at and uses it in the error/notice/notification/command-tag parsers; describeprepared and the DataRow value loop now bounds-check every offset before reading. - send CancelRequest over TLS when the connection being cancelled uses TLS (the cancel key is a credential valid for that backend's lifetime), and refuse to send it in the clear under sslmode=require/verify-full - bound the numeric exponent so bogus server text can't drive an enormous BigInt scaling - reject embedded NULs in escape_identifier/escape_literal, and document that escape_literal assumes standard_conforming_strings=on - document that sslservername is also the name verified against the certificate under verify-full (not merely an SNI override), that require encrypts without authenticating the server, and that query_logger receives bound parameter values Co-Authored-By: Claude Fable 5 --- README.md | 4 +- docs/src/index.md | 4 +- docs/src/manual.md | 2 + src/Postgres.jl | 30 ++++++++++--- src/api/API.jl | 110 +++++++++++++++++++++++++++++++++------------ src/api/types.jl | 9 ++++ test/runtests.jl | 22 +++++++++ 7 files changed, 142 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 4bd6704..9716186 100644 --- a/README.md +++ b/README.md @@ -36,8 +36,8 @@ Connection options support: - libpq-style keyword strings such as `host=127.0.0.1 port=5432 user=postgres dbname=postgres`. - PostgreSQL URIs such as `postgresql://postgres:postgres@127.0.0.1:5432/postgres`. - Environment defaults: `PGHOST`, `PGPORT`, `PGUSER`, `PGPASSWORD`, `PGDATABASE`, `PGAPPNAME`, `PGCONNECT_TIMEOUT`, and TLS-related `PGSSL*` variables. -- `sslmode` values: `disable`, `prefer`, `require`, `verify-full` (only `verify-full` enforces certificate verification). -- TLS files: `sslrootcert`, `sslcert`, `sslkey`, `sslcapath`; `sslservername` overrides the TLS SNI hostname when connecting to a pre-resolved address. +- `sslmode` values: `disable`, `prefer`, `require`, `verify-full`. Only `verify-full` verifies the server's certificate; `require` encrypts without authenticating the server. +- TLS files: `sslrootcert`, `sslcert`, `sslkey`, `sslcapath`. `sslservername` overrides the TLS server name when connecting to a pre-resolved address; under `verify-full` it is also the name the certificate is verified against, so it must name the server you intend to authenticate. - `connect_timeout` (seconds) and `statement_timeout` (milliseconds). - `application_name` and `statement_cache_maxsize`. diff --git a/docs/src/index.md b/docs/src/index.md index 4916e6c..a20a556 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -17,8 +17,8 @@ Postgres.jl accepts DSN strings or PostgreSQL URIs and supports: - libpq-style keyword strings such as `host=127.0.0.1 port=5432 user=postgres dbname=postgres`. - Environment defaults from `PGHOST`, `PGPORT`, `PGUSER`, `PGPASSWORD`, `PGDATABASE`, `PGAPPNAME`, `PGCONNECT_TIMEOUT`, and TLS-related `PGSSL*` variables. -- `sslmode` values: `disable`, `prefer`, `require`, `verify-full` (only `verify-full` verifies certificates). -- TLS files: `sslrootcert`, `sslcert`, `sslkey`, `sslcapath`; `sslservername` overrides the TLS SNI hostname when connecting to a pre-resolved address. +- `sslmode` values: `disable`, `prefer`, `require`, `verify-full`. Only `verify-full` verifies the server's certificate; `require` encrypts without authenticating the server. +- TLS files: `sslrootcert`, `sslcert`, `sslkey`, `sslcapath`. `sslservername` overrides the TLS server name when connecting to a pre-resolved address; under `verify-full` it is also the name the certificate is verified against, so it must name the server you intend to authenticate. - `connect_timeout` (seconds), `statement_timeout` (milliseconds). - `application_name` and `statement_cache_maxsize`. diff --git a/docs/src/manual.md b/docs/src/manual.md index f7d2662..0382f54 100644 --- a/docs/src/manual.md +++ b/docs/src/manual.md @@ -127,6 +127,8 @@ Postgres.notification_callback(::LoggingStyle, notification) = @info "notificati conn = DBInterface.connect(Postgres.Connection, "host=127.0.0.1 user=postgres dbname=postgres"; style=LoggingStyle()) ``` +`query_logger`'s `info` includes the SQL and the bound parameter values, so a logger that writes them out records whatever sensitive data those queries carry. Redact or omit `info.params` when the log destination is less trusted than the database itself. + ```@docs Postgres.AbstractPostgresStyle Postgres.PostgresStyle diff --git a/src/Postgres.jl b/src/Postgres.jl index d9c1ed6..7b10ecf 100644 --- a/src/Postgres.jl +++ b/src/Postgres.jl @@ -44,8 +44,12 @@ options): - `dbname`, `port`, `application_name` - `connect_timeout` (seconds), `statement_timeout` (milliseconds) - `sslmode` (`"disable"`, `"prefer"` (default), `"require"`, `"verify-full"`), - `sslrootcert`, `sslcert`, `sslkey`, `sslcapath`, and `sslservername` - (TLS SNI override for pre-resolved hosts) + `sslrootcert`, `sslcert`, `sslkey`, `sslcapath`, and `sslservername`. + Only `verify-full` verifies the server's certificate; `require` encrypts + without authenticating the server. `sslservername` overrides the TLS server + name when the host is a pre-resolved address — note that under + `verify-full` this is also the name the certificate is verified against, + so it must name the server you intend to authenticate. - `statement_cache_maxsize`: LRU prepared-statement cache size (default 100; `0` disables) - `reconnect`: automatically reconnect and re-prepare statements if the connection is found dead (default `false`; never reconnects mid-transaction) @@ -249,12 +253,16 @@ function set_statement_timeout!(conn::Connection, timeout::Union{Integer, Nothin return conn end +@noinline _reject_nul(what::String) = throw(PostgresInterfaceError("$what cannot contain a NUL byte")) + """ Postgres.escape_identifier(name) -> String -Quote a string for use as a SQL identifier (double-quoted, embedded quotes doubled). +Quote a string for use as a SQL identifier (double-quoted, embedded quotes +doubled). Throws if `name` contains a NUL byte. """ function escape_identifier(name::AbstractString) + occursin('\0', name) && _reject_nul("identifier") return string("\"", replace(name, "\"" => "\"\""), "\"") end @@ -262,10 +270,17 @@ end Postgres.escape_literal(val) -> String Quote a string for use as a SQL literal (single-quoted, embedded quotes -doubled). Prefer query parameters (`\$1`, `\$2`, ...) over literal interpolation -whenever possible. +doubled). Throws if `val` contains a NUL byte. + +Prefer query parameters (`\$1`, `\$2`, ...) over literal interpolation +whenever possible — parameters are never parsed as SQL. This helper assumes +the server's `standard_conforming_strings` is `on` (the default since +PostgreSQL 9.1); with it turned off, backslashes in the literal are escape +characters and doubling quotes alone is not sufficient to make interpolation +safe. """ function escape_literal(val::AbstractString) + occursin('\0', val) && _reject_nul("literal") return string("'", replace(val, "'" => "''"), "'") end @@ -551,6 +566,9 @@ Send a PostgreSQL CancelRequest for the query currently running on `conn` (over a separate, short-lived connection, so it works while `conn` is busy). The cancelled query fails with a [`Postgres.Error`](@ref Postgres.API.Error) with SQLSTATE `57014`. + +The cancel connection uses the same TLS settings as `conn`, since the cancel +key it carries is a credential. """ function cancel_query!(conn::Connection) host = conn.host @@ -567,7 +585,7 @@ function cancel_query!(conn::Connection) unlock(conn.lock) end end - API.cancel_request(host, port, pid, skey, debug) + API.cancel_request(host, port, pid, skey, debug, conn.sslmode, conn.sslrootcert, conn.sslcert, conn.sslkey, conn.sslcapath, conn.sslservername, conn.connect_timeout) return conn end diff --git a/src/api/API.jl b/src/api/API.jl index c2d68a3..da5c62b 100644 --- a/src/api/API.jl +++ b/src/api/API.jl @@ -70,6 +70,21 @@ function Base.showerror(io::IO, e::Error) return end +# Read a NUL-terminated string from `buf` starting at `pos`, bounded by the +# buffer's actual length: `read` returns a short buffer at EOF, and a hostile +# or failing server can send an unterminated or truncated field, so an +# unbounded scan (plain `unsafe_string(pointer(buf, pos))`) would read past +# the allocation. Returns (string, next_pos). +function cstring_at(buf::Vector{UInt8}, pos::Int) + n = length(buf) + pos > n && return "", n + 1 + stop = findnext(isequal(UInt8(0)), buf, pos) + if stop === nothing + return GC.@preserve(buf, unsafe_string(pointer(buf, pos), n - pos + 1)), n + 1 + end + return GC.@preserve(buf, unsafe_string(pointer(buf, pos), stop - pos)), stop + 1 +end + function errorResponse(len, socket, debug) buf = read(socket, len) # parse error fields @@ -91,11 +106,12 @@ function errorResponse(len, socket, debug) file = nothing line = nothing routine = nothing - GC.@preserve buf while i < len + while i <= length(buf) ccode = Char(buf[i]) + # the field list is terminated by a zero byte + ccode == '\0' && break i += 1 - val = unsafe_string(pointer(buf, i)) - i += sizeof(val) + 1 + val, i = cstring_at(buf, i) if ccode == 'S' severity = val elseif ccode == 'C' @@ -141,11 +157,11 @@ function noticeResponse(len, socket) buf = read(socket, len) i = 1 notice = Dict{String, String}() - GC.@preserve buf while i < length(buf) + while i <= length(buf) ccode = Char(buf[i]) + ccode == '\0' && break i += 1 - val = unsafe_string(pointer(buf, i)) - i += sizeof(val) + 1 + val, i = cstring_at(buf, i) notice[string(ccode)] = val end return notice @@ -158,11 +174,8 @@ function notificationResponse(len, socket) channel = "" payload = "" if !isempty(buf) - GC.@preserve buf begin - channel = unsafe_string(pointer(buf, i)) - i += sizeof(channel) + 1 - i <= length(buf) && (payload = unsafe_string(pointer(buf, i))) - end + channel, i = cstring_at(buf, i) + i <= length(buf) && ((payload, i) = cstring_at(buf, i)) end return Notification(pid, channel, payload) end @@ -510,7 +523,11 @@ function authRequest(debug, len, socket, user, password, client::Union{Nothing, client = SASLAuth.SCRAMSHA256Client(user, password) msg, _ = SASLAuth.step!(client, nothing) bytes = Vector{UInt8}(msg) - writemessage(socket, debug, 'p', "SCRAM-SHA-256", Int32(length(bytes)), bytes) + # SASL messages carry the client nonce and (in the client-final message) + # the client proof, from which the password is brute-forcible offline: + # write them with debug=false and log a redacted line instead + debug && @info "sending message: p, (SASL initial response redacted)" + writemessage(socket, false, 'p', "SCRAM-SHA-256", Int32(length(bytes)), bytes) mt, len = readheader(socket, debug) expect_auth_message(socket, debug, mt, len) return authRequest(debug, len, socket, user, password, client) @@ -518,7 +535,8 @@ function authRequest(debug, len, socket, user, password, client::Union{Nothing, # SASL Challenge challenge = String(read(socket, len - 4)) msg, _ = SASLAuth.step!(client, challenge) - writemessage(socket, debug, 'p', Vector{UInt8}(msg)) + debug && @info "sending message: p, (SASL response redacted)" + writemessage(socket, false, 'p', Vector{UInt8}(msg)) mt, len = readheader(socket, debug) expect_auth_message(socket, debug, mt, len) return authRequest(debug, len, socket, user, password, client) @@ -712,20 +730,19 @@ function describeprepared(socket, name::String, debug::Bool) ncols = Int(ntoh(read(socket, Int16))) buf = read(socket, len - 2) i = 1 - GC.@preserve buf while i < len - 2 - ptr = pointer(buf, i) - plen = Int(@ccall strlen(ptr::Ptr{Cvoid})::Csize_t) - name = _symbol(ptr, plen) - i += plen + 1 - i += 4 # skip table oid - i += 2 # skip column number - typeId = Int(ntoh(unsafe_load(Ptr{Int32}(pointer(buf, i))))) + # each field: name (cstring), table oid (4), column number (2), + # type oid (4), type length (2), type modifier (4), format code (2). + # All offsets are bounds-checked against the buffer actually received: + # a short read or a malformed RowDescription must not read past it. + GC.@preserve buf while i <= length(buf) + stop = findnext(isequal(UInt8(0)), buf, i) + stop === nothing && break + name = _symbol(pointer(buf, i), stop - i) + i = stop + 1 + i + 17 <= length(buf) || break + typeId = Int(ntoh(unsafe_load(Ptr{Int32}(pointer(buf, i + 6))))) + i += 18 push!(types, typeId) - i += 4 - i += 2 # skip type length - # typeModifier = Int(ntoh(unsafe_load(Ptr{Int32}(pointer(buf, i))))) - i += 4 - i += 2 # skip format code push!(cols, name) end waitfor(socket, debug, 'Z') @@ -755,16 +772,23 @@ end function StructUtils.applyeach(::AbstractPostgresStyle, f, dr::DataRow) buf = dr.buf + nbuf = length(buf) GC.@preserve buf begin + nbuf >= 2 || throw(Error("truncated DataRow message from server")) ncols = Int(ntoh(unsafe_load(Ptr{Int16}(pointer(buf))))) + ncols <= length(dr.names) || throw(Error("DataRow column count exceeds the row description")) pos = 3 for i = 1:ncols + # column lengths come off the wire: validate each against the + # message actually received before reading the value + pos + 3 <= nbuf || throw(Error("truncated DataRow message from server")) len = Int(ntoh(unsafe_load(Ptr{Int32}(pointer(buf, pos))))) pos += 4 if len == -1 # null f(dr.names[i], nothing) else + (len >= 0 && pos + len - 1 <= nbuf) || throw(Error("truncated DataRow message from server")) str = unsafe_string(pointer(buf, pos), len) pos += len @inbounds applycast(f, dr.names[i], dr.typeIds[i], str, dr.type_registry) @@ -788,7 +812,8 @@ end function commandComplete(len, socket) buf = read(socket, len) isempty(buf) && return "" - return GC.@preserve buf unsafe_string(pointer(buf)) + tag, _ = cstring_at(buf, 1) + return tag end function rows_affected_from_command_tag(tag::String) @@ -1090,9 +1115,36 @@ function close_statement(socket, name::String, debug::Bool) return end -function cancel_request(host::String, port::Int, pid::Int32, skey::Int32, debug::Bool=false) - socket = connectsocket(host, port) +# The cancel key is a credential: anyone holding it can cancel that backend's +# queries for the life of the connection, so the CancelRequest goes over TLS +# whenever the connection it cancels uses TLS. +function cancel_request(host::String, port::Int, pid::Int32, skey::Int32, debug::Bool=false, + @nospecialize(sslmode::Union{String, Nothing}=nothing), + @nospecialize(sslrootcert::Union{String, Nothing}=nothing), + @nospecialize(sslcert::Union{String, Nothing}=nothing), + @nospecialize(sslkey::Union{String, Nothing}=nothing), + @nospecialize(sslcapath::Union{String, Nothing}=nothing), + @nospecialize(sslservername::Union{String, Nothing}=nothing), + @nospecialize(connect_timeout::Union{Int, Nothing}=nothing)) + sslmode_v = sslmode::Union{String, Nothing} + connect_timeout_v = connect_timeout::Union{Int, Nothing} + socket = connectsocket(host, port, connect_timeout_v) try + sslmode_str = sslmode_v === nothing ? "prefer" : lowercase(String(sslmode_v)) + if sslmode_str != "disable" + writemessage(socket, debug, '\0', Int32(80877103)) + mt = read(socket, UInt8) + if mt == UInt8('S') + socket = tlsupgrade(socket, connect_timeout_v, + sslservername isa String ? sslservername::String : host, + sslmode_str == "verify-full", + sslcert::Union{String, Nothing}, sslkey::Union{String, Nothing}, + sslrootcert::Union{String, Nothing}, sslcapath::Union{String, Nothing}) + elseif sslmode_str == "require" || sslmode_str == "verify-full" + # never send the cancel key in the clear when TLS was required + return false + end + end buf = IOBuffer(Vector{UInt8}(undef, 16); write=true) write(buf, hton(Int32(16))) write(buf, hton(Int32(80877102))) # CancelRequest code diff --git a/src/api/types.jl b/src/api/types.jl index 6616495..87fa830 100644 --- a/src/api/types.jl +++ b/src/api/types.jl @@ -12,6 +12,11 @@ behavior interface on it: Postgres.notice_callback(::MyStyle, notice) = ... Postgres.notification_callback(::MyStyle, notification) = ... +`query_logger`'s `info` carries the SQL and the bound parameter values, so a +logger that writes them out will record whatever sensitive data those queries +carry — redact or omit `info.params` if the log is not as trusted as the +database. + Custom styles inherit the default row-materialization traits (lift/structlike/...), which dispatch on `AbstractPostgresStyle`, and are used as the StructUtils style when materializing query results — so `StructUtils.lift` overloads on a custom style apply @@ -346,6 +351,10 @@ function parse_numeric(val::String) exp_val = 0 if exp_index !== nothing exp_val = parse(Int, stripped[exp_index + 1:end]) + # postgres numeric tops out at 16383 digits either side of the point; + # bound the exponent so a bogus value can't drive an enormous BigInt + # scaling below + abs(exp_val) <= 100_000 || throw(PostgresInterfaceError("postgres numeric exponent out of range: $stripped")) stripped = stripped[1:exp_index - 1] end parts = split(stripped, '.'; limit=2) diff --git a/test/runtests.jl b/test/runtests.jl index f08706b..ee3bcaa 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -546,6 +546,23 @@ end @test_throws Postgres.PostgresInterfaceError Postgres.API.parse_numeric("NaN") @test_throws Postgres.PostgresInterfaceError Postgres.API.parse_numeric("Infinity") @test_throws Postgres.PostgresInterfaceError Postgres.API.parse_numeric("-Infinity") + # an absurd exponent must be rejected, not turned into a huge BigInt + @test_throws Postgres.PostgresInterfaceError Postgres.API.parse_numeric("1e999999999999") + + # message-field parsing is bounded by the buffer actually received: a + # truncated or unterminated field must not read past the allocation + @test Postgres.API.cstring_at(UInt8[], 1) == ("", 1) + @test Postgres.API.cstring_at(UInt8['a', 'b', 0x00], 1) == ("ab", 4) + @test Postgres.API.cstring_at(UInt8['a', 'b'], 1) == ("ab", 3) + @test Postgres.API.cstring_at(UInt8['a', 0x00, 'c', 0x00], 3) == ("c", 5) + @test Postgres.API.cstring_at(UInt8['a', 0x00], 5) == ("", 3) + + # escaping helpers reject embedded NULs rather than emitting SQL the + # server would truncate mid-statement + @test Postgres.escape_identifier("a\"b") == "\"a\"\"b\"" + @test Postgres.escape_literal("a'b") == "'a''b'" + @test_throws Postgres.PostgresInterfaceError Postgres.escape_identifier("a\0b") + @test_throws Postgres.PostgresInterfaceError Postgres.escape_literal("a\0b") @test Postgres.API.parse_value(1184, "2024-02-13 05:28:17+02", registry) == DateTime(2024, 2, 13, 3, 28, 17) @test Postgres.API.parse_value(1184, "2024-02-13 05:28:17+02:30", registry) == DateTime(2024, 2, 13, 2, 58, 17) @@ -1203,6 +1220,11 @@ end @test result isa Postgres.API.Error @test result.code == "57014" DBInterface.close!(cancel_conn) + + # the cancel key must never go out in the clear when the + # connection it cancels required TLS + @test !Postgres.API.cancel_request(cfg.host, cfg.port, Int32(1), Int32(1), false, "require") + @test !Postgres.API.cancel_request(cfg.host, cfg.port, Int32(1), Int32(1), false, "verify-full") end @testset "Interval Types" begin From 3e362b9e3c4f3f9c7050da3eb3dbb86e1742952f Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 6 Aug 2026 00:39:07 -0600 Subject: [PATCH 05/23] Address security verification findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - cancel_query! no longer fails silently. cancel_request's refusal to send the key in cleartext now throws instead of returning false into a discarded return value, and cancel_query! throws when the request could not be delivered at all — previously a refused or failed cancel left the caller believing a runaway query had been cancelled. - the cancel connection requires TLS when the connection being cancelled actually negotiated TLS, not merely when sslmode said so: under the default "prefer" the main session can be on TLS while the cancel connection silently fell back to cleartext. - bound the server-declared message length in readheader (and in the pre-TLS, pre-auth SSLRequest error path, which bypasses it) to PostgreSQL's own 1 GiB protocol maximum. Reseau's read allocates the requested size up front, so a 5-byte header claiming ~2 GB committed that much memory before a single body byte arrived — reachable by an on-path attacker before authentication. - parse_numeric reports an out-of-range exponent consistently instead of letting an oversized one surface as OverflowError - document that debug logging redacts authentication messages but not bind parameter values Adds coverage that cancellation works over TLS against the SSL fixture server, and that the cleartext refusal throws. Co-Authored-By: Claude Fable 5 --- src/Postgres.jl | 18 ++++++++++++++--- src/api/API.jl | 52 ++++++++++++++++++++++++++++++++++++------------ src/api/types.jl | 9 ++++++--- test/runtests.jl | 36 ++++++++++++++++++++++++++++++--- 4 files changed, 93 insertions(+), 22 deletions(-) diff --git a/src/Postgres.jl b/src/Postgres.jl index 7b10ecf..8262b89 100644 --- a/src/Postgres.jl +++ b/src/Postgres.jl @@ -55,7 +55,9 @@ options): connection is found dead (default `false`; never reconnects mid-transaction) - `style`: a custom [`AbstractPostgresStyle`](@ref Postgres.API.AbstractPostgresStyle) for query logging / notice / notification behavior -- `debug`: log wire protocol messages +- `debug`: log wire protocol messages. Authentication messages are redacted, + but bind parameter values are not — treat a debug log as sensitive as the + data the connection carries. Connections are safe for concurrent use from multiple tasks: operations are serialized on an internal lock. Close with `DBInterface.close!(conn)` or @@ -568,7 +570,10 @@ The cancelled query fails with a [`Postgres.Error`](@ref Postgres.API.Error) with SQLSTATE `57014`. The cancel connection uses the same TLS settings as `conn`, since the cancel -key it carries is a credential. +key it carries is a credential: if `conn` itself is on TLS, the cancel +connection requires TLS too. Throws a `PostgresInterfaceError` if the cancel +request could not be delivered (rather than failing silently, which would +leave the query running). """ function cancel_query!(conn::Connection) host = conn.host @@ -576,16 +581,23 @@ function cancel_query!(conn::Connection) pid = conn.pid skey = conn.skey debug = conn.debug + sslmode = conn.sslmode if trylock(conn.lock) try !isopen(conn.socket) && throw(PostgresInterfaceError("cannot cancel query: connection not open")) pid = conn.pid skey = conn.skey + # the connection actually negotiated TLS, so require it for the + # cancel connection too even under the permissive default + if conn.socket isa Reseau.TLS.Conn && (sslmode === nothing || lowercase(sslmode) == "prefer") + sslmode = "require" + end finally unlock(conn.lock) end end - API.cancel_request(host, port, pid, skey, debug, conn.sslmode, conn.sslrootcert, conn.sslcert, conn.sslkey, conn.sslcapath, conn.sslservername, conn.connect_timeout) + ok = API.cancel_request(host, port, pid, skey, debug, sslmode, conn.sslrootcert, conn.sslcert, conn.sslkey, conn.sslcapath, conn.sslservername, conn.connect_timeout) + ok || throw(PostgresInterfaceError("failed to deliver the cancel request to $(host):$(port)")) return conn end diff --git a/src/api/API.jl b/src/api/API.jl index da5c62b..accd1ad 100644 --- a/src/api/API.jl +++ b/src/api/API.jl @@ -316,10 +316,22 @@ function skipbytes!(io::IO, n::Integer) return nothing end +# PostgreSQL's own protocol maximum (PQ_LARGE_MESSAGE_LIMIT): no valid message +# body exceeds 1 GiB. The length is server-supplied and the transport allocates +# it up front, so bound it here — the single point every message passes through +# — rather than letting a bogus header commit gigabytes. This is reachable +# before authentication (an ErrorResponse to the SSLRequest), so it must not +# depend on a trusted peer. +const MAX_MESSAGE_LEN = Int32(1) << 30 + +@noinline _bad_message_length(len) = + throw(Error("invalid message length $len from server; connection protocol state is corrupted")) + function readheader(socket, debug=false) mt = read(socket, UInt8) len = ntoh(read(socket, Int32)) - 4 debug && @info "readheader: $(Char(mt)), $len" + (len < 0 || len > MAX_MESSAGE_LEN) && _bad_message_length(len) return mt, len end @@ -674,8 +686,11 @@ function connect(host::String, port::Integer, dbname::String, user::String, @nos elseif mt == UInt8('N') (sslmode_str == "require" || sslmode_str == "verify-full") && throw(Error("server does not support SSL")) elseif mt == UInt8('E') - # server may answer SSLRequest with a full ErrorResponse + # server may answer SSLRequest with a full ErrorResponse. This is + # pre-TLS and pre-auth, so bound the length like readheader does + # before handing it to the allocating read. len = ntoh(read(socket, Int32)) - 4 + (len < 0 || len > MAX_MESSAGE_LEN) && close_and_throw(socket, Error("invalid message length $len from server")) close_and_throw_error_response(socket, len, debug) else close_and_throw(socket, Error("unexpected response to SSLRequest: $(Char(mt))")) @@ -1128,9 +1143,13 @@ function cancel_request(host::String, port::Int, pid::Int32, skey::Int32, debug: @nospecialize(connect_timeout::Union{Int, Nothing}=nothing)) sslmode_v = sslmode::Union{String, Nothing} connect_timeout_v = connect_timeout::Union{Int, Nothing} + sslmode_str = sslmode_v === nothing ? "prefer" : lowercase(String(sslmode_v)) + tls_required = sslmode_str == "require" || sslmode_str == "verify-full" socket = connectsocket(host, port, connect_timeout_v) + refused_cleartext = false + sent = false try - sslmode_str = sslmode_v === nothing ? "prefer" : lowercase(String(sslmode_v)) + send_key = true if sslmode_str != "disable" writemessage(socket, debug, '\0', Int32(80877103)) mt = read(socket, UInt8) @@ -1140,24 +1159,31 @@ function cancel_request(host::String, port::Int, pid::Int32, skey::Int32, debug: sslmode_str == "verify-full", sslcert::Union{String, Nothing}, sslkey::Union{String, Nothing}, sslrootcert::Union{String, Nothing}, sslcapath::Union{String, Nothing}) - elseif sslmode_str == "require" || sslmode_str == "verify-full" + elseif tls_required # never send the cancel key in the clear when TLS was required - return false + refused_cleartext = true + send_key = false end end - buf = IOBuffer(Vector{UInt8}(undef, 16); write=true) - write(buf, hton(Int32(16))) - write(buf, hton(Int32(80877102))) # CancelRequest code - write(buf, hton(pid)) - write(buf, hton(skey)) - write(socket, take!(buf)) - flush(socket) - return true + if send_key + buf = IOBuffer(Vector{UInt8}(undef, 16); write=true) + write(buf, hton(Int32(16))) + write(buf, hton(Int32(80877102))) # CancelRequest code + write(buf, hton(pid)) + write(buf, hton(skey)) + write(socket, take!(buf)) + flush(socket) + sent = true + end catch - return false + sent = false finally close(socket) end + # refusing to send is a hard failure the caller must hear about, not a + # silent no-op: the query they asked to cancel is still running + refused_cleartext && throw(PostgresInterfaceError("server refused TLS on the cancel connection; not sending the cancel key in cleartext under sslmode=$sslmode_str")) + return sent end include("../array_parsing.jl") diff --git a/src/api/types.jl b/src/api/types.jl index 87fa830..4ce9b85 100644 --- a/src/api/types.jl +++ b/src/api/types.jl @@ -350,11 +350,14 @@ function parse_numeric(val::String) exp_index = findfirst(c -> c == 'e' || c == 'E', stripped) exp_val = 0 if exp_index !== nothing - exp_val = parse(Int, stripped[exp_index + 1:end]) # postgres numeric tops out at 16383 digits either side of the point; # bound the exponent so a bogus value can't drive an enormous BigInt - # scaling below - abs(exp_val) <= 100_000 || throw(PostgresInterfaceError("postgres numeric exponent out of range: $stripped")) + # scaling below (tryparse so an oversized exponent reports the same + # error as an out-of-range one, rather than an OverflowError) + parsed_exp = tryparse(Int, stripped[exp_index + 1:end]) + (parsed_exp === nothing || abs(parsed_exp) > 100_000) && + throw(PostgresInterfaceError("postgres numeric exponent out of range: $stripped")) + exp_val = parsed_exp stripped = stripped[1:exp_index - 1] end parts = split(stripped, '.'; limit=2) diff --git a/test/runtests.jl b/test/runtests.jl index ee3bcaa..5a3045b 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -548,6 +548,10 @@ end @test_throws Postgres.PostgresInterfaceError Postgres.API.parse_numeric("-Infinity") # an absurd exponent must be rejected, not turned into a huge BigInt @test_throws Postgres.PostgresInterfaceError Postgres.API.parse_numeric("1e999999999999") + @test_throws Postgres.PostgresInterfaceError Postgres.API.parse_numeric("1e99999999999999999999999999") + # ordinary scientific notation still round-trips + @test string(Postgres.API.parse_numeric("1.5e2")) == "150" + @test string(Postgres.API.parse_numeric("1.5e-2")) == "0.015" # message-field parsing is bounded by the buffer actually received: a # truncated or unterminated field must not read past the allocation @@ -1222,9 +1226,13 @@ end DBInterface.close!(cancel_conn) # the cancel key must never go out in the clear when the - # connection it cancels required TLS - @test !Postgres.API.cancel_request(cfg.host, cfg.port, Int32(1), Int32(1), false, "require") - @test !Postgres.API.cancel_request(cfg.host, cfg.port, Int32(1), Int32(1), false, "verify-full") + # connection it cancels required TLS; the refusal is a + # thrown error, not a silent no-op (this server has no SSL, + # so it answers the SSLRequest with 'N') + @test_throws Postgres.PostgresInterfaceError Postgres.API.cancel_request(cfg.host, cfg.port, Int32(1), Int32(1), false, "require") + @test_throws Postgres.PostgresInterfaceError Postgres.API.cancel_request(cfg.host, cfg.port, Int32(1), Int32(1), false, "verify-full") + # a cleartext-allowed cancel still delivers + @test Postgres.API.cancel_request(cfg.host, cfg.port, Int32(1), Int32(1), false, "disable") end @testset "Interval Types" begin @@ -1265,6 +1273,28 @@ end @test connection_error(ssl_cfg.host, ssl_cfg; sslmode="verify-full") !== nothing @test connection_error(ssl_cfg.host, ssl_cfg; sslmode="verify-full", sslrootcert=tls.wrongrootcert) !== nothing + # against a TLS-capable server the cancel key goes over TLS + # and the request is delivered + @test Postgres.API.cancel_request(ssl_cfg.host, ssl_cfg.port, Int32(1), Int32(1), false, "require") + ssl_cancel_conn = DBInterface.connect(Postgres.Connection, ssl_cfg.host, ssl_cfg.user, ssl_cfg.password; dbname=ssl_cfg.dbname, port=ssl_cfg.port, sslmode="require") + try + ssl_task = errormonitor(Threads.@spawn begin + try + DBInterface.execute(ssl_cancel_conn, "SELECT pg_sleep(5)") + return :completed + catch err + return err + end + end) + sleep(0.5) + Postgres.cancel_query!(ssl_cancel_conn) + ssl_result = fetch(ssl_task) + @test ssl_result isa Postgres.API.Error + @test ssl_result.code == "57014" + finally + isopen(ssl_cancel_conn) && DBInterface.close!(ssl_cancel_conn) + end + localhost_require_err = connection_error("localhost", ssl_cfg; sslmode="require") if localhost_require_err === nothing @test connection_error("localhost", ssl_cfg; sslmode="verify-full", sslrootcert=tls.rootcert) !== nothing From 18cc378e17fe8986297b0a6f14e473189039eacc Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 6 Aug 2026 00:47:23 -0600 Subject: [PATCH 06/23] Fix cancel TLS upgrade being skipped, and desync on bad message length MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs in the previous commit, plus adjacent hardening: - the prefer->require TLS upgrade for the cancel connection was inside `if trylock(conn.lock)`, so it was skipped in cancel_query!'s primary use case: another task holds the lock running the very query being cancelled. A default-sslmode connection that had negotiated TLS would still let the cancel key fall back to cleartext — exactly the downgrade the upgrade was added to prevent. The socket check needs no lock (host/pid/skey are read unlocked too), so it now happens before the trylock. - the new message-length check threw API.Error, which describeprepared treats as "the stream is clean, at ReadyForQuery" — so a bogus length left a desynchronized socket open and reusable by the pool. It now closes the socket before throwing, like the other protocol-corruption paths. - DataRow's column count is signed on the wire: a negative passed the upper-bound check and produced an unfilled row (UndefRefError downstream) instead of a clean protocol error. Also bounds against typeIds, not just names. - the remaining ParameterStatus field loop is bounded by the buffer length rather than the server-declared length, matching the others. - the three COPY mid-stream catches surface an already-received server error instead of the raw IO error, matching the execute path. Adds DataRow malformed-message unit tests, and makes the TLS cancel test use the default sslmode so it covers the upgrade path with the lock held. Co-Authored-By: Claude Fable 5 --- src/Postgres.jl | 15 ++++++++++----- src/api/API.jl | 22 ++++++++++++++++++---- test/runtests.jl | 30 ++++++++++++++++++++++++++++-- 3 files changed, 56 insertions(+), 11 deletions(-) diff --git a/src/Postgres.jl b/src/Postgres.jl index 8262b89..e5c6b49 100644 --- a/src/Postgres.jl +++ b/src/Postgres.jl @@ -582,16 +582,21 @@ function cancel_query!(conn::Connection) skey = conn.skey debug = conn.debug sslmode = conn.sslmode + # If the connection actually negotiated TLS, require it for the cancel + # connection too, even under the permissive default — otherwise a server + # answering 'N' downgrades the cancel key to cleartext. Checked outside + # the lock deliberately: cancel_query! is called precisely when another + # task holds it running the query being cancelled, so a trylock-guarded + # check would be skipped in the case that matters. Reading the socket + # field unlocked matches how host/pid/skey are read below. + if conn.socket isa Reseau.TLS.Conn && (sslmode === nothing || lowercase(sslmode) == "prefer") + sslmode = "require" + end if trylock(conn.lock) try !isopen(conn.socket) && throw(PostgresInterfaceError("cannot cancel query: connection not open")) pid = conn.pid skey = conn.skey - # the connection actually negotiated TLS, so require it for the - # cancel connection too even under the permissive default - if conn.socket isa Reseau.TLS.Conn && (sslmode === nothing || lowercase(sslmode) == "prefer") - sslmode = "require" - end finally unlock(conn.lock) end diff --git a/src/api/API.jl b/src/api/API.jl index accd1ad..df769d5 100644 --- a/src/api/API.jl +++ b/src/api/API.jl @@ -324,14 +324,20 @@ end # depend on a trusted peer. const MAX_MESSAGE_LEN = Int32(1) << 30 -@noinline _bad_message_length(len) = +# A bogus length means the stream is desynchronized, so the socket must be +# closed before throwing — callers such as describeprepared treat a surviving +# `Error` as "the stream is clean, at ReadyForQuery" and would otherwise keep +# using a connection whose position is unknowable. +@noinline function _bad_message_length(socket, len) + close(socket) throw(Error("invalid message length $len from server; connection protocol state is corrupted")) +end function readheader(socket, debug=false) mt = read(socket, UInt8) len = ntoh(read(socket, Int32)) - 4 debug && @info "readheader: $(Char(mt)), $len" - (len < 0 || len > MAX_MESSAGE_LEN) && _bad_message_length(len) + (len < 0 || len > MAX_MESSAGE_LEN) && _bad_message_length(socket, len) return mt, len end @@ -414,7 +420,7 @@ function waitfor(socket, debug::Bool, codes::Vararg{Char, N}) where {N} # parameter status buf = read(socket, len) i = 1 - GC.@preserve buf while i < len + GC.@preserve buf while i <= length(buf) j = findnext(isequal(UInt8(0)), buf, i) j === nothing && break key = unsafe_string(pointer(buf, i), j - i) @@ -791,7 +797,10 @@ function StructUtils.applyeach(::AbstractPostgresStyle, f, dr::DataRow) GC.@preserve buf begin nbuf >= 2 || throw(Error("truncated DataRow message from server")) ncols = Int(ntoh(unsafe_load(Ptr{Int16}(pointer(buf))))) - ncols <= length(dr.names) || throw(Error("DataRow column count exceeds the row description")) + # the count is signed on the wire: a negative would pass an upper-bound + # check and silently yield an unfilled row (UndefRefError downstream) + (0 <= ncols <= length(dr.names) && ncols <= length(dr.typeIds)) || + throw(Error("DataRow column count does not match the row description")) pos = 3 for i = 1:ncols # column lengths come off the wire: validate each against the @@ -1011,6 +1020,9 @@ function copy_in(style::S, socket, query::String, source::IO, debug::Bool) where catch # bailed mid-stream: the position is unknowable, never reuse the socket close(socket) + # if the server reported an error before the connection died, surface + # it over the raw IO error — it explains what actually went wrong + error_msg === nothing || throw(error_msg) rethrow() end error_msg === nothing || throw(error_msg) @@ -1066,6 +1078,7 @@ function copy_in(style::S, socket, query::String, source::IO, debug::Bool) where catch # bailed mid-stream: the position is unknowable, never reuse the socket close(socket) + error_msg === nothing || throw(error_msg) rethrow() end second_copy && throw(PostgresInterfaceError("copy_from supports a single COPY ... FROM STDIN statement per call")) @@ -1116,6 +1129,7 @@ function copy_out(style::S, socket, query::String, dest::IO, debug::Bool) where # bailed mid-stream (socket failure, or the user's dest IO threw): # the position is unknowable, never reuse the socket close(socket) + error_msg === nothing || throw(error_msg) rethrow() end wrong_direction && throw(PostgresInterfaceError("statement initiated COPY ... FROM STDIN; use Postgres.copy_from")) diff --git a/test/runtests.jl b/test/runtests.jl index 5a3045b..c099ad0 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -561,6 +561,27 @@ end @test Postgres.API.cstring_at(UInt8['a', 0x00, 'c', 0x00], 3) == ("c", 5) @test Postgres.API.cstring_at(UInt8['a', 0x00], 5) == ("", 3) + # a malformed DataRow must fail with a clear protocol error rather + # than reading past the buffer or leaving the row partly unfilled + let nms = Symbol[:a, :b], tids = Int[23, 23], + mk = b -> Postgres.API.DataRow(b, nms, tids, registry), + consume = row -> StructUtils.applyeach(Postgres.API.PostgresStyle(), (k, v) -> nothing, row) + # ncols is signed on the wire: -1 must not pass an upper-bound check + @test_throws Postgres.API.Error consume(mk(UInt8[0xff, 0xff])) + # more columns than the row description declared + @test_throws Postgres.API.Error consume(mk(UInt8[0x00, 0x09])) + # truncated header, and a column length running past the body + @test_throws Postgres.API.Error consume(mk(UInt8[0x00])) + @test_throws Postgres.API.Error consume(mk(UInt8[0x00, 0x01, 0x00, 0x00, 0x00, 0x7f])) + # a well-formed row still parses, including a NULL column + vals = Any[] + consume_ok = Postgres.API.DataRow( + UInt8[0x00, 0x02, 0x00, 0x00, 0x00, 0x01, UInt8('5'), 0xff, 0xff, 0xff, 0xff], + nms, tids, registry) + StructUtils.applyeach(Postgres.API.PostgresStyle(), (k, v) -> push!(vals, v), consume_ok) + @test vals == Any[Int32(5), nothing] + end + # escaping helpers reject embedded NULs rather than emitting SQL the # server would truncate mid-statement @test Postgres.escape_identifier("a\"b") == "\"a\"\"b\"" @@ -1274,9 +1295,14 @@ end @test connection_error(ssl_cfg.host, ssl_cfg; sslmode="verify-full", sslrootcert=tls.wrongrootcert) !== nothing # against a TLS-capable server the cancel key goes over TLS - # and the request is delivered + # and the request is delivered. The connection uses the + # default sslmode ("prefer") but negotiates TLS, so this + # also covers the cancel path upgrading itself to require + # TLS — which has to happen while the query being cancelled + # holds the connection lock. @test Postgres.API.cancel_request(ssl_cfg.host, ssl_cfg.port, Int32(1), Int32(1), false, "require") - ssl_cancel_conn = DBInterface.connect(Postgres.Connection, ssl_cfg.host, ssl_cfg.user, ssl_cfg.password; dbname=ssl_cfg.dbname, port=ssl_cfg.port, sslmode="require") + ssl_cancel_conn = DBInterface.connect(Postgres.Connection, ssl_cfg.host, ssl_cfg.user, ssl_cfg.password; dbname=ssl_cfg.dbname, port=ssl_cfg.port) + @test ssl_cancel_conn.socket isa Postgres.Reseau.TLS.Conn try ssl_task = errormonitor(Threads.@spawn begin try From 136d060b3596342841400f1a65e9331f0c14bcb1 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 6 Aug 2026 01:07:34 -0600 Subject: [PATCH 07/23] Fix async-message desync in waitfor, and notification read deadline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - waitfor's message loop had no fallback branch, so any message type it wasn't waiting for had its header read and its body left on the wire — the body was then read as the next message header. Reproducible against a real server: LISTEN on a connection, have another connection NOTIFY, then run a query on the listener; the NotificationResponse arrives interleaved with the query's messages and destroys the connection. Now discards the body like every other read loop. Covered by a regression test (verified by mutation: removing the fix fails that test). - wait_for_notification's read deadline now covers only the first byte of a message. Previously it covered the body too, so a message straddling the 100ms poll boundary left the stream parked mid-message, and the swallowed DeadlineExceededError meant the loop resumed reading body bytes as a header. It also no longer runs _clear_read_deadline! on a socket that the message-length check just closed, which replaced the protocol diagnostic with a Reseau-internal NetClosingError. - DataRow's column count must equal the described column count, not merely not exceed it: a short count left the caller's row partly unfilled. - extract the cancel TLS-upgrade decision into cancel_sslmode and test it directly — the end-to-end test could not pin it (mutation-verified: the SSL fixture always offers TLS, so removing the upgrade changed nothing). Co-Authored-By: Claude Fable 5 --- src/Postgres.jl | 79 +++++++++++++++++++++++++++++------------------- src/api/API.jl | 13 ++++++-- test/runtests.jl | 23 ++++++++++++++ 3 files changed, 81 insertions(+), 34 deletions(-) diff --git a/src/Postgres.jl b/src/Postgres.jl index e5c6b49..3157357 100644 --- a/src/Postgres.jl +++ b/src/Postgres.jl @@ -375,29 +375,42 @@ function wait_for_notification(conn::Connection; timeout::Union{Real, Nothing}=n remaining_s <= 0 && return nothing Int64(time_ns()) + min(NOTIFICATION_POLL_INTERVAL_NS, round(Int64, remaining_s * 1_000_000_000)) end + # The deadline covers only the first byte: if it expires there, + # nothing of a message has been consumed and polling again is safe. + # Once a byte arrives the rest of the message is read without a + # deadline, so a message straddling the poll boundary can never + # leave the stream parked mid-message. _set_read_deadline!(conn.socket, deadline_ns) - try - mt, len = API.readheader(conn.socket, conn.debug) - if mt == UInt8('A') - notification = API.notificationResponse(len, conn.socket) - API.notification_callback(conn.style, notification) - return notification - elseif mt == UInt8('N') - notice = API.noticeResponse(len, conn.socket) - API.notice_callback(conn.style, notice) - elseif mt == UInt8('S') - buf = read(conn.socket, len) - update_server_parameters!(conn, buf) - elseif mt == UInt8('E') - err = API.errorResponse(len, conn.socket, conn.debug) - throw(err) - else - API.skipbytes!(conn.socket, len) - end + mt = try + read(conn.socket, UInt8) catch err err isa Reseau.IOPoll.DeadlineExceededError || rethrow() - finally - _clear_read_deadline!(conn.socket) + isopen(conn.socket) && _clear_read_deadline!(conn.socket) + continue + end + isopen(conn.socket) && _clear_read_deadline!(conn.socket) + # no deadline is in effect from here on, so the rest of the message + # is read to completion and any failure is a real one + len = ntoh(read(conn.socket, Int32)) - 4 + if len < 0 || len > API.MAX_MESSAGE_LEN + close(conn.socket) + throw(API.Error("invalid message length $len from server; connection protocol state is corrupted")) + end + conn.debug && @info "readheader: $(Char(mt)), $len" + if mt == UInt8('A') + notification = API.notificationResponse(len, conn.socket) + API.notification_callback(conn.style, notification) + return notification + elseif mt == UInt8('N') + notice = API.noticeResponse(len, conn.socket) + API.notice_callback(conn.style, notice) + elseif mt == UInt8('S') + buf = read(conn.socket, len) + update_server_parameters!(conn, buf) + elseif mt == UInt8('E') + throw(API.errorResponse(len, conn.socket, conn.debug)) + else + API.skipbytes!(conn.socket, len) end end end @@ -561,6 +574,16 @@ function register_range!(conn::Connection, name::AbstractString; schema::Abstrac return conn end +# If the connection actually negotiated TLS, the cancel connection must +# require it too, even under the permissive default — otherwise a server +# answering 'N' to the cancel connection's SSLRequest downgrades the cancel +# key to cleartext. +function cancel_sslmode(socket_is_tls::Bool, sslmode::Union{String, Nothing}) + socket_is_tls || return sslmode + (sslmode === nothing || lowercase(sslmode) == "prefer") && return "require" + return sslmode +end + """ Postgres.cancel_query!(conn) @@ -581,17 +604,11 @@ function cancel_query!(conn::Connection) pid = conn.pid skey = conn.skey debug = conn.debug - sslmode = conn.sslmode - # If the connection actually negotiated TLS, require it for the cancel - # connection too, even under the permissive default — otherwise a server - # answering 'N' downgrades the cancel key to cleartext. Checked outside - # the lock deliberately: cancel_query! is called precisely when another - # task holds it running the query being cancelled, so a trylock-guarded - # check would be skipped in the case that matters. Reading the socket - # field unlocked matches how host/pid/skey are read below. - if conn.socket isa Reseau.TLS.Conn && (sslmode === nothing || lowercase(sslmode) == "prefer") - sslmode = "require" - end + # Checked outside the lock deliberately: cancel_query! is called precisely + # when another task holds it running the query being cancelled, so a + # trylock-guarded check would be skipped in the case that matters. Reading + # the socket field unlocked matches how host/pid/skey are read below. + sslmode = cancel_sslmode(conn.socket isa Reseau.TLS.Conn, conn.sslmode) if trylock(conn.lock) try !isopen(conn.socket) && throw(PostgresInterfaceError("cannot cancel query: connection not open")) diff --git a/src/api/API.jl b/src/api/API.jl index df769d5..3b42cfb 100644 --- a/src/api/API.jl +++ b/src/api/API.jl @@ -442,6 +442,12 @@ function waitfor(socket, debug::Bool, codes::Vararg{Char, N}) where {N} skipbytes!(socket, len) end found == 0 && break + else + # any other message (notices, notifications, ...): discard the + # body. Without this the body is read as the next header and + # the stream desynchronizes — e.g. a NOTIFY delivered on a + # connection that is also running queries. + skipbytes!(socket, len) end end catch @@ -797,9 +803,10 @@ function StructUtils.applyeach(::AbstractPostgresStyle, f, dr::DataRow) GC.@preserve buf begin nbuf >= 2 || throw(Error("truncated DataRow message from server")) ncols = Int(ntoh(unsafe_load(Ptr{Int16}(pointer(buf))))) - # the count is signed on the wire: a negative would pass an upper-bound - # check and silently yield an unfilled row (UndefRefError downstream) - (0 <= ncols <= length(dr.names) && ncols <= length(dr.typeIds)) || + # the protocol mandates one value per described column; anything else + # (including a negative count, which is signed on the wire) would leave + # the caller's row partly unfilled — an UndefRefError downstream + (ncols == length(dr.names) && ncols == length(dr.typeIds)) || throw(Error("DataRow column count does not match the row description")) pos = 3 for i = 1:ncols diff --git a/test/runtests.jl b/test/runtests.jl index c099ad0..8d7a570 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -582,6 +582,18 @@ end @test vals == Any[Int32(5), nothing] end + # a cancel connection must never downgrade to cleartext when the + # connection being cancelled actually negotiated TLS + @test Postgres.cancel_sslmode(true, nothing) == "require" + @test Postgres.cancel_sslmode(true, "prefer") == "require" + @test Postgres.cancel_sslmode(true, "PREFER") == "require" + @test Postgres.cancel_sslmode(true, "verify-full") == "verify-full" + @test Postgres.cancel_sslmode(true, "require") == "require" + # an explicitly plaintext connection is left alone, as are non-TLS ones + @test Postgres.cancel_sslmode(true, "disable") == "disable" + @test Postgres.cancel_sslmode(false, nothing) === nothing + @test Postgres.cancel_sslmode(false, "prefer") == "prefer" + # escaping helpers reject embedded NULs rather than emitting SQL the # server would truncate mid-statement @test Postgres.escape_identifier("a\"b") == "\"a\"\"b\"" @@ -1130,6 +1142,17 @@ end @test notification !== nothing @test notification.channel == "notify_test" @test notification.payload == "payload" + # A notification delivered while the same connection runs + # queries must not desync the stream: the async message + # arrives interleaved with the query's own messages, and + # its body has to be consumed rather than read as the next + # message header. + Postgres.notify!(notifier, "notify_test", "interleaved") + sleep(0.2) + @test Tables.rowtable(DBInterface.execute(listener, "SELECT 2 AS a"))[1].a == 2 + @test Tables.rowtable(DBInterface.execute(listener, "SELECT 3 AS a"))[1].a == 3 + @test isopen(listener) + DBInterface.close!(notifier) DBInterface.close!(listener) end From 302aa975ec30dbda8eeb77ebe9913a07d252a7af Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 6 Aug 2026 01:30:20 -0600 Subject: [PATCH 08/23] Fix leaked read deadline, and notification waiting over TLS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - wait_for_notification could leave an expired read deadline set on the socket when the first-byte read failed with anything other than a bare DeadlineExceededError. Reseau keeps failing every subsequent read until a deadline is cleared or moved, so the connection was permanently poisoned: the wait threw, and every later query on that connection threw too. The clear is back in a finally, covering the rethrow path. - wait_for_notification never worked over TLS. The TLS layer wraps transport failures, so a poll-interval expiry arrives as a TLSError carrying the DeadlineExceededError as its cause, never matching the bare-type guard — so the very first 100ms tick threw instead of continuing to wait. The guard now recognizes both forms. This path had no test coverage at all; there is now a LISTEN/NOTIFY-over-TLS test (mutation-verified: it errors without the fix). - the numeric exponent bound compared abs(exp), which wraps at typemin(Int64) and let that one value through to the BigInt scaling - DataRow's exact-count guard is now pinned by a too-few-columns test, and the interleaved-NOTIFY test repeats so it can't pass vacuously - wait_for_notification's docstring now says what timeout actually bounds Co-Authored-By: Claude Fable 5 --- src/Postgres.jl | 27 ++++++++++++++++++++++----- src/api/types.jl | 4 +++- test/runtests.jl | 45 ++++++++++++++++++++++++++++++++++++++------- 3 files changed, 63 insertions(+), 13 deletions(-) diff --git a/src/Postgres.jl b/src/Postgres.jl index 3157357..0c1f094 100644 --- a/src/Postgres.jl +++ b/src/Postgres.jl @@ -354,14 +354,25 @@ end const NOTIFICATION_POLL_INTERVAL_NS = Int64(100_000_000) +# A read deadline surfaces directly as DeadlineExceededError on a plain TCP +# connection, but the TLS layer wraps transport failures, so over TLS the same +# expiry arrives as a TLSError carrying it as the cause. +function _is_read_deadline_error(err) + err isa Reseau.IOPoll.DeadlineExceededError && return true + err isa Reseau.TLS.TLSError && return err.cause isa Reseau.IOPoll.DeadlineExceededError + return false +end + """ Postgres.wait_for_notification(conn; timeout=nothing) -> Union{Notification, Nothing} Block until a `NOTIFY` message arrives on the connection (see [`listen!`](@ref Postgres.listen!)) and return it as a [`Notification`](@ref Postgres.API.Notification). With a `timeout` (seconds), -return `nothing` if no notification arrives in time. The connection lock is -held while waiting, so use a dedicated connection for listening. +return `nothing` if no message begins arriving in that window; once a message +starts, it is always read to completion so the connection is never left parked +mid-message. The connection lock is held while waiting, so use a dedicated +connection for listening. """ function wait_for_notification(conn::Connection; timeout::Union{Real, Nothing}=nothing) start_time = time() @@ -384,11 +395,17 @@ function wait_for_notification(conn::Connection; timeout::Union{Real, Nothing}=n mt = try read(conn.socket, UInt8) catch err - err isa Reseau.IOPoll.DeadlineExceededError || rethrow() + _is_read_deadline_error(err) || rethrow() + nothing + finally + # the deadline must be cleared on every path, including a + # rethrow: an expired deadline left set on the socket makes + # every later read on this connection fail isopen(conn.socket) && _clear_read_deadline!(conn.socket) - continue end - isopen(conn.socket) && _clear_read_deadline!(conn.socket) + # nothing arrived within this poll interval; nothing of a message + # has been consumed, so it is safe to loop and re-check the timeout + mt === nothing && continue # no deadline is in effect from here on, so the rest of the message # is read to completion and any failure is a real one len = ntoh(read(conn.socket, Int32)) - 4 diff --git a/src/api/types.jl b/src/api/types.jl index 4ce9b85..877849c 100644 --- a/src/api/types.jl +++ b/src/api/types.jl @@ -355,7 +355,9 @@ function parse_numeric(val::String) # scaling below (tryparse so an oversized exponent reports the same # error as an out-of-range one, rather than an OverflowError) parsed_exp = tryparse(Int, stripped[exp_index + 1:end]) - (parsed_exp === nothing || abs(parsed_exp) > 100_000) && + # compared without abs: abs(typemin(Int)) wraps back to itself and + # would slip past the bound + (parsed_exp === nothing || parsed_exp < -100_000 || parsed_exp > 100_000) && throw(PostgresInterfaceError("postgres numeric exponent out of range: $stripped")) exp_val = parsed_exp stripped = stripped[1:exp_index - 1] diff --git a/test/runtests.jl b/test/runtests.jl index 8d7a570..e724e3a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -549,6 +549,8 @@ end # an absurd exponent must be rejected, not turned into a huge BigInt @test_throws Postgres.PostgresInterfaceError Postgres.API.parse_numeric("1e999999999999") @test_throws Postgres.PostgresInterfaceError Postgres.API.parse_numeric("1e99999999999999999999999999") + # abs(typemin(Int)) wraps to itself, so the bound must not use abs + @test_throws Postgres.PostgresInterfaceError Postgres.API.parse_numeric("1e-9223372036854775808") # ordinary scientific notation still round-trips @test string(Postgres.API.parse_numeric("1.5e2")) == "150" @test string(Postgres.API.parse_numeric("1.5e-2")) == "0.015" @@ -568,8 +570,10 @@ end consume = row -> StructUtils.applyeach(Postgres.API.PostgresStyle(), (k, v) -> nothing, row) # ncols is signed on the wire: -1 must not pass an upper-bound check @test_throws Postgres.API.Error consume(mk(UInt8[0xff, 0xff])) - # more columns than the row description declared + # the count must equal the described column count exactly: too few + # would leave the caller's row partly unfilled @test_throws Postgres.API.Error consume(mk(UInt8[0x00, 0x09])) + @test_throws Postgres.API.Error consume(mk(UInt8[0x00, 0x01, 0x00, 0x00, 0x00, 0x01, UInt8('7')])) # truncated header, and a column length running past the body @test_throws Postgres.API.Error consume(mk(UInt8[0x00])) @test_throws Postgres.API.Error consume(mk(UInt8[0x00, 0x01, 0x00, 0x00, 0x00, 0x7f])) @@ -1146,12 +1150,17 @@ end # queries must not desync the stream: the async message # arrives interleaved with the query's own messages, and # its body has to be consumed rather than read as the next - # message header. - Postgres.notify!(notifier, "notify_test", "interleaved") - sleep(0.2) - @test Tables.rowtable(DBInterface.execute(listener, "SELECT 2 AS a"))[1].a == 2 - @test Tables.rowtable(DBInterface.execute(listener, "SELECT 3 AS a"))[1].a == 3 - @test isopen(listener) + # message header. Repeated so the notification is unlikely + # to land after every query and pass vacuously. + interleaved_ok = true + for i in 1:5 + Postgres.notify!(notifier, "notify_test", "interleaved $i") + sleep(0.1) + interleaved_ok &= Tables.rowtable(DBInterface.execute(listener, "SELECT $i AS a"))[1].a == i + interleaved_ok &= isopen(listener) + interleaved_ok || break + end + @test interleaved_ok DBInterface.close!(notifier) DBInterface.close!(listener) @@ -1324,6 +1333,28 @@ end # TLS — which has to happen while the query being cancelled # holds the connection lock. @test Postgres.API.cancel_request(ssl_cfg.host, ssl_cfg.port, Int32(1), Int32(1), false, "require") + # LISTEN/NOTIFY over TLS: the read deadline surfaces as a + # wrapped TLSError rather than a bare DeadlineExceededError, + # so the poll loop must recognize it — and must not leave an + # expired deadline set, which would kill the connection. + ssl_listener = wait_for_connection(ssl_cfg; sslmode="require") + ssl_notifier = wait_for_connection(ssl_cfg; sslmode="require") + try + Postgres.listen!(ssl_listener, "tls_notify_test") + @test Postgres.wait_for_notification(ssl_listener; timeout=0.3) === nothing + # the connection survives an elapsed poll deadline + @test Tables.rowtable(DBInterface.execute(ssl_listener, "SELECT 1 AS a"))[1].a == 1 + Postgres.notify!(ssl_notifier, "tls_notify_test", "over-tls") + tls_notification = Postgres.wait_for_notification(ssl_listener; timeout=5.0) + @test tls_notification !== nothing + @test tls_notification.channel == "tls_notify_test" + @test tls_notification.payload == "over-tls" + @test Tables.rowtable(DBInterface.execute(ssl_listener, "SELECT 2 AS a"))[1].a == 2 + finally + isopen(ssl_notifier) && DBInterface.close!(ssl_notifier) + isopen(ssl_listener) && DBInterface.close!(ssl_listener) + end + ssl_cancel_conn = DBInterface.connect(Postgres.Connection, ssl_cfg.host, ssl_cfg.user, ssl_cfg.password; dbname=ssl_cfg.dbname, port=ssl_cfg.port) @test ssl_cancel_conn.socket isa Postgres.Reseau.TLS.Conn try From 6323566399a721170fc55412a449346a492f3335 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 6 Aug 2026 02:08:45 -0600 Subject: [PATCH 09/23] Fix silent array data loss, connection-string handling, and fd leak MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The final review sweep surfaced several pre-existing bugs serious enough to block a 1.0, plus one defect in the previous commit. - PostgreSQL prefixes an array literal with explicit dimensions whenever a lower bound isn't 1 ("[0:2]={a,b,c}"). The parser took the '[' for an array-open bracket, so array_fill with a lower bound, slices like arr[0:2], and array_prepend results silently returned the dimension string instead of the elements ("[0:2]={x,y,z}" parsed to ["0:2"]) — or threw for numeric element types. Now skipped before parsing, with tests against a live server. - an unrecognized connection parameter now errors instead of being dropped: "ssl_mode=verify-full" (a typo for sslmode) previously left sslmode unset, silently falling back to an unverified, possibly unencrypted connection. - an empty port value — an unset PGPORT expanded by a process manager — no longer aborts every parse_dsn call with an opaque ArgumentError. - debug= and reconnect= in a DSN were parsed and then ignored, though the ConnectionParams docstring advertises them. - a failed connection attempt leaked its file descriptor: nothing else holds the socket and the transport has no finalizer, so a pool or reconnect loop against a flapping server walked into EMFILE. Now closed on every failure path out of connect. - wait_for_notification closes the connection when it fails after consuming part of a message, like every other message loop. This also converts the one case a poll deadline cannot make safe — a TLS record split across the poll boundary, which the record layer cannot resume — from silent stream corruption into a closed connection and a raised error. Documented, along with which notifications a busy connection can drop. - timeout=Inf threw InexactError instead of waiting. - closing a cursor no longer strands the transaction it opened when the portal close fails, which would block reconnects on that connection forever. - Tables.schema now widens to the type actually parsed whenever a value doesn't fit the OID-derived column type, instead of reporting a type the data doesn't satisfy. - docs: the last unescaped $1 in a non-raw string (a copy-paste syntax error), the sslmode default and its fallback behavior, and sslcapath being loaded as a CA file rather than a hashed directory. Nested arrays are left as Vector{Any} deliberately: narrowing them needs a runtime-typed Vector construction that juliac --trim cannot resolve (it took the verifier from 74 to 103 errors). Co-Authored-By: Claude Fable 5 --- README.md | 4 +-- docs/src/index.md | 6 ++-- src/Postgres.jl | 75 ++++++++++++++++++++++++++++------------ src/api/API.jl | 9 +++++ src/array_parsing.jl | 36 +++++++++++++++++++ src/connection_string.jl | 34 +++++++++++++++--- src/execute.jl | 30 +++++++++++----- test/runtests.jl | 38 ++++++++++++++++++++ 8 files changed, 193 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 9716186..faeeefb 100644 --- a/README.md +++ b/README.md @@ -36,8 +36,8 @@ Connection options support: - libpq-style keyword strings such as `host=127.0.0.1 port=5432 user=postgres dbname=postgres`. - PostgreSQL URIs such as `postgresql://postgres:postgres@127.0.0.1:5432/postgres`. - Environment defaults: `PGHOST`, `PGPORT`, `PGUSER`, `PGPASSWORD`, `PGDATABASE`, `PGAPPNAME`, `PGCONNECT_TIMEOUT`, and TLS-related `PGSSL*` variables. -- `sslmode` values: `disable`, `prefer`, `require`, `verify-full`. Only `verify-full` verifies the server's certificate; `require` encrypts without authenticating the server. -- TLS files: `sslrootcert`, `sslcert`, `sslkey`, `sslcapath`. `sslservername` overrides the TLS server name when connecting to a pre-resolved address; under `verify-full` it is also the name the certificate is verified against, so it must name the server you intend to authenticate. +- `sslmode` values: `disable`, `prefer` (the default), `require`, `verify-full`. Only `verify-full` verifies the server's certificate; `require` encrypts without authenticating the server, and the default `prefer` falls back to an unencrypted connection if the server declines TLS. Use `verify-full` with `sslrootcert` when the connection needs to be authenticated. +- TLS files: `sslrootcert`, `sslcert`, `sslkey`, and `sslcapath` (loaded as an additional CA *file*; libpq-style hashed CA directories are not supported). `sslservername` overrides the TLS server name when connecting to a pre-resolved address; under `verify-full` it is also the name the certificate is verified against, so it must name the server you intend to authenticate. - `connect_timeout` (seconds) and `statement_timeout` (milliseconds). - `application_name` and `statement_cache_maxsize`. diff --git a/docs/src/index.md b/docs/src/index.md index a20a556..d1b006d 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -17,8 +17,8 @@ Postgres.jl accepts DSN strings or PostgreSQL URIs and supports: - libpq-style keyword strings such as `host=127.0.0.1 port=5432 user=postgres dbname=postgres`. - Environment defaults from `PGHOST`, `PGPORT`, `PGUSER`, `PGPASSWORD`, `PGDATABASE`, `PGAPPNAME`, `PGCONNECT_TIMEOUT`, and TLS-related `PGSSL*` variables. -- `sslmode` values: `disable`, `prefer`, `require`, `verify-full`. Only `verify-full` verifies the server's certificate; `require` encrypts without authenticating the server. -- TLS files: `sslrootcert`, `sslcert`, `sslkey`, `sslcapath`. `sslservername` overrides the TLS server name when connecting to a pre-resolved address; under `verify-full` it is also the name the certificate is verified against, so it must name the server you intend to authenticate. +- `sslmode` values: `disable`, `prefer` (the default), `require`, `verify-full`. Only `verify-full` verifies the server's certificate; `require` encrypts without authenticating the server, and the default `prefer` falls back to an unencrypted connection if the server declines TLS. Use `verify-full` with `sslrootcert` when the connection needs to be authenticated. +- TLS files: `sslrootcert`, `sslcert`, `sslkey`, and `sslcapath` (loaded as an additional CA *file*; libpq-style hashed CA directories are not supported). `sslservername` overrides the TLS server name when connecting to a pre-resolved address; under `verify-full` it is also the name the certificate is verified against, so it must name the server you intend to authenticate. - `connect_timeout` (seconds), `statement_timeout` (milliseconds). - `application_name` and `statement_cache_maxsize`. @@ -84,7 +84,7 @@ Prepared statements are cached with LRU eviction; disable caching via `statement ```julia using Postgres, DBInterface, Tables conn = DBInterface.connect(Postgres.Connection, "host=127.0.0.1;user=postgres;password=postgres;dbname=postgres"; statement_cache_maxsize=5) -stmt = DBInterface.prepare(conn, "SELECT $1::int AS val") +stmt = DBInterface.prepare(conn, raw"SELECT $1::int AS val") rows = Tables.rowtable(DBInterface.execute(stmt, (7,))) DBInterface.close!(stmt) DBInterface.close!(conn) diff --git a/src/Postgres.jl b/src/Postgres.jl index 0c1f094..d8adf20 100644 --- a/src/Postgres.jl +++ b/src/Postgres.jl @@ -46,7 +46,10 @@ options): - `sslmode` (`"disable"`, `"prefer"` (default), `"require"`, `"verify-full"`), `sslrootcert`, `sslcert`, `sslkey`, `sslcapath`, and `sslservername`. Only `verify-full` verifies the server's certificate; `require` encrypts - without authenticating the server. `sslservername` overrides the TLS server + without authenticating the server, and the default `prefer` falls back to an + unencrypted connection if the server declines TLS. `sslcapath` is loaded as + an additional CA *file*; libpq-style hashed CA directories are not + supported. `sslservername` overrides the TLS server name when the host is a pre-resolved address — note that under `verify-full` this is also the name the certificate is verified against, so it must name the server you intend to authenticate. @@ -372,7 +375,17 @@ Block until a `NOTIFY` message arrives on the connection (see return `nothing` if no message begins arriving in that window; once a message starts, it is always read to completion so the connection is never left parked mid-message. The connection lock is held while waiting, so use a dedicated -connection for listening. +connection for listening — that is also the only way to receive every +notification, since notifications that arrive while the connection is busy +with a query are delivered to +[`notification_callback`](@ref Postgres.API.notification_callback) only during +the phases of a query that read result data. + +Over TLS the poll interval bounds a read on the underlying transport rather +than on the TLS record layer, so a record that arrives split across a poll +boundary cannot be resumed. That is detected on the following poll and closes +the connection with an error rather than returning corrupt data; a blocking +wait (no `timeout`) is not affected. """ function wait_for_notification(conn::Connection; timeout::Union{Real, Nothing}=nothing) start_time = time() @@ -384,7 +397,10 @@ function wait_for_notification(conn::Connection; timeout::Union{Real, Nothing}=n else remaining_s = timeout - (time() - start_time) remaining_s <= 0 && return nothing - Int64(time_ns()) + min(NOTIFICATION_POLL_INTERVAL_NS, round(Int64, remaining_s * 1_000_000_000)) + # clamp before converting: an Inf or very large timeout would + # overflow the nanosecond conversion + remaining_ns = remaining_s >= 10.0 ? NOTIFICATION_POLL_INTERVAL_NS : round(Int64, remaining_s * 1_000_000_000) + Int64(time_ns()) + min(NOTIFICATION_POLL_INTERVAL_NS, remaining_ns) end # The deadline covers only the first byte: if it expires there, # nothing of a message has been consumed and polling again is safe. @@ -395,7 +411,14 @@ function wait_for_notification(conn::Connection; timeout::Union{Real, Nothing}=n mt = try read(conn.socket, UInt8) catch err - _is_read_deadline_error(err) || rethrow() + if !_is_read_deadline_error(err) + # the stream position is unknowable, so the connection + # must never be reused (see the TLS caveat in the + # docstring: a deadline that expires partway through a TLS + # record surfaces here on the following poll) + close(conn.socket) + rethrow() + end nothing finally # the deadline must be cleared on every path, including a @@ -406,28 +429,36 @@ function wait_for_notification(conn::Connection; timeout::Union{Real, Nothing}=n # nothing arrived within this poll interval; nothing of a message # has been consumed, so it is safe to loop and re-check the timeout mt === nothing && continue - # no deadline is in effect from here on, so the rest of the message - # is read to completion and any failure is a real one - len = ntoh(read(conn.socket, Int32)) - 4 - if len < 0 || len > API.MAX_MESSAGE_LEN + # A byte of a message has been consumed, so from here any failure + # leaves the stream at an unknowable position: close the connection + # rather than hand back one that still looks healthy. No deadline is + # in effect, so the message is read to completion. + notification = try + len = ntoh(read(conn.socket, Int32)) - 4 + (len < 0 || len > API.MAX_MESSAGE_LEN) && + throw(API.Error("invalid message length $len from server; connection protocol state is corrupted")) + conn.debug && @info "readheader: $(Char(mt)), $len" + if mt == UInt8('A') + API.notificationResponse(len, conn.socket) + elseif mt == UInt8('N') + API.notice_callback(conn.style, API.noticeResponse(len, conn.socket)) + nothing + elseif mt == UInt8('S') + update_server_parameters!(conn, read(conn.socket, len)) + nothing + elseif mt == UInt8('E') + throw(API.errorResponse(len, conn.socket, conn.debug)) + else + API.skipbytes!(conn.socket, len) + nothing + end + catch close(conn.socket) - throw(API.Error("invalid message length $len from server; connection protocol state is corrupted")) + rethrow() end - conn.debug && @info "readheader: $(Char(mt)), $len" - if mt == UInt8('A') - notification = API.notificationResponse(len, conn.socket) + if notification !== nothing API.notification_callback(conn.style, notification) return notification - elseif mt == UInt8('N') - notice = API.noticeResponse(len, conn.socket) - API.notice_callback(conn.style, notice) - elseif mt == UInt8('S') - buf = read(conn.socket, len) - update_server_parameters!(conn, buf) - elseif mt == UInt8('E') - throw(API.errorResponse(len, conn.socket, conn.debug)) - else - API.skipbytes!(conn.socket, len) end end end diff --git a/src/api/API.jl b/src/api/API.jl index 3b42cfb..7da3369 100644 --- a/src/api/API.jl +++ b/src/api/API.jl @@ -683,6 +683,11 @@ function connect(host::String, port::Integer, dbname::String, user::String, @nos sslservername_v = sslservername::Union{String, Nothing} statement_timeout_v = statement_timeout::Union{Int, Nothing} socket = connectsocket(host, port, connect_timeout_v) + # Any failure from here on must close the socket: nothing else holds a + # reference to it, and the transport has no finalizer, so an escaping + # exception would leak the descriptor for the life of the process — + # a pool or reconnect loop against a flapping server would hit EMFILE. + try sslmode_str = sslmode_v === nothing ? "prefer" : lowercase(String(sslmode_v)) sslmode_str == "disable" || sslmode_str == "prefer" || sslmode_str == "require" || sslmode_str == "verify-full" || throw(Error("invalid sslmode: $sslmode_str")) if sslmode_str != "disable" @@ -727,6 +732,10 @@ function connect(host::String, port::Integer, dbname::String, user::String, @nos end pid, skey, server_params = waitfor(socket, debug, 'K', 'Z') return socket, pid, skey, server_params + catch + close(socket) + rethrow() + end end function prepare(socket, sql::String, debug::Bool; name::Union{Nothing, String}=nothing) diff --git a/src/array_parsing.jl b/src/array_parsing.jl index 0ab1608..9da60d2 100644 --- a/src/array_parsing.jl +++ b/src/array_parsing.jl @@ -101,6 +101,9 @@ function coerce_array(values::Vector{Any}, inner_type::Type{T}) where {T} value isa AbstractVector && (has_nested = true; break) value === missing && (has_missing = true) end + # nested arrays stay Vector{Any}: narrowing to the runtime element type + # would need a runtime-typed Vector construction, which `juliac --trim` + # cannot resolve. The values themselves are fully parsed either way. has_nested && return values if has_missing dest = Union{inner_type, Missing}[] @@ -159,6 +162,34 @@ function parse_array_inner(code::Base.CodeUnits{UInt8, String}, pos::Ref{Int}, i return coerce_array(values, inner_type) end +const EQUALS = UInt8('=') + +# PostgreSQL prefixes an array literal with explicit dimensions whenever a +# lower bound isn't 1: "[0:2]={a,b,c}", "[1:2][1:2]={{1,2},{3,4}}". Skip the +# prefix so the rest parses as an ordinary array literal — otherwise the '[' +# is taken for an array-open bracket and the real elements are lost. +function skip_dimension_prefix!(code::Base.CodeUnits{UInt8, String}, pos::Ref{Int}) + start = pos[] + n = length(code) + while pos[] <= n && code[pos[]] == BRACKET_OPEN + close_idx = pos[] + while close_idx <= n && code[close_idx] != BRACKET_CLOSE + close_idx += 1 + end + if close_idx > n + pos[] = start + return + end + pos[] = close_idx + 1 + end + if pos[] > start && pos[] <= n && code[pos[]] == EQUALS + pos[] += 1 + return + end + pos[] = start + return +end + function parse_array(str::String, inner_type::Type{T}) where {T} code = codeunits(str) pos = Ref{Int}(1) @@ -166,6 +197,11 @@ function parse_array(str::String, inner_type::Type{T}) where {T} if pos[] > length(code) return inner_type[] end + skip_dimension_prefix!(code, pos) + skip_ws(code, pos) + if pos[] > length(code) + return inner_type[] + end c = code[pos[]] if c == BRACE_OPEN || c == BRACKET_OPEN return parse_array_inner(code, pos, inner_type) diff --git a/src/connection_string.jl b/src/connection_string.jl index ef94b9d..287633f 100644 --- a/src/connection_string.jl +++ b/src/connection_string.jl @@ -77,7 +77,28 @@ function apply_env_defaults!(values::Dict{String, String}) return values end +const KNOWN_PARAMS = Set([ + "host", "port", "user", "password", "dbname", "application_name", + "connect_timeout", "sslmode", "sslrootcert", "sslcert", "sslkey", + "sslcapath", "sslservername", "statement_timeout", + "statement_cache_maxsize", "debug", "reconnect", +]) + +parse_bool_param(value::Union{String, Nothing}, default::Bool) = value === nothing ? default : lowercase(value) in ("1", "on", "true", "yes") + +# An unrecognized key is almost always a typo, and silently dropping it is +# dangerous: "ssl_mode=verify-full" would leave sslmode unset and fall back to +# an unverified connection while the caller believes otherwise. libpq errors +# on unknown keywords for the same reason. +function check_known_params(values::Dict{String, String}) + for key in keys(values) + key in KNOWN_PARAMS || throw(ArgumentError("unrecognized connection parameter \"$key\"; recognized parameters are $(join(sort!(collect(KNOWN_PARAMS)), ", "))")) + end + return values +end + function params_from_values(values::Dict{String, String}) + check_known_params(values) merged = connection_defaults() merge!(merged, values) apply_env_defaults!(merged) @@ -86,7 +107,9 @@ function params_from_values(values::Dict{String, String}) return ConnectionParams( ; host=get(merged, "host", "localhost"), - port=parse(Int, get(merged, "port", "5432")), + # empty values (an unset PGPORT expanded into the environment) fall + # back to the default rather than failing to parse + port=something(parse_optional_int(get(merged, "port", nothing)), 5432), user=user, password=get(merged, "password", nothing), dbname=dbname, @@ -99,7 +122,9 @@ function params_from_values(values::Dict{String, String}) sslcapath=get(merged, "sslcapath", nothing), sslservername=get(merged, "sslservername", nothing), statement_timeout=parse_optional_int(get(merged, "statement_timeout", nothing)), - statement_cache_maxsize=parse(Int, get(merged, "statement_cache_maxsize", "100")), + statement_cache_maxsize=something(parse_optional_int(get(merged, "statement_cache_maxsize", nothing)), 100), + debug=parse_bool_param(get(merged, "debug", nothing), false), + reconnect=parse_bool_param(get(merged, "reconnect", nothing), false), ) end @@ -204,8 +229,9 @@ function parse_uri(uri::String) query = String(parsed.query) if !isempty(query) params = URIs.queryparams(query) - for key in ("host", "port", "user", "password", "dbname", "application_name", "connect_timeout", "sslmode", "sslrootcert", "sslcert", "sslkey", "sslcapath", "sslservername", "statement_timeout", "statement_cache_maxsize") - haskey(params, key) && (values[key] = params[key]) + for (key, value) in params + key in KNOWN_PARAMS || throw(ArgumentError("unrecognized connection parameter \"$key\" in URI")) + values[key] = value end end return params_from_values(values) diff --git a/src/execute.jl b/src/execute.jl index b8ffd72..e995b0c 100644 --- a/src/execute.jl +++ b/src/execute.jl @@ -168,15 +168,26 @@ end function DBInterface.close!(cursor::Cursor) owns_transaction = cursor.owns_transaction - @lock cursor.conn.lock begin - if !cursor.done - API.writemessages(cursor.conn.socket, cursor.conn.debug, ('C', UInt8('P'), cursor.portal), ('S',)) - API.waitfor(cursor.conn.socket, cursor.conn.debug, '3', 'Z') + try + @lock cursor.conn.lock begin + if !cursor.done + API.writemessages(cursor.conn.socket, cursor.conn.debug, ('C', UInt8('P'), cursor.portal), ('S',)) + API.waitfor(cursor.conn.socket, cursor.conn.debug, '3', 'Z') + end + cursor.done = true + empty!(cursor.buffer) + end + finally + # close the transaction this cursor opened even if closing the portal + # failed: leaving in_transaction set would block reconnects forever + if owns_transaction && isopen(cursor.conn) && in_transaction(cursor.conn) + try + commit(cursor.conn) + catch + # the connection is already failing; don't mask the original error + end end - cursor.done = true - empty!(cursor.buffer) end - owns_transaction && in_transaction(cursor.conn) && commit(cursor.conn) return end Base.close(cursor::Cursor) = DBInterface.close!(cursor) @@ -258,7 +269,10 @@ end @inbounds f.types[f.i] = Union{f.types[f.i], Missing} @inbounds f.data[f.i] = missing else - if v isa AbstractVector{>:Missing} + # the OID-derived column type is a default; widen the schema whenever + # the parsed value doesn't fit it (nullable or nested array elements, + # values from a custom parser), so Tables.schema stays truthful + @inbounds if !(v isa f.types[f.i]) @inbounds f.types[f.i] = typeof(v) end @inbounds f.data[f.i] = v diff --git a/test/runtests.jl b/test/runtests.jl index e724e3a..9b1380f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -510,6 +510,26 @@ end @test Postgres.API.hostport_address("db.example.com", 6432) == "db.example.com:6432" @test_throws Postgres.PostgresInterfaceError Postgres.API.hostport_address("/var/run/postgresql", 5432) + # debug/reconnect are accepted from a DSN and actually applied + flag_params = Postgres.parse_dsn("host=h debug=true reconnect=on") + @test flag_params.debug + @test flag_params.reconnect + @test !Postgres.parse_dsn("host=h").debug + @test !Postgres.parse_dsn("host=h").reconnect + + # an unrecognized parameter is a typo, not something to silently drop: + # "ssl_mode=verify-full" would otherwise leave sslmode unset and + # quietly fall back to an unverified connection + @test_throws ArgumentError Postgres.parse_dsn("host=h ssl_mode=verify-full") + @test_throws ArgumentError Postgres.parse_dsn("postgresql://u@h/db?ssl_mode=require") + + # an empty value (an unset PGPORT expanded by a process manager) falls + # back to the default instead of failing to parse + withenv("PGPORT" => "") do + @test Postgres.parse_dsn("host=h").port == 5432 + @test Postgres.parse_dsn(nothing).port == 5432 + end + withenv( "PGHOST" => "envhost", "PGPORT" => "5544", @@ -632,6 +652,18 @@ end fields = Postgres.API.parse_composite_fields("(\"a,b\",,\"a\\\"b\",\"c\\\\d\",plain)") @test isequal(fields, Union{String, Missing}["a,b", missing, "a\"b", "c\\d", "plain"]) + # postgres prefixes the literal with explicit dimensions whenever a + # lower bound isn't 1; without handling it the elements are silently + # dropped (text) or the parse throws (numeric) + @test Postgres.API.parse_array_by_oid("[0:2]={x,y,z}", 25, registry) == ["x", "y", "z"] + @test Postgres.API.parse_array_by_oid("[0:2]={1,2,3}", 23, registry) == [1, 2, 3] + @test Postgres.API.parse_value(1009, "[0:1]={a,b}", registry) == ["a", "b"] + @test Postgres.API.ArrayParsing.parse_array("[1:2][1:2]={{1,2},{3,4}}", Int64) == [[1, 2], [3, 4]] + # a bare '[' that isn't a dimension prefix is still treated as an array + @test Postgres.API.ArrayParsing.parse_array("[1,2]", Int64) == [1, 2] + + @test Postgres.API.ArrayParsing.parse_array("{1,2}", Int64) isa Vector{Int64} + rng = MersenneTwister(0x5097) for _ in 1:200 values = [random_array_string(rng) for _ in 1:rand(rng, 0:8)] @@ -834,6 +866,12 @@ end @test isequal(array_row.arr, Union{Missing, Int32}[1, missing, 3]) nested_row = only(Tables.rowtable(DBInterface.execute(conn, "SELECT '{{1,2},{3,4}}'::int[] AS arr"))) @test nested_row.arr == [Int32[1, 2], Int32[3, 4]] + # arrays whose lower bound isn't 1 come back with an explicit + # dimension prefix; the elements must survive it + lb_row = only(Tables.rowtable(DBInterface.execute(conn, "SELECT array_fill(7, ARRAY[3], ARRAY[0]) AS arr"))) + @test lb_row.arr == [7, 7, 7] + lb_text_row = only(Tables.rowtable(DBInterface.execute(conn, "SELECT array_fill('x'::text, ARRAY[3], ARRAY[0]) AS arr"))) + @test lb_text_row.arr == ["x", "x", "x"] typed = DBInterface.execute(conn, raw"SELECT * FROM types_test WHERE id = $1", (id,), TypeRow) @test typed isa TypeRow @test typed.uuid_col == expected[12] From a9c348a47bdb1cfabfde7fd91f263a05e55f9302 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 6 Aug 2026 02:14:17 -0600 Subject: [PATCH 10/23] Document the driver-style behavior interface The docs build failed on an unresolvable cross-reference to notification_callback, which is public API but had no docstring. Documents all four style hooks (query_logging_enabled, query_logger, notice_callback, notification_callback) and includes them in the manual, rather than dropping the link. Notes on notification_callback which notifications a connection can miss, and on query_logger that info.params carries parameter values. Co-Authored-By: Claude Fable 5 --- docs/src/manual.md | 4 ++++ src/api/types.jl | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/docs/src/manual.md b/docs/src/manual.md index 0382f54..f7c440e 100644 --- a/docs/src/manual.md +++ b/docs/src/manual.md @@ -132,6 +132,10 @@ conn = DBInterface.connect(Postgres.Connection, "host=127.0.0.1 user=postgres db ```@docs Postgres.AbstractPostgresStyle Postgres.PostgresStyle +Postgres.query_logging_enabled +Postgres.query_logger +Postgres.notice_callback +Postgres.notification_callback ``` ## Parameters And Prepared Statements diff --git a/src/api/types.jl b/src/api/types.jl index 877849c..b10d029 100644 --- a/src/api/types.jl +++ b/src/api/types.jl @@ -29,13 +29,55 @@ abstract type AbstractPostgresStyle <: StructUtils.StructStyle end struct PostgresStyle <: AbstractPostgresStyle end # behavior interface (style-first; overload on your own style) + +""" + Postgres.query_logging_enabled(style) -> Bool + +Whether [`query_logger`](@ref Postgres.API.query_logger) should be called for +queries on connections using `style`. `false` by default, which also skips the +timing work the logger would need. +""" query_logging_enabled(::AbstractPostgresStyle) = false + +""" + Postgres.query_logger(style, event::Symbol, info::NamedTuple) + +Called after each query when +[`query_logging_enabled`](@ref Postgres.API.query_logging_enabled) is true for +`style`. `event` is `:execute`, `:copy_from`, or `:copy_to`; `info` carries +`sql`, `duration_ns`, `success`, the bound `params` (for `:execute`), and +`error` when the query failed. + +`info.params` holds the query's parameter values, so a logger that records +them will record whatever sensitive data those queries carry. +""" query_logger(::AbstractPostgresStyle, event::Symbol, info::NamedTuple) = nothing + +""" + Postgres.notice_callback(style, notice) + +Called for each `NoticeResponse` the server sends. `notice` is a `Dict` of the +raw notice fields, keyed by their single-character protocol codes (`"M"` is +the message, `"S"` the severity). Emits the message as a `@warn` by default. +""" function notice_callback(::AbstractPostgresStyle, notice) msg = get(notice, "M", "") !isempty(msg) && @warn msg return nothing end + +""" + Postgres.notification_callback(style, notification::Notification) + +Called for each asynchronous `NOTIFY` ([`Notification`](@ref +Postgres.API.Notification)) received while reading query results. Does nothing +by default. + +A connection only observes notifications while it is reading from the server, +so a connection that is idle or busy in another phase of a query may not see +one. Use [`wait_for_notification`](@ref Postgres.wait_for_notification) on a +dedicated connection to receive every notification on a channel. +""" notification_callback(::AbstractPostgresStyle, notification) = nothing StructUtils.fieldtagkey(::AbstractPostgresStyle) = :postgres From fa9bad1b13f5207d7432fbb479dd477faa5b7ec6 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 6 Aug 2026 02:34:12 -0600 Subject: [PATCH 11/23] Fix array truncation on ']', schema narrowing, and cursor transaction cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ']' is not an array terminator. PostgreSQL only quotes elements containing '"', '\', '{', '}', ',' or whitespace, so an unquoted ']' — file paths, "x[1]", embedded JSON — was treated as the end of the array: {a]b,x[1],plain} parsed to ["a"], silently dropping two elements. Bracket-delimited arrays are no longer accepted at all; postgres never emits them, and the one legitimate leading '[' (the dimension prefix) is handled separately. - Tables.schema widening was replacing rather than widening, so a column mixing null-free and null-bearing arrays ended up reporting whichever type the last mismatching row had — Tables.columntable then failed to convert, and the outcome depended on row order. Now unions, like the missing branch above it. (A regression from the previous commit; the predicate it replaced was monotone.) - closing a cursor no longer swallows a COMMIT failure on the success path, which discarded the caller's writes while reporting success and left in_transaction set against an idle backend. - when the connection is already dead, cursor cleanup clears the transaction bookkeeping directly instead of attempting a COMMIT that cannot work: that state otherwise blocks reconnect on that connection permanently, which is the case the previous commit meant to fix but skipped via its isopen guard. - connection-string strictness no longer rejects real libpq keywords this driver doesn't implement (channel_binding, target_session_attrs, options, hostaddr, ...). Providers put these in the URIs they hand users, so they are accepted and ignored; only genuinely unknown keys (the ssl_mode typo case) still error. - invalid values for a recognized parameter now name that parameter, and a bad boolean errors instead of silently reading as false. - wait_for_notification no longer closes the connection when a user notice_callback throws or when the server sends an asynchronous ErrorResponse: both leave the stream at a clean message boundary, so only failures while reading a message are treated as fatal. Co-Authored-By: Claude Fable 5 --- src/Postgres.jl | 21 ++++++++++++------ src/array_parsing.jl | 13 +++++++---- src/connection_string.jl | 48 ++++++++++++++++++++++++++++++---------- src/execute.jl | 42 +++++++++++++++++++++++++++-------- test/runtests.jl | 32 +++++++++++++++++++++++++-- 5 files changed, 122 insertions(+), 34 deletions(-) diff --git a/src/Postgres.jl b/src/Postgres.jl index d8adf20..96f9ed9 100644 --- a/src/Postgres.jl +++ b/src/Postgres.jl @@ -433,7 +433,11 @@ function wait_for_notification(conn::Connection; timeout::Union{Real, Nothing}=n # leaves the stream at an unknowable position: close the connection # rather than hand back one that still looks healthy. No deadline is # in effect, so the message is read to completion. - notification = try + # Read the message off the socket. Only the reading is guarded: + # once a message is fully consumed the stream is back at a clean + # boundary, so user callbacks and server errors are surfaced + # without destroying the connection. + message = try len = ntoh(read(conn.socket, Int32)) - 4 (len < 0 || len > API.MAX_MESSAGE_LEN) && throw(API.Error("invalid message length $len from server; connection protocol state is corrupted")) @@ -441,13 +445,12 @@ function wait_for_notification(conn::Connection; timeout::Union{Real, Nothing}=n if mt == UInt8('A') API.notificationResponse(len, conn.socket) elseif mt == UInt8('N') - API.notice_callback(conn.style, API.noticeResponse(len, conn.socket)) - nothing + API.noticeResponse(len, conn.socket) elseif mt == UInt8('S') update_server_parameters!(conn, read(conn.socket, len)) nothing elseif mt == UInt8('E') - throw(API.errorResponse(len, conn.socket, conn.debug)) + API.errorResponse(len, conn.socket, conn.debug) else API.skipbytes!(conn.socket, len) nothing @@ -456,9 +459,13 @@ function wait_for_notification(conn::Connection; timeout::Union{Real, Nothing}=n close(conn.socket) rethrow() end - if notification !== nothing - API.notification_callback(conn.style, notification) - return notification + if message isa API.Notification + API.notification_callback(conn.style, message) + return message + elseif message isa API.Error + throw(message) + elseif message !== nothing + API.notice_callback(conn.style, message) end end end diff --git a/src/array_parsing.jl b/src/array_parsing.jl index 9da60d2..aab461e 100644 --- a/src/array_parsing.jl +++ b/src/array_parsing.jl @@ -50,7 +50,12 @@ function parse_unquoted(code::Base.CodeUnits{UInt8, String}, pos::Ref{Int}) buf = UInt8[] while pos[] <= length(code) c = code[pos[]] - if c == COMMA || c == BRACE_CLOSE || c == BRACKET_CLOSE + # ']' is NOT a terminator: postgres only quotes elements containing + # '"', '\\', '{', '}', ',' or whitespace, so an unquoted ']' (file + # paths, "x[1]", embedded JSON) is ordinary element data. Treating it + # as a terminator silently truncated the element and dropped every + # element after it. + if c == COMMA || c == BRACE_CLOSE break elseif c == BACKSLASH pos[] += 1 @@ -130,7 +135,7 @@ end function parse_array_value(code::Base.CodeUnits{UInt8, String}, pos::Ref{Int}, inner_type::Type{T}) where {T} c = code[pos[]] - if c == BRACE_OPEN || c == BRACKET_OPEN + if c == BRACE_OPEN return parse_array_inner(code, pos, inner_type) elseif c == QUOTE token = parse_quoted(code, pos) @@ -148,7 +153,7 @@ function parse_array_inner(code::Base.CodeUnits{UInt8, String}, pos::Ref{Int}, i skip_ws(code, pos) pos[] > length(code) && break c = code[pos[]] - if c == BRACE_CLOSE || c == BRACKET_CLOSE + if c == BRACE_CLOSE pos[] += 1 break end @@ -203,7 +208,7 @@ function parse_array(str::String, inner_type::Type{T}) where {T} return inner_type[] end c = code[pos[]] - if c == BRACE_OPEN || c == BRACKET_OPEN + if c == BRACE_OPEN return parse_array_inner(code, pos, inner_type) end value = parse_scalar(str, inner_type, true) diff --git a/src/connection_string.jl b/src/connection_string.jl index 287633f..9408cad 100644 --- a/src/connection_string.jl +++ b/src/connection_string.jl @@ -43,10 +43,12 @@ end default_user() = get(ENV, "PGUSER", get(ENV, "USER", get(ENV, "USERNAME", ""))) -function parse_optional_int(value::Union{String, Nothing}) +function parse_optional_int(value::Union{String, Nothing}, key::String="") value === nothing && return nothing isempty(value) && return nothing - return parse(Int, value) + parsed = tryparse(Int, value) + parsed === nothing && throw(ArgumentError("invalid value \"$value\" for connection parameter \"$key\"; expected an integer")) + return parsed end function connection_defaults() @@ -84,7 +86,26 @@ const KNOWN_PARAMS = Set([ "statement_cache_maxsize", "debug", "reconnect", ]) -parse_bool_param(value::Union{String, Nothing}, default::Bool) = value === nothing ? default : lowercase(value) in ("1", "on", "true", "yes") +# libpq keywords this driver doesn't implement. They are accepted and ignored +# rather than rejected: managed-PostgreSQL providers routinely include them in +# the connection URI they hand users, and failing on a DSN that names a real +# libpq option would be worse than not honoring it. +const IGNORED_PARAMS = Set([ + "channel_binding", "target_session_attrs", "options", "gssencmode", + "gsslib", "krbsrvname", "sslnegotiation", "sslcompression", "sslcrl", + "sslcrldir", "sslpassword", "requiressl", "requirepeer", "hostaddr", + "client_encoding", "passfile", "service", "fallback_application_name", + "keepalives", "keepalives_idle", "keepalives_interval", "keepalives_count", + "tcp_user_timeout", "load_balance_hosts", "replication", +]) + +function parse_bool_param(value::Union{String, Nothing}, default::Bool, key::String) + value === nothing && return default + lowered = lowercase(value) + lowered in ("1", "on", "true", "yes") && return true + lowered in ("0", "off", "false", "no") && return false + throw(ArgumentError("invalid value \"$value\" for connection parameter \"$key\"; expected a boolean (on/off, true/false, yes/no, 1/0)")) +end # An unrecognized key is almost always a typo, and silently dropping it is # dangerous: "ssl_mode=verify-full" would leave sslmode unset and fall back to @@ -92,7 +113,8 @@ parse_bool_param(value::Union{String, Nothing}, default::Bool) = value === nothi # on unknown keywords for the same reason. function check_known_params(values::Dict{String, String}) for key in keys(values) - key in KNOWN_PARAMS || throw(ArgumentError("unrecognized connection parameter \"$key\"; recognized parameters are $(join(sort!(collect(KNOWN_PARAMS)), ", "))")) + (key in KNOWN_PARAMS || key in IGNORED_PARAMS) || + throw(ArgumentError("unrecognized connection parameter \"$key\"; recognized parameters are $(join(sort!(collect(KNOWN_PARAMS)), ", "))")) end return values end @@ -109,22 +131,22 @@ function params_from_values(values::Dict{String, String}) host=get(merged, "host", "localhost"), # empty values (an unset PGPORT expanded into the environment) fall # back to the default rather than failing to parse - port=something(parse_optional_int(get(merged, "port", nothing)), 5432), + port=something(parse_optional_int(get(merged, "port", nothing), "port"), 5432), user=user, password=get(merged, "password", nothing), dbname=dbname, application_name=get(merged, "application_name", nothing), - connect_timeout=parse_optional_int(get(merged, "connect_timeout", nothing)), + connect_timeout=parse_optional_int(get(merged, "connect_timeout", nothing), "connect_timeout"), sslmode=haskey(merged, "sslmode") ? lowercase(merged["sslmode"]) : nothing, sslrootcert=get(merged, "sslrootcert", nothing), sslcert=get(merged, "sslcert", nothing), sslkey=get(merged, "sslkey", nothing), sslcapath=get(merged, "sslcapath", nothing), sslservername=get(merged, "sslservername", nothing), - statement_timeout=parse_optional_int(get(merged, "statement_timeout", nothing)), - statement_cache_maxsize=something(parse_optional_int(get(merged, "statement_cache_maxsize", nothing)), 100), - debug=parse_bool_param(get(merged, "debug", nothing), false), - reconnect=parse_bool_param(get(merged, "reconnect", nothing), false), + statement_timeout=parse_optional_int(get(merged, "statement_timeout", nothing), "statement_timeout"), + statement_cache_maxsize=something(parse_optional_int(get(merged, "statement_cache_maxsize", nothing), "statement_cache_maxsize"), 100), + debug=parse_bool_param(get(merged, "debug", nothing), false, "debug"), + reconnect=parse_bool_param(get(merged, "reconnect", nothing), false, "reconnect"), ) end @@ -230,8 +252,10 @@ function parse_uri(uri::String) if !isempty(query) params = URIs.queryparams(query) for (key, value) in params - key in KNOWN_PARAMS || throw(ArgumentError("unrecognized connection parameter \"$key\" in URI")) - values[key] = value + (key in KNOWN_PARAMS || key in IGNORED_PARAMS) || + throw(ArgumentError("unrecognized connection parameter \"$key\" in URI; recognized parameters are $(join(sort!(collect(KNOWN_PARAMS)), ", "))")) + # keys we accept but don't implement must not reach params_from_values + key in KNOWN_PARAMS && (values[key] = value) end end return params_from_values(values) diff --git a/src/execute.jl b/src/execute.jl index e995b0c..b90b198 100644 --- a/src/execute.jl +++ b/src/execute.jl @@ -166,8 +166,25 @@ function DBInterface.close!(stmt::Statement) return end +# Finish the transaction a cursor opened for itself. If the connection died, +# clear the bookkeeping directly rather than trying to COMMIT: leaving +# in_transaction set would make checkconn refuse to reconnect forever, and the +# server-side transaction is already gone with the session. +function finish_cursor_transaction!(conn::Connection) + if !isopen(conn) + @lock conn.lock begin + conn.in_transaction = false + conn.transaction_depth = 0 + end + return + end + in_transaction(conn) && commit(conn) + return +end + function DBInterface.close!(cursor::Cursor) owns_transaction = cursor.owns_transaction + closed_cleanly = false try @lock cursor.conn.lock begin if !cursor.done @@ -177,14 +194,19 @@ function DBInterface.close!(cursor::Cursor) cursor.done = true empty!(cursor.buffer) end + closed_cleanly = true finally - # close the transaction this cursor opened even if closing the portal - # failed: leaving in_transaction set would block reconnects forever - if owns_transaction && isopen(cursor.conn) && in_transaction(cursor.conn) - try - commit(cursor.conn) - catch - # the connection is already failing; don't mask the original error + if owns_transaction + if closed_cleanly + # a COMMIT failure here means the caller's writes did not land, + # so it must propagate rather than be swallowed + finish_cursor_transaction!(cursor.conn) + else + try + finish_cursor_transaction!(cursor.conn) + catch + # already unwinding; don't mask the original error + end end end end @@ -271,9 +293,11 @@ end else # the OID-derived column type is a default; widen the schema whenever # the parsed value doesn't fit it (nullable or nested array elements, - # values from a custom parser), so Tables.schema stays truthful + # values from a custom parser), so Tables.schema stays truthful. + # Widening must be monotone — replacing would let a later row narrow + # the schema back to a type earlier rows don't satisfy. @inbounds if !(v isa f.types[f.i]) - @inbounds f.types[f.i] = typeof(v) + @inbounds f.types[f.i] = Union{f.types[f.i], typeof(v)} end @inbounds f.data[f.i] = v end diff --git a/test/runtests.jl b/test/runtests.jl index 9b1380f..e9355f5 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -523,6 +523,17 @@ end @test_throws ArgumentError Postgres.parse_dsn("host=h ssl_mode=verify-full") @test_throws ArgumentError Postgres.parse_dsn("postgresql://u@h/db?ssl_mode=require") + # real libpq keywords this driver doesn't implement are accepted and + # ignored: providers routinely put them in the URI they hand users + @test Postgres.parse_dsn("postgresql://u:p@h/db?sslmode=require&channel_binding=require").sslmode == "require" + @test Postgres.parse_dsn("postgresql://u@h/db?target_session_attrs=read-write").dbname == "db" + @test Postgres.parse_dsn("host=h options=-csearch_path=x").host == "h" + + # invalid values for a recognized parameter are reported against that + # parameter rather than silently defaulting + @test_throws ArgumentError Postgres.parse_dsn("host=h reconnect=ture") + @test_throws ArgumentError Postgres.parse_dsn("host=h port=abc") + # an empty value (an unset PGPORT expanded by a process manager) falls # back to the default instead of failing to parse withenv("PGPORT" => "") do @@ -659,8 +670,12 @@ end @test Postgres.API.parse_array_by_oid("[0:2]={1,2,3}", 23, registry) == [1, 2, 3] @test Postgres.API.parse_value(1009, "[0:1]={a,b}", registry) == ["a", "b"] @test Postgres.API.ArrayParsing.parse_array("[1:2][1:2]={{1,2},{3,4}}", Int64) == [[1, 2], [3, 4]] - # a bare '[' that isn't a dimension prefix is still treated as an array - @test Postgres.API.ArrayParsing.parse_array("[1,2]", Int64) == [1, 2] + # ']' is ordinary element data: postgres doesn't quote it, so treating + # it as a terminator silently truncated the element and dropped every + # element after it + @test Postgres.API.parse_array_by_oid("{a]b,x[1],plain}", 25, registry) == ["a]b", "x[1]", "plain"] + @test Postgres.API.parse_array_by_oid("{]}", 25, registry) == ["]"] + @test Postgres.API.parse_array_by_oid("{/var/log/x[1].txt,b}", 25, registry) == ["/var/log/x[1].txt", "b"] @test Postgres.API.ArrayParsing.parse_array("{1,2}", Int64) isa Vector{Int64} @@ -862,6 +877,19 @@ end @test ismissing(row.nullable_text) @test row.int_array == expected[23] @test eltype(row.int_array) == Int32 + # a column mixing null-free and null-bearing arrays must report + # a schema type every row satisfies, whatever the row order + for order in ("'{1,2}'::int[]), ('{1,NULL}'::int[]), ('{3,4}'::int[]", + "'{1,NULL}'::int[]), ('{1,2}'::int[]), ('{3,4}'::int[]") + mixed = DBInterface.execute(conn, "SELECT a FROM (VALUES ($order)) t(a)") + schema_type = Tables.schema(mixed).types[1] + @test all(row -> Tables.getcolumn(row, 1) isa schema_type, mixed) + @test length(Tables.columntable(mixed).a) == 3 + end + # a text array element containing ']' must survive the round trip + bracket_param = ["a]b", "x[1]", "]", "plain"] + bracket_row = only(Tables.rowtable(DBInterface.execute(conn, raw"SELECT $1::text[] AS arr", (bracket_param,)))) + @test bracket_row.arr == bracket_param array_row = only(Tables.rowtable(DBInterface.execute(conn, "SELECT '{1,NULL,3}'::int[] AS arr"))) @test isequal(array_row.arr, Union{Missing, Int32}[1, missing, 3]) nested_row = only(Tables.rowtable(DBInterface.execute(conn, "SELECT '{{1,2},{3,4}}'::int[] AS arr"))) From 2cf22bfd51a48e3f44f727b1fd5295c7c416aa2b Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 6 Aug 2026 02:55:23 -0600 Subject: [PATCH 12/23] Fix transaction state surviving a failed commit, and cursor double-close MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - commit/rollback left in_transaction set when the statement failed, but the server ends the transaction either way. The stale flag then made checkconn refuse to reconnect ("reconnect disabled during transaction") forever, and made the next cursor skip its BEGIN and die with "portal does not exist". State is now cleared whether the statement succeeds, fails, or can't be sent at all because the socket is already gone — the last case has to be handled before checkconn, which would otherwise throw first. - closing an already-closed cursor committed whatever transaction the caller had open at that moment: owns_transaction was never cleared, so a second close! ran the cleanup again, silently turning a later rollback into a commit. The cursor now takes responsibility exactly once. - an asynchronous FATAL error during wait_for_notification closes the socket again, so the next use reports that error instead of a bare EOFError from a connection the server had already dropped. Non-fatal errors and notice callbacks still leave the connection usable. - an empty debug=/reconnect= value means "unset", matching how the integer parameters already treat an empty value from an unexpanded ${VAR}. Co-Authored-By: Claude Fable 5 --- src/Postgres.jl | 42 ++++++++++++++++++++++++++++++++++------ src/connection_string.jl | 2 ++ src/execute.jl | 3 +++ test/runtests.jl | 40 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 6 deletions(-) diff --git a/src/Postgres.jl b/src/Postgres.jl index 96f9ed9..bcec5f0 100644 --- a/src/Postgres.jl +++ b/src/Postgres.jl @@ -463,6 +463,10 @@ function wait_for_notification(conn::Connection; timeout::Union{Real, Nothing}=n API.notification_callback(conn.style, message) return message elseif message isa API.Error + # a FATAL error means the server is terminating this session; + # close now so the next use reports the error rather than a + # bare EOF from a socket the server has already dropped + message.severity == "FATAL" && close(conn.socket) throw(message) elseif message !== nothing API.notice_callback(conn.style, message) @@ -885,12 +889,27 @@ Commit the current transaction (or release one level of transaction nesting). """ function commit(conn::Connection) @lock conn.lock begin - checkconn(conn) !conn.in_transaction && throw(PostgresInterfaceError("no transaction in progress")) - if conn.transaction_depth == 1 - execute_simple(conn, "COMMIT") + # a dead socket took the transaction with it: clear the bookkeeping + # before reporting, or checkconn will refuse to reconnect forever + # ("reconnect disabled during transaction") + if !isopen(conn.socket) conn.in_transaction = false conn.transaction_depth = 0 + disconnected() + end + checkconn(conn) + if conn.transaction_depth == 1 + # COMMIT ends the transaction server-side whether it succeeds or + # fails (and a dead connection ends it too), so the client's + # transaction state must be cleared either way — leaving it set + # would block reconnects and make the next cursor skip its BEGIN + try + execute_simple(conn, "COMMIT") + finally + conn.in_transaction = false + conn.transaction_depth = 0 + end else # Release SAVEPOINT for nested transaction conn.transaction_depth -= 1 @@ -908,12 +927,23 @@ the enclosing savepoint). """ function rollback(conn::Connection) @lock conn.lock begin - checkconn(conn) !conn.in_transaction && throw(PostgresInterfaceError("no transaction in progress")) - if conn.transaction_depth == 1 - execute_simple(conn, "ROLLBACK") + # as in commit: a dead session already ended the transaction + if !isopen(conn.socket) conn.in_transaction = false conn.transaction_depth = 0 + disconnected() + end + checkconn(conn) + if conn.transaction_depth == 1 + # as in commit: the transaction is over server-side regardless of + # how ROLLBACK fares, so don't leave client state describing it + try + execute_simple(conn, "ROLLBACK") + finally + conn.in_transaction = false + conn.transaction_depth = 0 + end else # Rollback to SAVEPOINT for nested transaction conn.transaction_depth -= 1 diff --git a/src/connection_string.jl b/src/connection_string.jl index 9408cad..16cc444 100644 --- a/src/connection_string.jl +++ b/src/connection_string.jl @@ -101,6 +101,8 @@ const IGNORED_PARAMS = Set([ function parse_bool_param(value::Union{String, Nothing}, default::Bool, key::String) value === nothing && return default + # an empty value means "unset", as it does for the integer parameters + isempty(value) && return default lowered = lowercase(value) lowered in ("1", "on", "true", "yes") && return true lowered in ("0", "off", "false", "no") && return false diff --git a/src/execute.jl b/src/execute.jl index b90b198..a82b3b0 100644 --- a/src/execute.jl +++ b/src/execute.jl @@ -196,6 +196,9 @@ function DBInterface.close!(cursor::Cursor) end closed_cleanly = true finally + # take responsibility exactly once: closing an already-closed cursor + # must not commit whatever transaction the caller has open now + cursor.owns_transaction = false if owns_transaction if closed_cleanly # a COMMIT failure here means the caller's writes did not land, diff --git a/test/runtests.jl b/test/runtests.jl index e9355f5..4257e8b 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -968,6 +968,20 @@ end @test_throws Postgres.PostgresInterfaceError Postgres.commit(conn) @test_throws Postgres.PostgresInterfaceError Postgres.rollback(conn) + # COMMIT/ROLLBACK end the transaction server-side even when + # they fail, so client state must not be left behind + fail_conn = DBInterface.connect(Postgres.Connection, cfg.host, cfg.user, cfg.password; dbname=cfg.dbname, port=cfg.port, reconnect=true) + Postgres.start_transaction(fail_conn) + close(fail_conn.socket) + try + Postgres.commit(fail_conn) + catch + # the connection is gone; the COMMIT cannot be delivered + end + @test !Postgres.in_transaction(fail_conn) + @test Tables.rowtable(DBInterface.execute(fail_conn, "SELECT 1 AS a"))[1].a == 1 + DBInterface.close!(fail_conn) + Postgres.start_transaction(conn) @test_throws Postgres.API.Error DBInterface.execute(conn, "INVALID SQL") Postgres.rollback(conn) @@ -1317,6 +1331,32 @@ end values = [row.n for row in cur] @test values == [1, 2, 3, 4, 5] DBInterface.close!(cur) + @test !Postgres.in_transaction(conn) + + # closing an already-closed cursor must not reach into a + # transaction the caller opened afterwards and commit it + DBInterface.execute(conn, "DROP TABLE IF EXISTS cursor_reclose") + DBInterface.execute(conn, "CREATE TABLE cursor_reclose (id int)") + Postgres.start_transaction(conn) + DBInterface.execute(conn, "INSERT INTO cursor_reclose VALUES (1)") + DBInterface.close!(cur) + @test Postgres.in_transaction(conn) + Postgres.rollback(conn) + @test isempty(Tables.rowtable(DBInterface.execute(conn, "SELECT * FROM cursor_reclose"))) + + # a cursor over a dead connection must not leave transaction + # state behind, which would block reconnect forever + dead_conn = DBInterface.connect(Postgres.Connection, cfg.host, cfg.user, cfg.password; dbname=cfg.dbname, port=cfg.port, reconnect=true) + dead_cur = Postgres.cursor(dead_conn, "SELECT generate_series(1, 100) AS n"; fetchsize=2) + close(dead_conn.socket) + try + DBInterface.close!(dead_cur) + catch + # closing the portal on a dead socket may throw + end + @test !Postgres.in_transaction(dead_conn) + @test Tables.rowtable(DBInterface.execute(dead_conn, "SELECT 1 AS a"))[1].a == 1 + DBInterface.close!(dead_conn) end @testset "Notice Callback (style)" begin From e8451c077887dfedbc96d40e60241d8c9e009fad Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 6 Aug 2026 02:57:49 -0600 Subject: [PATCH 13/23] Use the non-localized severity from ErrorResponse The 'S' severity field is translated per the server's lc_messages, so comparing it against "FATAL" (as the notification path now does to decide whether the server is terminating the session) only works on an English server. PostgreSQL 9.6+ also sends 'V', the non-localized severity, which was being discarded. Error.severity now prefers it when present. Co-Authored-By: Claude Fable 5 --- src/api/API.jl | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/api/API.jl b/src/api/API.jl index 7da3369..7b1a114 100644 --- a/src/api/API.jl +++ b/src/api/API.jl @@ -12,7 +12,9 @@ const SKIP_BUFFER_SIZE = 8192 Postgres.Error <: Exception A PostgreSQL server error (an `ErrorResponse` message). Carries the fields the -server reported: `severity`, `code` (the SQLSTATE, e.g. `"23505"`), `message`, +server reported: `severity` (non-localized when the server supplies it, so it +can be compared against `"FATAL"`, `"ERROR"`, ... regardless of the server's +`lc_messages`), `code` (the SQLSTATE, e.g. `"23505"`), `message`, and optional context such as `detail`, `hint`, `position`, `schema`, `table`, `column`, and `constraint`. A small number of protocol-level failures detected client-side (unsupported authentication methods, protocol desync) also use @@ -90,6 +92,9 @@ function errorResponse(len, socket, debug) # parse error fields i = 1 severity = "" + # 'V' is the non-localized severity (PostgreSQL 9.6+); 'S' is translated + # per the server's lc_messages, so it can't be compared against literals + severity_nonlocalized = "" code = "" message = "" detail = nothing @@ -114,6 +119,8 @@ function errorResponse(len, socket, debug) val, i = cstring_at(buf, i) if ccode == 'S' severity = val + elseif ccode == 'V' + severity_nonlocalized = val elseif ccode == 'C' code = val elseif ccode == 'M' @@ -148,6 +155,9 @@ function errorResponse(len, socket, debug) routine = val end end + # prefer the non-localized severity so callers can compare it to "FATAL" + # and friends regardless of the server's locale + isempty(severity_nonlocalized) || (severity = severity_nonlocalized) err = Error(severity, code, message, detail, hint, position, internal_position, internal_query, where, schema, table, column, datatype, constraint, file, line, routine) debug && @error err return err From 007ceaa98de45a872303f79b88d00ac2582c853a Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 6 Aug 2026 03:39:48 -0600 Subject: [PATCH 14/23] Fix lost server errors on commit failure, and pooled transaction leakage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - transaction/@transaction/DBInterface.transaction rolled back unguarded in their catch. Since the previous commit clears transaction state when COMMIT fails, that rollback threw "no transaction in progress" from inside the catch and replaced the server's error — so a deferred-constraint violation or a 40001 serialization failure raised at commit reached the caller as an opaque client error with no SQLSTATE to retry on. Guarded on in_transaction. (Regression from the previous commit; covered now by a DEFERRABLE INITIALLY DEFERRED test across all three wrappers, which is the test gap that let it through.) - releasing a connection to the pool mid-transaction handed that transaction to the next borrower: their start_transaction issued a SAVEPOINT instead of BEGIN and their commit only decremented the depth, so their writes were silently discarded and the backend sat idle-in-transaction holding locks. The pool now rolls back before reuse, and drops the connection if it can't. - cursor setup failure on a dead connection cleared no transaction state, permanently blocking reconnect; it now clears it like the close path does. - PANIC terminates the session as FATAL does, so it closes the socket too. - an empty sslmode= (unexpanded ${PGSSLMODE}) means unset rather than an invalid mode. - docs: reconnect and debug ARE available as DSN options; sslcapath is a fallback CA file ignored when sslrootcert is set. - tests: the empty-boolean DSN case, the non-localized severity preference, and a SQLSTATE check in place of a localized-message match. Co-Authored-By: Claude Fable 5 --- README.md | 2 +- docs/src/index.md | 2 +- src/Postgres.jl | 72 +++++++++++++++++++++++------ src/connection_string.jl | 4 +- src/execute.jl | 26 +++++++++-- test/runtests.jl | 99 +++++++++++++++++++++++++++++++++++++++- 6 files changed, 182 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index faeeefb..9ae61ea 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Connection options support: - PostgreSQL URIs such as `postgresql://postgres:postgres@127.0.0.1:5432/postgres`. - Environment defaults: `PGHOST`, `PGPORT`, `PGUSER`, `PGPASSWORD`, `PGDATABASE`, `PGAPPNAME`, `PGCONNECT_TIMEOUT`, and TLS-related `PGSSL*` variables. - `sslmode` values: `disable`, `prefer` (the default), `require`, `verify-full`. Only `verify-full` verifies the server's certificate; `require` encrypts without authenticating the server, and the default `prefer` falls back to an unencrypted connection if the server declines TLS. Use `verify-full` with `sslrootcert` when the connection needs to be authenticated. -- TLS files: `sslrootcert`, `sslcert`, `sslkey`, and `sslcapath` (loaded as an additional CA *file*; libpq-style hashed CA directories are not supported). `sslservername` overrides the TLS server name when connecting to a pre-resolved address; under `verify-full` it is also the name the certificate is verified against, so it must name the server you intend to authenticate. +- TLS files: `sslrootcert`, `sslcert`, `sslkey`, and `sslcapath` (a *fallback* CA file, used only when `sslrootcert` is unset and ignored otherwise; libpq-style hashed CA directories are not supported). `sslservername` overrides the TLS server name when connecting to a pre-resolved address; under `verify-full` it is also the name the certificate is verified against, so it must name the server you intend to authenticate. - `connect_timeout` (seconds) and `statement_timeout` (milliseconds). - `application_name` and `statement_cache_maxsize`. diff --git a/docs/src/index.md b/docs/src/index.md index d1b006d..bb6e011 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -18,7 +18,7 @@ Postgres.jl accepts DSN strings or PostgreSQL URIs and supports: - libpq-style keyword strings such as `host=127.0.0.1 port=5432 user=postgres dbname=postgres`. - Environment defaults from `PGHOST`, `PGPORT`, `PGUSER`, `PGPASSWORD`, `PGDATABASE`, `PGAPPNAME`, `PGCONNECT_TIMEOUT`, and TLS-related `PGSSL*` variables. - `sslmode` values: `disable`, `prefer` (the default), `require`, `verify-full`. Only `verify-full` verifies the server's certificate; `require` encrypts without authenticating the server, and the default `prefer` falls back to an unencrypted connection if the server declines TLS. Use `verify-full` with `sslrootcert` when the connection needs to be authenticated. -- TLS files: `sslrootcert`, `sslcert`, `sslkey`, and `sslcapath` (loaded as an additional CA *file*; libpq-style hashed CA directories are not supported). `sslservername` overrides the TLS server name when connecting to a pre-resolved address; under `verify-full` it is also the name the certificate is verified against, so it must name the server you intend to authenticate. +- TLS files: `sslrootcert`, `sslcert`, `sslkey`, and `sslcapath` (a *fallback* CA file, used only when `sslrootcert` is unset and ignored otherwise; libpq-style hashed CA directories are not supported). `sslservername` overrides the TLS server name when connecting to a pre-resolved address; under `verify-full` it is also the name the certificate is verified against, so it must name the server you intend to authenticate. - `connect_timeout` (seconds), `statement_timeout` (milliseconds). - `application_name` and `statement_cache_maxsize`. diff --git a/src/Postgres.jl b/src/Postgres.jl index bcec5f0..a9cc274 100644 --- a/src/Postgres.jl +++ b/src/Postgres.jl @@ -38,8 +38,8 @@ A single connection to a PostgreSQL server, created via `dsn` may be a libpq-style keyword string (`"host=127.0.0.1 user=postgres dbname=postgres"`) or a PostgreSQL URI (`"postgresql://user:pass@host:5432/dbname?sslmode=require"`). -Supported keyword arguments (all but the last three also available as DSN/URI -options): +Supported keyword arguments. All are also available as DSN/URI options except +`style`, which is Julia-only: - `dbname`, `port`, `application_name` - `connect_timeout` (seconds), `statement_timeout` (milliseconds) @@ -47,9 +47,10 @@ options): `sslrootcert`, `sslcert`, `sslkey`, `sslcapath`, and `sslservername`. Only `verify-full` verifies the server's certificate; `require` encrypts without authenticating the server, and the default `prefer` falls back to an - unencrypted connection if the server declines TLS. `sslcapath` is loaded as - an additional CA *file*; libpq-style hashed CA directories are not - supported. `sslservername` overrides the TLS server + unencrypted connection if the server declines TLS. `sslcapath` is a + *fallback* CA file used only when `sslrootcert` is unset (it is ignored + otherwise); libpq-style hashed CA directories are not supported. + `sslservername` overrides the TLS server name when the host is a pre-resolved address — note that under `verify-full` this is also the name the certificate is verified against, so it must name the server you intend to authenticate. @@ -463,10 +464,10 @@ function wait_for_notification(conn::Connection; timeout::Union{Real, Nothing}=n API.notification_callback(conn.style, message) return message elseif message isa API.Error - # a FATAL error means the server is terminating this session; - # close now so the next use reports the error rather than a - # bare EOF from a socket the server has already dropped - message.severity == "FATAL" && close(conn.socket) + # FATAL and PANIC both terminate the session; close now so the + # next use reports the error rather than a bare EOF from a + # socket the server has already dropped + (message.severity == "FATAL" || message.severity == "PANIC") && close(conn.socket) throw(message) elseif message !== nothing API.notice_callback(conn.style, message) @@ -790,14 +791,44 @@ function acquire(pool::ConnectionPool; forcenew::Bool=false) return conn end +# A connection going back into the pool must not carry a transaction with it: +# the next borrower's `start_transaction` would issue a SAVEPOINT instead of +# BEGIN, and their commit would only decrement the depth — their writes would +# be silently discarded when the connection is later reset. Roll it back; if +# that can't be done, drop the connection instead of handing it on. +function reset_pooled_connection!(conn::Connection) + in_transaction(conn) || return true + try + @lock conn.lock begin + checkconn(conn) + try + execute_simple(conn, "ROLLBACK") + finally + conn.in_transaction = false + conn.transaction_depth = 0 + end + end + return true + catch + try + DBInterface.close!(conn) + catch + # already unusable; nothing more to do + end + return false + end +end + """ Postgres.release(pool, conn) Return a connection previously taken with [`acquire`](@ref Postgres.acquire) -to the pool. +to the pool. A connection still inside a transaction is rolled back first, so +the next borrower starts from a clean session; if it can't be rolled back it +is closed rather than reused. """ function release(pool::ConnectionPool, conn::Connection) - if pool_isvalid(conn) + if pool_isvalid(conn) && reset_pooled_connection!(conn) Pools.release(pool.pool, conn) else Pools.release(pool.pool) @@ -882,6 +913,16 @@ Whether the connection currently has an open transaction. """ in_transaction(conn::Connection) = @lock conn.lock conn.in_transaction +# Forget a transaction whose session is already gone. Nothing can be sent to +# end it, and leaving the flags set makes checkconn refuse to reconnect. +function clear_transaction_state!(conn::Connection) + @lock conn.lock begin + conn.in_transaction = false + conn.transaction_depth = 0 + end + return +end + """ Postgres.commit(conn) @@ -971,7 +1012,10 @@ function transaction(f::F, conn::Connection) where {F} commit(conn) return result catch - rollback(conn) + # only roll back if the transaction is still open: a failed COMMIT has + # already ended it, and rolling back then would throw "no transaction + # in progress" from this catch and destroy the server's error + in_transaction(conn) && rollback(conn) rethrow() end end @@ -983,7 +1027,7 @@ function DBInterface.transaction(f::F, conn::Connection) where {F} commit(conn) return result catch - rollback(conn) + in_transaction(conn) && rollback(conn) rethrow() end end @@ -1004,7 +1048,7 @@ macro transaction(conn, expr) success = true result catch - !success && rollback($(esc(conn))) + !success && in_transaction($(esc(conn))) && rollback($(esc(conn))) rethrow() end end diff --git a/src/connection_string.jl b/src/connection_string.jl index 16cc444..e06a120 100644 --- a/src/connection_string.jl +++ b/src/connection_string.jl @@ -139,7 +139,9 @@ function params_from_values(values::Dict{String, String}) dbname=dbname, application_name=get(merged, "application_name", nothing), connect_timeout=parse_optional_int(get(merged, "connect_timeout", nothing), "connect_timeout"), - sslmode=haskey(merged, "sslmode") ? lowercase(merged["sslmode"]) : nothing, + # an empty value means unset, as it does for the numeric and boolean + # parameters (an unexpanded ${PGSSLMODE} must not become an invalid mode) + sslmode=(haskey(merged, "sslmode") && !isempty(merged["sslmode"])) ? lowercase(merged["sslmode"]) : nothing, sslrootcert=get(merged, "sslrootcert", nothing), sslcert=get(merged, "sslcert", nothing), sslkey=get(merged, "sslkey", nothing), diff --git a/src/execute.jl b/src/execute.jl index a82b3b0..86698c3 100644 --- a/src/execute.jl +++ b/src/execute.jl @@ -172,16 +172,23 @@ end # server-side transaction is already gone with the session. function finish_cursor_transaction!(conn::Connection) if !isopen(conn) - @lock conn.lock begin - conn.in_transaction = false - conn.transaction_depth = 0 - end + clear_transaction_state!(conn) return end in_transaction(conn) && commit(conn) return end +# same, for the failure path: roll back rather than commit +function abort_cursor_transaction!(conn::Connection) + if !isopen(conn) + clear_transaction_state!(conn) + return + end + in_transaction(conn) && rollback(conn) + return +end + function DBInterface.close!(cursor::Cursor) owns_transaction = cursor.owns_transaction closed_cleanly = false @@ -519,7 +526,16 @@ function cursor(conn::Connection, sql::AbstractString, params=nothing; fetchsize return cursor(stmt, params; fetchsize=fetchsize, owns_transaction=owns_transaction) catch # don't leave the transaction we started dangling on a failed cursor - owns_transaction && isopen(conn) && in_transaction(conn) && rollback(conn) + # don't leave the transaction we started dangling on a failed cursor; + # if the connection died, clear the state directly (a ROLLBACK can't be + # delivered, and leaving it set would block reconnect forever) + if owns_transaction + try + abort_cursor_transaction!(conn) + catch + # already unwinding; don't mask the original error + end + end rethrow() end end diff --git a/test/runtests.jl b/test/runtests.jl index 4257e8b..dc3da4c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -540,6 +540,14 @@ end @test Postgres.parse_dsn("host=h").port == 5432 @test Postgres.parse_dsn(nothing).port == 5432 end + # ... and the same for the boolean and TLS-mode parameters (quoted so + # the empty value can't swallow the next key: a bare "reconnect=" takes + # the following token as its value, as libpq does) + @test !Postgres.parse_dsn("host=h reconnect='' debug=''").reconnect + @test !Postgres.parse_dsn("host=h reconnect='' debug=''").debug + withenv("PGSSLMODE" => "") do + @test Postgres.parse_dsn("host=h").sslmode === nothing + end withenv( "PGHOST" => "envhost", @@ -636,6 +644,28 @@ end @test_throws Postgres.PostgresInterfaceError Postgres.escape_identifier("a\0b") @test_throws Postgres.PostgresInterfaceError Postgres.escape_literal("a\0b") + # severity must come from the non-localized 'V' field when the server + # sends it: 'S' is translated, so comparing it to "FATAL" would depend + # on the server's lc_messages + let socket = IOBuffer(Vector{UInt8}(vcat( + UInt8('S'), Vector{UInt8}("SCHWERWIEGEND"), 0x00, + UInt8('V'), Vector{UInt8}("FATAL"), 0x00, + UInt8('C'), Vector{UInt8}("57P01"), 0x00, + UInt8('M'), Vector{UInt8}("terminating connection"), 0x00, + 0x00))) + err = Postgres.API.errorResponse(bytesavailable(socket), socket, false) + @test err.severity == "FATAL" + @test err.code == "57P01" + end + # without 'V' the localized 'S' is still reported + let socket = IOBuffer(Vector{UInt8}(vcat( + UInt8('S'), Vector{UInt8}("ERROR"), 0x00, + UInt8('C'), Vector{UInt8}("42601"), 0x00, + 0x00))) + err = Postgres.API.errorResponse(bytesavailable(socket), socket, false) + @test err.severity == "ERROR" + end + @test Postgres.API.parse_value(1184, "2024-02-13 05:28:17+02", registry) == DateTime(2024, 2, 13, 3, 28, 17) @test Postgres.API.parse_value(1184, "2024-02-13 05:28:17+02:30", registry) == DateTime(2024, 2, 13, 2, 58, 17) @test Postgres.API.parse_value(1184, "2024-02-13 05:28:17Z", registry) == DateTime(2024, 2, 13, 5, 28, 17) @@ -821,7 +851,8 @@ end DBInterface.execute(connp, "SELECT pg_terminate_backend($(conn_victim.pid))") err = fetch(victim_task) @test err isa Postgres.API.Error - @test occursin("terminat", err.message) + # 57P01: admin_shutdown — the code, not the localized message + @test err.code == "57P01" @test !isopen(conn_victim.socket) DBInterface.close!(conn_victim) @@ -968,6 +999,44 @@ end @test_throws Postgres.PostgresInterfaceError Postgres.commit(conn) @test_throws Postgres.PostgresInterfaceError Postgres.rollback(conn) + # A COMMIT that fails server-side (deferred constraint) must + # surface the server's error with its SQLSTATE — a retry + # loop keys on that — and must not leave the transaction + # open on the client after the server has ended it. + DBInterface.execute(conn, "DROP TABLE IF EXISTS deferred_child") + DBInterface.execute(conn, "DROP TABLE IF EXISTS deferred_parent") + DBInterface.execute(conn, "CREATE TABLE deferred_parent (id int PRIMARY KEY)") + DBInterface.execute(conn, """ + CREATE TABLE deferred_child ( + id int, + parent_id int REFERENCES deferred_parent(id) DEFERRABLE INITIALLY DEFERRED + ) + """) + for wrapper in (:plain, :helper, :macro) + err = try + if wrapper === :plain + Postgres.start_transaction(conn) + DBInterface.execute(conn, "INSERT INTO deferred_child VALUES (1, 999)") + Postgres.commit(conn) + elseif wrapper === :helper + Postgres.transaction(conn) do tx + DBInterface.execute(tx, "INSERT INTO deferred_child VALUES (1, 999)") + end + else + Postgres.@transaction conn begin + DBInterface.execute(conn, "INSERT INTO deferred_child VALUES (1, 999)") + end + end + nothing + catch e + e + end + @test err isa Postgres.API.Error + @test err.code == "23503" + @test !Postgres.in_transaction(conn) + @test isempty(Tables.rowtable(DBInterface.execute(conn, "SELECT * FROM deferred_child"))) + end + # COMMIT/ROLLBACK end the transaction server-side even when # they fail, so client state must not be left behind fail_conn = DBInterface.connect(Postgres.Connection, cfg.host, cfg.user, cfg.password; dbname=cfg.dbname, port=cfg.port, reconnect=true) @@ -1187,6 +1256,34 @@ end return rows[1].a end @test pooled_result == 1 + + # A connection returned to the pool mid-transaction must not + # hand that transaction to the next borrower: their BEGIN + # would become a SAVEPOINT and their commit would be lost. + Postgres.with_connection(pool) do pooled_conn + DBInterface.execute(pooled_conn, "DROP TABLE IF EXISTS pool_tx_test") + DBInterface.execute(pooled_conn, "CREATE TABLE pool_tx_test (id int)") + end + try + Postgres.with_connection(pool) do pooled_conn + Postgres.start_transaction(pooled_conn) + DBInterface.execute(pooled_conn, "INSERT INTO pool_tx_test VALUES (1)") + error("abandon the block mid-transaction") + end + catch + # the caller's error propagates; the pool must still be clean + end + Postgres.with_connection(pool) do pooled_conn + @test !Postgres.in_transaction(pooled_conn) + Postgres.transaction(pooled_conn) do tx + DBInterface.execute(tx, "INSERT INTO pool_tx_test VALUES (2)") + end + end + # the abandoned insert rolled back; the committed one landed + Postgres.with_connection(pool) do pooled_conn + ids = [row.id for row in Tables.rowtable(DBInterface.execute(pooled_conn, "SELECT id FROM pool_tx_test ORDER BY id"))] + @test ids == [2] + end DBInterface.close!(pool) @test !isopen(conn_a) end From 0da51f2ca6415d5464505619a9f97b57002fb134 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 6 Aug 2026 03:43:41 -0600 Subject: [PATCH 15/23] Fix timestamp ranges, "char" zero values, and pin DateStyle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by an adversarial sweep against a live server; all pre-existing. - tsrange/tstzrange were completely undecodable. PostgreSQL quotes any range bound containing whitespace, so every timestamp bound arrives quoted, and the quotes were passed straight to the element parser: selecting any tsrange threw "Month: 2530 out of range". Bounds are now unquoted (with their backslash escapes) first. The registry advertises these types, so they were advertised-but-broken. - a "char" column holding the zero value renders as an empty string, and the parser indexed position 1 unconditionally — so ordinary catalog queries like SELECT attidentity FROM pg_attribute threw BoundsError. Both the OID path and the typed-struct lift now yield '\0'. - the session now pins DateStyle=ISO,MDY and IntervalStyle=postgres. The text parsers only understand those formats; against a server or role configured otherwise every date and timestamp failed with an error that pointed nowhere near the cause. - infinity/-infinity timestamps and dates report that they can't be represented, matching how numeric NaN/Infinity is handled, instead of "invalid postgres timestamp". - a NotificationResponse shorter than its 4-byte pid no longer reads into the following message. Co-Authored-By: Claude Fable 5 --- src/api/API.jl | 12 +++++++++++- src/api/types.jl | 42 +++++++++++++++++++++++++++++++++++++++--- test/runtests.jl | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 4 deletions(-) diff --git a/src/api/API.jl b/src/api/API.jl index 7b1a114..6f49c04 100644 --- a/src/api/API.jl +++ b/src/api/API.jl @@ -178,6 +178,9 @@ function noticeResponse(len, socket) end function notificationResponse(len, socket) + # the body is at least the 4-byte pid; a shorter one would make the + # channel/payload read consume the next message + len < 4 && throw(Error("truncated NotificationResponse from server")) pid = ntoh(read(socket, Int32)) buf = read(socket, len - 4) i = 1 @@ -278,7 +281,14 @@ function writestartupmessage( application_name::Union{Nothing, String}, statement_timeout::Union{Nothing, Int}, )::Nothing - timeout_options = statement_timeout === nothing ? nothing : string("-c statement_timeout=", statement_timeout) + # The text-format date/time parsers only understand ISO dates and + # postgres-style intervals, so pin them for the session: a server or role + # configured with a different DateStyle/IntervalStyle would otherwise send + # values that decode into silently wrong dates or fail with an error that + # points nowhere near the cause. + timeout_options = statement_timeout === nothing ? + "-c DateStyle=ISO,MDY -c IntervalStyle=postgres" : + string("-c DateStyle=ISO,MDY -c IntervalStyle=postgres -c statement_timeout=", statement_timeout) len = 8 + msgsizeof(("user", user)) + msgsizeof(("database", dbname)) + 1 application_name !== nothing && (len += msgsizeof(("application_name", application_name))) timeout_options !== nothing && (len += msgsizeof(("options", timeout_options))) diff --git a/src/api/types.jl b/src/api/types.jl index b10d029..a17e8dc 100644 --- a/src/api/types.jl +++ b/src/api/types.jl @@ -305,6 +305,7 @@ end function pg_parse_date(s::AbstractString)::Date c = codeunits(s) + _check_temporal_special(s, "date") length(c) >= 10 || throw(ArgumentError("invalid postgres date")) return _pg_date_at(c, 1) end @@ -316,8 +317,17 @@ function pg_parse_time(s::AbstractString)::Time return Time(h, mi, se, ms) end +@noinline _reject_temporal_special(s::AbstractString, what::String) = + throw(PostgresInterfaceError("postgres $what value \"$s\" cannot be represented as a Julia $(what == "date" ? "Date" : "DateTime")")) + +@inline function _check_temporal_special(s::AbstractString, what::String) + (s == "infinity" || s == "-infinity") && _reject_temporal_special(s, what) + return +end + function pg_parse_datetime(s::AbstractString)::DateTime c = codeunits(s) + _check_temporal_special(s, "timestamp") length(c) >= 19 || throw(ArgumentError("invalid postgres timestamp")) d = _pg_date_at(c, 1) h, mi, se, ms = _pg_hms_at(c, 12) @@ -507,9 +517,34 @@ function split_range_values(val::String) return left, right end +# A range bound is quoted whenever it contains whitespace, a comma, a quote, a +# backslash or a bracket — which every timestamp bound does. The quotes and +# their backslash escapes have to come off before the element parser sees it. +function unquote_range_bound(token::String) + cu = codeunits(token) + (length(cu) >= 2 && cu[1] == UInt8('"') && cu[end] == UInt8('"')) || return token + out = IOBuffer() + i = 2 + last = length(cu) - 1 + while i <= last + c = cu[i] + if c == UInt8('\\') && i < last + i += 1 + write(out, cu[i]) + elseif c == UInt8('"') && i < last && cu[i + 1] == UInt8('"') + write(out, UInt8('"')) + i += 1 + else + write(out, c) + end + i += 1 + end + return String(take!(out)) +end + function parse_range_value(token::String, typeId::Int, registry::Dict{Int, TypeInfo}) token == "" && return missing - return parse_value(typeId, token, registry) + return parse_value(typeId, unquote_range_bound(token), registry) end # construct over the standard range element types explicitly: PostgresRange{T} @@ -708,7 +743,8 @@ function parse_value(typeId::Int, val::String, registry::Dict{Int, TypeInfo}) end return val == "t" elseif T == Char - return val[1] + # the "char" type renders its zero value as an empty string + return isempty(val) ? '\0' : val[1] elseif T == DateTime if typeId == 1184 return parse_timestamptz(val) @@ -780,7 +816,7 @@ end StructUtils.lift(::AbstractPostgresStyle, ::Type{Int8}, s::String) = Parsers.parse(Int8, s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Bool}, s::String) = (s == "t" || s == "1"), nothing -StructUtils.lift(::AbstractPostgresStyle, ::Type{Char}, s::String) = s[1], nothing +StructUtils.lift(::AbstractPostgresStyle, ::Type{Char}, s::String) = (isempty(s) ? '\0' : s[1]), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Int16}, s::String) = Parsers.parse(Int16, s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Int32}, s::String) = Parsers.parse(Int32, s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Int64}, s::String) = Parsers.parse(Int64, s), nothing diff --git a/test/runtests.jl b/test/runtests.jl index dc3da4c..30d4107 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -678,6 +678,25 @@ end @test_throws ArgumentError Postgres.API.decode_bytea(raw"\xabc") @test_throws ArgumentError Postgres.API.decode_bytea(raw"\xzz") + # timestamp range bounds are quoted on the wire; without unquoting them + # every tsrange/tstzrange value fails to decode + ts_range = Postgres.API.parse_range("[\"2020-01-01 00:00:00\",\"2020-01-02 00:00:00\")", 1114, registry) + @test ts_range.lower == DateTime(2020, 1, 1) + @test ts_range.upper == DateTime(2020, 1, 2) + @test ts_range.lower_inclusive + @test !ts_range.upper_inclusive + @test Postgres.API.unquote_range_bound("\"a\\\"b\"") == "a\"b" + @test Postgres.API.unquote_range_bound("plain") == "plain" + + # the "char" type renders its zero value as an empty string + @test Postgres.API.parse_value(18, "", registry) == '\0' + @test Postgres.API.parse_value(18, "Z", registry) == 'Z' + + # infinite timestamps/dates can't be represented and must say so + @test_throws Postgres.PostgresInterfaceError Postgres.API.pg_parse_datetime("infinity") + @test_throws Postgres.PostgresInterfaceError Postgres.API.pg_parse_datetime("-infinity") + @test_throws Postgres.PostgresInterfaceError Postgres.API.pg_parse_date("infinity") + range = Postgres.API.parse_range("[1,5)", 23, registry) @test range == Postgres.PostgresRange{Int32}(1, 5, true, false, false) unbounded = Postgres.API.parse_range("(,5]", 23, registry) @@ -1491,6 +1510,20 @@ end @test Postgres.API.cancel_request(cfg.host, cfg.port, Int32(1), Int32(1), false, "disable") end + @testset "Timestamp Ranges And char" begin + ts_row = only(Tables.rowtable(DBInterface.execute(conn, "SELECT '[2020-01-01 00:00:00,2020-01-02 00:00:00)'::tsrange AS r"))) + @test ts_row.r.lower == DateTime(2020, 1, 1) + @test ts_row.r.upper == DateTime(2020, 1, 2) + # "char" columns holding the zero value appear throughout + # the system catalogs + cat_rows = Tables.rowtable(DBInterface.execute(conn, "SELECT attidentity FROM pg_attribute LIMIT 5")) + @test length(cat_rows) == 5 + # the session pins ISO dates and postgres intervals, so a + # server default of something else can't break parsing + @test Postgres.get_server_parameter(conn, "DateStyle") == "ISO, MDY" + @test Postgres.get_server_parameter(conn, "IntervalStyle") == "postgres" + end + @testset "Interval Types" begin interval_row = only(Tables.rowtable(DBInterface.execute(conn, "SELECT '1 day'::interval AS interval_col"))) @test interval_row.interval_col == Dates.Day(1) From c1ae7ba152a60ba6b2740a4f3e791d4b44043b72 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 6 Aug 2026 03:46:59 -0600 Subject: [PATCH 16/23] Align session date formats without new startup parameters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit pinned DateStyle/IntervalStyle via the startup `options` parameter, which is a deployment regression: connections that set no statement_timeout previously sent no `options` at all, and poolers such as pgbouncer reject `options` unless it is explicitly allowlisted — so working setups would start failing at connect. The server already reports both settings in its startup ParameterStatus, so the driver now corrects them with a SET only when they actually differ from what the text parsers require. A default server pays nothing and its startup packet is unchanged; a server or role configured otherwise is fixed up on the one connection that needs it. Covered by a test that sets the database default to 'German, DMY' / 'sql_standard' and asserts a fresh connection still decodes timestamps and intervals correctly. Co-Authored-By: Claude Fable 5 --- src/api/API.jl | 30 ++++++++++++++++++++++-------- test/runtests.jl | 27 +++++++++++++++++++++++---- 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/src/api/API.jl b/src/api/API.jl index 6f49c04..e08a72e 100644 --- a/src/api/API.jl +++ b/src/api/API.jl @@ -281,14 +281,7 @@ function writestartupmessage( application_name::Union{Nothing, String}, statement_timeout::Union{Nothing, Int}, )::Nothing - # The text-format date/time parsers only understand ISO dates and - # postgres-style intervals, so pin them for the session: a server or role - # configured with a different DateStyle/IntervalStyle would otherwise send - # values that decode into silently wrong dates or fail with an error that - # points nowhere near the cause. - timeout_options = statement_timeout === nothing ? - "-c DateStyle=ISO,MDY -c IntervalStyle=postgres" : - string("-c DateStyle=ISO,MDY -c IntervalStyle=postgres -c statement_timeout=", statement_timeout) + timeout_options = statement_timeout === nothing ? nothing : string("-c statement_timeout=", statement_timeout) len = 8 + msgsizeof(("user", user)) + msgsizeof(("database", dbname)) + 1 application_name !== nothing && (len += msgsizeof(("application_name", application_name))) timeout_options !== nothing && (len += msgsizeof(("options", timeout_options))) @@ -751,6 +744,12 @@ function connect(host::String, port::Integer, dbname::String, user::String, @nos close_and_throw(socket, Error("server version too old")) end pid, skey, server_params = waitfor(socket, debug, 'K', 'Z') + # socket-union isa split so the call resolves under --trim, as above + if socket isa Reseau.TCP.Conn + align_session_formats!(socket::Reseau.TCP.Conn, server_params, debug) + else + align_session_formats!(socket::Reseau.TLS.Conn, server_params, debug) + end return socket, pid, skey, server_params catch close(socket) @@ -758,6 +757,21 @@ function connect(host::String, port::Integer, dbname::String, user::String, @nos end end +# The text-format parsers only understand ISO dates and postgres-style +# intervals; against any other setting values decode into silently wrong dates +# or fail with an error that points nowhere near the cause. The server reports +# both in its startup ParameterStatus, so correct them only when they actually +# differ: a default server pays nothing, and no extra startup parameters are +# sent (poolers such as pgbouncer reject `options` unless it is allowlisted). +function align_session_formats!(socket, server_params::Dict{String, String}, debug::Bool) + datestyle = get(server_params, "DateStyle", "") + startswith(datestyle, "ISO") || exec(PostgresStyle(), socket, "SET DateStyle = 'ISO, MDY'", debug) + intervalstyle = get(server_params, "IntervalStyle", "") + (isempty(intervalstyle) || intervalstyle == "postgres") || + exec(PostgresStyle(), socket, "SET IntervalStyle = 'postgres'", debug) + return +end + function prepare(socket, sql::String, debug::Bool; name::Union{Nothing, String}=nothing) stmtname = name === nothing ? randstring(Random.RandomDevice(), 36) : String(name) writemessages(socket, debug, ('P', stmtname, sql, Int16(0)), ('S',)) diff --git a/test/runtests.jl b/test/runtests.jl index 30d4107..aa6bdfd 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1518,10 +1518,29 @@ end # the system catalogs cat_rows = Tables.rowtable(DBInterface.execute(conn, "SELECT attidentity FROM pg_attribute LIMIT 5")) @test length(cat_rows) == 5 - # the session pins ISO dates and postgres intervals, so a - # server default of something else can't break parsing - @test Postgres.get_server_parameter(conn, "DateStyle") == "ISO, MDY" - @test Postgres.get_server_parameter(conn, "IntervalStyle") == "postgres" + # the session uses ISO dates and postgres intervals, which + # the text parsers require + style_row = only(Tables.rowtable(DBInterface.execute(conn, "SELECT current_setting('DateStyle') AS ds, current_setting('IntervalStyle') AS is"))) + @test startswith(style_row.ds, "ISO") + @test style_row.is == "postgres" + + # a server whose default is not ISO is corrected at connect + # rather than silently producing unparseable dates + DBInterface.execute(conn, "ALTER DATABASE $(cfg.dbname) SET DateStyle = 'German, DMY'") + DBInterface.execute(conn, "ALTER DATABASE $(cfg.dbname) SET IntervalStyle = 'sql_standard'") + try + german_conn = DBInterface.connect(Postgres.Connection, cfg.host, cfg.user, cfg.password; dbname=cfg.dbname, port=cfg.port) + try + row = only(Tables.rowtable(DBInterface.execute(german_conn, "SELECT '2020-03-04 05:06:07'::timestamp AS t, '1 day'::interval AS i"))) + @test row.t == DateTime(2020, 3, 4, 5, 6, 7) + @test row.i == Dates.Day(1) + finally + DBInterface.close!(german_conn) + end + finally + DBInterface.execute(conn, "ALTER DATABASE $(cfg.dbname) RESET DateStyle") + DBInterface.execute(conn, "ALTER DATABASE $(cfg.dbname) RESET IntervalStyle") + end end @testset "Interval Types" begin From eab05bdd5f578457d83f98d131f409254cc2f93f Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 6 Aug 2026 04:07:57 -0600 Subject: [PATCH 17/23] Fix interval-style gap, raw-SQL transactions in the pool, empty sslmode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - align_session_formats! treated an unreported IntervalStyle as already correct while treating an unreported DateStyle as needing a fix. A pooler that doesn't forward IntervalStyle (pgbouncer before 1.21, or without it in track_extra_parameters) against a server set to sql_standard therefore left every interval decoding to zero, silently — in exactly the deployment the ParameterStatus approach was chosen to support. Both now correct an absent value, and the recorded server parameters are updated to match. - the pool reset only saw transactions opened through start_transaction, so a raw DBInterface.execute(conn, "BEGIN") sailed through: the next borrower inherited the open transaction and its commit committed the previous borrower's abandoned writes. The connection now also tracks the server's own ReadyForQuery transaction status, which sees transactions however they were opened, and the pool consults both. - empty sslmode is no longer treated as unset. libpq rejects it, and the previous commit's "empty means unset" reasoning is wrong here specifically: an unexpanded ${PGSSLMODE} intended as verify-full would have silently become the unauthenticated default. It fails loudly again. - a failing rollback in the transaction wrappers no longer replaces the error that caused it, which was still losing the SQLSTATE when the body failed because the session died. - @transaction binds its connection expression once: `@transaction acquire(pool) ...` previously acquired a different connection for the BEGIN, the COMMIT and the ROLLBACK, and leaked pool permits. Co-Authored-By: Claude Fable 5 --- src/Postgres.jl | 42 +++++++++++++++++++++++++++++----------- src/api/API.jl | 38 ++++++++++++++++++++++++++++-------- src/connection_string.jl | 7 ++++--- src/execute.jl | 2 ++ test/runtests.jl | 28 ++++++++++++++++++++++++++- 5 files changed, 94 insertions(+), 23 deletions(-) diff --git a/src/Postgres.jl b/src/Postgres.jl index a9cc274..1b041bb 100644 --- a/src/Postgres.jl +++ b/src/Postgres.jl @@ -103,6 +103,9 @@ mutable struct Connection{T, S <: API.AbstractPostgresStyle} <: DBInterface.Conn in_transaction::Bool # track transaction state transaction_depth::Int # track nested transactions (SAVEPOINTs) generation::Int # increment on reconnect to invalidate statements + # the server's own ReadyForQuery transaction status: unlike in_transaction + # it also sees a transaction opened by raw SQL (`execute(conn, "BEGIN")`) + server_in_transaction::Bool function Connection(; host::AbstractString="", user::AbstractString="", password::Union{AbstractString, Nothing}=nothing, dbname::AbstractString="", port::Integer=5432, debug::Bool=false, reconnect::Bool=false, application_name::Union{AbstractString, Nothing}=nothing, connect_timeout::Union{Integer, Nothing}=nothing, sslmode::Union{AbstractString, Nothing}=nothing, sslrootcert::Union{AbstractString, Nothing}=nothing, sslcert::Union{AbstractString, Nothing}=nothing, sslkey::Union{AbstractString, Nothing}=nothing, sslcapath::Union{AbstractString, Nothing}=nothing, sslservername::Union{AbstractString, Nothing}=nothing, statement_timeout::Union{Integer, Nothing}=nothing, statement_cache_maxsize::Integer=100, style::API.AbstractPostgresStyle=PostgresStyle()) host = String(host) @@ -122,7 +125,7 @@ mutable struct Connection{T, S <: API.AbstractPostgresStyle} <: DBInterface.Conn maxsize = max(0, Int(statement_cache_maxsize)) socket, pid, skey, server_params = API.connect(host, port, dbname, user, password, debug, app_name, timeout, sslmode_val, sslrootcert_val, sslcert_val, sslkey_val, sslcapath_val, sslservername_val, statement_timeout_val) registry = Dict(API.DEFAULT_TYPE_REGISTRY) - return new{Statement{typeof(style)}, typeof(style)}(ReentrantLock(), socket, host, user, password, dbname, port, app_name, timeout, sslmode_val, sslrootcert_val, sslcert_val, sslkey_val, sslcapath_val, sslservername_val, statement_timeout_val, pid, skey, Dict{String, Statement{typeof(style)}}(), maxsize, 0, server_params, registry, false, reconnect, debug, style, false, 0, 1) + return new{Statement{typeof(style)}, typeof(style)}(ReentrantLock(), socket, host, user, password, dbname, port, app_name, timeout, sslmode_val, sslrootcert_val, sslcert_val, sslkey_val, sslcapath_val, sslservername_val, statement_timeout_val, pid, skey, Dict{String, Statement{typeof(style)}}(), maxsize, 0, server_params, registry, false, reconnect, debug, style, false, 0, 1, false) end end @@ -797,7 +800,9 @@ end # be silently discarded when the connection is later reset. Roll it back; if # that can't be done, drop the connection instead of handing it on. function reset_pooled_connection!(conn::Connection) - in_transaction(conn) || return true + # the client flag misses a transaction opened by raw SQL, so trust the + # server's ReadyForQuery status too + (in_transaction(conn) || (@lock conn.lock conn.server_in_transaction)) || return true try @lock conn.lock begin checkconn(conn) @@ -876,7 +881,8 @@ include("execute.jl") # does not exist"). Also one network round trip instead of three. Callers # must hold conn.lock. function execute_simple(conn::Connection, sql::String) - API.exec(conn.style, conn.socket, sql, conn.debug) + status = API.exec(conn.style, conn.socket, sql, conn.debug) + conn.server_in_transaction = API.in_transaction_status(status) return conn end @@ -995,6 +1001,19 @@ function rollback(conn::Connection) return conn end +# Roll back after the body (or the COMMIT) failed. Only acts if a transaction +# is still open — a failed COMMIT has already ended it — and never lets its own +# failure replace the original error, which is what the caller needs to see. +function rollback_for_failed_transaction!(conn::Connection) + in_transaction(conn) || return + try + rollback(conn) + catch + # the connection is already failing; the original error is the useful one + end + return +end + """ Postgres.transaction(f, conn) @@ -1012,10 +1031,7 @@ function transaction(f::F, conn::Connection) where {F} commit(conn) return result catch - # only roll back if the transaction is still open: a failed COMMIT has - # already ended it, and rolling back then would throw "no transaction - # in progress" from this catch and destroy the server's error - in_transaction(conn) && rollback(conn) + rollback_for_failed_transaction!(conn) rethrow() end end @@ -1027,7 +1043,7 @@ function DBInterface.transaction(f::F, conn::Connection) where {F} commit(conn) return result catch - in_transaction(conn) && rollback(conn) + rollback_for_failed_transaction!(conn) rethrow() end end @@ -1040,15 +1056,19 @@ throws. Evaluates to `expr`'s value. """ macro transaction(conn, expr) quote + # bind once: the connection expression may have side effects + # (`@transaction acquire(pool) ...` would otherwise take a different + # connection for the BEGIN, the COMMIT and the ROLLBACK) + local c = $(esc(conn)) local success = false - start_transaction($(esc(conn))) + start_transaction(c) try result = $(esc(expr)) - commit($(esc(conn))) + commit(c) success = true result catch - !success && in_transaction($(esc(conn))) && rollback($(esc(conn))) + !success && rollback_for_failed_transaction!(c) rethrow() end end diff --git a/src/api/API.jl b/src/api/API.jl index e08a72e..93a3cd9 100644 --- a/src/api/API.jl +++ b/src/api/API.jl @@ -764,11 +764,17 @@ end # differ: a default server pays nothing, and no extra startup parameters are # sent (poolers such as pgbouncer reject `options` unless it is allowlisted). function align_session_formats!(socket, server_params::Dict{String, String}, debug::Bool) - datestyle = get(server_params, "DateStyle", "") - startswith(datestyle, "ISO") || exec(PostgresStyle(), socket, "SET DateStyle = 'ISO, MDY'", debug) - intervalstyle = get(server_params, "IntervalStyle", "") - (isempty(intervalstyle) || intervalstyle == "postgres") || + # A parameter the server didn't report (a pooler may not forward it) must + # be treated as unknown, i.e. corrected — assuming it is already right is + # how intervals silently decode to zero. + if !startswith(get(server_params, "DateStyle", ""), "ISO") + exec(PostgresStyle(), socket, "SET DateStyle = 'ISO, MDY'", debug) + server_params["DateStyle"] = "ISO, MDY" + end + if get(server_params, "IntervalStyle", "") != "postgres" exec(PostgresStyle(), socket, "SET IntervalStyle = 'postgres'", debug) + server_params["IntervalStyle"] = "postgres" + end return end @@ -881,8 +887,23 @@ struct Exec{S <: AbstractPostgresStyle} debug::Bool command_tag::Base.RefValue{Union{Nothing, String}} rows_affected::Base.RefValue{Union{Nothing, Int}} + # ReadyForQuery's transaction status: 'I' idle, 'T' in a transaction, + # 'E' in a failed transaction. The server's own view, which is the only + # thing that knows about a transaction opened by raw SQL. + tx_status::Base.RefValue{UInt8} +end + +# read ReadyForQuery's one-byte transaction status (older/odd servers may send +# an empty body; treat that as unknown-but-idle) +function read_ready_status(socket, len) + len < 1 && (skipbytes!(socket, len); return UInt8('I')) + status = read(socket, UInt8) + skipbytes!(socket, len - 1) + return status end +in_transaction_status(status::UInt8) = status == UInt8('T') || status == UInt8('E') + function commandComplete(len, socket) buf = read(socket, len) isempty(buf) && return "" @@ -913,7 +934,7 @@ function StructUtils.applyeach(::AbstractPostgresStyle, f, e::Exec) # error; keep reading until ready-for-query, thrown below server_error = errorResponse(len, e.socket, e.debug) elseif mt == UInt8('Z') - skipbytes!(e.socket, len) + e.tx_status[] = read_ready_status(e.socket, len) break elseif mt == UInt8('D') nrows += 1 @@ -1000,12 +1021,13 @@ function exec(style::S, socket::ReseauConn, stmtname::String, params::Vector{Uni # bind, then execute, then sync writemessages(socket, debug, ('B', "", stmtname, npformats, nparams, Params(params), Int16(0)), ('E', "", Int32(rowlimit)), ('S',)) waitfor(socket, debug, '2') - return Exec{S}(style, socket, names, typeIds, type_registry, debug, Ref{Union{Nothing, String}}(nothing), Ref{Union{Nothing, Int}}(nothing)) + return Exec{S}(style, socket, names, typeIds, type_registry, debug, Ref{Union{Nothing, String}}(nothing), Ref{Union{Nothing, Int}}(nothing), Ref{UInt8}(UInt8('I'))) end function exec(style::S, socket::ReseauConn, query::String, debug::Bool) where {S <: AbstractPostgresStyle} writemessages(socket, debug, ('Q', query)) server_error = nothing + tx_status = UInt8('I') try while true mt, len = readheader(socket, debug) @@ -1014,7 +1036,7 @@ function exec(style::S, socket::ReseauConn, query::String, debug::Bool) where {S # server error so the connection remains reusable. server_error = errorResponse(len, socket, debug) elseif mt == UInt8('Z') - skipbytes!(socket, len) + tx_status = read_ready_status(socket, len) break elseif mt == UInt8('N') notice_callback(style, noticeResponse(len, socket)) @@ -1034,7 +1056,7 @@ function exec(style::S, socket::ReseauConn, query::String, debug::Bool) where {S rethrow() end server_error === nothing || throw(server_error) - return + return tx_status end exec(socket::ReseauConn, query::String, debug::Bool) = exec(PostgresStyle(), socket, query, debug) diff --git a/src/connection_string.jl b/src/connection_string.jl index e06a120..0ccad2e 100644 --- a/src/connection_string.jl +++ b/src/connection_string.jl @@ -139,9 +139,10 @@ function params_from_values(values::Dict{String, String}) dbname=dbname, application_name=get(merged, "application_name", nothing), connect_timeout=parse_optional_int(get(merged, "connect_timeout", nothing), "connect_timeout"), - # an empty value means unset, as it does for the numeric and boolean - # parameters (an unexpanded ${PGSSLMODE} must not become an invalid mode) - sslmode=(haskey(merged, "sslmode") && !isempty(merged["sslmode"])) ? lowercase(merged["sslmode"]) : nothing, + # deliberately NOT empty-tolerant, matching libpq: an unexpanded + # ${PGSSLMODE} that was meant to be verify-full must fail loudly + # rather than fall back to the unauthenticated default + sslmode=haskey(merged, "sslmode") ? lowercase(merged["sslmode"]) : nothing, sslrootcert=get(merged, "sslrootcert", nothing), sslcert=get(merged, "sslcert", nothing), sslkey=get(merged, "sslkey", nothing), diff --git a/src/execute.jl b/src/execute.jl index 86698c3..b5daaff 100644 --- a/src/execute.jl +++ b/src/execute.jl @@ -456,6 +456,7 @@ function DBInterface.execute(stmt::Statement, params=nothing, ::Type{T}=Any; deb API.exec(style, socket::Reseau.TLS.Conn, stmt.name, stmt.params, stmt.names, stmt.typeIds, stmt.conn.type_registry, debug, 0) end result = T === Any ? makeresult(e) : StructUtils.arraylike(T) ? StructUtils.make(T, e, style) : only(StructUtils.make(Vector{T}, e, style)) + stmt.conn.server_in_transaction = API.in_transaction_status(e.tx_status[]) end log_enabled && API.query_logger(style, :execute, (sql=stmt.sql, params=params, duration_ns=time_ns() - start_ns, success=true)) return result @@ -485,6 +486,7 @@ function DBInterface.execute(conn::Connection, sql::AbstractString, params=nothi API.exec(style, socket::Reseau.TLS.Conn, stmtname, params_vec, names, types, conn.type_registry, debug, 0) end result = T === Any ? makeresult(e) : StructUtils.arraylike(T) ? StructUtils.make(T, e, style) : only(StructUtils.make(Vector{T}, e, style)) + conn.server_in_transaction = API.in_transaction_status(e.tx_status[]) end log_enabled && API.query_logger(style, :execute, (sql=sql_str, params=params, duration_ns=time_ns() - start_ns, success=true)) return result diff --git a/test/runtests.jl b/test/runtests.jl index aa6bdfd..bdb5bf7 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -545,8 +545,11 @@ end # the following token as its value, as libpq does) @test !Postgres.parse_dsn("host=h reconnect='' debug=''").reconnect @test !Postgres.parse_dsn("host=h reconnect='' debug=''").debug + # sslmode is deliberately NOT empty-tolerant (libpq rejects it too): + # an unexpanded ${PGSSLMODE} meant to be verify-full must fail loudly + # rather than fall back to the unauthenticated default withenv("PGSSLMODE" => "") do - @test Postgres.parse_dsn("host=h").sslmode === nothing + @test Postgres.parse_dsn("host=h").sslmode == "" end withenv( @@ -1090,6 +1093,14 @@ end DBInterface.execute(conn, "CREATE TABLE macro_test (id SERIAL PRIMARY KEY, value INTEGER)") DBInterface.execute(conn, "INSERT INTO macro_test (value) VALUES (1)") + # the macro must evaluate its connection expression once + conn_evals = Ref(0) + eval_conn = () -> (conn_evals[] += 1; conn) + Postgres.@transaction eval_conn() begin + DBInterface.execute(conn, "SELECT 1") + end + @test conn_evals[] == 1 + result = Postgres.@transaction conn begin DBInterface.execute(conn, "INSERT INTO macro_test (value) VALUES (2)") length(Tables.rowtable(DBInterface.execute(conn, "SELECT * FROM macro_test"))) @@ -1298,6 +1309,21 @@ end DBInterface.execute(tx, "INSERT INTO pool_tx_test VALUES (2)") end end + + # ... including a transaction opened by raw SQL, which the + # client-side flag never sees + try + Postgres.with_connection(pool) do pooled_conn + DBInterface.execute(pooled_conn, "BEGIN") + DBInterface.execute(pooled_conn, "INSERT INTO pool_tx_test VALUES (99)") + error("abandon a raw-SQL transaction") + end + catch + end + Postgres.with_connection(pool) do pooled_conn + rows = Tables.rowtable(DBInterface.execute(pooled_conn, "SELECT id FROM pool_tx_test ORDER BY id")) + @test [row.id for row in rows] == [2] + end # the abandoned insert rolled back; the committed one landed Postgres.with_connection(pool) do pooled_conn ids = [row.id for row in Tables.rowtable(DBInterface.execute(pooled_conn, "SELECT id FROM pool_tx_test ORDER BY id"))] From 826babf3cce009b392c2d1fe0a6e2fbd889243f0 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 6 Aug 2026 08:39:39 -0600 Subject: [PATCH 18/23] Fix range registration, BC/wide dates, char escapes, DateStyle order Round-12 review findings, all verified against a live server: - register_range! was advertised-but-broken for element types outside the six builtins: registration succeeded, then every value threw. The parser now binds the element type discovered at registration time. - The DateStyle correction set 'ISO, MDY' unconditionally, silently reinterpreting ambiguous input literals like '01/02/2020' for a DMY-configured database (and hard-erroring on '13/02/2020'). Only the output-format half is corrected now; the configured field order is kept. - statement_timeout moved from the startup `options` parameter to a post-connect SET: pgbouncer rejects unknown startup options outright, so sending it there failed the whole connection. - parse_interval silently returned a zero interval for text it didn't understand (sql_standard, iso_8601, postgres_verbose renderings). Only a genuine "00:00:00" decodes to zero; anything unrecognized now throws, naming the IntervalStyle requirement. - BC dates decoded silently as AD -- a different year numbering with no year zero. They now throw, including the timestamptz rendering where " BC" follows the zone offset, out of sight of the datetime parser. - Years beyond 9999 misparsed into "Month: NNNN out of range" errors; the year field is scanned rather than assumed 4 digits wide. - "char" values for high-bit bytes arrive as backslash-octal escapes ("\377") and decoded to '\\'; they now decode to the byte value, on the OID path, the typed-struct lift, and the array element path alike. "char"[] (oid 1002) was unregistered entirely and returned raw literals. - libpq keywords that are accepted-but-ignored now warn when set to a value that requests a security or connection-selection behavior (channel_binding=require, sslcrl, sslcrldir, requiressl, target_session_attrs, options); silently dropping them left callers believing a protection was in place. - A raw-SQL transaction's tracked server status survived reconnect, triggering a spurious ROLLBACK warning on the fresh session. - Documented the session-format requirement (ISO / postgres), the values with no Julia representation (infinity, BC, numeric NaN), and that a '\0' read from a "char" column cannot be bound back as text. Co-Authored-By: Claude Fable 5 --- docs/src/manual.md | 23 ++++++++ src/Postgres.jl | 9 ++- src/api/API.jl | 35 +++++++++--- src/api/types.jl | 101 +++++++++++++++++++++++++++++---- src/array_parsing.jl | 3 +- src/connection_string.jl | 25 ++++++++- test/runtests.jl | 118 +++++++++++++++++++++++++++++++++++++-- 7 files changed, 286 insertions(+), 28 deletions(-) diff --git a/docs/src/manual.md b/docs/src/manual.md index f7c440e..ee56c66 100644 --- a/docs/src/manual.md +++ b/docs/src/manual.md @@ -259,3 +259,26 @@ row = only(Tables.rowtable(DBInterface.execute(conn, "SELECT 'happy'::mood AS mo ``` Registering composite and range types follows the same pattern. + +### Session Formats + +Text decoding assumes the server renders dates and timestamps in the `ISO` +`DateStyle` output format and intervals in the `postgres` `IntervalStyle`. The +driver checks the server-reported settings at connect time and issues a `SET` +for any that differ, preserving the configured date field order (`MDY`/`DMY`/ +`YMD`) since it decides how ambiguous input literals like `'01/02/2020'` are +read. The alignment is re-applied on reconnect, but not if the session is +changed afterwards: running `SET DateStyle = ...` or `SET IntervalStyle = ...` +mid-session breaks decoding — intervals and unparseable dates raise errors +rather than silently returning wrong values. + +### Values Without A Julia Representation + +A few PostgreSQL values have no faithful Julia equivalent and raise +`Postgres.PostgresInterfaceError` when decoded rather than silently returning +a wrong value: `infinity`/`-infinity` dates and timestamps, dates in the BC +era, and `numeric` `NaN`/`infinity`. `"char"` columns (the 1-byte internal +catalog type) decode to `Char`, including the zero byte (`'\0'`) and high-bit +bytes; note that a `'\0'` read from such a column cannot be bound back as a +text parameter, because PostgreSQL rejects NUL bytes in text — write it with +an explicit cast such as `0::"char"` instead. diff --git a/src/Postgres.jl b/src/Postgres.jl index 1b041bb..f6c3424 100644 --- a/src/Postgres.jl +++ b/src/Postgres.jl @@ -632,7 +632,10 @@ function register_range!(conn::Connection, name::AbstractString; schema::Abstrac oid = Int(rows[1].oid) subtype_oid = Int(rows[1].rngsubtype) subtype_type = API.type_info(conn.type_registry, subtype_oid).julia_type - parser = (val, registry) -> API.parse_range(val, subtype_oid, registry) + # bind the element type here rather than rediscovering it per value: the + # builtin parse_range only knows a fixed set of element types, so a range + # over anything else would register successfully and then fail on every value + parser = (val, registry) -> API.parse_range_of(subtype_type, val, subtype_oid, registry) register_type!(conn, oid, PostgresRange{subtype_type}; parser=parser) return conn end @@ -698,6 +701,9 @@ function checkconn(conn::Connection) empty!(conn.statements) conn.in_transaction = false conn.transaction_depth = 0 + # the server-side transaction died with the old socket; a stale flag + # would trigger a spurious ROLLBACK on the fresh session + conn.server_in_transaction = false conn.generation += 1 conn.server_parameters = server_params @warn "postgres connection was closed; reconnected" @@ -925,6 +931,7 @@ function clear_transaction_state!(conn::Connection) @lock conn.lock begin conn.in_transaction = false conn.transaction_depth = 0 + conn.server_in_transaction = false end return end diff --git a/src/api/API.jl b/src/api/API.jl index 93a3cd9..2cd520b 100644 --- a/src/api/API.jl +++ b/src/api/API.jl @@ -281,10 +281,11 @@ function writestartupmessage( application_name::Union{Nothing, String}, statement_timeout::Union{Nothing, Int}, )::Nothing - timeout_options = statement_timeout === nothing ? nothing : string("-c statement_timeout=", statement_timeout) + # statement_timeout is applied with a SET after connect rather than through + # the startup `options` parameter: poolers (pgbouncer) reject unknown + # startup options outright, so sending it here fails the whole connection. len = 8 + msgsizeof(("user", user)) + msgsizeof(("database", dbname)) + 1 application_name !== nothing && (len += msgsizeof(("application_name", application_name))) - timeout_options !== nothing && (len += msgsizeof(("options", timeout_options))) debug && @info "sending startup message" buf = IOBuffer(Vector{UInt8}(undef, len); write=true) write(buf, hton(Int32(len))) @@ -292,7 +293,6 @@ function writestartupmessage( _write_startup_param(buf, "user", user) _write_startup_param(buf, "database", dbname) application_name !== nothing && _write_startup_param(buf, "application_name", application_name) - timeout_options !== nothing && _write_startup_param(buf, "options", timeout_options) write(buf, UInt8(0)) write(socket, take!(buf)) flush(socket) @@ -746,9 +746,9 @@ function connect(host::String, port::Integer, dbname::String, user::String, @nos pid, skey, server_params = waitfor(socket, debug, 'K', 'Z') # socket-union isa split so the call resolves under --trim, as above if socket isa Reseau.TCP.Conn - align_session_formats!(socket::Reseau.TCP.Conn, server_params, debug) + align_session_formats!(socket::Reseau.TCP.Conn, server_params, debug, statement_timeout_v) else - align_session_formats!(socket::Reseau.TLS.Conn, server_params, debug) + align_session_formats!(socket::Reseau.TLS.Conn, server_params, debug, statement_timeout_v) end return socket, pid, skey, server_params catch @@ -763,13 +763,30 @@ end # both in its startup ParameterStatus, so correct them only when they actually # differ: a default server pays nothing, and no extra startup parameters are # sent (poolers such as pgbouncer reject `options` unless it is allowlisted). -function align_session_formats!(socket, server_params::Dict{String, String}, debug::Bool) +# The field-order half of DateStyle ("MDY"/"DMY"/"YMD") decides how ambiguous +# *input* like '01/02/2020' is read; only the output half has to be ISO. Keep +# whatever order the server was configured with so correcting the output format +# doesn't silently change what the user's literals mean. +function date_order(datestyle::AbstractString) + for part in split(datestyle, ',') + order = uppercase(strip(part)) + (order == "MDY" || order == "DMY" || order == "YMD") && return order + end + return "MDY" +end + +function align_session_formats!(socket, server_params::Dict{String, String}, debug::Bool, @nospecialize(statement_timeout::Union{Int, Nothing})=nothing) + if statement_timeout !== nothing + exec(PostgresStyle(), socket, string("SET statement_timeout = ", statement_timeout::Int), debug) + end # A parameter the server didn't report (a pooler may not forward it) must # be treated as unknown, i.e. corrected — assuming it is already right is # how intervals silently decode to zero. - if !startswith(get(server_params, "DateStyle", ""), "ISO") - exec(PostgresStyle(), socket, "SET DateStyle = 'ISO, MDY'", debug) - server_params["DateStyle"] = "ISO, MDY" + datestyle = get(server_params, "DateStyle", "") + if !startswith(datestyle, "ISO") + wanted = string("ISO, ", date_order(datestyle)) + exec(PostgresStyle(), socket, string("SET DateStyle = '", wanted, "'"), debug) + server_params["DateStyle"] = wanted end if get(server_params, "IntervalStyle", "") != "postgres" exec(PostgresStyle(), socket, "SET IntervalStyle = 'postgres'", debug) diff --git a/src/api/types.jl b/src/api/types.jl index a17e8dc..8e9de93 100644 --- a/src/api/types.jl +++ b/src/api/types.jl @@ -215,6 +215,7 @@ function _populate_default_type_registry!() 1560 => TypeInfo(Bool, nothing), 1000 => TypeInfo(Vector{Bool}, nothing), 1001 => TypeInfo(Vector{Vector{UInt8}}, (val, registry) -> parse_array_by_oid(val, 17, registry)), + 1002 => TypeInfo(Vector{Char}, (val, registry) -> parse_array_by_oid(val, 18, registry)), 1005 => TypeInfo(Vector{Int16}, nothing), 1007 => TypeInfo(Vector{Int32}, nothing), 1016 => TypeInfo(Vector{Int64}, nothing), @@ -276,13 +277,27 @@ end @inline _pg_digit(b::UInt8)::Int = Int(b - UInt8('0')) @inline _pg_isdigit(b::UInt8)::Bool = UInt8('0') <= b <= UInt8('9') -@inline function _pg_date_at(c, o::Int)::Date - y = _pg_digit(c[o]) * 1000 + _pg_digit(c[o+1]) * 100 + _pg_digit(c[o+2]) * 10 + _pg_digit(c[o+3]) - m = _pg_digit(c[o+5]) * 10 + _pg_digit(c[o+6]) - d = _pg_digit(c[o+8]) * 10 + _pg_digit(c[o+9]) - return Date(y, m, d) +# The year is normally 4 digits but PostgreSQL emits more beyond year 9999, so +# scan it rather than assuming a fixed width. Returns the date and the offset +# just past it. +@inline function _pg_date_at_end(c, o::Int) + y = 0 + i = o + n = length(c) + while i <= n && _pg_isdigit(c[i]) + y = y * 10 + _pg_digit(c[i]) + i += 1 + end + (i > n || c[i] != UInt8('-')) && throw(ArgumentError("invalid postgres date")) + i += 1 + m = _pg_digit(c[i]) * 10 + _pg_digit(c[i+1]) + i += 3 + d = _pg_digit(c[i]) * 10 + _pg_digit(c[i+1]) + return Date(y, m, d), i + 2 end +@inline _pg_date_at(c, o::Int)::Date = first(_pg_date_at_end(c, o)) + @inline function _pg_hms_at(c, o::Int) h = _pg_digit(c[o]) * 10 + _pg_digit(c[o+1]) mi = _pg_digit(c[o+3]) * 10 + _pg_digit(c[o+4]) @@ -303,6 +318,20 @@ end return h, mi, se, ms end +# `"char"` output: byte 0 renders as an empty string, bytes with the high bit +# set render as a backslash-octal escape ("\\200".."\\377"), and anything else +# is the raw byte. A backslash byte itself renders as a lone "\\", so only the +# exact 4-byte escape shape is decoded. +function pg_parse_char(s::String) + isempty(s) && return '\0' + c = codeunits(s) + if length(c) == 4 && c[1] == UInt8('\\') && + UInt8('0') <= c[2] <= UInt8('3') && UInt8('0') <= c[3] <= UInt8('7') && UInt8('0') <= c[4] <= UInt8('7') + return Char((_pg_digit(c[2]) << 6) | (_pg_digit(c[3]) << 3) | _pg_digit(c[4])) + end + return s[1] +end + function pg_parse_date(s::AbstractString)::Date c = codeunits(s) _check_temporal_special(s, "date") @@ -322,6 +351,9 @@ end @inline function _check_temporal_special(s::AbstractString, what::String) (s == "infinity" || s == "-infinity") && _reject_temporal_special(s, what) + # BC years are a different numbering than Julia's (proleptic, no year 0), + # so decoding them as AD would silently produce the wrong date + endswith(s, " BC") && throw(PostgresInterfaceError("postgres BC $what value \"$s\" is not supported")) return end @@ -329,8 +361,9 @@ function pg_parse_datetime(s::AbstractString)::DateTime c = codeunits(s) _check_temporal_special(s, "timestamp") length(c) >= 19 || throw(ArgumentError("invalid postgres timestamp")) - d = _pg_date_at(c, 1) - h, mi, se, ms = _pg_hms_at(c, 12) + # the time starts one space past the date, whose year may be wider than 4 + d, after_date = _pg_date_at_end(c, 1) + h, mi, se, ms = _pg_hms_at(c, after_date + 1) return DateTime(Dates.year(d), Dates.month(d), Dates.day(d), h, mi, se, ms) end @@ -346,6 +379,10 @@ end @inline function parse_timestamptz(val::String) lastindex(val) == 0 && throw(ArgumentError("invalid postgres timestamptz")) + # timestamptz output puts " BC" after the zone offset, so the check inside + # pg_parse_datetime (which sees only the offset-stripped prefix) can't + # catch it; check the full value here + _check_temporal_special(val, "timestamp") if val[end] == 'Z' ts = SubString(val, 1, prevind(val, lastindex(val))) return pg_parse_datetime(ts) @@ -454,20 +491,34 @@ function parse_interval_time(token::AbstractString) return periods end +@noinline _reject_interval(val::String) = + throw(PostgresInterfaceError("unrecognized postgres interval \"$val\"; this driver requires IntervalStyle=postgres")) + function parse_interval(val::String) tokens = split(strip(val)) periods = Dates.Period[] + # whether any token was understood; distinguishes a real zero interval + # ("00:00:00") from text in a style this parser can't read + recognized = false i = 1 while i <= length(tokens) token = tokens[i] if occursin(':', token) + # a postgres time component is exactly h:m:s; anything else with a + # colon comes from a different IntervalStyle + count(isequal(':'), token) == 2 && (recognized = true) append!(periods, parse_interval_time(token)) i += 1 continue end i == length(tokens) && break - amount = parse(Int, token) + amount = tryparse(Int, token) + # a non-numeric token ("@", sql_standard year-month like "1-2") means a + # different IntervalStyle; fail with the actionable error, not a raw + # integer-parse failure + amount === nothing && _reject_interval(val) unit = lowercase(tokens[i + 1]) + nbefore = length(periods) if startswith(unit, "year") push!(periods, Dates.Year(amount)) elseif startswith(unit, "mon") @@ -481,9 +532,19 @@ function parse_interval(val::String) elseif startswith(unit, "sec") push!(periods, Dates.Second(amount)) end + # every understood unit pushes a period, so growth means this token + # pair was genuinely matched + length(periods) > nbefore && (recognized = true) i += 2 end - isempty(periods) && return Dates.Millisecond(0) + if isempty(periods) + # a genuine zero interval renders as "00:00:00"; anything else that + # yielded no periods is a format this parser doesn't understand (a + # mid-session `SET IntervalStyle` to sql_standard or iso_8601), and + # silently returning zero would be wrong data + recognized || _reject_interval(val) + return Dates.Millisecond(0) + end length(periods) == 1 && return only(periods) # n=0 construction skips CompoundPeriod's canonicalize loop (whose Period + # merge is dynamic dispatch under --trim); pg interval text is already @@ -578,6 +639,22 @@ function parse_range(val::String, typeId::Int, registry::Dict{Int, TypeInfo}) return _range_typed(T, lower, upper, lower_inclusive, upper_inclusive, false) end +# Range parsing for an element type discovered at registration time +# (`register_range!`). The builtin registry keeps using `parse_range`'s fixed +# dispatch, which is what stays resolvable under `--trim`; this path is only +# reachable once a user registers a range type at runtime. +function parse_range_of(::Type{T}, val::String, typeId::Int, registry::Dict{Int, TypeInfo}) where {T} + lowercase(val) == "empty" && return PostgresRange{T}(missing, missing, false, false, true) + lower_inclusive = val[1] == '[' + upper_inclusive = val[end] == ']' + left, right = split_range_values(val[2:end - 1]) + lower = parse_range_value(left, typeId, registry) + upper = parse_range_value(right, typeId, registry) + l = lower === missing ? missing : convert(T, lower) + u = upper === missing ? missing : convert(T, upper) + return PostgresRange{T}(l, u, lower_inclusive, upper_inclusive, false) +end + parse_array_scalar(typeId::Int, registry::Dict{Int, TypeInfo}, value::Missing) = missing parse_array_scalar(typeId::Int, registry::Dict{Int, TypeInfo}, value::AbstractString) = parse_value(typeId, String(value), registry) # explicit two-level walk: the self-recursive AbstractVector method is an @@ -743,8 +820,7 @@ function parse_value(typeId::Int, val::String, registry::Dict{Int, TypeInfo}) end return val == "t" elseif T == Char - # the "char" type renders its zero value as an empty string - return isempty(val) ? '\0' : val[1] + return pg_parse_char(val) elseif T == DateTime if typeId == 1184 return parse_timestamptz(val) @@ -816,7 +892,7 @@ end StructUtils.lift(::AbstractPostgresStyle, ::Type{Int8}, s::String) = Parsers.parse(Int8, s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Bool}, s::String) = (s == "t" || s == "1"), nothing -StructUtils.lift(::AbstractPostgresStyle, ::Type{Char}, s::String) = (isempty(s) ? '\0' : s[1]), nothing +StructUtils.lift(::AbstractPostgresStyle, ::Type{Char}, s::String) = pg_parse_char(s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Int16}, s::String) = Parsers.parse(Int16, s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Int32}, s::String) = Parsers.parse(Int32, s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Int64}, s::String) = Parsers.parse(Int64, s), nothing @@ -843,6 +919,7 @@ StructUtils.lift(::AbstractPostgresStyle, ::Type{Vector{Time}}, s::String) = par StructUtils.lift(::AbstractPostgresStyle, ::Type{Vector{DateTime}}, s::String) = parse_array(s, DateTime), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Vector{UUID}}, s::String) = parse_array(s, UUID), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Vector{Numeric}}, s::String) = parse_array(s, Numeric), nothing +StructUtils.lift(::AbstractPostgresStyle, ::Type{Vector{Char}}, s::String) = parse_array(s, Char), nothing # For array-typed fields the generic `make` takes its arraylike branch (applyeach # over the source) before consulting lifts — but our source is the wire STRING, diff --git a/src/array_parsing.jl b/src/array_parsing.jl index aab461e..ac39195 100644 --- a/src/array_parsing.jl +++ b/src/array_parsing.jl @@ -1,7 +1,7 @@ module ArrayParsing using Parsers, Dates, UUIDs -import ..pg_parse_date, ..pg_parse_time, ..pg_parse_datetime_any, ..parse_numeric, ..Numeric +import ..pg_parse_date, ..pg_parse_time, ..pg_parse_datetime_any, ..parse_numeric, ..Numeric, ..pg_parse_char const BRACKET_OPEN = UInt8('[') const BRACKET_CLOSE = UInt8(']') @@ -95,6 +95,7 @@ function parse_scalar(token::String, inner_type::Type{T}, quoted::Bool) where {T inner_type === DateTime && return pg_parse_datetime_any(token) inner_type === UUID && return UUID(token) inner_type === Numeric && return parse_numeric(token) + inner_type === Char && return pg_parse_char(token) return token end diff --git a/src/connection_string.jl b/src/connection_string.jl index 0ccad2e..eafe255 100644 --- a/src/connection_string.jl +++ b/src/connection_string.jl @@ -109,14 +109,36 @@ function parse_bool_param(value::Union{String, Nothing}, default::Bool, key::Str throw(ArgumentError("invalid value \"$value\" for connection parameter \"$key\"; expected a boolean (on/off, true/false, yes/no, 1/0)")) end +# Ignored keywords that change security or connection-selection behavior when +# set: silently dropping "channel_binding=require" or a CRL file would leave +# the caller believing a protection is in place. The values listed are the +# no-op defaults for each keyword; any other value draws a warning. +const SECURITY_SENSITIVE_IGNORED = Dict( + "channel_binding" => ("", "prefer", "disable"), + "target_session_attrs" => ("", "any"), + "options" => ("",), + "sslcrl" => ("",), + "sslcrldir" => ("",), + "requiressl" => ("", "0"), +) + +function warn_ignored_param(key::String, value::String) + inert = get(SECURITY_SENSITIVE_IGNORED, key, nothing) + inert === nothing && return + value in inert && return + @warn "connection parameter \"$key=$value\" is not supported by Postgres.jl and is ignored" + return +end + # An unrecognized key is almost always a typo, and silently dropping it is # dangerous: "ssl_mode=verify-full" would leave sslmode unset and fall back to # an unverified connection while the caller believes otherwise. libpq errors # on unknown keywords for the same reason. function check_known_params(values::Dict{String, String}) - for key in keys(values) + for (key, value) in values (key in KNOWN_PARAMS || key in IGNORED_PARAMS) || throw(ArgumentError("unrecognized connection parameter \"$key\"; recognized parameters are $(join(sort!(collect(KNOWN_PARAMS)), ", "))")) + key in IGNORED_PARAMS && warn_ignored_param(key, value) end return values end @@ -259,6 +281,7 @@ function parse_uri(uri::String) for (key, value) in params (key in KNOWN_PARAMS || key in IGNORED_PARAMS) || throw(ArgumentError("unrecognized connection parameter \"$key\" in URI; recognized parameters are $(join(sort!(collect(KNOWN_PARAMS)), ", "))")) + key in IGNORED_PARAMS && warn_ignored_param(key, value) # keys we accept but don't implement must not reach params_from_values key in KNOWN_PARAMS && (values[key] = value) end diff --git a/test/runtests.jl b/test/runtests.jl index bdb5bf7..7398701 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -524,10 +524,19 @@ end @test_throws ArgumentError Postgres.parse_dsn("postgresql://u@h/db?ssl_mode=require") # real libpq keywords this driver doesn't implement are accepted and - # ignored: providers routinely put them in the URI they hand users - @test Postgres.parse_dsn("postgresql://u:p@h/db?sslmode=require&channel_binding=require").sslmode == "require" - @test Postgres.parse_dsn("postgresql://u@h/db?target_session_attrs=read-write").dbname == "db" - @test Postgres.parse_dsn("host=h options=-csearch_path=x").host == "h" + # ignored: providers routinely put them in the URI they hand users. + # The ones that request a security or connection-selection behavior + # warn, so the caller isn't left believing a protection is in place + @test (@test_logs (:warn, r"channel_binding=require.*ignored") Postgres.parse_dsn("postgresql://u:p@h/db?sslmode=require&channel_binding=require")).sslmode == "require" + @test (@test_logs (:warn, r"target_session_attrs=read-write.*ignored") Postgres.parse_dsn("postgresql://u@h/db?target_session_attrs=read-write")).dbname == "db" + @test (@test_logs (:warn, r"options=.*ignored") Postgres.parse_dsn("host=h options=-csearch_path=x")).host == "h" + @test_logs (:warn, r"sslcrl=.*ignored") Postgres.parse_dsn("host=h sslcrl=/tmp/crl.pem") + @test_logs (:warn, r"requiressl=1.*ignored") Postgres.parse_dsn("host=h requiressl=1") + # the no-op defaults for those keywords stay silent, as do keywords + # with no security consequence + @test_logs Postgres.parse_dsn("host=h channel_binding=prefer target_session_attrs=any requiressl=0") + @test_logs Postgres.parse_dsn("host=h keepalives=1 client_encoding=UTF8") + @test_logs Postgres.parse_dsn("postgresql://u@h/db?channel_binding=disable") # invalid values for a recognized parameter are reported against that # parameter rather than silently defaulting @@ -675,6 +684,23 @@ end @test Postgres.API.parse_interval("1 year 2 mons 3 days 04:05:06.789") == Dates.CompoundPeriod(Dates.Year(1), Dates.Month(2), Dates.Day(3), Dates.Hour(4), Dates.Minute(5), Dates.Second(6), Dates.Millisecond(789)) @test Postgres.API.parse_interval("-04:05:06.789") == Dates.CompoundPeriod(Dates.Hour(-4), Dates.Minute(-5), Dates.Second(-6), Dates.Millisecond(-789)) + # a genuine zero interval renders as "00:00:00" in the postgres style + @test Postgres.API.parse_interval("00:00:00") == Dates.Millisecond(0) + # text in an IntervalStyle this parser can't read (a mid-session SET + # to sql_standard or iso_8601) must fail loudly, not silently decode + # to a zero interval + for foreign in ("+1 +2:00:00", "1 2:00:00", "P1DT2H", "PT0S", "@ 1 day 2 hours", "1-2") + @test_throws Postgres.PostgresInterfaceError Postgres.API.parse_interval(foreign) + end + + # the field-order half of DateStyle survives the correction to ISO + @test Postgres.API.date_order("German, DMY") == "DMY" + @test Postgres.API.date_order("SQL, MDY") == "MDY" + @test Postgres.API.date_order("Postgres, YMD") == "YMD" + @test Postgres.API.date_order("ISO, DMY") == "DMY" + # unreported or unrecognized styles fall back to the postgres default + @test Postgres.API.date_order("") == "MDY" + @test Postgres.API.date_order("German") == "MDY" @test Postgres.API.parse_value(17, raw"\xDEADBEEF", registry) == UInt8[0xde, 0xad, 0xbe, 0xef] @test Postgres.API.decode_bytea(raw"\141\\") == UInt8['a', '\\'] @@ -694,11 +720,32 @@ end # the "char" type renders its zero value as an empty string @test Postgres.API.parse_value(18, "", registry) == '\0' @test Postgres.API.parse_value(18, "Z", registry) == 'Z' + # ... and high-bit bytes as backslash-octal escapes + @test Postgres.API.parse_value(18, "\\200", registry) == Char(0x80) + @test Postgres.API.parse_value(18, "\\377", registry) == Char(0xff) + # a backslash byte renders as a lone backslash, not an escape + @test Postgres.API.parse_value(18, "\\", registry) == '\\' + @test Postgres.API.pg_parse_char("\\310") == Char(0xc8) + # "char"[] (oid 1002) decodes elements, escapes included; on the wire + # the escape's backslash is itself array-quoted as "\\200" + @test isequal(Postgres.API.parse_value(1002, "{a,\"\\\\200\",NULL}", registry), Any['a', Char(0x80), missing]) + @test Postgres.API.ArrayParsing.parse_array("{a,b}", Char) == ['a', 'b'] # infinite timestamps/dates can't be represented and must say so @test_throws Postgres.PostgresInterfaceError Postgres.API.pg_parse_datetime("infinity") @test_throws Postgres.PostgresInterfaceError Postgres.API.pg_parse_datetime("-infinity") @test_throws Postgres.PostgresInterfaceError Postgres.API.pg_parse_date("infinity") + # BC dates are a different year numbering than Julia's (no year zero); + # decoding them as AD would be silent corruption + @test_throws Postgres.PostgresInterfaceError Postgres.API.pg_parse_date("0044-03-15 BC") + @test_throws Postgres.PostgresInterfaceError Postgres.API.pg_parse_datetime("0044-03-15 12:00:00 BC") + # timestamptz puts " BC" after the zone offset, past where the + # offset-stripped datetime parse can see it + @test_throws Postgres.PostgresInterfaceError Postgres.API.parse_timestamptz("0044-03-15 12:00:00+00 BC") + # years beyond 9999 widen the year field rather than misparsing + @test Postgres.API.pg_parse_date("10000-01-01") == Date(10000, 1, 1) + @test Postgres.API.pg_parse_datetime("294276-12-31 23:59:59") == DateTime(294276, 12, 31, 23, 59, 59) + @test Postgres.API.parse_value(1184, "10000-01-02 03:04:05+02", registry) == DateTime(10000, 1, 2, 1, 4, 5) range = Postgres.API.parse_range("[1,5)", 23, registry) @test range == Postgres.PostgresRange{Int32}(1, 5, true, false, false) @@ -1000,6 +1047,22 @@ end @test row.span.upper == 5 @test row.span.lower_inclusive @test !row.span.upper_inclusive + + # a range over an element type outside the builtin set must + # decode after registration, not throw on every value + DBInterface.execute(conn, "DROP TYPE IF EXISTS textrange CASCADE") + DBInterface.execute(conn, "CREATE TYPE textrange AS RANGE (subtype = text)") + Postgres.register_range!(conn, "textrange"; schema="public") + trow = only(Tables.rowtable(DBInterface.execute(conn, "SELECT textrange('a','z') AS r, textrange('x','y','[]') AS s, 'empty'::textrange AS e"))) + @test trow.r isa Postgres.PostgresRange{String} + @test trow.r.lower == "a" + @test trow.r.upper == "z" + @test trow.r.lower_inclusive + @test !trow.r.upper_inclusive + @test trow.s.lower == "x" + @test trow.s.upper_inclusive + @test trow.e.empty + DBInterface.execute(conn, "DROP TYPE textrange CASCADE") end @testset "Transactions" begin DBInterface.execute(conn, "DROP TABLE IF EXISTS trans_test") @@ -1347,10 +1410,25 @@ end rows = Tables.rowtable(DBInterface.execute(app_conn, "SELECT current_setting('application_name') AS app_name")) @test rows[1].app_name == "reconnect_test" DBInterface.close!(app_conn) + + # a raw-SQL transaction open at disconnect died with the + # session; the tracked server status must not survive the + # reconnect, or the next pool release issues a spurious + # ROLLBACK on the fresh session + tx_conn = DBInterface.connect(Postgres.Connection, cfg.host, cfg.user, cfg.password; dbname=cfg.dbname, port=cfg.port, reconnect=true) + DBInterface.execute(tx_conn, "BEGIN") + @test @lock tx_conn.lock tx_conn.server_in_transaction + close(tx_conn.socket) + @test only(Tables.rowtable(DBInterface.execute(tx_conn, "SELECT 1 AS a"))).a == 1 + @test !(@lock tx_conn.lock tx_conn.server_in_transaction) + DBInterface.close!(tx_conn) end @testset "Statement Timeout" begin timeout_conn = DBInterface.connect(Postgres.Connection, cfg.host, cfg.user, cfg.password; dbname=cfg.dbname, port=cfg.port, statement_timeout=200) + # applied with a post-connect SET (poolers reject it as a + # startup option), so confirm it actually took effect + @test only(Tables.rowtable(DBInterface.execute(timeout_conn, "SELECT current_setting('statement_timeout') AS t"))).t == "200ms" @test_throws Postgres.API.Error DBInterface.execute(timeout_conn, "SELECT pg_sleep(1)") Postgres.set_statement_timeout!(timeout_conn, 0) rows = Tables.rowtable(DBInterface.execute(timeout_conn, "SELECT 1 AS a")) @@ -1544,6 +1622,32 @@ end # the system catalogs cat_rows = Tables.rowtable(DBInterface.execute(conn, "SELECT attidentity FROM pg_attribute LIMIT 5")) @test length(cat_rows) == 5 + # high-bit "char" values arrive as backslash-octal escapes + oct_row = only(Tables.rowtable(DBInterface.execute(conn, "SELECT (-1)::\"char\" AS c1, (-128)::\"char\" AS c2, 'a'::\"char\" AS c3, 0::\"char\" AS c4"))) + @test oct_row.c1 == Char(0xff) + @test oct_row.c2 == Char(0x80) + @test oct_row.c3 == 'a' + @test oct_row.c4 == '\0' + # "char"[] round-trips too, escapes and zero included + chararr_row = only(Tables.rowtable(DBInterface.execute(conn, "SELECT ARRAY['a'::\"char\", 0::\"char\", (-1)::\"char\"] AS a"))) + @test isequal(collect(chararr_row.a), Any['a', '\0', Char(0xff)]) + + # years beyond 9999 and BC dates: wide years decode, BC + # fails loudly, and neither poisons the connection + big_row = only(Tables.rowtable(DBInterface.execute(conn, "SELECT '10000-01-01'::date AS d, '10000-01-02 03:04:05'::timestamp AS t"))) + @test big_row.d == Date(10000, 1, 1) + @test big_row.t == DateTime(10000, 1, 2, 3, 4, 5) + @test_throws Postgres.PostgresInterfaceError Tables.rowtable(DBInterface.execute(conn, "SELECT '0044-03-15 BC'::date AS d")) + @test only(Tables.rowtable(DBInterface.execute(conn, "SELECT 1 AS ok"))).ok == 1 + # an interval in a foreign IntervalStyle fails loudly too, + # at a message boundary the connection can recover from + DBInterface.execute(conn, "SET IntervalStyle = 'sql_standard'") + try + @test_throws Postgres.PostgresInterfaceError Tables.rowtable(DBInterface.execute(conn, "SELECT '1 day 2 hours'::interval AS i")) + finally + DBInterface.execute(conn, "SET IntervalStyle = 'postgres'") + end + @test only(Tables.rowtable(DBInterface.execute(conn, "SELECT '1 day'::interval AS i"))).i == Dates.Day(1) # the session uses ISO dates and postgres intervals, which # the text parsers require style_row = only(Tables.rowtable(DBInterface.execute(conn, "SELECT current_setting('DateStyle') AS ds, current_setting('IntervalStyle') AS is"))) @@ -1560,6 +1664,12 @@ end row = only(Tables.rowtable(DBInterface.execute(german_conn, "SELECT '2020-03-04 05:06:07'::timestamp AS t, '1 day'::interval AS i"))) @test row.t == DateTime(2020, 3, 4, 5, 6, 7) @test row.i == Dates.Day(1) + # the correction must keep the configured DMY field + # order: reading '01/02/2020' as MDY would silently + # turn the user's 1 February into January 2 + order_row = only(Tables.rowtable(DBInterface.execute(german_conn, "SELECT current_setting('DateStyle') AS ds, '01/02/2020'::date AS d"))) + @test order_row.ds == "ISO, DMY" + @test order_row.d == Date(2020, 2, 1) finally DBInterface.close!(german_conn) end From 6f65316ba7fd887923011243556c4a8050d69ae3 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 6 Aug 2026 09:23:33 -0600 Subject: [PATCH 19/23] Fix raw-SQL transaction safety, multibyte range bounds, offset seconds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-13 review findings, all verified against a live server: - Driver transaction helpers consulted only the client-side flag, so over a transaction opened with raw SQL (execute(conn, "BEGIN")) they issued a BEGIN the server ignored and their commit committed the caller's transaction -- the caller's own ROLLBACK then silently no-oped. start_transaction now nests inside a raw transaction with a savepoint (tracked by owns_base_transaction); the outermost driver commit releases that savepoint and rollback rolls back to it, leaving the caller's transaction open and under their control. cursor treats the server-reported transaction as "already in one", as its docstring says. - Range bounds ending in a multibyte character threw StringIndexError (byte-index slicing): every textrange('α','ω')-style value was undecodable after a successful registration. - Timezone offsets carrying a seconds field ("+05:21:10", the rendering of LMT-era timestamps in named zones) silently lost the seconds. - The variable-width year scan accepted fewer than 4 year digits, so "03-04-2020" -- Postgres-style output after a mid-session SET DateStyle -- silently decoded as year 3 where the old fixed-width parser threw. The year is now bounded to 4..9 digits and both separators are checked, which also closes an Int-overflow and short-input BoundsErrors on adversarial input. - An unreported DateStyle (a pooler not forwarding ParameterStatus) was corrected to 'ISO, MDY', force-flipping DMY sessions; setting just 'ISO' preserves the server-side field order we can't see. - server_in_transaction went stale on every failed statement: error paths skipped the status copy, so a failed COMMIT left it true and drew a spurious ROLLBACK ("no transaction in progress") on the next pool release. The status is recorded in a finally on the prepared path and published through a Ref on the simple-query path. - postgres's valid time '24:00:00' threw a raw ArgumentError mid-decode; it now reports PostgresInterfaceError like other unrepresentable values. - register_enum!/register_composite!/register_range! register the type's array OID alongside it, so mood[]/composite[]/range[] values decode instead of returning the raw literal string. register_range! documents that element types must be registered before ranges over them. - The reconnect test now triggers checkconn directly: the previous version passed even without the fix because the follow-up statement refreshed the flag from its own ReadyForQuery. Mutation-tested: the raw-transaction savepoint base, the cursor ownership check, and the checkconn reset each fail their new tests when reverted. Co-Authored-By: Claude Fable 5 --- src/Postgres.jl | 81 ++++++++++++++++++++++++++++++----- src/api/API.jl | 16 ++++++- src/api/types.jl | 34 ++++++++++----- src/execute.jl | 24 ++++++++--- test/runtests.jl | 108 ++++++++++++++++++++++++++++++++++++++++++++++- 5 files changed, 234 insertions(+), 29 deletions(-) diff --git a/src/Postgres.jl b/src/Postgres.jl index f6c3424..9dc552d 100644 --- a/src/Postgres.jl +++ b/src/Postgres.jl @@ -106,6 +106,11 @@ mutable struct Connection{T, S <: API.AbstractPostgresStyle} <: DBInterface.Conn # the server's own ReadyForQuery transaction status: unlike in_transaction # it also sees a transaction opened by raw SQL (`execute(conn, "BEGIN")`) server_in_transaction::Bool + # whether the outermost driver-managed transaction level issued the BEGIN: + # when start_transaction found a raw-SQL transaction already open, the base + # level is a savepoint inside the caller's transaction, and the final + # commit/rollback must not COMMIT/ROLLBACK the caller's work + owns_base_transaction::Bool function Connection(; host::AbstractString="", user::AbstractString="", password::Union{AbstractString, Nothing}=nothing, dbname::AbstractString="", port::Integer=5432, debug::Bool=false, reconnect::Bool=false, application_name::Union{AbstractString, Nothing}=nothing, connect_timeout::Union{Integer, Nothing}=nothing, sslmode::Union{AbstractString, Nothing}=nothing, sslrootcert::Union{AbstractString, Nothing}=nothing, sslcert::Union{AbstractString, Nothing}=nothing, sslkey::Union{AbstractString, Nothing}=nothing, sslcapath::Union{AbstractString, Nothing}=nothing, sslservername::Union{AbstractString, Nothing}=nothing, statement_timeout::Union{Integer, Nothing}=nothing, statement_cache_maxsize::Integer=100, style::API.AbstractPostgresStyle=PostgresStyle()) host = String(host) @@ -125,7 +130,7 @@ mutable struct Connection{T, S <: API.AbstractPostgresStyle} <: DBInterface.Conn maxsize = max(0, Int(statement_cache_maxsize)) socket, pid, skey, server_params = API.connect(host, port, dbname, user, password, debug, app_name, timeout, sslmode_val, sslrootcert_val, sslcert_val, sslkey_val, sslcapath_val, sslservername_val, statement_timeout_val) registry = Dict(API.DEFAULT_TYPE_REGISTRY) - return new{Statement{typeof(style)}, typeof(style)}(ReentrantLock(), socket, host, user, password, dbname, port, app_name, timeout, sslmode_val, sslrootcert_val, sslcert_val, sslkey_val, sslcapath_val, sslservername_val, statement_timeout_val, pid, skey, Dict{String, Statement{typeof(style)}}(), maxsize, 0, server_params, registry, false, reconnect, debug, style, false, 0, 1, false) + return new{Statement{typeof(style)}, typeof(style)}(ReentrantLock(), socket, host, user, password, dbname, port, app_name, timeout, sslmode_val, sslrootcert_val, sslcert_val, sslkey_val, sslcapath_val, sslservername_val, statement_timeout_val, pid, skey, Dict{String, Statement{typeof(style)}}(), maxsize, 0, server_params, registry, false, reconnect, debug, style, false, 0, 1, false, true) end end @@ -555,13 +560,22 @@ end function lookup_type_oid(conn::Connection, name::AbstractString, schema::AbstractString) rows = Tables.rowtable(DBInterface.execute(conn, """ - SELECT t.oid + SELECT t.oid, t.typarray FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace WHERE t.typname = \$1 AND n.nspname = \$2 """, (name, schema))) isempty(rows) && throw(PostgresInterfaceError("type $(schema).$(name) not found")) - return Int(rows[1].oid) + return Int(rows[1].oid), Int(rows[1].typarray) +end + +# Register the type's array OID alongside it, so `ARRAY[...]::name[]` values +# decode as arrays of the registered type instead of the raw literal string. +function register_array_type!(conn::Connection, arrayoid::Int, oid::Int, @nospecialize(julia_type::Type)) + arrayoid == 0 && return conn + register_type!(conn, arrayoid, Vector{julia_type}; + parser=(val, registry) -> API.parse_array_by_oid(val, oid, registry)) + return conn end """ @@ -571,9 +585,10 @@ Look up the enum type `schema.name` on the server and register it so values are returned as `julia_type` (by default `Symbol`). """ function register_enum!(conn::Connection, name::AbstractString; schema::AbstractString="public", julia_type::Type=Symbol) - oid = lookup_type_oid(conn, name, schema) + oid, arrayoid = lookup_type_oid(conn, name, schema) parser = julia_type === Symbol ? (val, registry) -> Symbol(val) : nothing register_type!(conn, oid, julia_type; parser=parser) + register_array_type!(conn, arrayoid, oid, julia_type) return conn end @@ -585,7 +600,7 @@ values are returned as `NamedTuple`s with the composite's field names. """ function register_composite!(conn::Connection, name::AbstractString; schema::AbstractString="public") rows = Tables.rowtable(DBInterface.execute(conn, """ - SELECT t.oid, a.attname, a.atttypid + SELECT t.oid, t.typarray, a.attname, a.atttypid FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace JOIN pg_class c ON c.oid = t.typrelid @@ -595,6 +610,7 @@ function register_composite!(conn::Connection, name::AbstractString; schema::Abs """, (name, schema))) isempty(rows) && throw(PostgresInterfaceError("composite type $(schema).$(name) not found")) oid = Int(rows[1].oid) + arrayoid = Int(rows[1].typarray) field_names = [Symbol(row.attname) for row in rows] field_oids = Int[row.atttypid for row in rows] tuple_type = NamedTuple{Tuple(field_names)} @@ -610,6 +626,7 @@ function register_composite!(conn::Connection, name::AbstractString; schema::Abs return tuple_type(Tuple(values)) end register_type!(conn, oid, tuple_type; parser=parser) + register_array_type!(conn, arrayoid, oid, tuple_type) return conn end @@ -619,10 +636,15 @@ end Look up the range type `schema.name` on the server and register it so values are returned as [`PostgresRange`](@ref Postgres.API.PostgresRange) of the range's element type. + +The element type is captured when the range is registered, so register it +first: a range over a custom enum or composite must come after the +corresponding [`register_enum!`](@ref Postgres.register_enum!) / +[`register_composite!`](@ref Postgres.register_composite!) call. """ function register_range!(conn::Connection, name::AbstractString; schema::AbstractString="public") rows = Tables.rowtable(DBInterface.execute(conn, """ - SELECT t.oid, r.rngsubtype + SELECT t.oid, t.typarray, r.rngsubtype FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace JOIN pg_range r ON r.rngtypid = t.oid @@ -630,6 +652,7 @@ function register_range!(conn::Connection, name::AbstractString; schema::Abstrac """, (name, schema))) isempty(rows) && throw(PostgresInterfaceError("range type $(schema).$(name) not found")) oid = Int(rows[1].oid) + arrayoid = Int(rows[1].typarray) subtype_oid = Int(rows[1].rngsubtype) subtype_type = API.type_info(conn.type_registry, subtype_oid).julia_type # bind the element type here rather than rediscovering it per value: the @@ -637,6 +660,7 @@ function register_range!(conn::Connection, name::AbstractString; schema::Abstrac # over anything else would register successfully and then fail on every value parser = (val, registry) -> API.parse_range_of(subtype_type, val, subtype_oid, registry) register_type!(conn, oid, PostgresRange{subtype_type}; parser=parser) + register_array_type!(conn, arrayoid, oid, PostgresRange{subtype_type}) return conn end @@ -887,8 +911,16 @@ include("execute.jl") # does not exist"). Also one network round trip instead of three. Callers # must hold conn.lock. function execute_simple(conn::Connection, sql::String) - status = API.exec(conn.style, conn.socket, sql, conn.debug) - conn.server_in_transaction = API.in_transaction_status(status) + # capture the ReadyForQuery status through a Ref so a failed statement + # (a COMMIT hitting a deferred constraint) still refreshes the tracking: + # the server ended the transaction either way, and a stale flag draws a + # spurious ROLLBACK on the next pool release + status_ref = Ref{UInt8}(UInt8('I')) + try + API.exec(conn.style, conn.socket, sql, conn.debug, status_ref) + finally + conn.server_in_transaction = API.in_transaction_status(status_ref[]) + end return conn end @@ -905,7 +937,17 @@ function start_transaction(conn::Connection) @lock conn.lock begin checkconn(conn) if !conn.in_transaction - execute_simple(conn, "BEGIN") + if conn.server_in_transaction + # a transaction opened with raw SQL (execute(conn, "BEGIN")) + # belongs to the caller: nest inside it with a savepoint, as + # for driver-owned nesting, so our commit can't commit — and + # our rollback can't destroy — their work + execute_simple(conn, "SAVEPOINT sp_0") + conn.owns_base_transaction = false + else + execute_simple(conn, "BEGIN") + conn.owns_base_transaction = true + end conn.in_transaction = true conn.transaction_depth = 1 else @@ -959,7 +1001,13 @@ function commit(conn::Connection) # transaction state must be cleared either way — leaving it set # would block reconnects and make the next cursor skip its BEGIN try - execute_simple(conn, "COMMIT") + if conn.owns_base_transaction + execute_simple(conn, "COMMIT") + else + # the base transaction is the caller's raw-SQL one: keep + # this level's work pending inside it and leave it open + execute_simple(conn, "RELEASE SAVEPOINT sp_0") + end finally conn.in_transaction = false conn.transaction_depth = 0 @@ -993,7 +1041,13 @@ function rollback(conn::Connection) # as in commit: the transaction is over server-side regardless of # how ROLLBACK fares, so don't leave client state describing it try - execute_simple(conn, "ROLLBACK") + if conn.owns_base_transaction + execute_simple(conn, "ROLLBACK") + else + # undo only this level; a plain ROLLBACK would destroy the + # caller's raw-SQL transaction along with it + execute_simple(conn, "ROLLBACK TO SAVEPOINT sp_0") + end finally conn.in_transaction = false conn.transaction_depth = 0 @@ -1027,6 +1081,11 @@ end Run `f(conn)` inside a transaction: committed if `f` returns normally, rolled back if it throws. Nested calls use savepoints. Returns `f`'s result. +A transaction already opened with raw SQL (`execute(conn, "BEGIN")`) is +treated as the enclosing level: the block nests inside it with a savepoint +and leaves it open, so the caller's own `COMMIT`/`ROLLBACK` stays in control +of their transaction. + Postgres.transaction(conn) do conn DBInterface.execute(conn, "INSERT INTO t VALUES (1)") end diff --git a/src/api/API.jl b/src/api/API.jl index 2cd520b..97cd78f 100644 --- a/src/api/API.jl +++ b/src/api/API.jl @@ -783,7 +783,14 @@ function align_session_formats!(socket, server_params::Dict{String, String}, deb # be treated as unknown, i.e. corrected — assuming it is already right is # how intervals silently decode to zero. datestyle = get(server_params, "DateStyle", "") - if !startswith(datestyle, "ISO") + if isempty(datestyle) + # unreported (a pooler may not forward ParameterStatus): setting just + # the format half preserves whatever field order is configured + # server-side, which we can't see — naming an order here would flip a + # DMY session to MDY + exec(PostgresStyle(), socket, "SET DateStyle = 'ISO'", debug) + server_params["DateStyle"] = "ISO" + elseif !startswith(datestyle, "ISO") wanted = string("ISO, ", date_order(datestyle)) exec(PostgresStyle(), socket, string("SET DateStyle = '", wanted, "'"), debug) server_params["DateStyle"] = wanted @@ -1041,7 +1048,7 @@ function exec(style::S, socket::ReseauConn, stmtname::String, params::Vector{Uni return Exec{S}(style, socket, names, typeIds, type_registry, debug, Ref{Union{Nothing, String}}(nothing), Ref{Union{Nothing, Int}}(nothing), Ref{UInt8}(UInt8('I'))) end -function exec(style::S, socket::ReseauConn, query::String, debug::Bool) where {S <: AbstractPostgresStyle} +function exec(style::S, socket::ReseauConn, query::String, debug::Bool, tx_status_ref::Union{Nothing, Base.RefValue{UInt8}}=nothing) where {S <: AbstractPostgresStyle} writemessages(socket, debug, ('Q', query)) server_error = nothing tx_status = UInt8('I') @@ -1054,6 +1061,11 @@ function exec(style::S, socket::ReseauConn, query::String, debug::Bool) where {S server_error = errorResponse(len, socket, debug) elseif mt == UInt8('Z') tx_status = read_ready_status(socket, len) + # publish through the Ref as well: a failed statement (a + # COMMIT hitting a deferred constraint) throws below, but its + # ReadyForQuery status is authoritative and the caller's + # transaction tracking must not go stale + tx_status_ref === nothing || (tx_status_ref[] = tx_status) break elseif mt == UInt8('N') notice_callback(style, noticeResponse(len, socket)) diff --git a/src/api/types.jl b/src/api/types.jl index 8e9de93..78d7315 100644 --- a/src/api/types.jl +++ b/src/api/types.jl @@ -264,7 +264,10 @@ end isempty(digits) && return 0 hours = parse(Int, digits[1:2]) mins = length(digits) >= 4 ? parse(Int, digits[3:4]) : 0 - return sign * (hours * 3600 + mins * 60) + # pre-standardization (LMT-era) timestamps in named zones carry a seconds + # field ("+05:21:10"); dropping it silently shifts the decoded value + secs = length(digits) >= 6 ? parse(Int, digits[5:6]) : 0 + return sign * (hours * 3600 + mins * 60 + secs) end # ── hand-rolled postgres text-format date/time parsing ────────────────────── @@ -279,7 +282,11 @@ end # The year is normally 4 digits but PostgreSQL emits more beyond year 9999, so # scan it rather than assuming a fixed width. Returns the date and the offset -# just past it. +# just past it. The digit-count bound and separator checks matter: an ISO year +# is zero-padded to at least 4 digits, so accepting fewer would silently +# mis-decode other DateStyle renderings ("03-04-2020" is Postgres-style for +# 2020-03-04, not year 3), and an unbounded scan would overflow Int on +# adversarial input. @inline function _pg_date_at_end(c, o::Int) y = 0 i = o @@ -288,12 +295,14 @@ end y = y * 10 + _pg_digit(c[i]) i += 1 end - (i > n || c[i] != UInt8('-')) && throw(ArgumentError("invalid postgres date")) - i += 1 - m = _pg_digit(c[i]) * 10 + _pg_digit(c[i+1]) - i += 3 - d = _pg_digit(c[i]) * 10 + _pg_digit(c[i+1]) - return Date(y, m, d), i + 2 + ndigits = i - o + (4 <= ndigits <= 9) || throw(ArgumentError("invalid postgres date")) + (i + 5 <= n && c[i] == UInt8('-') && c[i+3] == UInt8('-') && + _pg_isdigit(c[i+1]) && _pg_isdigit(c[i+2]) && _pg_isdigit(c[i+4]) && _pg_isdigit(c[i+5])) || + throw(ArgumentError("invalid postgres date")) + m = _pg_digit(c[i+1]) * 10 + _pg_digit(c[i+2]) + d = _pg_digit(c[i+4]) * 10 + _pg_digit(c[i+5]) + return Date(y, m, d), i + 6 end @inline _pg_date_at(c, o::Int)::Date = first(_pg_date_at_end(c, o)) @@ -343,6 +352,8 @@ function pg_parse_time(s::AbstractString)::Time c = codeunits(s) length(c) >= 8 || throw(ArgumentError("invalid postgres time")) h, mi, se, ms = _pg_hms_at(c, 1) + # postgres permits '24:00:00' as a time value; Julia's Time does not + h == 24 && throw(PostgresInterfaceError("postgres time value \"$s\" cannot be represented as a Julia Time")) return Time(h, mi, se, ms) end @@ -632,7 +643,9 @@ function parse_range(val::String, typeId::Int, registry::Dict{Int, TypeInfo}) lowercase(val) == "empty" && return _range_typed(T, missing, missing, false, false, true) lower_inclusive = val[1] == '[' upper_inclusive = val[end] == ']' - inner = val[2:end - 1] + # the brackets are ASCII but the bounds may not be: slice by character + # index, or a bound ending in a multibyte character throws StringIndexError + inner = val[2:prevind(val, lastindex(val))] left, right = split_range_values(inner) lower = parse_range_value(left, typeId, registry) upper = parse_range_value(right, typeId, registry) @@ -647,7 +660,8 @@ function parse_range_of(::Type{T}, val::String, typeId::Int, registry::Dict{Int, lowercase(val) == "empty" && return PostgresRange{T}(missing, missing, false, false, true) lower_inclusive = val[1] == '[' upper_inclusive = val[end] == ']' - left, right = split_range_values(val[2:end - 1]) + # character slicing, as in parse_range: bounds may end in multibyte text + left, right = split_range_values(val[2:prevind(val, lastindex(val))]) lower = parse_range_value(left, typeId, registry) upper = parse_range_value(right, typeId, registry) l = lower === missing ? missing : convert(T, lower) diff --git a/src/execute.jl b/src/execute.jl index b5daaff..981e54e 100644 --- a/src/execute.jl +++ b/src/execute.jl @@ -455,8 +455,14 @@ function DBInterface.execute(stmt::Statement, params=nothing, ::Type{T}=Any; deb else API.exec(style, socket::Reseau.TLS.Conn, stmt.name, stmt.params, stmt.names, stmt.typeIds, stmt.conn.type_registry, debug, 0) end - result = T === Any ? makeresult(e) : StructUtils.arraylike(T) ? StructUtils.make(T, e, style) : only(StructUtils.make(Vector{T}, e, style)) - stmt.conn.server_in_transaction = API.in_transaction_status(e.tx_status[]) + # in a finally: a failed statement still drained to ReadyForQuery + # and its status is authoritative — skipping the copy on the error + # path leaves the transaction tracking stale + try + result = T === Any ? makeresult(e) : StructUtils.arraylike(T) ? StructUtils.make(T, e, style) : only(StructUtils.make(Vector{T}, e, style)) + finally + stmt.conn.server_in_transaction = API.in_transaction_status(e.tx_status[]) + end end log_enabled && API.query_logger(style, :execute, (sql=stmt.sql, params=params, duration_ns=time_ns() - start_ns, success=true)) return result @@ -485,8 +491,12 @@ function DBInterface.execute(conn::Connection, sql::AbstractString, params=nothi else API.exec(style, socket::Reseau.TLS.Conn, stmtname, params_vec, names, types, conn.type_registry, debug, 0) end - result = T === Any ? makeresult(e) : StructUtils.arraylike(T) ? StructUtils.make(T, e, style) : only(StructUtils.make(Vector{T}, e, style)) - conn.server_in_transaction = API.in_transaction_status(e.tx_status[]) + # in a finally, as in the statement-execute method above + try + result = T === Any ? makeresult(e) : StructUtils.arraylike(T) ? StructUtils.make(T, e, style) : only(StructUtils.make(Vector{T}, e, style)) + finally + conn.server_in_transaction = API.in_transaction_status(e.tx_status[]) + end end log_enabled && API.query_logger(style, :execute, (sql=sql_str, params=params, duration_ns=time_ns() - start_ns, success=true)) return result @@ -522,7 +532,11 @@ started (and committed on close) if the connection isn't already in one. """ function cursor(conn::Connection, sql::AbstractString, params=nothing; fetchsize::Integer=1000, debug::Bool=false) owns_transaction = false - in_transaction(conn) || (start_transaction(conn); owns_transaction = true) + # a transaction opened with raw SQL counts as "already in one": the server + # status sees it even though the client flag doesn't, and owning it here + # would mean committing the caller's transaction on cursor close + already_in_tx = @lock conn.lock (conn.in_transaction || conn.server_in_transaction) + already_in_tx || (start_transaction(conn); owns_transaction = true) try stmt = DBInterface.prepare(conn, sql; debug=debug) return cursor(stmt, params; fetchsize=fetchsize, owns_transaction=owns_transaction) diff --git a/test/runtests.jl b/test/runtests.jl index 7398701..9c34814 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -681,6 +681,10 @@ end @test Postgres.API.parse_value(1184, "2024-02-13 05:28:17+02", registry) == DateTime(2024, 2, 13, 3, 28, 17) @test Postgres.API.parse_value(1184, "2024-02-13 05:28:17+02:30", registry) == DateTime(2024, 2, 13, 2, 58, 17) @test Postgres.API.parse_value(1184, "2024-02-13 05:28:17Z", registry) == DateTime(2024, 2, 13, 5, 28, 17) + # LMT-era offsets in named zones carry a seconds field; dropping it + # silently shifted the decoded value + @test Postgres.API.parse_value(1184, "1880-01-01 05:21:10+05:21:10", registry) == DateTime(1880, 1, 1, 0, 0, 0) + @test Postgres.API.parse_value(1184, "1879-12-31 18:38:50-05:21:10", registry) == DateTime(1880, 1, 1, 0, 0, 0) @test Postgres.API.parse_interval("1 year 2 mons 3 days 04:05:06.789") == Dates.CompoundPeriod(Dates.Year(1), Dates.Month(2), Dates.Day(3), Dates.Hour(4), Dates.Minute(5), Dates.Second(6), Dates.Millisecond(789)) @test Postgres.API.parse_interval("-04:05:06.789") == Dates.CompoundPeriod(Dates.Hour(-4), Dates.Minute(-5), Dates.Second(-6), Dates.Millisecond(-789)) @@ -746,6 +750,27 @@ end @test Postgres.API.pg_parse_date("10000-01-01") == Date(10000, 1, 1) @test Postgres.API.pg_parse_datetime("294276-12-31 23:59:59") == DateTime(294276, 12, 31, 23, 59, 59) @test Postgres.API.parse_value(1184, "10000-01-02 03:04:05+02", registry) == DateTime(10000, 1, 2, 1, 4, 5) + # ... but only genuine ISO renderings: an ISO year is zero-padded to + # at least 4 digits, so "03-04-2020" (Postgres-style for 2020-03-04 + # after a mid-session SET DateStyle) must throw, not decode as year 3 + @test_throws ArgumentError Postgres.API.pg_parse_date("03-04-2020") + @test_throws ArgumentError Postgres.API.pg_parse_date("123-01-01") + # adversarial input: bounded digits (no Int overflow), checked layout + @test_throws ArgumentError Postgres.API.pg_parse_date("99999999999999999999-01-01") + @test_throws ArgumentError Postgres.API.pg_parse_date("12345678-9") + @test_throws ArgumentError Postgres.API.pg_parse_datetime("999999999- 03:04:05 ") + + # postgres permits time '24:00:00'; Julia's Time does not + @test_throws Postgres.PostgresInterfaceError Postgres.API.pg_parse_time("24:00:00") + @test Postgres.API.pg_parse_time("23:59:59.999") == Time(23, 59, 59, 999) + + # range bounds may end in multibyte text; byte-index slicing threw + # StringIndexError on every such value + uni_range = Postgres.API.parse_range_of(String, "[α,ω)", 25, registry) + @test uni_range.lower == "α" + @test uni_range.upper == "ω" + @test uni_range.lower_inclusive + @test !uni_range.upper_inclusive range = Postgres.API.parse_range("[1,5)", 23, registry) @test range == Postgres.PostgresRange{Int32}(1, 5, true, false, false) @@ -1062,6 +1087,25 @@ end @test trow.s.lower == "x" @test trow.s.upper_inclusive @test trow.e.empty + # bounds ending in multibyte text arrive unquoted + uni_row = only(Tables.rowtable(DBInterface.execute(conn, "SELECT textrange('α','ω') AS r"))) + @test uni_row.r.lower == "α" + @test uni_row.r.upper == "ω" + + # arrays of registered custom types decode as arrays, not + # raw literal strings + enumarr = only(Tables.rowtable(DBInterface.execute(conn, "SELECT ARRAY['sad','happy']::mood[] AS a, ARRAY['ok',NULL]::mood[] AS b"))) + @test isequal(collect(enumarr.a), Any[:sad, :happy]) + @test isequal(collect(enumarr.b), Any[:ok, missing]) + comparr = only(Tables.rowtable(DBInterface.execute(conn, "SELECT ARRAY[ROW('A',1)::address, ROW('B',2)::address] AS a"))) + @test comparr.a[1] == (street="A", number=1) + @test comparr.a[2] == (street="B", number=2) + rangearr = only(Tables.rowtable(DBInterface.execute(conn, "SELECT ARRAY['[1,3)'::int4range, '[5,7)'::int4range] AS a"))) + @test rangearr.a[1].lower == 1 + @test rangearr.a[2].upper == 7 + textrangearr = only(Tables.rowtable(DBInterface.execute(conn, "SELECT ARRAY[textrange('a','c'), textrange('α','ω')] AS a"))) + @test textrangearr.a[1].upper == "c" + @test textrangearr.a[2].lower == "α" DBInterface.execute(conn, "DROP TYPE textrange CASCADE") end @testset "Transactions" begin @@ -1119,9 +1163,55 @@ end @test err isa Postgres.API.Error @test err.code == "23503" @test !Postgres.in_transaction(conn) + # the failed COMMIT's own ReadyForQuery says the + # transaction is over; the tracked server status must + # not stay stale-true from the preceding INSERT + @test !(@lock conn.lock conn.server_in_transaction) @test isempty(Tables.rowtable(DBInterface.execute(conn, "SELECT * FROM deferred_child"))) end + # a transaction opened with raw SQL belongs to the caller: + # driver helpers must nest inside it (savepoints), never + # commit it, and never destroy it on their rollback + DBInterface.execute(conn, "DROP TABLE IF EXISTS rawtx_test") + DBInterface.execute(conn, "CREATE TABLE rawtx_test (id int)") + DBInterface.execute(conn, "BEGIN") + DBInterface.execute(conn, "INSERT INTO rawtx_test VALUES (1)") + Postgres.transaction(conn) do tx + DBInterface.execute(tx, "INSERT INTO rawtx_test VALUES (2)") + end + # a failing block rolls back only its own level, leaving + # the caller's transaction alive and usable + raw_err = try + Postgres.transaction(conn) do tx + DBInterface.execute(tx, "INSERT INTO rawtx_test VALUES (3)") + DBInterface.execute(tx, "SELECT 1/0") + end + nothing + catch e + e + end + @test raw_err isa Postgres.API.Error + # a cursor sees the open transaction and must not own + # (and so commit) it on close — nor start any driver-level + # nesting of its own + raw_cur = Postgres.cursor(conn, "SELECT id FROM rawtx_test ORDER BY id"; fetchsize=1) + @test !Postgres.in_transaction(conn) + @test [row.id for row in raw_cur] == [1, 2] + DBInterface.close!(raw_cur) + @test @lock conn.lock conn.server_in_transaction + # the caller's ROLLBACK is still in control of all of it + DBInterface.execute(conn, "ROLLBACK") + @test isempty(Tables.rowtable(DBInterface.execute(conn, "SELECT id FROM rawtx_test"))) + # ... and after a raw COMMIT, the driver-level work sticks + DBInterface.execute(conn, "BEGIN") + Postgres.transaction(conn) do tx + DBInterface.execute(tx, "INSERT INTO rawtx_test VALUES (4)") + end + DBInterface.execute(conn, "COMMIT") + @test only(Tables.rowtable(DBInterface.execute(conn, "SELECT id FROM rawtx_test"))).id == 4 + DBInterface.execute(conn, "DROP TABLE rawtx_test") + # COMMIT/ROLLBACK end the transaction server-side even when # they fail, so client state must not be left behind fail_conn = DBInterface.connect(Postgres.Connection, cfg.host, cfg.user, cfg.password; dbname=cfg.dbname, port=cfg.port, reconnect=true) @@ -1419,8 +1509,12 @@ end DBInterface.execute(tx_conn, "BEGIN") @test @lock tx_conn.lock tx_conn.server_in_transaction close(tx_conn.socket) - @test only(Tables.rowtable(DBInterface.execute(tx_conn, "SELECT 1 AS a"))).a == 1 + # trigger the reconnect via checkconn directly: a full + # statement would refresh the flag from its own + # ReadyForQuery and mask a missing reset + @lock tx_conn.lock Postgres.checkconn(tx_conn) @test !(@lock tx_conn.lock tx_conn.server_in_transaction) + @test only(Tables.rowtable(DBInterface.execute(tx_conn, "SELECT 1 AS a"))).a == 1 DBInterface.close!(tx_conn) end @@ -1648,6 +1742,18 @@ end DBInterface.execute(conn, "SET IntervalStyle = 'postgres'") end @test only(Tables.rowtable(DBInterface.execute(conn, "SELECT '1 day'::interval AS i"))).i == Dates.Day(1) + + # postgres's time '24:00:00' has no Julia representation + @test_throws Postgres.PostgresInterfaceError Tables.rowtable(DBInterface.execute(conn, "SELECT '24:00:00'::time AS t")) + # LMT-era timestamptz offsets carry seconds ("+05:21:10"); + # dropping them silently shifted the value + DBInterface.execute(conn, "SET TimeZone = 'Asia/Kolkata'") + try + lmt_row = only(Tables.rowtable(DBInterface.execute(conn, "SELECT '1880-01-01 00:00:00+00'::timestamptz AS t"))) + @test lmt_row.t == DateTime(1880, 1, 1, 0, 0, 0) + finally + DBInterface.execute(conn, "RESET TimeZone") + end # the session uses ISO dates and postgres intervals, which # the text parsers require style_row = only(Tables.rowtable(DBInterface.execute(conn, "SELECT current_setting('DateStyle') AS ds, current_setting('IntervalStyle') AS is"))) From 4fea291ceb6a6e7ea1be3a482d94a6c9bf3561a2 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 6 Aug 2026 11:46:03 -0600 Subject: [PATCH 20/23] fix: close 1.0 release blockers Fix transaction and statement ownership, make unnamed execution safe through transaction-mode PgBouncer, harden protocol and TLS handling, enforce the supported type and DSN contracts, and add the release validation matrix and support policy. --- .github/workflows/CI.yml | 54 ++++ LICENSE | 21 ++ LICENSE.md | 22 -- Project.toml | 18 +- README.md | 18 +- docs/make.jl | 1 + docs/src/index.md | 8 +- docs/src/manual.md | 11 +- docs/src/support.md | 58 ++++ src/Postgres.jl | 269 +++++++++++++---- src/api/API.jl | 295 ++++++++++++++----- src/api/types.jl | 24 +- src/connection_string.jl | 70 +++-- src/execute.jl | 234 +++++++++++---- test/postgres_trim_queries.jl | 176 ----------- test/runtests.jl | 533 +++++++++++++++++++++++++++++++++- test/trim_compile_tests.jl | 263 ----------------- 17 files changed, 1379 insertions(+), 696 deletions(-) create mode 100644 LICENSE delete mode 100644 LICENSE.md create mode 100644 docs/src/support.md delete mode 100644 test/postgres_trim_queries.jl delete mode 100644 test/trim_compile_tests.jl diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index c4eeb0b..03f92ee 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -70,6 +70,60 @@ jobs: - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 + lower-bounds: + name: Lower dependency bounds - Julia 1.10 - ubuntu-latest + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v6 + - uses: julia-actions/setup-julia@v3 + with: + version: '1.10' + arch: x64 + - uses: julia-actions/cache@v3 + - name: Test declared dependency floors + run: | + using Pkg + mktempdir() do env + Pkg.activate(env) + Pkg.develop(path=ENV["GITHUB_WORKSPACE"]) + Pkg.add([ + Pkg.PackageSpec(name="ConcurrentUtilities", version="2.1.0"), + Pkg.PackageSpec(name="DBInterface", version="2.5.0"), + Pkg.PackageSpec(name="JSON", version="1.0.0"), + Pkg.PackageSpec(name="MD5", version="0.2.0"), + Pkg.PackageSpec(name="Parsers", version="2.5.4"), + Pkg.PackageSpec(name="Reseau", version="1.1.1"), + Pkg.PackageSpec(name="SASLAuth", version="1.0.0"), + Pkg.PackageSpec(name="StructUtils", version="2.3.0"), + Pkg.PackageSpec(name="Tables", version="1.0.0"), + Pkg.PackageSpec(name="URIs", version="1.0.0"), + ]) + Pkg.test("Postgres") + end + shell: julia --color=yes {0} + + postgres-versions: + name: PostgreSQL ${{ matrix.postgres }} - Julia 1 - ubuntu-latest + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + # PostgreSQL 16 is covered by the main OS and Julia-version matrix. + postgres: ['14', '15', '17', '18'] + env: + POSTGRES_IMAGE: postgres:${{ matrix.postgres }} + steps: + - uses: actions/checkout@v6 + - uses: julia-actions/setup-julia@v3 + with: + version: '1' + arch: x64 + - uses: julia-actions/cache@v3 + - uses: julia-actions/julia-buildpkg@v1 + - uses: julia-actions/julia-runtest@v1 + docs: name: Documentation runs-on: ubuntu-latest diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..4aa305a --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Jacob Quinn and contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/LICENSE.md b/LICENSE.md deleted file mode 100644 index 2639a23..0000000 --- a/LICENSE.md +++ /dev/null @@ -1,22 +0,0 @@ -The Postgres.jl package is licensed under the MIT "Expat" License: - -> Copyright (c) 2022: Jacob Quinn and contributors -> -> Permission is hereby granted, free of charge, to any person obtaining -> a copy of this software and associated documentation files (the -> "Software"), to deal in the Software without restriction, including -> without limitation the rights to use, copy, modify, merge, publish, -> distribute, sublicense, and/or sell copies of the Software, and to -> permit persons to whom the Software is furnished to do so, subject to -> the following conditions: -> -> The above copyright notice and this permission notice shall be -> included in all copies or substantial portions of the Software. -> -> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -> EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -> MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -> IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -> CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -> TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -> SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Project.toml b/Project.toml index 210cc27..41257e1 100644 --- a/Project.toml +++ b/Project.toml @@ -18,24 +18,30 @@ URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] -ConcurrentUtilities = "2" +Aqua = "0.8" +ConcurrentUtilities = "2.1" DBInterface = "2.5" +Dates = "1.10" Harbor = "1" JSON = "1" MD5 = "0.2" -Parsers = "2" -Reseau = "1.1" +Parsers = "2.5.4" +Random = "1.10" +Reseau = "1.1.1" SASLAuth = "1" -StructUtils = "2" +Sockets = "1.10" +StructUtils = "2.3" Tables = "1" +Test = "1.10" URIs = "1" +UUIDs = "1.10" julia = "1.10" [extras] +Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" Harbor = "af79dbb9-1a80-47ad-8928-192a4af69376" -JuliaC = "acedd4c2-ced6-4a15-accc-2607eb759ba2" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Harbor", "JuliaC", "Sockets", "Test"] +test = ["Aqua", "Harbor", "Sockets", "Test"] diff --git a/README.md b/README.md index 9ae61ea..24021ba 100644 --- a/README.md +++ b/README.md @@ -16,10 +16,10 @@ Pkg.add("Postgres") ## Quick start ```julia -using Postgres, DBInterface, Tables +using Postgres DBInterface.connect(Postgres.Connection, "host=127.0.0.1;port=5432;user=postgres;password=postgres;dbname=postgres") do conn - rows = Tables.rowtable(DBInterface.execute(conn, "SELECT 1 AS a")) - @show rows[1].a + row = only(DBInterface.execute(conn, "SELECT 1 AS a")) + @show row.a end ``` @@ -37,10 +37,14 @@ Connection options support: - PostgreSQL URIs such as `postgresql://postgres:postgres@127.0.0.1:5432/postgres`. - Environment defaults: `PGHOST`, `PGPORT`, `PGUSER`, `PGPASSWORD`, `PGDATABASE`, `PGAPPNAME`, `PGCONNECT_TIMEOUT`, and TLS-related `PGSSL*` variables. - `sslmode` values: `disable`, `prefer` (the default), `require`, `verify-full`. Only `verify-full` verifies the server's certificate; `require` encrypts without authenticating the server, and the default `prefer` falls back to an unencrypted connection if the server declines TLS. Use `verify-full` with `sslrootcert` when the connection needs to be authenticated. -- TLS files: `sslrootcert`, `sslcert`, `sslkey`, and `sslcapath` (a *fallback* CA file, used only when `sslrootcert` is unset and ignored otherwise; libpq-style hashed CA directories are not supported). `sslservername` overrides the TLS server name when connecting to a pre-resolved address; under `verify-full` it is also the name the certificate is verified against, so it must name the server you intend to authenticate. +- TLS files: `sslrootcert`, `sslcert`, `sslkey`, and `sslcapath` (`sslcapath` is a fallback CA bundle or directory, used only when `sslrootcert` is unset and ignored otherwise). `sslservername` overrides the TLS server name when connecting to a pre-resolved address; under `verify-full` it is also the name the certificate is verified against, so it must name the server you intend to authenticate. - `connect_timeout` (seconds) and `statement_timeout` (milliseconds). - `application_name` and `statement_cache_maxsize`. +See the [1.0 support policy](https://JuliaDatabases.github.io/Postgres.jl/dev/support/) +for tested Julia and PostgreSQL versions, TLS limits, and transaction-pooler +requirements. + You can also use `ConnectionParams`: ```julia @@ -107,7 +111,8 @@ profiles = DBInterface.execute(conn, """ `Postgres.command_tag(result)` and `Postgres.rows_affected(result)` expose PostgreSQL command completion metadata. -Statement caching is LRU-based. Set `statement_cache_maxsize=0` to disable caching. +Explicit named prepared statements use an LRU backend cache. Caller handles are +independent. Set `statement_cache_maxsize=0` to disable this cache. ```julia using Postgres, DBInterface @@ -187,6 +192,9 @@ DBInterface.close!(conn) ``` `Numeric` values are returned as `Postgres.Numeric`, `interval` values as `Dates.Period` or `Dates.CompoundPeriod`, and range types as `Postgres.PostgresRange{T}`. +Custom enum, composite, and range registration controls result decoding. Those +custom Julia values are not accepted as direct query parameters in 1.0; bind a +PostgreSQL text representation with an explicit SQL cast instead. ## Query logging and driver styles diff --git a/docs/make.jl b/docs/make.jl index 83c2d65..9278acf 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -6,6 +6,7 @@ makedocs( pages = [ "Home" => "index.md", "Manual" => "manual.md", + "Support Policy" => "support.md", ], ) diff --git a/docs/src/index.md b/docs/src/index.md index bb6e011..fa84b34 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -3,6 +3,7 @@ Postgres.jl is a PostgreSQL client that speaks the v3 wire protocol with `DBInterface` and `Tables` integration. See the [Manual](@ref) for a guided walk through connections, queries, prepared statements, transactions, cancellation, notifications, and type translation. +See the [1.0 Support Policy](@ref) for tested versions and explicit limits. ## Installation @@ -18,10 +19,13 @@ Postgres.jl accepts DSN strings or PostgreSQL URIs and supports: - libpq-style keyword strings such as `host=127.0.0.1 port=5432 user=postgres dbname=postgres`. - Environment defaults from `PGHOST`, `PGPORT`, `PGUSER`, `PGPASSWORD`, `PGDATABASE`, `PGAPPNAME`, `PGCONNECT_TIMEOUT`, and TLS-related `PGSSL*` variables. - `sslmode` values: `disable`, `prefer` (the default), `require`, `verify-full`. Only `verify-full` verifies the server's certificate; `require` encrypts without authenticating the server, and the default `prefer` falls back to an unencrypted connection if the server declines TLS. Use `verify-full` with `sslrootcert` when the connection needs to be authenticated. -- TLS files: `sslrootcert`, `sslcert`, `sslkey`, and `sslcapath` (a *fallback* CA file, used only when `sslrootcert` is unset and ignored otherwise; libpq-style hashed CA directories are not supported). `sslservername` overrides the TLS server name when connecting to a pre-resolved address; under `verify-full` it is also the name the certificate is verified against, so it must name the server you intend to authenticate. +- TLS files: `sslrootcert`, `sslcert`, `sslkey`, and `sslcapath` (`sslcapath` is a fallback CA bundle or directory, used only when `sslrootcert` is unset and ignored otherwise). `sslservername` overrides the TLS server name when connecting to a pre-resolved address; under `verify-full` it is also the name the certificate is verified against, so it must name the server you intend to authenticate. - `connect_timeout` (seconds), `statement_timeout` (milliseconds). - `application_name` and `statement_cache_maxsize`. +Options that request unsupported security or server-selection behavior are +rejected. They are not silently ignored. + ```julia using Postgres, DBInterface conn = DBInterface.connect(Postgres.Connection, "postgresql://postgres:postgres@127.0.0.1:5432/postgres?sslmode=disable") @@ -79,7 +83,7 @@ profiles = DBInterface.execute(conn, """ """, (), Vector{ProfileSummary}) ``` -Prepared statements are cached with LRU eviction; disable caching via `statement_cache_maxsize=0`. +Explicit named prepared statements use an LRU backend cache; disable it via `statement_cache_maxsize=0`. ```julia using Postgres, DBInterface, Tables diff --git a/docs/src/manual.md b/docs/src/manual.md index ee56c66..fe6baf5 100644 --- a/docs/src/manual.md +++ b/docs/src/manual.md @@ -147,7 +147,9 @@ rows = Tables.rowtable(DBInterface.execute(conn, raw"SELECT $1::int + $2::int AS @show only(rows).total ``` -Prepared statements can be created explicitly. Postgres.jl also caches prepared statements internally with LRU eviction; set `statement_cache_maxsize=0` to disable caching. +Prepared statements can be created explicitly. Postgres.jl caches their named +backend statements with LRU eviction while returning an independent handle to +each caller. Set `statement_cache_maxsize=0` to disable this cache. ```julia stmt = DBInterface.prepare(conn, raw"SELECT $1::text AS value") @@ -260,6 +262,10 @@ row = only(Tables.rowtable(DBInterface.execute(conn, "SELECT 'happy'::mood AS mo Registering composite and range types follows the same pattern. +Registration controls result decoding. Direct parameter binding for registered +enum, composite, and range Julia values is not part of the 1.0 interface. Bind +their PostgreSQL text representation and add an explicit SQL cast when needed. + ### Session Formats Text decoding assumes the server renders dates and timestamps in the `ISO` @@ -272,6 +278,9 @@ changed afterwards: running `SET DateStyle = ...` or `SET IntervalStyle = ...` mid-session breaks decoding — intervals and unparseable dates raise errors rather than silently returning wrong values. +Transaction-mode poolers cannot preserve session settings between logical +connections. See the [1.0 Support Policy](@ref) before using this mode. + ### Values Without A Julia Representation A few PostgreSQL values have no faithful Julia equivalent and raise diff --git a/docs/src/support.md b/docs/src/support.md new file mode 100644 index 0000000..94b6b43 --- /dev/null +++ b/docs/src/support.md @@ -0,0 +1,58 @@ +# 1.0 Support Policy + +Postgres.jl 1.0 supports Julia 1.10 and later. The release test matrix covers +PostgreSQL 14 through 18 on TCP connections. Unix-domain sockets are not +supported. + +## TLS + +`sslmode=verify-full` verifies the certificate chain and server name. Use a DNS +name as `host`, or set `sslservername` to the DNS name when dialing a resolved +address. IP-address subject-alternative-name matching on a TLS 1.2-only server +is not supported in 1.0. + +Client certificates require both `sslcert` and `sslkey`. Postgres.jl 1.0 limits +client-certificate connections to TLS 1.2 because the current Reseau 1.x mixed +TLS 1.2/1.3 client path does not send the certificate reliably. Server-only TLS +connections can negotiate TLS 1.2 or TLS 1.3. + +`connect_timeout` bounds the TCP connection and TLS handshake. It does not +bound PostgreSQL authentication or query execution. Use `statement_timeout` +for query execution on a direct or session-pooled connection. + +Keep a manual transaction or streaming cursor on the task that created it. +Do not run unrelated operations on that connection until the scope ends. Use +`ConnectionPool` to give concurrent tasks separate connections. + +## Transaction Poolers + +Connection-form `DBInterface.execute(conn, sql, params)` is safe through a +transaction-mode PgBouncer endpoint. Postgres.jl sends each unnamed extended +query as one dependent protocol segment. Explicit named prepared statements +require PgBouncer prepared-statement tracking, such as +`max_prepared_statements > 0`. + +A transaction pooler does not preserve session state between logical clients. +Do not use connection-level `statement_timeout`, `set_statement_timeout!`, +`LISTEN`, temporary tables, session advisory locks, or arbitrary `SET` state in +transaction mode. Configure PostgreSQL or the pooler defaults with +`DateStyle=ISO` and `IntervalStyle=postgres`; Postgres.jl needs these text +formats for correct decoding. Use direct connections or session pooling when +the application needs session state. + +`set_statement_timeout!` is rejected while a transaction is open. This keeps +the durable reconnect setting consistent with PostgreSQL's transactional `SET` +semantics. + +## Types + +Built-in scalar types, byte arrays, and one-dimensional PostgreSQL arrays can +be bound as parameters. Custom enum, composite, and range registration is a +result-decoding feature in 1.0. Bind a text representation with an explicit SQL +cast when writing those custom values. Multidimensional Julia arrays are not a +supported parameter form in 1.0. + +## Native Compilation + +JuliaC `--trim` compilation is not supported in Postgres.jl 1.0. Normal Julia +package precompilation is supported and is part of CI. diff --git a/src/Postgres.jl b/src/Postgres.jl index 9dc552d..cbba5ba 100644 --- a/src/Postgres.jl +++ b/src/Postgres.jl @@ -17,6 +17,8 @@ struct PostgresInterfaceError <: Exception end Base.showerror(io::IO, e::PostgresInterfaceError) = print(io, e.msg) +@noinline _reject_nul(what::String) = throw(PostgresInterfaceError("$what cannot contain a NUL byte")) + include("api/API.jl") using .API include("connection_string.jl") @@ -25,6 +27,17 @@ using .ConnectionString const Pools = ConcurrentUtilities.Pools const ReseauConn = Union{Reseau.TCP.Conn, Reseau.TLS.Conn} +# Observability must not change whether a database operation succeeds. A +# callback failure is reported, but never replaces the query result or error. +function query_log_safely(style, event::Symbol, info::NamedTuple) + try + API.query_logger(style, event, info) + catch err + @warn "Postgres query logger failed" event exception=(err, catch_backtrace()) + end + return nothing +end + """ Postgres.Connection @@ -48,13 +61,14 @@ Supported keyword arguments. All are also available as DSN/URI options except Only `verify-full` verifies the server's certificate; `require` encrypts without authenticating the server, and the default `prefer` falls back to an unencrypted connection if the server declines TLS. `sslcapath` is a - *fallback* CA file used only when `sslrootcert` is unset (it is ignored - otherwise); libpq-style hashed CA directories are not supported. + fallback CA bundle or directory used only when `sslrootcert` is unset (it is + ignored otherwise). `sslservername` overrides the TLS server name when the host is a pre-resolved address — note that under `verify-full` this is also the name the certificate is verified against, so it must name the server you intend to authenticate. -- `statement_cache_maxsize`: LRU prepared-statement cache size (default 100; `0` disables) +- `statement_cache_maxsize`: LRU backend cache size for explicit named prepared + statements (default 100; `0` disables) - `reconnect`: automatically reconnect and re-prepare statements if the connection is found dead (default `false`; never reconnects mid-transaction) - `style`: a custom [`AbstractPostgresStyle`](@ref Postgres.API.AbstractPostgresStyle) @@ -63,10 +77,12 @@ Supported keyword arguments. All are also available as DSN/URI options except but bind parameter values are not — treat a debug log as sensitive as the data the connection carries. -Connections are safe for concurrent use from multiple tasks: operations are -serialized on an internal lock. Close with `DBInterface.close!(conn)` or -`close(conn)`; the do-block form `DBInterface.connect(f, Postgres.Connection, ...)` -closes automatically. +Individual operations are serialized on an internal lock. Keep each manual +transaction and streaming cursor on one task and do not run unrelated work on +that connection until the scope ends. Use a [`ConnectionPool`](@ref) for +concurrent work. `cancel_query!` is the intentional cross-task exception. +Close with `DBInterface.close!(conn)` or `close(conn)`; the do-block form +`DBInterface.connect(f, Postgres.Connection, ...)` closes automatically. """ mutable struct Connection{T, S <: API.AbstractPostgresStyle} <: DBInterface.Connection const lock::ReentrantLock @@ -102,6 +118,7 @@ mutable struct Connection{T, S <: API.AbstractPostgresStyle} <: DBInterface.Conn const style::S in_transaction::Bool # track transaction state transaction_depth::Int # track nested transactions (SAVEPOINTs) + transaction_savepoints::Vector{String} # driver-owned names, outermost first generation::Int # increment on reconnect to invalidate statements # the server's own ReadyForQuery transaction status: unlike in_transaction # it also sees a transaction opened by raw SQL (`execute(conn, "BEGIN")`) @@ -127,10 +144,18 @@ mutable struct Connection{T, S <: API.AbstractPostgresStyle} <: DBInterface.Conn sslcapath_val = sslcapath === nothing ? nothing : String(sslcapath) statement_timeout_val = statement_timeout === nothing ? nothing : Int(statement_timeout) sslservername_val = sslservername === nothing ? nothing : String(sslservername) + occursin('\0', host) && _reject_nul("host") + occursin('\0', user) && _reject_nul("user") + occursin('\0', dbname) && _reject_nul("dbname") + password !== nothing && occursin('\0', password) && _reject_nul("password") + app_name !== nothing && occursin('\0', app_name) && _reject_nul("application_name") + sslservername_val !== nothing && occursin('\0', sslservername_val) && _reject_nul("sslservername") + xor(sslcert_val === nothing, sslkey_val === nothing) && + throw(PostgresInterfaceError("sslcert and sslkey must be provided together")) maxsize = max(0, Int(statement_cache_maxsize)) socket, pid, skey, server_params = API.connect(host, port, dbname, user, password, debug, app_name, timeout, sslmode_val, sslrootcert_val, sslcert_val, sslkey_val, sslcapath_val, sslservername_val, statement_timeout_val) registry = Dict(API.DEFAULT_TYPE_REGISTRY) - return new{Statement{typeof(style)}, typeof(style)}(ReentrantLock(), socket, host, user, password, dbname, port, app_name, timeout, sslmode_val, sslrootcert_val, sslcert_val, sslkey_val, sslcapath_val, sslservername_val, statement_timeout_val, pid, skey, Dict{String, Statement{typeof(style)}}(), maxsize, 0, server_params, registry, false, reconnect, debug, style, false, 0, 1, false, true) + return new{Statement{typeof(style)}, typeof(style)}(ReentrantLock(), socket, host, user, password, dbname, port, app_name, timeout, sslmode_val, sslrootcert_val, sslcert_val, sslkey_val, sslcapath_val, sslservername_val, statement_timeout_val, pid, skey, Dict{String, Statement{typeof(style)}}(), maxsize, 0, server_params, registry, false, reconnect, debug, style, false, 0, String[], 1, false, true) end end @@ -173,7 +198,10 @@ function evict_lru_statement!(conn::Connection) end oldest_sql === nothing && return stmt = pop!(conn.statements, oldest_sql) - !stmt.closed && API.close_statement(conn.socket, stmt.name, conn.debug) + if !stmt.closed + API.close_statement(conn.socket, stmt.name, conn.debug, conn.server_parameters) + stmt.closed = true + end return end @@ -183,7 +211,7 @@ end Return a copy of the connection's prepared-statement cache, keyed by SQL text. """ function get_cached_statements(conn::Connection) - @lock conn.lock copy(conn.statements) + @lock conn.lock Dict(sql => statement_handle(stmt) for (sql, stmt) in conn.statements) end """ @@ -194,7 +222,10 @@ Close all server-side prepared statements in the connection's cache and empty it function clear_statement_cache!(conn::Connection) @lock conn.lock begin for (sql, stmt) in conn.statements - !stmt.closed && API.close_statement(conn.socket, stmt.name, conn.debug) + if !stmt.closed + API.close_statement(conn.socket, stmt.name, conn.debug, conn.server_parameters) + stmt.closed = true + end end empty!(conn.statements) end @@ -212,7 +243,10 @@ function set_statement_cache_maxsize!(conn::Connection, maxsize::Integer) conn.statement_cache_maxsize = max(0, Int(maxsize)) if conn.statement_cache_maxsize == 0 for (sql, stmt) in conn.statements - !stmt.closed && API.close_statement(conn.socket, stmt.name, conn.debug) + if !stmt.closed + API.close_statement(conn.socket, stmt.name, conn.debug, conn.server_parameters) + stmt.closed = true + end end empty!(conn.statements) return conn @@ -258,17 +292,22 @@ end Postgres.set_statement_timeout!(conn, timeout) Set the server `statement_timeout` for the connection, in milliseconds. -`nothing` or `0` disables the timeout. +`nothing` or `0` disables the timeout. This setting is session state and cannot +be changed while a transaction is open. An explicit disable is retained across +automatic reconnects as `0`. """ function set_statement_timeout!(conn::Connection, timeout::Union{Integer, Nothing}) timeout_val = timeout === nothing ? 0 : max(0, Int(timeout)) - DBInterface.execute(conn, "SET statement_timeout = $timeout_val") - @lock conn.lock conn.statement_timeout = timeout === nothing ? nothing : timeout_val + @lock conn.lock begin + checkconn(conn) + (conn.in_transaction || conn.server_in_transaction) && + throw(PostgresInterfaceError("statement_timeout cannot be changed while a transaction is open")) + DBInterface.execute(conn, "SET statement_timeout = $timeout_val") + conn.statement_timeout = timeout_val + end return conn end -@noinline _reject_nul(what::String) = throw(PostgresInterfaceError("$what cannot contain a NUL byte")) - """ Postgres.escape_identifier(name) -> String @@ -287,15 +326,15 @@ Quote a string for use as a SQL literal (single-quoted, embedded quotes doubled). Throws if `val` contains a NUL byte. Prefer query parameters (`\$1`, `\$2`, ...) over literal interpolation -whenever possible — parameters are never parsed as SQL. This helper assumes -the server's `standard_conforming_strings` is `on` (the default since -PostgreSQL 9.1); with it turned off, backslashes in the literal are escape -characters and doubling quotes alone is not sufficient to make interpolation -safe. +whenever possible — parameters are never parsed as SQL. Inputs containing a +backslash use PostgreSQL's explicit escape-string syntax, with backslashes and +quotes escaped so the result is independent of `standard_conforming_strings`. """ function escape_literal(val::AbstractString) occursin('\0', val) && _reject_nul("literal") - return string("'", replace(val, "'" => "''"), "'") + escaped = replace(val, "'" => "''") + occursin('\\', escaped) || return string("'", escaped, "'") + return string("E'", replace(escaped, "\\" => "\\\\"), "'") end """ @@ -500,9 +539,9 @@ function copy_from(conn::Connection, sql::AbstractString, data::IO; debug::Bool= checkconn(conn) API.copy_in(conn.style, conn.socket, sql_str, data, debug || conn.debug) end - log_enabled && API.query_logger(conn.style, :copy_from, (sql=sql_str, duration_ns=time_ns() - start_ns, success=true)) + log_enabled && query_log_safely(conn.style, :copy_from, (sql=sql_str, duration_ns=time_ns() - start_ns, success=true)) catch err - log_enabled && API.query_logger(conn.style, :copy_from, (sql=sql_str, duration_ns=time_ns() - start_ns, success=false, error=err)) + log_enabled && query_log_safely(conn.style, :copy_from, (sql=sql_str, duration_ns=time_ns() - start_ns, success=false, error=err)) rethrow() end return conn @@ -529,9 +568,9 @@ function copy_to(conn::Connection, sql::AbstractString, dest::IO; debug::Bool=fa checkconn(conn) API.copy_out(conn.style, conn.socket, sql_str, dest, debug || conn.debug) end - log_enabled && API.query_logger(conn.style, :copy_to, (sql=sql_str, duration_ns=time_ns() - start_ns, success=true)) + log_enabled && query_log_safely(conn.style, :copy_to, (sql=sql_str, duration_ns=time_ns() - start_ns, success=true)) catch err - log_enabled && API.query_logger(conn.style, :copy_to, (sql=sql_str, duration_ns=time_ns() - start_ns, success=false, error=err)) + log_enabled && query_log_safely(conn.style, :copy_to, (sql=sql_str, duration_ns=time_ns() - start_ns, success=false, error=err)) rethrow() end return dest @@ -548,12 +587,14 @@ end Register a mapping from PostgreSQL type `oid` to `julia_type` in the connection's type registry. `parser` is a `(val::String, registry) -> value` -function that converts the wire text; without one, values are returned as -`String`. See also [`register_enum!`](@ref Postgres.register_enum!), +function that converts the wire text. If `parser` is omitted, `julia_type` +must be `String`. See also [`register_enum!`](@ref Postgres.register_enum!), [`register_composite!`](@ref Postgres.register_composite!), and [`register_range!`](@ref Postgres.register_range!). """ function register_type!(conn::Connection, oid::Integer, julia_type::Type; parser::Union{Function, Nothing}=nothing) + parser === nothing && julia_type !== String && + throw(ArgumentError("parser is required when julia_type is not String")) @lock conn.lock API.register_type!(conn.type_registry, oid, julia_type; parser=parser) return conn end @@ -582,9 +623,12 @@ end Postgres.register_enum!(conn, name; schema="public", julia_type=Symbol) Look up the enum type `schema.name` on the server and register it so values -are returned as `julia_type` (by default `Symbol`). +are returned as `julia_type` (by default `Symbol`). The supported types are +`Symbol` and `String`. """ function register_enum!(conn::Connection, name::AbstractString; schema::AbstractString="public", julia_type::Type=Symbol) + (julia_type === Symbol || julia_type === String) || + throw(ArgumentError("julia_type for register_enum! must be Symbol or String")) oid, arrayoid = lookup_type_oid(conn, name, schema) parser = julia_type === Symbol ? (val, registry) -> Symbol(val) : nothing register_type!(conn, oid, julia_type; parser=parser) @@ -725,6 +769,7 @@ function checkconn(conn::Connection) empty!(conn.statements) conn.in_transaction = false conn.transaction_depth = 0 + empty!(conn.transaction_savepoints) # the server-side transaction died with the old socket; a stale flag # would trigger a spurious ROLLBACK on the fresh session conn.server_in_transaction = false @@ -774,7 +819,9 @@ Base.close(conn::Connection) = DBInterface.close!(conn) A pool of [`Connection`](@ref Postgres.Connection)s, created lazily up to `limit` and reused across [`acquire`](@ref Postgres.acquire)/[`release`](@ref -Postgres.release) cycles (dead connections are replaced transparently): +Postgres.release) cycles. Locally closed connections are replaced. A peer close +that the local socket has not observed can surface on the borrower's first +operation; an ambiguous failed operation is never retried automatically. ConnectionPool(Postgres.Connection, host, user, password; limit=10, kwargs...) ConnectionPool(dsn::String; limit=10, kwargs...) @@ -787,13 +834,17 @@ acquire/release. Close all pooled connections with `DBInterface.close!(pool)`. struct ConnectionPool pool::Pools.Pool connector::Function + closed::Threads.Atomic{Bool} + lifecycle_lock::ReentrantLock end function ConnectionPool(connector::Function; limit::Integer=10) pool = Pools.Pool{Connection}(max(1, Int(limit))) - return ConnectionPool(pool, connector) + return ConnectionPool(pool, connector, Threads.Atomic{Bool}(false), ReentrantLock()) end +Base.isopen(pool::ConnectionPool) = !pool.closed[] + function ConnectionPool(::Type{Connection}, host::AbstractString, user::AbstractString, passwd::Union{AbstractString, Nothing}; dbname::AbstractString="", port::Integer=5432, debug::Bool=false, reconnect::Bool=false, application_name::Union{AbstractString, Nothing}=nothing, connect_timeout::Union{Integer, Nothing}=nothing, sslmode::Union{AbstractString, Nothing}=nothing, sslrootcert::Union{AbstractString, Nothing}=nothing, sslcert::Union{AbstractString, Nothing}=nothing, sslkey::Union{AbstractString, Nothing}=nothing, sslcapath::Union{AbstractString, Nothing}=nothing, sslservername::Union{AbstractString, Nothing}=nothing, statement_timeout::Union{Integer, Nothing}=nothing, statement_cache_maxsize::Integer=100, limit::Integer=10, style::API.AbstractPostgresStyle=PostgresStyle()) connector = () -> DBInterface.connect(Connection, host, user, passwd; dbname=dbname, port=port, debug=debug, reconnect=reconnect, application_name=application_name, connect_timeout=connect_timeout, sslmode=sslmode, sslrootcert=sslrootcert, sslcert=sslcert, sslkey=sslkey, sslcapath=sslcapath, sslservername=sslservername, statement_timeout=statement_timeout, statement_cache_maxsize=statement_cache_maxsize, style=style) return ConnectionPool(connector; limit=limit) @@ -820,7 +871,17 @@ Take a connection from the pool, creating one if none is available (blocking if the pool is at its limit). Return it with [`release`](@ref Postgres.release). """ function acquire(pool::ConnectionPool; forcenew::Bool=false) + pool.closed[] && throw(PostgresInterfaceError("connection pool is closed")) conn = Pools.acquire(pool.connector, pool.pool; forcenew=forcenew, isvalid=pool_isvalid) + closed = @lock pool.lifecycle_lock pool.closed[] + if closed + try + DBInterface.close!(conn) + finally + Pools.release(pool.pool) + end + throw(PostgresInterfaceError("connection pool is closed")) + end return conn end @@ -841,6 +902,7 @@ function reset_pooled_connection!(conn::Connection) finally conn.in_transaction = false conn.transaction_depth = 0 + empty!(conn.transaction_savepoints) end end return true @@ -863,9 +925,23 @@ the next borrower starts from a clean session; if it can't be rolled back it is closed rather than reused. """ function release(pool::ConnectionPool, conn::Connection) - if pool_isvalid(conn) && reset_pooled_connection!(conn) - Pools.release(pool.pool, conn) - else + reusable = !pool.closed[] && pool_isvalid(conn) && reset_pooled_connection!(conn) + if reusable + returned = @lock pool.lifecycle_lock begin + if pool.closed[] + false + else + Pools.release(pool.pool, conn) + true + end + end + returned && return pool + end + try + DBInterface.close!(conn) + catch + # already unusable; the permit still has to be returned + finally Pools.release(pool.pool) end return pool @@ -887,15 +963,18 @@ function with_connection(f::Function, pool::ConnectionPool; forcenew::Bool=false end function DBInterface.close!(pool::ConnectionPool) - Base.@lock pool.pool.lock begin - for conn in pool.pool.values - try - DBInterface.close!(conn) - catch - # ignore close errors for pooled connections + Base.@lock pool.lifecycle_lock begin + pool.closed[] = true + Base.@lock pool.pool.lock begin + for conn in pool.pool.values + try + DBInterface.close!(conn) + catch + # ignore close errors for pooled connections + end end + empty!(pool.pool.values) end - empty!(pool.pool.values) end return pool end @@ -910,20 +989,30 @@ include("execute.jl") # mid-sequence and drop the unnamed statement ("unnamed prepared statement # does not exist"). Also one network round trip instead of three. Callers # must hold conn.lock. -function execute_simple(conn::Connection, sql::String) +function execute_simple(conn::Connection, sql::String; expected_tag::Union{Nothing, String}=nothing) # capture the ReadyForQuery status through a Ref so a failed statement # (a COMMIT hitting a deferred constraint) still refreshes the tracking: # the server ended the transaction either way, and a stale flag draws a # spurious ROLLBACK on the next pool release status_ref = Ref{UInt8}(UInt8('I')) + command_tag_ref = Ref{Union{Nothing, String}}(nothing) try - API.exec(conn.style, conn.socket, sql, conn.debug, status_ref) + API.exec(conn.style, conn.socket, sql, conn.debug, status_ref, + command_tag_ref, conn.server_parameters) finally conn.server_in_transaction = API.in_transaction_status(status_ref[]) end + if expected_tag !== nothing && command_tag_ref[] != expected_tag + actual = something(command_tag_ref[], "no command tag") + throw(PostgresInterfaceError("expected $expected_tag but PostgreSQL completed $actual")) + end return conn end +function new_transaction_savepoint!() + return string("postgres_jl_", replace(string(UUIDs.uuid4()), "-" => "")) +end + """ Postgres.start_transaction(conn) @@ -942,18 +1031,21 @@ function start_transaction(conn::Connection) # belongs to the caller: nest inside it with a savepoint, as # for driver-owned nesting, so our commit can't commit — and # our rollback can't destroy — their work - execute_simple(conn, "SAVEPOINT sp_0") + savepoint = new_transaction_savepoint!() + execute_simple(conn, "SAVEPOINT $savepoint"; expected_tag="SAVEPOINT") + push!(conn.transaction_savepoints, savepoint) conn.owns_base_transaction = false else - execute_simple(conn, "BEGIN") + execute_simple(conn, "BEGIN"; expected_tag="BEGIN") conn.owns_base_transaction = true end conn.in_transaction = true conn.transaction_depth = 1 else # Start a SAVEPOINT for nested transactions - savepoint = "sp_$(conn.transaction_depth)" - execute_simple(conn, "SAVEPOINT $savepoint") + savepoint = new_transaction_savepoint!() + execute_simple(conn, "SAVEPOINT $savepoint"; expected_tag="SAVEPOINT") + push!(conn.transaction_savepoints, savepoint) conn.transaction_depth += 1 end end @@ -973,6 +1065,7 @@ function clear_transaction_state!(conn::Connection) @lock conn.lock begin conn.in_transaction = false conn.transaction_depth = 0 + empty!(conn.transaction_savepoints) conn.server_in_transaction = false end return @@ -992,6 +1085,7 @@ function commit(conn::Connection) if !isopen(conn.socket) conn.in_transaction = false conn.transaction_depth = 0 + empty!(conn.transaction_savepoints) disconnected() end checkconn(conn) @@ -1002,20 +1096,24 @@ function commit(conn::Connection) # would block reconnects and make the next cursor skip its BEGIN try if conn.owns_base_transaction - execute_simple(conn, "COMMIT") + execute_simple(conn, "COMMIT"; expected_tag="COMMIT") else # the base transaction is the caller's raw-SQL one: keep # this level's work pending inside it and leave it open - execute_simple(conn, "RELEASE SAVEPOINT sp_0") + savepoint = only(conn.transaction_savepoints) + execute_simple(conn, "RELEASE SAVEPOINT $savepoint"; expected_tag="RELEASE") end finally conn.in_transaction = false conn.transaction_depth = 0 + empty!(conn.transaction_savepoints) end else # Release SAVEPOINT for nested transaction + savepoint = last(conn.transaction_savepoints) + execute_simple(conn, "RELEASE SAVEPOINT $savepoint"; expected_tag="RELEASE") + pop!(conn.transaction_savepoints) conn.transaction_depth -= 1 - # Don't need to release SAVEPOINT explicitly, just commit will handle it end end return conn @@ -1034,6 +1132,7 @@ function rollback(conn::Connection) if !isopen(conn.socket) conn.in_transaction = false conn.transaction_depth = 0 + empty!(conn.transaction_savepoints) disconnected() end checkconn(conn) @@ -1042,21 +1141,26 @@ function rollback(conn::Connection) # how ROLLBACK fares, so don't leave client state describing it try if conn.owns_base_transaction - execute_simple(conn, "ROLLBACK") + execute_simple(conn, "ROLLBACK"; expected_tag="ROLLBACK") else # undo only this level; a plain ROLLBACK would destroy the # caller's raw-SQL transaction along with it - execute_simple(conn, "ROLLBACK TO SAVEPOINT sp_0") + savepoint = only(conn.transaction_savepoints) + execute_simple(conn, "ROLLBACK TO SAVEPOINT $savepoint"; expected_tag="ROLLBACK") + execute_simple(conn, "RELEASE SAVEPOINT $savepoint"; expected_tag="RELEASE") end finally conn.in_transaction = false conn.transaction_depth = 0 + empty!(conn.transaction_savepoints) end else # Rollback to SAVEPOINT for nested transaction + savepoint = last(conn.transaction_savepoints) + execute_simple(conn, "ROLLBACK TO SAVEPOINT $savepoint"; expected_tag="ROLLBACK") + execute_simple(conn, "RELEASE SAVEPOINT $savepoint"; expected_tag="RELEASE") + pop!(conn.transaction_savepoints) conn.transaction_depth -= 1 - savepoint = "sp_$(conn.transaction_depth)" - execute_simple(conn, "ROLLBACK TO SAVEPOINT $savepoint") end end return conn @@ -1114,6 +1218,24 @@ function DBInterface.transaction(f::F, conn::Connection) where {F} end end +struct TransactionReturn{T} <: Exception + value::T +end + +function rewrite_transaction_returns(expr) + expr isa Expr || return expr + if expr.head === :return + value = isempty(expr.args) ? nothing : rewrite_transaction_returns(only(expr.args)) + marker = GlobalRef(@__MODULE__, :TransactionReturn) + return Expr(:call, GlobalRef(Core, :throw), Expr(:call, marker, value)) + elseif expr.head === :function || expr.head === :(->) || expr.head === :quote + # A return in a nested function belongs to that function, not to the + # scope that contains this transaction macro. + return expr + end + return Expr(expr.head, map(rewrite_transaction_returns, expr.args)...) +end + """ Postgres.@transaction conn expr @@ -1121,6 +1243,7 @@ Run `expr` inside a transaction: committed if it completes, rolled back if it throws. Evaluates to `expr`'s value. """ macro transaction(conn, expr) + body = rewrite_transaction_returns(expr) quote # bind once: the connection expression may have side effects # (`@transaction acquire(pool) ...` would otherwise take a different @@ -1129,7 +1252,17 @@ macro transaction(conn, expr) local success = false start_transaction(c) try - result = $(esc(expr)) + local result + try + result = $(esc(body)) + catch err + if err isa TransactionReturn + commit(c) + success = true + return err.value + end + rethrow() + end commit(c) success = true result @@ -1203,13 +1336,29 @@ function describe(conn::Connection, table::AbstractString; schema::String="publi FROM information_schema.columns c LEFT JOIN - information_schema.key_column_usage kcu ON c.table_name = kcu.table_name AND c.column_name = kcu.column_name + information_schema.key_column_usage kcu ON + c.table_catalog = kcu.table_catalog AND + c.table_schema = kcu.table_schema AND + c.table_name = kcu.table_name AND + c.column_name = kcu.column_name LEFT JOIN - information_schema.table_constraints tc ON kcu.constraint_name = tc.constraint_name + information_schema.table_constraints tc ON + kcu.constraint_catalog = tc.constraint_catalog AND + kcu.constraint_schema = tc.constraint_schema AND + kcu.constraint_name = tc.constraint_name AND + kcu.table_schema = tc.table_schema AND + kcu.table_name = tc.table_name LEFT JOIN - information_schema.referential_constraints rc ON tc.constraint_name = rc.constraint_name + information_schema.referential_constraints rc ON + tc.constraint_catalog = rc.constraint_catalog AND + tc.constraint_schema = rc.constraint_schema AND + tc.constraint_name = rc.constraint_name LEFT JOIN - information_schema.key_column_usage fk ON rc.unique_constraint_name = fk.constraint_name AND fk.table_schema = c.table_schema + information_schema.key_column_usage fk ON + rc.unique_constraint_catalog = fk.constraint_catalog AND + rc.unique_constraint_schema = fk.constraint_schema AND + rc.unique_constraint_name = fk.constraint_name AND + kcu.position_in_unique_constraint = fk.ordinal_position WHERE c.table_name = \$1 AND c.table_schema = \$2 ) diff --git a/src/api/API.jl b/src/api/API.jl index 97cd78f..fb62870 100644 --- a/src/api/API.jl +++ b/src/api/API.jl @@ -193,6 +193,21 @@ function notificationResponse(len, socket) return Notification(pid, channel, payload) end +function parameterStatus!(parameters::Dict{String, String}, len, socket) + buf = read(socket, len) + length(buf) == len || close_and_throw(socket, Error("truncated ParameterStatus message from server")) + first_nul = findfirst(isequal(UInt8(0)), buf) + first_nul === nothing && close_and_throw(socket, Error("invalid ParameterStatus message from server")) + second_nul = findnext(isequal(UInt8(0)), buf, first_nul + 1) + (second_nul === length(buf) && first_nul > 1) || + close_and_throw(socket, Error("invalid ParameterStatus message from server")) + key = GC.@preserve buf unsafe_string(pointer(buf), first_nul - 1) + value_start = first_nul + 1 + value = GC.@preserve buf unsafe_string(pointer(buf, value_start), second_nul - value_start) + parameters[key] = value + return nothing +end + include("types.jl") struct Params @@ -210,11 +225,14 @@ _msgsizeof_parts(parts::Tuple) = msgsizeof(first(parts)) + _msgsizeof_parts(Base writepart(io, x) = write(io, x) function writepart(io, x::String) + occursin('\0', x) && throw(Postgres.PostgresInterfaceError("PostgreSQL protocol strings cannot contain a NUL byte")) write(io, x) write(io, UInt8(0)) end writepart(io, x::Integer) = write(io, hton(x)) function writepart(io, x::Tuple{String, String}) + (occursin('\0', x[1]) || occursin('\0', x[2])) && + throw(Postgres.PostgresInterfaceError("PostgreSQL startup parameters cannot contain a NUL byte")) write(io, x[1]) write(io, UInt8(0)) write(io, x[2]) @@ -284,7 +302,8 @@ function writestartupmessage( # statement_timeout is applied with a SET after connect rather than through # the startup `options` parameter: poolers (pgbouncer) reject unknown # startup options outright, so sending it here fails the whole connection. - len = 8 + msgsizeof(("user", user)) + msgsizeof(("database", dbname)) + 1 + len = 8 + msgsizeof(("user", user)) + msgsizeof(("database", dbname)) + + msgsizeof(("client_encoding", "UTF8")) + 1 application_name !== nothing && (len += msgsizeof(("application_name", application_name))) debug && @info "sending startup message" buf = IOBuffer(Vector{UInt8}(undef, len); write=true) @@ -292,6 +311,7 @@ function writestartupmessage( write(buf, hton(Int32(196608))) _write_startup_param(buf, "user", user) _write_startup_param(buf, "database", dbname) + _write_startup_param(buf, "client_encoding", "UTF8") application_name !== nothing && _write_startup_param(buf, "application_name", application_name) write(buf, UInt8(0)) write(socket, take!(buf)) @@ -323,8 +343,9 @@ function skipbytes!(io::IO, n::Integer) buf = Vector{UInt8}(undef, min(SKIP_BUFFER_SIZE, remaining)) while remaining > 0 nb = min(length(buf), remaining) - readbytes!(io, buf, nb) - remaining -= nb + nr = readbytes!(io, buf, nb) + nr == nb || throw(EOFError()) + remaining -= nr end return nothing end @@ -336,6 +357,10 @@ end # before authentication (an ErrorResponse to the SSLRequest), so it must not # depend on a trusted peer. const MAX_MESSAGE_LEN = Int32(1) << 30 +# Authentication and startup messages are small. A separate bound prevents an +# unauthenticated peer from forcing a process-sized allocation before the +# connection is established. +const MAX_PREAUTH_MESSAGE_LEN = Int32(1) << 20 # A bogus length means the stream is desynchronized, so the socket must be # closed before throwing — callers such as describeprepared treat a surviving @@ -346,11 +371,11 @@ const MAX_MESSAGE_LEN = Int32(1) << 30 throw(Error("invalid message length $len from server; connection protocol state is corrupted")) end -function readheader(socket, debug=false) +function readheader(socket, debug=false, max_message_len::Int32=MAX_MESSAGE_LEN) mt = read(socket, UInt8) len = ntoh(read(socket, Int32)) - 4 debug && @info "readheader: $(Char(mt)), $len" - (len < 0 || len > MAX_MESSAGE_LEN) && _bad_message_length(socket, len) + (len < 0 || len > max_message_len) && _bad_message_length(socket, len) return mt, len end @@ -387,7 +412,8 @@ end # ErrorResponse is read fully, the stream drained through ReadyForQuery (the # connection stays usable), and thrown as a Postgres.Error. Any other message # type means the stream is desynchronized: close the connection and throw. -function read_expected(socket, debug, expected::Char...) +function read_expected(socket, debug, expected::Char...; + server_parameters::Union{Nothing, Dict{String, String}}=nothing) while true mt, len = readheader(socket, debug) if any(c -> mt == UInt8(c), expected) @@ -396,7 +422,10 @@ function read_expected(socket, debug, expected::Char...) err = errorResponse(len, socket, debug) drain_to_ready!(socket, debug) throw(err) - elseif mt == UInt8('S') || mt == UInt8('N') || mt == UInt8('A') + elseif mt == UInt8('S') + server_parameters === nothing ? skipbytes!(socket, len) : + parameterStatus!(server_parameters, len, socket) + elseif mt == UInt8('N') || mt == UInt8('A') skipbytes!(socket, len) else close_and_throw(socket, Error("unexpected message type '$(Char(mt))' from server; connection protocol state is corrupted")) @@ -411,7 +440,9 @@ function expect_auth_message(socket, debug, mt, len) end # wait for code, then ready -function waitfor(socket, debug::Bool, codes::Vararg{Char, N}) where {N} +function waitfor(socket, debug::Bool, codes::Vararg{Char, N}; + max_message_len::Int32=MAX_MESSAGE_LEN, + server_parameters::Union{Nothing, Dict{String, String}}=nothing) where {N} error = false error_msg = nothing found = _sum_codes(codes) @@ -420,7 +451,7 @@ function waitfor(socket, debug::Bool, codes::Vararg{Char, N}) where {N} debug && @info "waitfor: $codes" try while true - mt, len = readheader(socket, debug) + mt, len = readheader(socket, debug, max_message_len) if mt == UInt8('E') # error error = true @@ -430,20 +461,12 @@ function waitfor(socket, debug::Bool, codes::Vararg{Char, N}) where {N} skipbytes!(socket, len) break elseif mt == UInt8('S') - # parameter status - buf = read(socket, len) - i = 1 - GC.@preserve buf while i <= length(buf) - j = findnext(isequal(UInt8(0)), buf, i) - j === nothing && break - key = unsafe_string(pointer(buf, i), j - i) - i = j + 1 - j = findnext(isequal(UInt8(0)), buf, i) - j === nothing && break - val = unsafe_string(pointer(buf, i), j - i) - server_params[key] = val - i = j + 1 - end + # ParameterStatus is part of the connection's public state. + # During startup, collect it in the return value. During later + # waits, update the connection-owned dictionary supplied by + # the caller. + target = server_parameters === nothing ? server_params : server_parameters + parameterStatus!(target, len, socket) elseif _contains_code(mt, codes) # found found -= mt @@ -498,7 +521,7 @@ function authRequest(debug, len, socket, user, password, client::Union{Nothing, elseif auth_code == 3 # send cleartext password message write_password_message(socket, debug, password) - mt, len = readheader(socket, debug) + mt, len = readheader(socket, debug, MAX_PREAUTH_MESSAGE_LEN) if mt == UInt8('E') # error close_and_throw_error_response(socket, len, debug) @@ -522,7 +545,7 @@ function authRequest(debug, len, socket, user, password, client::Union{Nothing, pass = string("md5", bytes2hex(md5(vcat(Vector{UInt8}(bytes2hex(md5(string(password, user)))), salt)))) # Send password message write_password_message(socket, debug, pass) - mt, len = readheader(socket, debug) + mt, len = readheader(socket, debug, MAX_PREAUTH_MESSAGE_LEN) if mt == UInt8('E') # error close_and_throw_error_response(socket, len, debug) @@ -565,7 +588,7 @@ function authRequest(debug, len, socket, user, password, client::Union{Nothing, # write them with debug=false and log a redacted line instead debug && @info "sending message: p, (SASL initial response redacted)" writemessage(socket, false, 'p', "SCRAM-SHA-256", Int32(length(bytes)), bytes) - mt, len = readheader(socket, debug) + mt, len = readheader(socket, debug, MAX_PREAUTH_MESSAGE_LEN) expect_auth_message(socket, debug, mt, len) return authRequest(debug, len, socket, user, password, client) elseif auth_code == 11 @@ -574,7 +597,7 @@ function authRequest(debug, len, socket, user, password, client::Union{Nothing, msg, _ = SASLAuth.step!(client, challenge) debug && @info "sending message: p, (SASL response redacted)" writemessage(socket, false, 'p', Vector{UInt8}(msg)) - mt, len = readheader(socket, debug) + mt, len = readheader(socket, debug, MAX_PREAUTH_MESSAGE_LEN) expect_auth_message(socket, debug, mt, len) return authRequest(debug, len, socket, user, password, client) elseif auth_code == 12 @@ -582,7 +605,7 @@ function authRequest(debug, len, socket, user, password, client::Union{Nothing, final_msg = String(read(socket, len - 4)) _, done = SASLAuth.step!(client, final_msg) done || close_and_throw(socket, Error("SASL authentication did not complete")) - mt, len = readheader(socket, debug) + mt, len = readheader(socket, debug, MAX_PREAUTH_MESSAGE_LEN) expect_auth_message(socket, debug, mt, len) auth_code = ntoh(read(socket, Int32)) auth_code == 0 || close_and_throw(socket, Error("SASL authentication failed: $auth_code")) @@ -663,9 +686,12 @@ function tlsupgrade(socket::Reseau.TCP.Conn, @nospecialize(connect_timeout::Unio nothing, nothing, ca_file, nothing, String[], UInt16[], handshake_timeout_ns, Reseau.TLS.TLS1_2_VERSION, nothing, false) else + # Reseau 1.3.x does not send a configured client certificate on its + # mixed TLS 1.2/1.3 client path. Its TLS 1.2 path does. Keep mTLS + # functional and fail-closed until the dependency fixes that path. Reseau.TLS.Config(sni, verify_peer, verify_peer, Reseau.TLS.ClientAuthMode.NoClientCert, ssl_cert::String, ssl_key::String, ca_file, nothing, String[], UInt16[], - handshake_timeout_ns, Reseau.TLS.TLS1_2_VERSION, nothing, false) + handshake_timeout_ns, Reseau.TLS.TLS1_2_VERSION, Reseau.TLS.TLS1_2_VERSION, false) end tls_conn = Reseau.TLS.client(socket, config) try @@ -720,7 +746,7 @@ function connect(host::String, port::Integer, dbname::String, user::String, @nos # pre-TLS and pre-auth, so bound the length like readheader does # before handing it to the allocating read. len = ntoh(read(socket, Int32)) - 4 - (len < 0 || len > MAX_MESSAGE_LEN) && close_and_throw(socket, Error("invalid message length $len from server")) + (len < 0 || len > MAX_PREAUTH_MESSAGE_LEN) && close_and_throw(socket, Error("invalid message length $len from server")) close_and_throw_error_response(socket, len, debug) else close_and_throw(socket, Error("unexpected response to SSLRequest: $(Char(mt))")) @@ -733,7 +759,7 @@ function connect(host::String, port::Integer, dbname::String, user::String, @nos writestartupmessage(socket::Reseau.TLS.Conn, debug, user, dbname, application_name_v, statement_timeout_v) end # read initial response - mt, len = readheader(socket, debug) + mt, len = readheader(socket, debug, MAX_PREAUTH_MESSAGE_LEN) if mt == UInt8('E') # error close_and_throw_error_response(socket, len, debug) @@ -743,7 +769,9 @@ function connect(host::String, port::Integer, dbname::String, user::String, @nos # server version too old close_and_throw(socket, Error("server version too old")) end - pid, skey, server_params = waitfor(socket, debug, 'K', 'Z') + pid, skey, server_params = waitfor(socket, debug, 'K', 'Z'; max_message_len=MAX_PREAUTH_MESSAGE_LEN) + uppercase(replace(get(server_params, "client_encoding", ""), "-" => "")) == "UTF8" || + close_and_throw(socket, Error("server did not confirm UTF8 client_encoding")) # socket-union isa split so the call resolves under --trim, as above if socket isa Reseau.TCP.Conn align_session_formats!(socket::Reseau.TCP.Conn, server_params, debug, statement_timeout_v) @@ -777,7 +805,8 @@ end function align_session_formats!(socket, server_params::Dict{String, String}, debug::Bool, @nospecialize(statement_timeout::Union{Int, Nothing})=nothing) if statement_timeout !== nothing - exec(PostgresStyle(), socket, string("SET statement_timeout = ", statement_timeout::Int), debug) + exec(PostgresStyle(), socket, string("SET statement_timeout = ", statement_timeout::Int), + debug, nothing, nothing, server_params) end # A parameter the server didn't report (a pooler may not forward it) must # be treated as unknown, i.e. corrected — assuming it is already right is @@ -788,47 +817,65 @@ function align_session_formats!(socket, server_params::Dict{String, String}, deb # the format half preserves whatever field order is configured # server-side, which we can't see — naming an order here would flip a # DMY session to MDY - exec(PostgresStyle(), socket, "SET DateStyle = 'ISO'", debug) + exec(PostgresStyle(), socket, "SET DateStyle = 'ISO'", debug, + nothing, nothing, server_params) server_params["DateStyle"] = "ISO" elseif !startswith(datestyle, "ISO") wanted = string("ISO, ", date_order(datestyle)) - exec(PostgresStyle(), socket, string("SET DateStyle = '", wanted, "'"), debug) + exec(PostgresStyle(), socket, string("SET DateStyle = '", wanted, "'"), + debug, nothing, nothing, server_params) server_params["DateStyle"] = wanted end if get(server_params, "IntervalStyle", "") != "postgres" - exec(PostgresStyle(), socket, "SET IntervalStyle = 'postgres'", debug) + exec(PostgresStyle(), socket, "SET IntervalStyle = 'postgres'", debug, + nothing, nothing, server_params) server_params["IntervalStyle"] = "postgres" end return end -function prepare(socket, sql::String, debug::Bool; name::Union{Nothing, String}=nothing) +function prepare(socket, sql::String, debug::Bool, + server_parameters::Union{Nothing, Dict{String, String}}=nothing; + name::Union{Nothing, String}=nothing) stmtname = name === nothing ? randstring(Random.RandomDevice(), 36) : String(name) writemessages(socket, debug, ('P', stmtname, sql, Int16(0)), ('S',)) - waitfor(socket, debug, '1', 'Z') + waitfor(socket, debug, '1', 'Z'; server_parameters=server_parameters) return stmtname end _symbol(ptr, len) = ccall(:jl_symbol_n, Ref{Symbol}, (Ptr{UInt8}, Int), ptr, len) -function describeprepared(socket, name::String, debug::Bool) +function describeprepared(socket, name::String, debug::Bool, + server_parameters::Union{Nothing, Dict{String, String}}=nothing) writemessages(socket, debug, ('D', UInt8('S'), name), ('S',)) + nparams, cols, types = readprepareddescription(socket, debug, server_parameters) + waitfor(socket, debug, 'Z'; server_parameters=server_parameters) + return nparams, cols, types +end + +function readprepareddescription(socket, debug::Bool, + server_parameters::Union{Nothing, Dict{String, String}}=nothing) nparams = 0 ncols = 0 cols = Symbol[] types = Int[] try - mt, len = read_expected(socket, debug, 't') + mt, len = read_expected(socket, debug, 't'; server_parameters=server_parameters) + len >= 2 || close_and_throw(socket, Error("truncated ParameterDescription message from server")) nparams = Int(ntoh(read(socket, Int16))) + nparams >= 0 || close_and_throw(socket, Error("invalid ParameterDescription count from server")) + len == 2 + 4nparams || close_and_throw(socket, Error("invalid ParameterDescription length from server")) skipbytes!(socket, len - 2) - mt, len = read_expected(socket, debug, 'T', 'n') + mt, len = read_expected(socket, debug, 'T', 'n'; server_parameters=server_parameters) if mt == UInt8('n') # no data - waitfor(socket, debug, 'Z') return nparams, cols, types end + len >= 2 || close_and_throw(socket, Error("truncated RowDescription message from server")) ncols = Int(ntoh(read(socket, Int16))) + ncols >= 0 || close_and_throw(socket, Error("invalid RowDescription column count from server")) buf = read(socket, len - 2) + length(buf) == len - 2 || close_and_throw(socket, Error("truncated RowDescription message from server")) i = 1 # each field: name (cstring), table oid (4), column number (2), # type oid (4), type length (2), type modifier (4), format code (2). @@ -836,16 +883,17 @@ function describeprepared(socket, name::String, debug::Bool) # a short read or a malformed RowDescription must not read past it. GC.@preserve buf while i <= length(buf) stop = findnext(isequal(UInt8(0)), buf, i) - stop === nothing && break + stop === nothing && close_and_throw(socket, Error("truncated RowDescription message from server")) name = _symbol(pointer(buf, i), stop - i) i = stop + 1 - i + 17 <= length(buf) || break + i + 17 <= length(buf) || close_and_throw(socket, Error("truncated RowDescription message from server")) typeId = Int(ntoh(unsafe_load(Ptr{Int32}(pointer(buf, i + 6))))) i += 18 push!(types, typeId) push!(cols, name) end - waitfor(socket, debug, 'Z') + length(cols) == ncols || close_and_throw(socket, Error("RowDescription column count does not match its fields")) + i == length(buf) + 1 || close_and_throw(socket, Error("RowDescription message has trailing bytes")) return nparams, cols, types catch err # a deliberately-thrown Error leaves the stream at ReadyForQuery (or @@ -898,16 +946,26 @@ function StructUtils.applyeach(::AbstractPostgresStyle, f, dr::DataRow) @inbounds applycast(f, dr.names[i], dr.typeIds[i], str, dr.type_registry) end end + pos == nbuf + 1 || throw(Error("DataRow message has trailing bytes")) end return end +# Resolve the intersection with StructUtils.applyeach(f, ::StructStyle, x). +# A StructStyle is not a row callback, so this argument order is invalid. +function StructUtils.applyeach(style::AbstractPostgresStyle, + callback::StructUtils.StructStyle, + dr::DataRow) + throw(MethodError(StructUtils.applyeach, (style, callback, dr))) +end + struct Exec{S <: AbstractPostgresStyle} style::S socket::ReseauConn names::Vector{Symbol} typeIds::Vector{Int} type_registry::Dict{Int, TypeInfo} + server_parameters::Dict{String, String} debug::Bool command_tag::Base.RefValue{Union{Nothing, String}} rows_affected::Base.RefValue{Union{Nothing, Int}} @@ -917,12 +975,12 @@ struct Exec{S <: AbstractPostgresStyle} tx_status::Base.RefValue{UInt8} end -# read ReadyForQuery's one-byte transaction status (older/odd servers may send -# an empty body; treat that as unknown-but-idle) +# Read ReadyForQuery's required one-byte transaction status. function read_ready_status(socket, len) - len < 1 && (skipbytes!(socket, len); return UInt8('I')) + len == 1 || close_and_throw(socket, Error("invalid ReadyForQuery message length from server")) status = read(socket, UInt8) - skipbytes!(socket, len - 1) + status in (UInt8('I'), UInt8('T'), UInt8('E')) || + close_and_throw(socket, Error("invalid ReadyForQuery transaction status from server")) return status end @@ -930,7 +988,10 @@ in_transaction_status(status::UInt8) = status == UInt8('T') || status == UInt8(' function commandComplete(len, socket) buf = read(socket, len) - isempty(buf) && return "" + length(buf) == len || throw(Error("truncated CommandComplete message from server")) + isempty(buf) && throw(Error("empty CommandComplete message from server")) + findfirst(isequal(UInt8(0)), buf) == length(buf) || + throw(Error("invalid CommandComplete message from server")) tag, _ = cstring_at(buf, 1) return tag end @@ -980,8 +1041,10 @@ function StructUtils.applyeach(::AbstractPostgresStyle, f, e::Exec) tag = commandComplete(len, e.socket) e.command_tag[] = tag e.rows_affected[] = rows_affected_from_command_tag(tag) - elseif mt == UInt8('T') || mt == UInt8('n') || mt == UInt8('I') || mt == UInt8('S') - # row description / no data / empty query response / parameter status + elseif mt == UInt8('S') + parameterStatus!(e.server_parameters, len, e.socket) + elseif mt == UInt8('T') || mt == UInt8('n') || mt == UInt8('I') + # row description / no data / empty query response skipbytes!(e.socket, len) elseif mt == UInt8('G') # CopyInResponse: the statement was a COPY ... FROM STDIN, which @@ -1038,17 +1101,59 @@ function StructUtils.applyeach(::AbstractPostgresStyle, f, e::Exec) return end -function exec(style::S, socket::ReseauConn, stmtname::String, params::Vector{Union{String, Missing}}, names, typeIds, type_registry::Dict{Int, TypeInfo}, debug::Bool, rowlimit::Int=0) where {S <: AbstractPostgresStyle} +function StructUtils.applyeach(style::AbstractPostgresStyle, + callback::StructUtils.StructStyle, + e::Exec) + throw(MethodError(StructUtils.applyeach, (style, callback, e))) +end + +function exec(style::S, socket::ReseauConn, stmtname::String, + params::Vector{Union{String, Missing}}, names, typeIds, + type_registry::Dict{Int, TypeInfo}, debug::Bool, rowlimit::Int=0, + server_parameters::Dict{String, String}=Dict{String, String}()) where {S <: AbstractPostgresStyle} #TODO: support binary format: here and in applycast npformats = Int16(0) # all params use text format nparams = Int16(length(params)) # bind, then execute, then sync writemessages(socket, debug, ('B', "", stmtname, npformats, nparams, Params(params), Int16(0)), ('E', "", Int32(rowlimit)), ('S',)) - waitfor(socket, debug, '2') - return Exec{S}(style, socket, names, typeIds, type_registry, debug, Ref{Union{Nothing, String}}(nothing), Ref{Union{Nothing, Int}}(nothing), Ref{UInt8}(UInt8('I'))) + waitfor(socket, debug, '2'; server_parameters=server_parameters) + return Exec{S}(style, socket, names, typeIds, type_registry, + server_parameters, debug, Ref{Union{Nothing, String}}(nothing), + Ref{Union{Nothing, Int}}(nothing), Ref{UInt8}(UInt8('I'))) end -function exec(style::S, socket::ReseauConn, query::String, debug::Bool, tx_status_ref::Union{Nothing, Base.RefValue{UInt8}}=nothing) where {S <: AbstractPostgresStyle} +# Execute an unnamed statement as one extended-query segment. There is one +# Sync, after Parse/Describe/Bind/Execute. A transaction-mode pooler therefore +# cannot return the backend between dependent protocol messages and replace the +# unnamed statement with another client's statement. +function exec_unnamed(style::S, socket::ReseauConn, sql::String, + params::Vector{Union{String, Missing}}, + type_registry::Dict{Int, TypeInfo}, debug::Bool, + rowlimit::Int=0, + server_parameters::Dict{String, String}=Dict{String, String}()) where {S <: AbstractPostgresStyle} + npformats = Int16(0) + nparams = Int16(length(params)) + writemessages(socket, debug, + ('P', "", sql, Int16(0)), + ('D', UInt8('S'), ""), + ('B', "", "", npformats, nparams, Params(params), Int16(0)), + ('E', "", Int32(rowlimit)), + ('S',)) + mt, len = read_expected(socket, debug, '1'; server_parameters=server_parameters) + skipbytes!(socket, len) + _, names, typeIds = readprepareddescription(socket, debug, server_parameters) + mt, len = read_expected(socket, debug, '2'; server_parameters=server_parameters) + skipbytes!(socket, len) + return Exec{S}(style, socket, names, typeIds, type_registry, server_parameters, debug, + Ref{Union{Nothing, String}}(nothing), + Ref{Union{Nothing, Int}}(nothing), + Ref{UInt8}(UInt8('I'))) +end + +function exec(style::S, socket::ReseauConn, query::String, debug::Bool, + tx_status_ref::Union{Nothing, Base.RefValue{UInt8}}=nothing, + command_tag_ref::Union{Nothing, Base.RefValue{Union{Nothing, String}}}=nothing, + server_parameters::Union{Nothing, Dict{String, String}}=nothing) where {S <: AbstractPostgresStyle} writemessages(socket, debug, ('Q', query)) server_error = nothing tx_status = UInt8('I') @@ -1071,9 +1176,14 @@ function exec(style::S, socket::ReseauConn, query::String, debug::Bool, tx_statu notice_callback(style, noticeResponse(len, socket)) elseif mt == UInt8('A') notification_callback(style, notificationResponse(len, socket)) - elseif mt == UInt8('C') || mt == UInt8('T') || mt == UInt8('D') || - mt == UInt8('I') || mt == UInt8('S') - # CommandComplete and any incidental simple-query result data. + elseif mt == UInt8('C') + tag = commandComplete(len, socket) + command_tag_ref === nothing || (command_tag_ref[] = tag) + elseif mt == UInt8('S') + server_parameters === nothing ? skipbytes!(socket, len) : + parameterStatus!(server_parameters, len, socket) + elseif mt == UInt8('T') || mt == UInt8('D') || mt == UInt8('I') + # Incidental simple-query result data. skipbytes!(socket, len) else close_and_throw(socket, Error("unexpected message type '$(Char(mt))' from server; connection protocol state is corrupted")) @@ -1094,12 +1204,15 @@ function copy_in(style::S, socket, query::String, source::IO, debug::Bool) where writemessage(socket, debug, 'Q', query) error_msg = nothing copy_started = false + extra_statement = false + saw_noncopy_response = false try while true mt, len = readheader(socket, debug) if mt == UInt8('G') skipbytes!(socket, len) copy_started = true + extra_statement |= saw_noncopy_response break elseif mt == UInt8('E') error_msg = errorResponse(len, socket, debug) @@ -1115,6 +1228,13 @@ function copy_in(style::S, socket, query::String, source::IO, debug::Bool) where notification = notificationResponse(len, socket) notification_callback(style, notification) else + # Any command response before CopyInResponse belongs to a + # separate statement in the simple-query string. Continue to + # the COPY so the server cannot be left waiting for input, but + # reject the call after the stream returns to ReadyForQuery. + mt in (UInt8('C'), UInt8('T'), UInt8('D'), UInt8('I'), + UInt8('n'), UInt8('H'), UInt8('d'), UInt8('c')) && + (saw_noncopy_response = true) skipbytes!(socket, len) end end @@ -1149,19 +1269,27 @@ function copy_in(style::S, socket, query::String, source::IO, debug::Bool) where end error_msg = nothing second_copy = false + copy_completed = false try while true mt, len = readheader(socket, debug) if mt == UInt8('E') + copy_completed && (extra_statement = true) error_msg = errorResponse(len, socket, debug) elseif mt == UInt8('C') - skipbytes!(socket, len) + commandComplete(len, socket) + if copy_completed + extra_statement = true + else + copy_completed = true + end elseif mt == UInt8('G') # a second CopyInResponse (multi-statement query string): the # server is waiting for more copy data, so abort with CopyFail # instead of deadlocking; a clear error is thrown below skipbytes!(socket, len) second_copy = true + extra_statement = true writemessage(socket, debug, 'f', "copy_from supports a single COPY FROM STDIN statement") elseif mt == UInt8('N') notice = noticeResponse(len, socket) @@ -1170,9 +1298,12 @@ function copy_in(style::S, socket, query::String, source::IO, debug::Bool) where notification = notificationResponse(len, socket) notification_callback(style, notification) elseif mt == UInt8('Z') - skipbytes!(socket, len) + read_ready_status(socket, len) break else + mt in (UInt8('T'), UInt8('D'), UInt8('I'), UInt8('n'), + UInt8('H'), UInt8('d'), UInt8('c')) && + (extra_statement = true) skipbytes!(socket, len) end end @@ -1182,8 +1313,10 @@ function copy_in(style::S, socket, query::String, source::IO, debug::Bool) where error_msg === nothing || throw(error_msg) rethrow() end - second_copy && throw(PostgresInterfaceError("copy_from supports a single COPY ... FROM STDIN statement per call")) + (extra_statement || second_copy) && + throw(PostgresInterfaceError("copy_from supports a single COPY ... FROM STDIN statement per call")) error_msg === nothing || throw(error_msg) + copy_completed || throw(PostgresInterfaceError("COPY ... FROM STDIN did not complete")) return end @@ -1191,27 +1324,44 @@ function copy_out(style::S, socket, query::String, dest::IO, debug::Bool) where writemessage(socket, debug, 'Q', query) error_msg = nothing copy_started = false + copy_completed = false + extra_statement = false + saw_noncopy_response = false wrong_direction = false try while true mt, len = readheader(socket, debug) if mt == UInt8('H') skipbytes!(socket, len) + (copy_started || copy_completed || saw_noncopy_response) && + (extra_statement = true) copy_started = true elseif mt == UInt8('d') write(dest, read(socket, len)) elseif mt == UInt8('c') skipbytes!(socket, len) elseif mt == UInt8('C') - skipbytes!(socket, len) + commandComplete(len, socket) + if copy_started && !copy_completed + copy_completed = true + elseif copy_completed + extra_statement = true + else + saw_noncopy_response = true + end elseif mt == UInt8('G') # CopyInResponse: the statement was COPY ... FROM STDIN. The server # is now waiting on us for data, so abort the copy with CopyFail to # return the stream to ready instead of deadlocking. skipbytes!(socket, len) - wrong_direction = true + if copy_started || copy_completed + extra_statement = true + else + wrong_direction = true + end writemessage(socket, debug, 'f', "COPY FROM STDIN is not supported via copy_to") elseif mt == UInt8('E') + copy_completed && (extra_statement = true) error_msg = errorResponse(len, socket, debug) elseif mt == UInt8('N') notice = noticeResponse(len, socket) @@ -1220,9 +1370,16 @@ function copy_out(style::S, socket, query::String, dest::IO, debug::Bool) where notification = notificationResponse(len, socket) notification_callback(style, notification) elseif mt == UInt8('Z') - skipbytes!(socket, len) + read_ready_status(socket, len) break else + if mt in (UInt8('T'), UInt8('D'), UInt8('I'), UInt8('n')) + if copy_started + extra_statement = true + else + saw_noncopy_response = true + end + end skipbytes!(socket, len) end end @@ -1233,15 +1390,19 @@ function copy_out(style::S, socket, query::String, dest::IO, debug::Bool) where error_msg === nothing || throw(error_msg) rethrow() end + (extra_statement || (copy_started && saw_noncopy_response)) && + throw(PostgresInterfaceError("copy_to supports a single COPY ... TO STDOUT statement per call")) wrong_direction && throw(PostgresInterfaceError("statement initiated COPY ... FROM STDIN; use Postgres.copy_from")) error_msg === nothing || throw(error_msg) copy_started || throw(PostgresInterfaceError("statement did not initiate COPY ... TO STDOUT")) + copy_completed || throw(PostgresInterfaceError("COPY ... TO STDOUT did not complete")) return dest end -function close_statement(socket, name::String, debug::Bool) +function close_statement(socket, name::String, debug::Bool, + server_parameters::Union{Nothing, Dict{String, String}}=nothing) writemessages(socket, debug, ('C', UInt8('S'), name), ('S',)) - waitfor(socket, debug, '3', 'Z') + waitfor(socket, debug, '3', 'Z'; server_parameters=server_parameters) return end diff --git a/src/api/types.jl b/src/api/types.jl index 78d7315..d41ab7a 100644 --- a/src/api/types.jl +++ b/src/api/types.jl @@ -120,6 +120,14 @@ struct PostgresRange{T} lower_inclusive::Bool upper_inclusive::Bool empty::Bool + + function PostgresRange{T}(lower, upper, lower_inclusive::Bool, + upper_inclusive::Bool, empty::Bool) where {T} + converted_lower = ismissing(lower) ? missing : convert(T, lower) + converted_upper = ismissing(upper) ? missing : convert(T, upper) + return new{T}(converted_lower, converted_upper, lower_inclusive, + upper_inclusive, empty) + end end # ── CastFn: trim-safe type-erased value caster (the Reseau TaskFn pattern) ── @@ -327,17 +335,25 @@ end return h, mi, se, ms end -# `"char"` output: byte 0 renders as an empty string, bytes with the high bit -# set render as a backslash-octal escape ("\\200".."\\377"), and anything else -# is the raw byte. A backslash byte itself renders as a lone "\\", so only the -# exact 4-byte escape shape is decoded. +# `"char"` output: byte 0 renders as an empty string. High bytes render as +# backslash-octal on current PostgreSQL releases, while PostgreSQL 14 can send +# the raw byte. A backslash byte itself renders as a lone "\\", so only the +# exact 4-byte escape shapes are decoded. function pg_parse_char(s::String) isempty(s) && return '\0' c = codeunits(s) + # Indexing a String that contains one raw high byte produces Julia's + # invalid-UTF8 Char sentinel. The PostgreSQL type is one byte, so decode + # that byte value directly. + length(c) == 1 && return Char(c[1]) if length(c) == 4 && c[1] == UInt8('\\') && UInt8('0') <= c[2] <= UInt8('3') && UInt8('0') <= c[3] <= UInt8('7') && UInt8('0') <= c[4] <= UInt8('7') return Char((_pg_digit(c[2]) << 6) | (_pg_digit(c[3]) << 3) | _pg_digit(c[4])) end + if length(c) == 4 && c[1] == UInt8('\\') && + (c[2] == UInt8('x') || c[2] == UInt8('X')) + return Char((hexnibble(c[3]) << 4) | hexnibble(c[4])) + end return s[1] end diff --git a/src/connection_string.jl b/src/connection_string.jl index eafe255..18f487d 100644 --- a/src/connection_string.jl +++ b/src/connection_string.jl @@ -41,6 +41,14 @@ function ConnectionParams(; host::String="localhost", port::Int=5432, user::Stri return ConnectionParams(host, port, user, password, dbname, application_name, connect_timeout, sslmode, sslrootcert, sslcert, sslkey, sslcapath, sslservername, statement_timeout, statement_cache_maxsize, debug, reconnect) end +function Base.show(io::IO, params::ConnectionParams) + password = params.password === nothing ? "nothing" : "***" + print(io, "Postgres.ConnectionParams(host=", repr(params.host), + ", port=", params.port, ", user=", repr(params.user), + ", password=", password, ", dbname=", repr(params.dbname), ")") +end +Base.show(io::IO, ::MIME"text/plain", params::ConnectionParams) = show(io, params) + default_user() = get(ENV, "PGUSER", get(ENV, "USER", get(ENV, "USERNAME", ""))) function parse_optional_int(value::Union{String, Nothing}, key::String="") @@ -112,22 +120,32 @@ end # Ignored keywords that change security or connection-selection behavior when # set: silently dropping "channel_binding=require" or a CRL file would leave # the caller believing a protection is in place. The values listed are the -# no-op defaults for each keyword; any other value draws a warning. +# no-op defaults for each keyword; any other value is rejected. const SECURITY_SENSITIVE_IGNORED = Dict( "channel_binding" => ("", "prefer", "disable"), "target_session_attrs" => ("", "any"), "options" => ("",), + "gssencmode" => ("", "prefer", "disable"), + "sslnegotiation" => ("", "postgres"), + "sslcompression" => ("", "0"), "sslcrl" => ("",), "sslcrldir" => ("",), + "sslpassword" => ("",), "requiressl" => ("", "0"), + "requirepeer" => ("",), + "hostaddr" => ("",), + "client_encoding" => ("", "UTF8", "UTF-8", "utf8", "utf-8"), + "passfile" => ("",), + "service" => ("",), + "load_balance_hosts" => ("", "disable"), + "replication" => ("", "0", "false", "off"), ) -function warn_ignored_param(key::String, value::String) +function check_ignored_param(key::String, value::String) inert = get(SECURITY_SENSITIVE_IGNORED, key, nothing) inert === nothing && return value in inert && return - @warn "connection parameter \"$key=$value\" is not supported by Postgres.jl and is ignored" - return + throw(ArgumentError("connection parameter \"$key=$value\" is not supported by Postgres.jl and cannot be safely ignored")) end # An unrecognized key is almost always a typo, and silently dropping it is @@ -138,7 +156,7 @@ function check_known_params(values::Dict{String, String}) for (key, value) in values (key in KNOWN_PARAMS || key in IGNORED_PARAMS) || throw(ArgumentError("unrecognized connection parameter \"$key\"; recognized parameters are $(join(sort!(collect(KNOWN_PARAMS)), ", "))")) - key in IGNORED_PARAMS && warn_ignored_param(key, value) + key in IGNORED_PARAMS && check_ignored_param(key, value) end return values end @@ -185,51 +203,67 @@ function parse_keyword_dsn(dsn::String) i = nextind(dsn, i) end i > lastindex(dsn) && break + key_start = i - while i <= lastindex(dsn) && dsn[i] != '=' + while i <= lastindex(dsn) && dsn[i] != '=' && + !isspace(dsn[i]) && dsn[i] != ';' i = nextind(dsn, i) end - i > lastindex(dsn) && break - key = lowercase(strip(dsn[key_start:prevind(dsn, i)])) + key_end = prevind(dsn, i) + key = key_end < key_start ? "" : lowercase(String(dsn[key_start:key_end])) + isempty(key) && throw(ArgumentError("empty connection parameter name")) + while i <= lastindex(dsn) && isspace(dsn[i]) + i = nextind(dsn, i) + end + (i <= lastindex(dsn) && dsn[i] == '=') || + throw(ArgumentError("connection parameter \"$key\" is missing '='")) i = nextind(dsn, i) while i <= lastindex(dsn) && isspace(dsn[i]) i = nextind(dsn, i) end + buf = IOBuffer() if i <= lastindex(dsn) && dsn[i] == '\'' i = nextind(dsn, i) + closed_quote = false while i <= lastindex(dsn) c = dsn[i] if c == '\\' i = nextind(dsn, i) - if i <= lastindex(dsn) - write(buf, dsn[i]) - i = nextind(dsn, i) - end + i <= lastindex(dsn) || + throw(ArgumentError("dangling escape in value for connection parameter \"$key\"")) + write(buf, dsn[i]) + i = nextind(dsn, i) elseif c == '\'' i = nextind(dsn, i) + closed_quote = true break else write(buf, c) i = nextind(dsn, i) end end + closed_quote || + throw(ArgumentError("unterminated quoted value for connection parameter \"$key\"")) + if i <= lastindex(dsn) && !isspace(dsn[i]) && dsn[i] != ';' + throw(ArgumentError("unexpected text after quoted value for connection parameter \"$key\"")) + end else while i <= lastindex(dsn) && !isspace(dsn[i]) && dsn[i] != ';' c = dsn[i] if c == '\\' i = nextind(dsn, i) - if i <= lastindex(dsn) - write(buf, dsn[i]) - i = nextind(dsn, i) - end + i <= lastindex(dsn) || + throw(ArgumentError("dangling escape in value for connection parameter \"$key\"")) + write(buf, dsn[i]) + i = nextind(dsn, i) else write(buf, c) i = nextind(dsn, i) end end end - !isempty(key) && (values[key] = String(take!(buf))) + values[key] = String(take!(buf)) end return values end @@ -281,7 +315,7 @@ function parse_uri(uri::String) for (key, value) in params (key in KNOWN_PARAMS || key in IGNORED_PARAMS) || throw(ArgumentError("unrecognized connection parameter \"$key\" in URI; recognized parameters are $(join(sort!(collect(KNOWN_PARAMS)), ", "))")) - key in IGNORED_PARAMS && warn_ignored_param(key, value) + key in IGNORED_PARAMS && check_ignored_param(key, value) # keys we accept but don't implement must not reach params_from_values key in KNOWN_PARAMS && (values[key] = value) end diff --git a/src/execute.jl b/src/execute.jl index 981e54e..41aa995 100644 --- a/src/execute.jl +++ b/src/execute.jl @@ -88,6 +88,8 @@ mutable struct Cursor{S <: API.AbstractPostgresStyle} index::Int done::Bool rowcount::Int + portal_closed::Bool + owned_statement::Union{Nothing, Statement{S}} owns_transaction::Bool end @@ -104,34 +106,52 @@ function checkstmt(stmt::Statement) checkconn(stmt.conn) stmt.closed && throw(PostgresInterfaceError("statement has been closed")) if stmt.cached - if stmt.generation != stmt.conn.generation || !haskey(stmt.conn.statements, stmt.sql) - # if the connection was reset, we need to re-prepare the statement - stmt.name = API.prepare(stmt.conn.socket, stmt.sql, stmt.conn.debug) - stmt.conn.statements[stmt.sql] = stmt + cached = get(stmt.conn.statements, stmt.sql, nothing) + if stmt.generation != stmt.conn.generation || cached === nothing || + cached.name != stmt.name || cached.generation != stmt.conn.generation + # The cache entry was evicted, cleared, replaced, or lost on + # reconnect. Keep this caller handle valid as an independent + # server statement; do not silently repopulate or overrun the + # current cache policy. + stmt.name = API.prepare(stmt.conn.socket, stmt.sql, stmt.conn.debug, + stmt.conn.server_parameters) stmt.generation = stmt.conn.generation + stmt.cached = false + else + touch_statement!(stmt.conn, cached) + stmt.last_used = cached.last_used end - touch_statement!(stmt.conn, stmt) elseif stmt.generation != stmt.conn.generation - stmt.name = API.prepare(stmt.conn.socket, stmt.sql, stmt.conn.debug) + stmt.name = API.prepare(stmt.conn.socket, stmt.sql, stmt.conn.debug, + stmt.conn.server_parameters) stmt.generation = stmt.conn.generation end - !stmt.cached && touch_statement!(stmt.conn, stmt) + stmt.cached || touch_statement!(stmt.conn, stmt) return end +function statement_handle(stmt::Statement) + params = Union{String, Missing}[missing for _ = 1:stmt.nparams] + return Statement{_style_type(stmt.conn)}( + stmt.conn, stmt.name, stmt.sql, stmt.nfields, stmt.names, stmt.typeIds, + stmt.nparams, params, false, true, stmt.generation, stmt.last_used) +end + function DBInterface.prepare(conn::Connection, sql::AbstractString; debug::Bool=false) sql_str = String(sql) + actual_debug = debug || conn.debug @lock conn.lock begin checkconn(conn) # check if we've already prepared this sql before if haskey(conn.statements, sql_str) - stmt = conn.statements[sql_str] - touch_statement!(conn, stmt) - return stmt + cached = conn.statements[sql_str] + touch_statement!(conn, cached) + return statement_handle(cached) end if conn.statement_cache_maxsize == 0 - name = API.prepare(conn.socket, sql_str, debug) - nparams, names, types = API.describeprepared(conn.socket, name, debug) + name = API.prepare(conn.socket, sql_str, actual_debug, conn.server_parameters) + nparams, names, types = API.describeprepared( + conn.socket, name, actual_debug, conn.server_parameters) params = Union{String, Missing}[missing for _ = 1:nparams] last_used = next_statement_clock!(conn) return Statement{_style_type(conn)}(conn, name, sql_str, length(names), names, types, nparams, params, false, false, conn.generation, last_used) @@ -141,13 +161,14 @@ function DBInterface.prepare(conn::Connection, sql::AbstractString; debug::Bool= evict_lru_statement!(conn) end # new statement to prepare - name = API.prepare(conn.socket, sql_str, debug) - nparams, names, types = API.describeprepared(conn.socket, name, debug) + name = API.prepare(conn.socket, sql_str, actual_debug, conn.server_parameters) + nparams, names, types = API.describeprepared( + conn.socket, name, actual_debug, conn.server_parameters) params = Union{String, Missing}[missing for _ = 1:nparams] last_used = next_statement_clock!(conn) - stmt = Statement{_style_type(conn)}(conn, name, sql_str, length(names), names, types, nparams, params, false, true, conn.generation, last_used) - conn.statements[sql_str] = stmt - return stmt + cached = Statement{_style_type(conn)}(conn, name, sql_str, length(names), names, types, nparams, params, false, true, conn.generation, last_used) + conn.statements[sql_str] = cached + return statement_handle(cached) end end @@ -155,12 +176,14 @@ function DBInterface.close!(stmt::Statement) @lock stmt.conn.lock begin stmt.closed && return if !isopen(stmt.conn.socket) - stmt.cached && haskey(stmt.conn.statements, stmt.sql) && delete!(stmt.conn.statements, stmt.sql) stmt.closed = true return end - stmt.cached && haskey(stmt.conn.statements, stmt.sql) && stmt.conn.statements[stmt.sql] === stmt && delete!(stmt.conn.statements, stmt.sql) - API.close_statement(stmt.conn.socket, stmt.name, stmt.conn.debug) + # Cached backend statements belong to the connection cache, not to a + # caller handle. Closing one handle must not invalidate independent + # handles or remove the cache entry. + stmt.cached || API.close_statement(stmt.conn.socket, stmt.name, stmt.conn.debug, + stmt.conn.server_parameters) stmt.closed = true end return @@ -191,15 +214,20 @@ end function DBInterface.close!(cursor::Cursor) owns_transaction = cursor.owns_transaction + owned_statement = cursor.owned_statement + cursor.owned_statement = nothing closed_cleanly = false try @lock cursor.conn.lock begin - if !cursor.done + if !cursor.portal_closed && isopen(cursor.conn.socket) API.writemessages(cursor.conn.socket, cursor.conn.debug, ('C', UInt8('P'), cursor.portal), ('S',)) - API.waitfor(cursor.conn.socket, cursor.conn.debug, '3', 'Z') + API.waitfor(cursor.conn.socket, cursor.conn.debug, '3', 'Z'; + server_parameters=cursor.conn.server_parameters) + cursor.portal_closed = true end cursor.done = true empty!(cursor.buffer) + owned_statement === nothing || DBInterface.close!(owned_statement) end closed_cleanly = true finally @@ -213,12 +241,19 @@ function DBInterface.close!(cursor::Cursor) finish_cursor_transaction!(cursor.conn) else try - finish_cursor_transaction!(cursor.conn) + abort_cursor_transaction!(cursor.conn) catch # already unwinding; don't mask the original error end end end + if !closed_cleanly && owned_statement !== nothing && !owned_statement.closed + try + DBInterface.close!(owned_statement) + catch + # Preserve the portal-close error already in flight. + end + end end return end @@ -286,6 +321,15 @@ function build_params(params, nparams::Int, sql::AbstractString) return dest end +function build_unchecked_params(params) + dest = Union{String, Missing}[] + params === nothing && return dest + for param in params + push!(dest, _param(param)) + end + return dest +end + mutable struct RowClosure data::Vector{Any} types::Vector{Type} @@ -380,10 +424,13 @@ function read_portal_batch!(cursor::Cursor) elseif mt == UInt8('A') notification = API.notificationResponse(len, conn.socket) API.notification_callback(conn.style, notification) + elseif mt == UInt8('S') + API.parameterStatus!(conn.server_parameters, len, conn.socket) elseif mt == UInt8('E') error_msg = API.errorResponse(len, conn.socket, conn.debug) elseif mt == UInt8('Z') - API.skipbytes!(conn.socket, len) + status = API.read_ready_status(conn.socket, len) + conn.server_in_transaction = API.in_transaction_status(status) break else API.skipbytes!(conn.socket, len) @@ -438,6 +485,7 @@ end function DBInterface.execute(stmt::Statement, params=nothing, ::Type{T}=Any; debug::Bool=false) where {T} style = stmt.conn.style + actual_debug = debug || stmt.conn.debug log_enabled = API.query_logging_enabled(style) start_ns = log_enabled ? time_ns() : 0 result = nothing @@ -445,29 +493,40 @@ function DBInterface.execute(stmt::Statement, params=nothing, ::Type{T}=Any; deb @lock stmt.conn.lock begin # check that connection/statement are ok checkstmt(stmt) - bind_params!(stmt.params, params, stmt.sql) - # isa-split the socket union with per-branch typeasserts (identical calls in - # both branches get tail-merged back into one dynamic call by the optimizer), - # so the exec call resolves statically under `juliac --trim` - socket = stmt.conn.socket - e = if socket isa Reseau.TCP.Conn - API.exec(style, socket::Reseau.TCP.Conn, stmt.name, stmt.params, stmt.names, stmt.typeIds, stmt.conn.type_registry, debug, 0) - else - API.exec(style, socket::Reseau.TLS.Conn, stmt.name, stmt.params, stmt.names, stmt.typeIds, stmt.conn.type_registry, debug, 0) + e = try + bind_params!(stmt.params, params, stmt.sql) + # isa-split the socket union with per-branch typeasserts + # (identical calls in both branches get tail-merged back into + # one dynamic call by the optimizer). + socket = stmt.conn.socket + e = if socket isa Reseau.TCP.Conn + API.exec(style, socket::Reseau.TCP.Conn, stmt.name, stmt.params, + stmt.names, stmt.typeIds, stmt.conn.type_registry, + actual_debug, 0, stmt.conn.server_parameters) + else + API.exec(style, socket::Reseau.TLS.Conn, stmt.name, stmt.params, + stmt.names, stmt.typeIds, stmt.conn.type_registry, + actual_debug, 0, stmt.conn.server_parameters) + end + e + finally + # Bound strings can contain credentials or personal data. The + # vector must be cleared even if local parameter validation or + # the server Bind fails. + fill!(stmt.params, missing) end # in a finally: a failed statement still drained to ReadyForQuery - # and its status is authoritative — skipping the copy on the error - # path leaves the transaction tracking stale + # and its status is authoritative. try result = T === Any ? makeresult(e) : StructUtils.arraylike(T) ? StructUtils.make(T, e, style) : only(StructUtils.make(Vector{T}, e, style)) finally stmt.conn.server_in_transaction = API.in_transaction_status(e.tx_status[]) end end - log_enabled && API.query_logger(style, :execute, (sql=stmt.sql, params=params, duration_ns=time_ns() - start_ns, success=true)) + log_enabled && query_log_safely(style, :execute, (sql=stmt.sql, params=params, duration_ns=time_ns() - start_ns, success=true)) return result catch err - log_enabled && API.query_logger(style, :execute, (sql=stmt.sql, params=params, duration_ns=time_ns() - start_ns, success=false, error=err)) + log_enabled && query_log_safely(style, :execute, (sql=stmt.sql, params=params, duration_ns=time_ns() - start_ns, success=false, error=err)) rethrow() end end @@ -475,21 +534,24 @@ end function DBInterface.execute(conn::Connection, sql::AbstractString, params=nothing, ::Type{T}=Any; debug::Bool=false) where {T} sql_str = String(sql) style = conn.style + actual_debug = debug || conn.debug log_enabled = API.query_logging_enabled(style) start_ns = log_enabled ? time_ns() : 0 result = nothing try @lock conn.lock begin checkconn(conn) - stmtname = API.prepare(conn.socket, sql_str, debug; name="") - nparams, names, types = API.describeprepared(conn.socket, stmtname, debug) - params_vec = build_params(params, nparams, sql_str) + params_vec = build_unchecked_params(params) # see the statement-execute method: socket union isa-split for --trim socket = conn.socket e = if socket isa Reseau.TCP.Conn - API.exec(style, socket::Reseau.TCP.Conn, stmtname, params_vec, names, types, conn.type_registry, debug, 0) + API.exec_unnamed(style, socket::Reseau.TCP.Conn, sql_str, params_vec, + conn.type_registry, actual_debug, 0, + conn.server_parameters) else - API.exec(style, socket::Reseau.TLS.Conn, stmtname, params_vec, names, types, conn.type_registry, debug, 0) + API.exec_unnamed(style, socket::Reseau.TLS.Conn, sql_str, params_vec, + conn.type_registry, actual_debug, 0, + conn.server_parameters) end # in a finally, as in the statement-execute method above try @@ -498,26 +560,75 @@ function DBInterface.execute(conn::Connection, sql::AbstractString, params=nothi conn.server_in_transaction = API.in_transaction_status(e.tx_status[]) end end - log_enabled && API.query_logger(style, :execute, (sql=sql_str, params=params, duration_ns=time_ns() - start_ns, success=true)) + log_enabled && query_log_safely(style, :execute, (sql=sql_str, params=params, duration_ns=time_ns() - start_ns, success=true)) return result catch err - log_enabled && API.query_logger(style, :execute, (sql=sql_str, params=params, duration_ns=time_ns() - start_ns, success=false, error=err)) + log_enabled && query_log_safely(style, :execute, (sql=sql_str, params=params, duration_ns=time_ns() - start_ns, success=false, error=err)) rethrow() end end -function cursor(stmt::Statement, params=nothing; fetchsize::Integer=1000, owns_transaction::Bool=false) +# DBInterface's generic connection overload prepares a statement without +# closing it. That leaks named server statements when this driver's cache is +# disabled. Keep the handle lifetime explicit for both bulk fallbacks. +function DBInterface.executemany(conn::Connection, sql::AbstractString, params) + stmt = DBInterface.prepare(conn, sql) + try + return DBInterface.executemany(stmt, params) + finally + DBInterface.close!(stmt) + end +end + +function DBInterface.executemultiple(conn::Connection, sql::AbstractString, params) + stmt = DBInterface.prepare(conn, sql) + try + return DBInterface.executemultiple(stmt, params) + finally + DBInterface.close!(stmt) + end +end + +function cursor(stmt::Statement, params=nothing; fetchsize::Integer=1000, + owns_transaction::Bool=false, owns_statement::Bool=false) conn = stmt.conn - @lock conn.lock begin - checkstmt(stmt) - bind_params!(stmt.params, params, stmt.sql) - portal = string(UUIDs.uuid4()) - types = Type[API.juliatype(x -> x, i, conn.type_registry) for i in stmt.typeIds] - lookup = Dict(x => i for (i, x) in enumerate(stmt.names)) - cursor = Cursor{_style_type(conn)}(conn, portal, stmt.names, stmt.typeIds, types, lookup, max(1, Int(fetchsize)), ResultRow[], 1, false, 0, owns_transaction) - API.writemessages(conn.socket, conn.debug, ('B', portal, stmt.name, Int16(0), Int16(length(stmt.params)), API.Params(stmt.params), Int16(0)), ('E', portal, Int32(cursor.fetchsize)), ('S',)) - read_portal_batch!(cursor) - return cursor + started_here = false + if !owns_transaction + already_in_tx = @lock conn.lock (conn.in_transaction || conn.server_in_transaction) + if !already_in_tx + start_transaction(conn) + owns_transaction = true + started_here = true + end + end + try + @lock conn.lock begin + checkstmt(stmt) + try + bind_params!(stmt.params, params, stmt.sql) + portal = string(UUIDs.uuid4()) + types = Type[API.juliatype(x -> x, i, conn.type_registry) for i in stmt.typeIds] + lookup = Dict(x => i for (i, x) in enumerate(stmt.names)) + owned_stmt = owns_statement ? stmt : nothing + cursor = Cursor{_style_type(conn)}(conn, portal, stmt.names, + stmt.typeIds, types, lookup, max(1, Int(fetchsize)), + ResultRow[], 1, false, 0, false, owned_stmt, + owns_transaction) + API.writemessages(conn.socket, conn.debug, ('B', portal, stmt.name, Int16(0), Int16(length(stmt.params)), API.Params(stmt.params), Int16(0)), ('E', portal, Int32(cursor.fetchsize)), ('S',)) + read_portal_batch!(cursor) + return cursor + finally + fill!(stmt.params, missing) + end + end + catch + if started_here + try + abort_cursor_transaction!(conn) + catch + end + end + rethrow() end end @@ -532,6 +643,7 @@ started (and committed on close) if the connection isn't already in one. """ function cursor(conn::Connection, sql::AbstractString, params=nothing; fetchsize::Integer=1000, debug::Bool=false) owns_transaction = false + stmt = nothing # a transaction opened with raw SQL counts as "already in one": the server # status sees it even though the client flag doesn't, and owning it here # would mean committing the caller's transaction on cursor close @@ -539,10 +651,16 @@ function cursor(conn::Connection, sql::AbstractString, params=nothing; fetchsize already_in_tx || (start_transaction(conn); owns_transaction = true) try stmt = DBInterface.prepare(conn, sql; debug=debug) - return cursor(stmt, params; fetchsize=fetchsize, owns_transaction=owns_transaction) + return cursor(stmt, params; fetchsize=fetchsize, owns_transaction=owns_transaction, + owns_statement=true) catch - # don't leave the transaction we started dangling on a failed cursor - # don't leave the transaction we started dangling on a failed cursor; + if stmt !== nothing && !stmt.closed + try + DBInterface.close!(stmt) + catch + end + end + # Don't leave the transaction we started dangling on a failed cursor; # if the connection died, clear the state directly (a ROLLBACK can't be # delivered, and leaving it set would block reconnect forever) if owns_transaction diff --git a/test/postgres_trim_queries.jl b/test/postgres_trim_queries.jl deleted file mode 100644 index 560502a..0000000 --- a/test/postgres_trim_queries.jl +++ /dev/null @@ -1,176 +0,0 @@ -using Dates -using DBInterface -using Postgres -using StructUtils -using UUIDs - -struct TrimId - profile_id::Int32 -end - -struct TrimCount - count::Int32 -end - -struct TrimName - display_name::String -end - -StructUtils.@tags struct TrimProfile - profileId::Int32 &(postgres=(name=:profile_id,),) - displayName::String &(postgres=(name=:display_name,),) - createdAt::DateTime &(postgres=(name=:created_at,),) - active::Bool - score::Union{Missing, Int32} - uid::UUID - flags::Vector{Int32} -end - -function _postgres_trim_connect() - host = get(ENV, "POSTGRES_TRIM_HOST", "127.0.0.1") - port = parse(Int, get(ENV, "POSTGRES_TRIM_PORT", "5432")) - user = get(ENV, "POSTGRES_TRIM_USER", "postgres") - dbname = get(ENV, "POSTGRES_TRIM_DBNAME", "postgres") - return DBInterface.connect( - Postgres.Connection, - host, - user, - nothing; - dbname=dbname, - port=port, - sslmode="disable", - connect_timeout=2, - application_name="postgres_trim", - statement_cache_maxsize=4, - ) -end - -function _assert_trim_profile(profile::TrimProfile, id::Int32, name::String)::Nothing - profile.profileId == id || error("unexpected profile id") - profile.displayName == name || error("unexpected profile name") - profile.active || error("expected active profile") - profile.uid == UUID("12345678-1234-5678-1234-567812345678") || id != 1 || error("unexpected UUID") - !isempty(profile.flags) || error("expected non-empty flags array") - return nothing -end - -function run_postgres_trim_queries()::Nothing - conn = _postgres_trim_connect() - try - DBInterface.execute(conn, """ - CREATE TEMP TABLE trim_compile_profiles ( - profile_id integer PRIMARY KEY, - display_name text NOT NULL, - created_at timestamp NOT NULL, - active boolean NOT NULL, - score integer, - uid uuid NOT NULL, - flags integer[] NOT NULL - ) - """) - - insert_sql = raw""" - INSERT INTO trim_compile_profiles ( - profile_id, - display_name, - created_at, - active, - score, - uid, - flags - ) VALUES ( - $1, - $2, - $3, - $4, - $5, - $6, - $7::integer[] - ) - RETURNING profile_id - """ - - first_id = DBInterface.execute( - conn, - insert_sql, - ( - Int32(1), - "Ada", - DateTime(2024, 1, 2, 3, 4, 5), - true, - Int32(99), - UUID("12345678-1234-5678-1234-567812345678"), - Int32[1, 2, 3], - ), - TrimId, - ) - first_id.profile_id == 1 || error("unexpected inserted id") - - DBInterface.transaction(conn) do - DBInterface.execute( - conn, - insert_sql, - ( - Int32(2), - "Grace", - DateTime(2024, 1, 3, 4, 5, 6), - true, - missing, - UUID("87654321-4321-8765-4321-876543218765"), - Int32[4, 5], - ), - ) - end - - stmt = DBInterface.prepare(conn, raw""" - SELECT profile_id, display_name, created_at, active, score, uid, flags - FROM trim_compile_profiles - WHERE profile_id = $1 - """) - try - profile = DBInterface.execute(stmt, (Int32(1),), TrimProfile) - _assert_trim_profile(profile, Int32(1), "Ada") - finally - DBInterface.close!(stmt) - end - - profiles = DBInterface.execute(conn, """ - SELECT profile_id, display_name, created_at, active, score, uid, flags - FROM trim_compile_profiles - ORDER BY profile_id - """, (), Vector{TrimProfile}) - length(profiles) == 2 || error("expected two profiles") - _assert_trim_profile(profiles[2], Int32(2), "Grace") - - count_row = DBInterface.execute( - conn, - "SELECT count(*)::integer AS count FROM trim_compile_profiles", - (), - TrimCount, - ) - count_row.count == 2 || error("unexpected typed count") - - name_row = DBInterface.execute( - conn, - "SELECT display_name FROM trim_compile_profiles WHERE profile_id = 1", - (), - TrimName, - ) - name_row.display_name == "Ada" || error("unexpected typed name") - - update_result = DBInterface.execute(conn, "UPDATE trim_compile_profiles SET score = coalesce(score, 0) + 1") - Postgres.rows_affected(update_result) == 2 || error("unexpected rows affected") - occursin("UPDATE", Postgres.command_tag(update_result)) || error("unexpected command tag") - finally - DBInterface.close!(conn) - end - return nothing -end - -function @main(args::Vector{String})::Cint - _ = args - run_postgres_trim_queries() - return 0 -end - -Base.Experimental.entrypoint(main, (Vector{String},)) diff --git a/test/runtests.jl b/test/runtests.jl index 9c34814..885cde7 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,4 +1,5 @@ using Test +using Aqua using Dates using UUIDs using DBInterface @@ -19,6 +20,14 @@ Postgres.API.query_logging_enabled(::LoggingStyle) = true Postgres.API.query_logger(::LoggingStyle, event::Symbol, info::NamedTuple) = (push!(LOGGED_EVENTS, (event=event, info=info)); nothing) Postgres.API.notice_callback(::LoggingStyle, notice) = (NOTICE_SEEN[] = true; nothing) +const FAILING_LOGGER_CALLS = Ref(0) +struct FailingLoggerStyle <: Postgres.API.AbstractPostgresStyle end +Postgres.API.query_logging_enabled(::FailingLoggerStyle) = true +function Postgres.API.query_logger(::FailingLoggerStyle, event::Symbol, info::NamedTuple) + FAILING_LOGGER_CALLS[] += 1 + error("logger failed") +end + # Integration tests for Postgres.jl protocol and API behavior. const JSONType = typeof(JSON.lazy("{}")) @@ -155,7 +164,7 @@ cp /certs/root.crt "\$certdir/root.crt" chown postgres:postgres "\$certdir/server.crt" "\$certdir/server.key" "\$certdir/root.crt" chmod 0644 "\$certdir/server.crt" "\$certdir/root.crt" chmod 0600 "\$certdir/server.key" -exec docker-entrypoint.sh postgres -c ssl=on -c ssl_cert_file="\$certdir/server.crt" -c ssl_key_file="\$certdir/server.key" -c ssl_ca_file="\$certdir/root.crt" +exec docker-entrypoint.sh postgres -c ssl=on -c ssl_cert_file="\$certdir/server.crt" -c ssl_key_file="\$certdir/server.key" -c ssl_ca_file="\$certdir/root.crt" -c hba_file=/certs/pg_hba.conf """ return ["sh", "-c", setup_script] end @@ -169,6 +178,11 @@ function generate_ssl_material(dir::AbstractString) server_csr = joinpath(dir, "server.csr") server_cert = joinpath(dir, "server.crt") server_config = joinpath(dir, "server-openssl.cnf") + client_key = joinpath(dir, "client.key") + client_csr = joinpath(dir, "client.csr") + client_cert = joinpath(dir, "client.crt") + client_config = joinpath(dir, "client-openssl.cnf") + hba_file = joinpath(dir, "pg_hba.conf") open(server_config, "w") do io write(io, """ @@ -191,13 +205,44 @@ IP.1 = 127.0.0.1 """) end + open(client_config, "w") do io + write(io, """ +[req] +distinguished_name = req_distinguished_name +prompt = no +req_extensions = v3_req + +[req_distinguished_name] +CN = postgres_mtls + +[v3_req] +basicConstraints = CA:FALSE +keyUsage = digitalSignature, keyEncipherment +extendedKeyUsage = clientAuth +""") + end + + open(hba_file, "w") do io + write(io, """ +local all all trust +hostssl all postgres_mtls 0.0.0.0/0 trust clientcert=verify-full +hostssl all postgres_mtls ::/0 trust clientcert=verify-full +host all all 0.0.0.0/0 trust +host all all ::/0 trust +""") + end + run_openssl("req", "-x509", "-newkey", "rsa:2048", "-days", "1", "-nodes", "-keyout", root_key, "-out", root_cert, "-subj", "/CN=Postgres.jl Test Root CA") run_openssl("req", "-x509", "-newkey", "rsa:2048", "-days", "1", "-nodes", "-keyout", wrong_root_key, "-out", wrong_root_cert, "-subj", "/CN=Postgres.jl Wrong Root CA") run_openssl("req", "-new", "-newkey", "rsa:2048", "-nodes", "-keyout", server_key, "-out", server_csr, "-config", server_config) run_openssl("x509", "-req", "-in", server_csr, "-CA", root_cert, "-CAkey", root_key, "-CAcreateserial", "-out", server_cert, "-days", "1", "-sha256", "-extensions", "v3_req", "-extfile", server_config) + run_openssl("req", "-new", "-newkey", "rsa:2048", "-nodes", "-keyout", client_key, "-out", client_csr, "-config", client_config) + run_openssl("x509", "-req", "-in", client_csr, "-CA", root_cert, "-CAkey", root_key, "-CAcreateserial", "-out", client_cert, "-days", "1", "-sha256", "-extensions", "v3_req", "-extfile", client_config) chmod(server_key, 0o600) + chmod(client_key, 0o600) - return (rootcert=root_cert, wrongrootcert=wrong_root_cert, certdir=dir) + return (rootcert=root_cert, wrongrootcert=wrong_root_cert, + clientcert=client_cert, clientkey=client_key, certdir=dir) end function with_ssl_postgres(f::Function) @@ -425,6 +470,8 @@ function random_array_string(rng::AbstractRNG) end @testset "Postgres" begin + Aqua.test_all(Postgres) + @testset "Export Surface" begin exported = Set([:DBInterface, :Postgres]) @static if VERSION >= v"1.11" @@ -523,21 +570,41 @@ end @test_throws ArgumentError Postgres.parse_dsn("host=h ssl_mode=verify-full") @test_throws ArgumentError Postgres.parse_dsn("postgresql://u@h/db?ssl_mode=require") - # real libpq keywords this driver doesn't implement are accepted and - # ignored: providers routinely put them in the URI they hand users. - # The ones that request a security or connection-selection behavior - # warn, so the caller isn't left believing a protection is in place - @test (@test_logs (:warn, r"channel_binding=require.*ignored") Postgres.parse_dsn("postgresql://u:p@h/db?sslmode=require&channel_binding=require")).sslmode == "require" - @test (@test_logs (:warn, r"target_session_attrs=read-write.*ignored") Postgres.parse_dsn("postgresql://u@h/db?target_session_attrs=read-write")).dbname == "db" - @test (@test_logs (:warn, r"options=.*ignored") Postgres.parse_dsn("host=h options=-csearch_path=x")).host == "h" - @test_logs (:warn, r"sslcrl=.*ignored") Postgres.parse_dsn("host=h sslcrl=/tmp/crl.pem") - @test_logs (:warn, r"requiressl=1.*ignored") Postgres.parse_dsn("host=h requiressl=1") + # A libpq option that requests behavior this driver cannot enforce is + # rejected. Warning and connecting would falsely report a security or + # routing guarantee to the caller. + @test_throws ArgumentError Postgres.parse_dsn("postgresql://u:p@h/db?sslmode=require&channel_binding=require") + @test_throws ArgumentError Postgres.parse_dsn("postgresql://u@h/db?target_session_attrs=read-write") + @test_throws ArgumentError Postgres.parse_dsn("host=h options=-csearch_path=x") + @test_throws ArgumentError Postgres.parse_dsn("host=h sslcrl=/tmp/crl.pem") + @test_throws ArgumentError Postgres.parse_dsn("host=h requiressl=1") + @test_throws ArgumentError Postgres.parse_dsn("host=h gssencmode=require") + @test_throws ArgumentError Postgres.parse_dsn("host=h requirepeer=postgres") + @test_throws ArgumentError Postgres.parse_dsn("host=h hostaddr=203.0.113.1") + @test_throws ArgumentError Postgres.parse_dsn("host=h client_encoding=LATIN1") # the no-op defaults for those keywords stay silent, as do keywords # with no security consequence @test_logs Postgres.parse_dsn("host=h channel_binding=prefer target_session_attrs=any requiressl=0") @test_logs Postgres.parse_dsn("host=h keepalives=1 client_encoding=UTF8") @test_logs Postgres.parse_dsn("postgresql://u@h/db?channel_binding=disable") + # Displaying structured connection options must never reveal a secret. + shown = repr(Postgres.ConnectionParams(host="h", user="u", password="top-secret", dbname="d")) + plain_shown = repr(MIME"text/plain"(), Postgres.ConnectionParams( + host="h", user="u", password="top-secret", dbname="d")) + @test !occursin("top-secret", shown) + @test !occursin("top-secret", plain_shown) + @test occursin("password=***", shown) + @test occursin("password=***", plain_shown) + + # Malformed keyword DSNs must never degrade to a usable partial + # configuration. In particular, a discarded security option could + # change which endpoint or transport is selected. + @test_throws ArgumentError Postgres.parse_dsn("host=h broken") + @test_throws ArgumentError Postgres.parse_dsn("host='unterminated") + @test_throws ArgumentError Postgres.parse_dsn("host=abc\\") + @test_throws ArgumentError Postgres.parse_dsn("host='h'trailing") + # invalid values for a recognized parameter are reported against that # parameter rather than silently defaulting @test_throws ArgumentError Postgres.parse_dsn("host=h reconnect=ture") @@ -613,6 +680,19 @@ end @test Postgres.API.cstring_at(UInt8['a', 'b'], 1) == ("ab", 3) @test Postgres.API.cstring_at(UInt8['a', 0x00, 'c', 0x00], 3) == ("c", 5) @test Postgres.API.cstring_at(UInt8['a', 0x00], 5) == ("", 3) + @test_throws EOFError Postgres.API.skipbytes!(IOBuffer(UInt8[0x01]), 2) + @test Postgres.API.read_ready_status(IOBuffer(UInt8['T']), 1) == UInt8('T') + @test_throws Postgres.API.Error Postgres.API.read_ready_status(IOBuffer(UInt8[]), 0) + @test Postgres.API.commandComplete(7, IOBuffer(UInt8[codeunits("SELECT\0")...])) == "SELECT" + @test_throws Postgres.API.Error Postgres.API.commandComplete(0, IOBuffer()) + @test_throws Postgres.API.Error Postgres.API.commandComplete(4, IOBuffer(UInt8['O', 'K', 0x00, 0x00])) + let socket = IOBuffer() + write(socket, UInt8('D')) + write(socket, hton(Postgres.API.MAX_PREAUTH_MESSAGE_LEN + Int32(5))) + seekstart(socket) + @test_throws Postgres.API.Error Postgres.API.readheader( + socket, false, Postgres.API.MAX_PREAUTH_MESSAGE_LEN) + end # a malformed DataRow must fail with a clear protocol error rather # than reading past the buffer or leaving the row partly unfilled @@ -653,8 +733,10 @@ end # server would truncate mid-statement @test Postgres.escape_identifier("a\"b") == "\"a\"\"b\"" @test Postgres.escape_literal("a'b") == "'a''b'" + @test Postgres.escape_literal("a\\b") == "E'a\\\\b'" @test_throws Postgres.PostgresInterfaceError Postgres.escape_identifier("a\0b") @test_throws Postgres.PostgresInterfaceError Postgres.escape_literal("a\0b") + @test_throws Postgres.PostgresInterfaceError Postgres.Connection(host="127.0.0.1", port=1, user="u\0x") # severity must come from the non-localized 'V' field when the server # sends it: 'S' is translated, so comparing it to "FATAL" would depend @@ -727,6 +809,8 @@ end # ... and high-bit bytes as backslash-octal escapes @test Postgres.API.parse_value(18, "\\200", registry) == Char(0x80) @test Postgres.API.parse_value(18, "\\377", registry) == Char(0xff) + @test Postgres.API.parse_value(18, "\\x80", registry) == Char(0x80) + @test Postgres.API.parse_value(18, "\\xFF", registry) == Char(0xff) # a backslash byte renders as a lone backslash, not an escape @test Postgres.API.parse_value(18, "\\", registry) == '\\' @test Postgres.API.pg_parse_char("\\310") == Char(0xc8) @@ -831,8 +915,6 @@ end end end - include("trim_compile_tests.jl") - if !docker_available() @info "Docker not available; skipping Postgres integration tests." @test true @@ -956,6 +1038,34 @@ end stmt = DBInterface.prepare(conn, raw"SELECT $1::int AS val") res = Tables.rowtable(DBInterface.execute(stmt, (1,))) @test res[1].val == 1 + @test all(ismissing, stmt.params) + + failing_stmt = DBInterface.prepare(conn, raw"SELECT 10 / $1::int AS val") + @test_throws Postgres.API.Error DBInterface.execute(failing_stmt, (0,)) + @test all(ismissing, failing_stmt.params) + mismatch_stmt = DBInterface.prepare(conn, + raw"SELECT $1::text AS a, $2::text AS b") + @test_throws Postgres.PostgresInterfaceError DBInterface.execute( + mismatch_stmt, ("must-not-remain",)) + @test all(ismissing, mismatch_stmt.params) + + # Each prepare call returns an independent caller handle, + # even when both handles share one cache-owned server + # statement. Closing one must not close the other. + held = DBInterface.prepare(conn, "SELECT 42 AS val") + alias = DBInterface.prepare(conn, "SELECT 42 AS val") + @test held !== alias + DBInterface.close!(alias) + @test only(DBInterface.execute(held)).val == 42 + function_value = DBInterface.execute(conn, "SELECT 42 AS val", nothing) do result + only(result).val + end + @test function_value == 42 + @test only(DBInterface.execute(held)).val == 42 + + DBInterface.close!(held) + DBInterface.close!(mismatch_stmt) + DBInterface.close!(failing_stmt) DBInterface.close!(stmt) @test_throws Postgres.PostgresInterfaceError DBInterface.execute(stmt, (1,)) end @@ -1106,8 +1216,56 @@ end textrangearr = only(Tables.rowtable(DBInterface.execute(conn, "SELECT ARRAY[textrange('a','c'), textrange('α','ω')] AS a"))) @test textrangearr.a[1].upper == "c" @test textrangearr.a[2].lower == "α" + + # Registry metadata must describe values the parser can + # actually produce, including empty and all-NULL results. + @test_throws ArgumentError Postgres.register_type!(conn, 900_000, Int) + @test_throws ArgumentError Postgres.register_enum!(conn, "mood"; julia_type=Int) + Postgres.register_enum!(conn, "mood"; julia_type=String) + empty_enum = DBInterface.execute(conn, "SELECT mood FROM custom_types WHERE false") + @test Tables.schema(empty_enum).types[1] === String + null_enum = DBInterface.execute(conn, "SELECT NULL::mood AS mood") + @test Tables.schema(null_enum).types[1] == Union{Missing, String} + @test ismissing(only(null_enum).mood) + string_enum = only(DBInterface.execute(conn, "SELECT 'happy'::mood AS mood")) + @test string_enum.mood == "happy" DBInterface.execute(conn, "DROP TYPE textrange CASCADE") end + + @testset "Describe Schema Isolation" begin + DBInterface.execute(conn, "DROP SCHEMA IF EXISTS postgres_describe_a CASCADE") + DBInterface.execute(conn, "DROP SCHEMA IF EXISTS postgres_describe_b CASCADE") + DBInterface.execute(conn, "CREATE SCHEMA postgres_describe_a") + DBInterface.execute(conn, "CREATE SCHEMA postgres_describe_b") + try + DBInterface.execute(conn, "CREATE TABLE postgres_describe_a.parent_a (id int PRIMARY KEY)") + DBInterface.execute(conn, "CREATE TABLE postgres_describe_b.parent_b (id int PRIMARY KEY)") + DBInterface.execute(conn, """ + CREATE TABLE postgres_describe_a.child ( + id int PRIMARY KEY, + parent_id int, + CONSTRAINT shared_fk FOREIGN KEY (parent_id) + REFERENCES postgres_describe_a.parent_a(id) + ) + """) + DBInterface.execute(conn, """ + CREATE TABLE postgres_describe_b.child ( + id int PRIMARY KEY, + parent_id int, + CONSTRAINT shared_fk FOREIGN KEY (parent_id) + REFERENCES postgres_describe_b.parent_b(id) + ) + """) + description = Postgres.describe(conn, "child"; schema="postgres_describe_a") + rows = Tables.rowtable(description.resultset) + @test length(rows) == 2 + parent_row = only(filter(row -> row.column_name == "parent_id", rows)) + @test parent_row.foreign_key_reference == "parent_a.id" + finally + DBInterface.execute(conn, "DROP SCHEMA IF EXISTS postgres_describe_a CASCADE") + DBInterface.execute(conn, "DROP SCHEMA IF EXISTS postgres_describe_b CASCADE") + end + end @testset "Transactions" begin DBInterface.execute(conn, "DROP TABLE IF EXISTS trans_test") DBInterface.execute(conn, "CREATE TABLE trans_test (id SERIAL PRIMARY KEY, value INTEGER)") @@ -1170,6 +1328,50 @@ end @test isempty(Tables.rowtable(DBInterface.execute(conn, "SELECT * FROM deferred_child"))) end + # PostgreSQL changes COMMIT to ROLLBACK when a statement + # error was caught inside the body. The helper must not + # return the body value as if the write committed. + DBInterface.execute(conn, "DROP TABLE IF EXISTS caught_error_tx") + DBInterface.execute(conn, "CREATE TABLE caught_error_tx (id int)") + for wrapper in (:plain, :helper, :macro) + err = try + if wrapper === :plain + Postgres.start_transaction(conn) + DBInterface.execute(conn, "INSERT INTO caught_error_tx VALUES (1)") + try + DBInterface.execute(conn, "SELECT 1/0") + catch + end + Postgres.commit(conn) + elseif wrapper === :helper + Postgres.transaction(conn) do tx + DBInterface.execute(tx, "INSERT INTO caught_error_tx VALUES (1)") + try + DBInterface.execute(tx, "SELECT 1/0") + catch + end + :body_value + end + else + Postgres.@transaction conn begin + DBInterface.execute(conn, "INSERT INTO caught_error_tx VALUES (1)") + try + DBInterface.execute(conn, "SELECT 1/0") + catch + end + :body_value + end + end + nothing + catch e + e + end + @test err isa Postgres.PostgresInterfaceError + @test occursin("completed ROLLBACK", sprint(showerror, err)) + @test !Postgres.in_transaction(conn) + @test isempty(Tables.rowtable(DBInterface.execute(conn, "SELECT * FROM caught_error_tx"))) + end + # a transaction opened with raw SQL belongs to the caller: # driver helpers must nest inside it (savepoints), never # commit it, and never destroy it on their rollback @@ -1203,6 +1405,20 @@ end # the caller's ROLLBACK is still in control of all of it DBInterface.execute(conn, "ROLLBACK") @test isempty(Tables.rowtable(DBInterface.execute(conn, "SELECT id FROM rawtx_test"))) + + # Driver savepoints must not shadow a caller savepoint with + # the same name. A failed helper must also release its own + # savepoint after rolling back to it. + DBInterface.execute(conn, "BEGIN") + DBInterface.execute(conn, "SAVEPOINT sp_0") + DBInterface.execute(conn, "INSERT INTO rawtx_test VALUES (10)") + @test_throws ErrorException Postgres.transaction(conn) do tx + DBInterface.execute(tx, "INSERT INTO rawtx_test VALUES (20)") + error("fail nested work") + end + DBInterface.execute(conn, "ROLLBACK TO SAVEPOINT sp_0") + @test isempty(Tables.rowtable(DBInterface.execute(conn, "SELECT id FROM rawtx_test"))) + DBInterface.execute(conn, "ROLLBACK") # ... and after a raw COMMIT, the driver-level work sticks DBInterface.execute(conn, "BEGIN") Postgres.transaction(conn) do tx @@ -1265,6 +1481,18 @@ end DBInterface.execute(conn, "INVALID SQL") end @test length(Tables.rowtable(DBInterface.execute(conn, "SELECT * FROM macro_test"))) == 2 + + # An early return must commit before it leaves the caller. + early_return = function(c) + Postgres.@transaction c begin + DBInterface.execute(c, "INSERT INTO macro_test (value) VALUES (4)") + return :early + end + return :late + end + @test early_return(conn) === :early + @test !Postgres.in_transaction(conn) + @test only(Tables.rowtable(DBInterface.execute(conn, "SELECT count(*)::int AS n FROM macro_test"))).n == 3 end @testset "Nested Transactions" begin @@ -1370,6 +1598,48 @@ end Postgres.clear_statement_cache!(conn_cache) @test length(Postgres.get_cached_statements(conn_cache)) == 0 DBInterface.close!(conn_cache) + + # Retained handles survive eviction and cache disablement, + # but private re-prepare must never exceed or repopulate + # the configured cache. + retained_conn = DBInterface.connect(Postgres.Connection, + cfg.host, cfg.user, cfg.password; dbname=cfg.dbname, + port=cfg.port, statement_cache_maxsize=1) + retained_a = DBInterface.prepare(retained_conn, "SELECT 101 AS n") + retained_b = DBInterface.prepare(retained_conn, "SELECT 102 AS n") + @test length(Postgres.get_cached_statements(retained_conn)) == 1 + @test only(DBInterface.execute(retained_a)).n == 101 + retained_cache = Postgres.get_cached_statements(retained_conn) + @test length(retained_cache) == 1 + @test haskey(retained_cache, "SELECT 102 AS n") + replacement_a = DBInterface.prepare(retained_conn, "SELECT 101 AS n") + @test only(DBInterface.execute(retained_a)).n == 101 + @test length(Postgres.get_cached_statements(retained_conn)) == 1 + Postgres.set_statement_cache_maxsize!(retained_conn, 0) + @test only(DBInterface.execute(retained_b)).n == 102 + @test isempty(Postgres.get_cached_statements(retained_conn)) + for retained in (retained_a, retained_b, replacement_a) + DBInterface.close!(retained) + end + DBInterface.close!(retained_conn) + + # Connection-form bulk helpers own and close the private + # statements they create when caching is disabled. + bulk_conn = DBInterface.connect(Postgres.Connection, + cfg.host, cfg.user, cfg.password; dbname=cfg.dbname, + port=cfg.port, statement_cache_maxsize=0) + DBInterface.execute(bulk_conn, "CREATE TEMP TABLE cache_bulk_test (id int)") + for i in 1:3 + DBInterface.executemany(bulk_conn, + raw"INSERT INTO cache_bulk_test VALUES ($1)", ([i],)) + resultsets = DBInterface.executemultiple( + bulk_conn, raw"SELECT $1::int AS n", (i,)) + @test only(only(resultsets)).n == i + prepared_count = only(DBInterface.execute(bulk_conn, + "SELECT count(*)::int AS n FROM pg_prepared_statements")).n + @test prepared_count == 0 + end + DBInterface.close!(bulk_conn) end @testset "Do-Block Helpers" begin @@ -1483,7 +1753,22 @@ end @test ids == [2] end DBInterface.close!(pool) + @test !isopen(pool) @test !isopen(conn_a) + @test_throws Postgres.PostgresInterfaceError Postgres.acquire(pool) + DBInterface.close!(pool) + + # Closing a pool is terminal even when a connection is + # still checked out. Its later release must close it, not + # add it back to the closed pool. + active_pool = Postgres.ConnectionPool(Postgres.Connection, cfg.host, cfg.user, cfg.password; + dbname=cfg.dbname, port=cfg.port, limit=1) + active_conn = Postgres.acquire(active_pool) + DBInterface.close!(active_pool) + @test isopen(active_conn) + Postgres.release(active_pool, active_conn) + @test !isopen(active_conn) + @test_throws Postgres.PostgresInterfaceError Postgres.acquire(active_pool) end @testset "Transaction Prevents Reconnect" begin @@ -1525,11 +1810,100 @@ end @test only(Tables.rowtable(DBInterface.execute(timeout_conn, "SELECT current_setting('statement_timeout') AS t"))).t == "200ms" @test_throws Postgres.API.Error DBInterface.execute(timeout_conn, "SELECT pg_sleep(1)") Postgres.set_statement_timeout!(timeout_conn, 0) + @test Postgres.get_statement_timeout(timeout_conn) == 0 + Postgres.start_transaction(timeout_conn) + @test_throws Postgres.PostgresInterfaceError Postgres.set_statement_timeout!(timeout_conn, 777) + @test Postgres.get_statement_timeout(timeout_conn) == 0 + Postgres.rollback(timeout_conn) rows = Tables.rowtable(DBInterface.execute(timeout_conn, "SELECT 1 AS a")) @test rows[1].a == 1 DBInterface.close!(timeout_conn) end + @testset "Server Parameters And UTF8 Startup" begin + original_app = only(DBInterface.execute(conn, + "SELECT current_setting('application_name') AS value")).value + @test Postgres.get_server_parameter(conn, "application_name") == original_app + try + DBInterface.execute(conn, "SET application_name = 'postgres_jl_changed'") + @test Postgres.get_server_parameter(conn, "application_name") == "postgres_jl_changed" + Postgres.start_transaction(conn) + DBInterface.execute(conn, "SET LOCAL application_name = 'postgres_jl_local'") + @test Postgres.get_server_parameter(conn, "application_name") == "postgres_jl_local" + Postgres.commit(conn) + @test Postgres.get_server_parameter(conn, "application_name") == "postgres_jl_changed" + finally + Postgres.in_transaction(conn) && Postgres.rollback(conn) + DBInterface.execute(conn, "RESET application_name") + end + @test Postgres.get_server_parameter(conn, "application_name") == original_app + + dangerous_literal = "\\' OR true --" + try + DBInterface.execute(conn, "SET standard_conforming_strings = off") + @test Postgres.get_server_parameter(conn, + "standard_conforming_strings") == "off" + literal_row = only(DBInterface.execute(conn, + "SELECT $(Postgres.escape_literal(dangerous_literal))::text AS value")) + @test literal_row.value == dangerous_literal + finally + DBInterface.execute(conn, "RESET standard_conforming_strings") + end + + role_name = "postgres_jl_utf8_" * replace(string(uuid4()), "-" => "") + role_ident = Postgres.escape_identifier(role_name) + role_conn = nothing + DBInterface.execute(conn, "CREATE ROLE $role_ident LOGIN PASSWORD 'postgres_jl_test'") + try + DBInterface.execute(conn, + "ALTER ROLE $role_ident SET client_encoding = 'LATIN1'") + DBInterface.execute(conn, + "ALTER ROLE $role_ident SET statement_timeout = '444ms'") + role_conn = DBInterface.connect(Postgres.Connection, cfg.host, + role_name, "postgres_jl_test"; dbname=cfg.dbname, + port=cfg.port, reconnect=true) + settings = only(DBInterface.execute(role_conn, """ + SELECT current_setting('client_encoding') AS encoding, + current_setting('statement_timeout') AS timeout + """)) + @test settings.encoding == "UTF8" + @test settings.timeout == "444ms" + @test Postgres.get_server_parameter(role_conn, + "client_encoding") == "UTF8" + @test only(DBInterface.execute(role_conn, + raw"SELECT $1::text AS value", ("雪",))).value == "雪" + + # An explicit disable must override the role default on + # this session and after automatic reconnect. + Postgres.set_statement_timeout!(role_conn, nothing) + @test Postgres.get_statement_timeout(role_conn) == 0 + close(role_conn.socket) + @test only(DBInterface.execute(role_conn, + "SELECT current_setting('statement_timeout') AS value")).value == "0" + finally + role_conn === nothing || DBInterface.close!(role_conn) + DBInterface.execute(conn, "DROP ROLE IF EXISTS $role_ident") + end + end + + + @testset "Query Logger Isolation" begin + FAILING_LOGGER_CALLS[] = 0 + log_conn = DBInterface.connect(Postgres.Connection, cfg.host, cfg.user, cfg.password; + dbname=cfg.dbname, port=cfg.port, style=FailingLoggerStyle()) + try + @test_logs (:warn, r"query logger failed") DBInterface.execute(log_conn, "CREATE TEMP TABLE logger_test (id int)") + calls = FAILING_LOGGER_CALLS[] + @test_logs (:warn, r"query logger failed") DBInterface.execute(log_conn, "INSERT INTO logger_test VALUES (1)") + @test FAILING_LOGGER_CALLS[] == calls + 1 + @test_logs (:warn, r"query logger failed") begin + @test only(Tables.rowtable(DBInterface.execute(log_conn, "SELECT count(*)::int AS n FROM logger_test"))).n == 1 + end + finally + DBInterface.close!(log_conn) + end + end + @testset "Listen/Notify" begin listener = DBInterface.connect(Postgres.Connection, cfg.host, cfg.user, cfg.password; dbname=cfg.dbname, port=cfg.port) notifier = DBInterface.connect(Postgres.Connection, cfg.host, cfg.user, cfg.password; dbname=cfg.dbname, port=cfg.port) @@ -1591,6 +1965,13 @@ end @test_throws Postgres.PostgresInterfaceError Postgres.copy_from(conn, "COPY copy_test TO STDOUT", "1\talpha\n") @test_throws Postgres.PostgresInterfaceError Postgres.copy_to(conn, "COPY copy_test FROM STDIN") @test_throws Postgres.PostgresInterfaceError Postgres.copy_from(conn, "COPY copy_test (id, name) FROM STDIN; COPY copy_test (id, name) FROM STDIN", "9\tomega\n") + @test_throws Postgres.PostgresInterfaceError Postgres.copy_from(conn, + "COPY copy_test (id, name) FROM STDIN; COPY copy_test TO STDOUT", + "10\tmixed\n") + @test_throws Postgres.PostgresInterfaceError Postgres.copy_to(conn, + "COPY (SELECT 7) TO STDOUT; COPY (SELECT 8) TO STDOUT") + @test_throws Postgres.PostgresInterfaceError Postgres.copy_to(conn, + "COPY (SELECT 7) TO STDOUT; SELECT 8") @test Tables.rowtable(DBInterface.execute(conn, "SELECT 2 AS a"))[1].a == 2 # a genuine mid-stream server error during copy-out wins @@ -1647,6 +2028,59 @@ end DBInterface.close!(cur) @test !Postgres.in_transaction(conn) + # The documented Statement overload must own a transaction + # when called outside one, or the first Sync drops a + # suspended multi-batch portal. + cursor_stmt = DBInterface.prepare(conn, + "SELECT generate_series(1, 5) AS n") + statement_cursor = Postgres.cursor(cursor_stmt; fetchsize=2) + @test [row.n for row in statement_cursor] == [1, 2, 3, 4, 5] + DBInterface.close!(statement_cursor) + @test !Postgres.in_transaction(conn) + @test first(DBInterface.execute(cursor_stmt)).n == 1 + DBInterface.close!(cursor_stmt) + + mismatch_cursor_stmt = DBInterface.prepare(conn, + raw"SELECT $1::text AS a, $2::text AS b") + @test_throws Postgres.PostgresInterfaceError Postgres.cursor( + mismatch_cursor_stmt, ("must-not-remain",); fetchsize=1) + @test all(ismissing, mismatch_cursor_stmt.params) + @test !Postgres.in_transaction(conn) + DBInterface.close!(mismatch_cursor_stmt) + + # A fully exhausted named portal still exists until Close + # or transaction end. Cursor close must release it now, + # even inside a caller-owned long transaction. + Postgres.start_transaction(conn) + portal_stmt = DBInterface.prepare(conn, + "SELECT generate_series(1, 5) AS n") + portal_cursor = Postgres.cursor(portal_stmt; fetchsize=2) + @test length(collect(portal_cursor)) == 5 + portal_name = portal_cursor.portal + @test only(DBInterface.execute(conn, + "SELECT count(*)::int AS n FROM pg_cursors WHERE name = \$1", + (portal_name,))).n == 1 + DBInterface.close!(portal_cursor) + @test only(DBInterface.execute(conn, + "SELECT count(*)::int AS n FROM pg_cursors WHERE name = \$1", + (portal_name,))).n == 0 + DBInterface.close!(portal_stmt) + Postgres.rollback(conn) + + # With caching disabled, the connection-form cursor owns + # its private prepared statement and closes it with the + # portal. + private_cursor_conn = DBInterface.connect(Postgres.Connection, + cfg.host, cfg.user, cfg.password; dbname=cfg.dbname, + port=cfg.port, statement_cache_maxsize=0) + private_cursor = Postgres.cursor(private_cursor_conn, + "SELECT generate_series(1, 3) AS n"; fetchsize=1) + @test length(collect(private_cursor)) == 3 + DBInterface.close!(private_cursor) + @test only(DBInterface.execute(private_cursor_conn, + "SELECT count(*)::int AS n FROM pg_prepared_statements")).n == 0 + DBInterface.close!(private_cursor_conn) + # closing an already-closed cursor must not reach into a # transaction the caller opened afterwards and commit it DBInterface.execute(conn, "DROP TABLE IF EXISTS cursor_reclose") @@ -1776,6 +2210,8 @@ end order_row = only(Tables.rowtable(DBInterface.execute(german_conn, "SELECT current_setting('DateStyle') AS ds, '01/02/2020'::date AS d"))) @test order_row.ds == "ISO, DMY" @test order_row.d == Date(2020, 2, 1) + @test Postgres.get_server_parameter(german_conn, "DateStyle") == "ISO, DMY" + @test Postgres.get_server_parameter(german_conn, "IntervalStyle") == "postgres" finally DBInterface.close!(german_conn) end @@ -1792,7 +2228,6 @@ end @test complex_interval.interval_col == Dates.CompoundPeriod(Dates.Year(1), Dates.Month(2), Dates.Day(3), Dates.Hour(4), Dates.Minute(5), Dates.Second(6), Dates.Millisecond(789)) end - run_postgres_trim_compile_tests(cfg) finally isopen(conn) && DBInterface.close!(conn) end @@ -1823,6 +2258,76 @@ end @test connection_error(ssl_cfg.host, ssl_cfg; sslmode="verify-full") !== nothing @test connection_error(ssl_cfg.host, ssl_cfg; sslmode="verify-full", sslrootcert=tls.wrongrootcert) !== nothing + ca_dir = joinpath(tls.certdir, "ca-directory") + mkpath(ca_dir) + cp(tls.rootcert, joinpath(ca_dir, "root.crt")) + capath_conn = DBInterface.connect(Postgres.Connection, + ssl_cfg.host, ssl_cfg.user, ssl_cfg.password; + dbname=ssl_cfg.dbname, port=ssl_cfg.port, + sslmode="verify-full", sslcapath=ca_dir) + try + @test connection_uses_ssl(capath_conn) + finally + DBInterface.close!(capath_conn) + end + + # Require and verify a client certificate for one role. + # This covers the TLS 1.2 mTLS path and the cancel request, + # which must present the same client identity. + mtls_admin = DBInterface.connect(Postgres.Connection, + ssl_cfg.host, ssl_cfg.user, ssl_cfg.password; + dbname=ssl_cfg.dbname, port=ssl_cfg.port, sslmode="require") + try + DBInterface.execute(mtls_admin, "DROP ROLE IF EXISTS postgres_mtls") + DBInterface.execute(mtls_admin, "CREATE ROLE postgres_mtls LOGIN") + no_cert_error = try + no_cert_conn = DBInterface.connect(Postgres.Connection, + ssl_cfg.host, "postgres_mtls", nothing; + dbname=ssl_cfg.dbname, port=ssl_cfg.port, + sslmode="verify-full", sslrootcert=tls.rootcert) + DBInterface.close!(no_cert_conn) + nothing + catch err + err + end + @test no_cert_error !== nothing + + mtls_conn = DBInterface.connect(Postgres.Connection, + ssl_cfg.host, "postgres_mtls", nothing; + dbname=ssl_cfg.dbname, port=ssl_cfg.port, + sslmode="verify-full", sslrootcert=tls.rootcert, + sslcert=tls.clientcert, sslkey=tls.clientkey) + try + tls_row = only(DBInterface.execute(mtls_conn, """ + SELECT ssl, version, client_dn + FROM pg_stat_ssl + WHERE pid = pg_backend_pid() + """)) + @test tls_row.ssl + @test tls_row.version == "TLSv1.2" + @test occursin("CN=postgres_mtls", tls_row.client_dn) + + mtls_task = errormonitor(Threads.@spawn begin + try + DBInterface.execute(mtls_conn, "SELECT pg_sleep(5)") + :completed + catch err + err + end + end) + sleep(0.5) + Postgres.cancel_query!(mtls_conn) + mtls_result = fetch(mtls_task) + @test mtls_result isa Postgres.API.Error + @test mtls_result.code == "57014" + finally + DBInterface.close!(mtls_conn) + end + finally + DBInterface.execute(mtls_admin, "DROP ROLE IF EXISTS postgres_mtls") + DBInterface.close!(mtls_admin) + end + # against a TLS-capable server the cancel key goes over TLS # and the request is delivered. The connection uses the # default sslmode ("prefer") but negotiates TLS, so this diff --git a/test/trim_compile_tests.jl b/test/trim_compile_tests.jl deleted file mode 100644 index bec75b3..0000000 --- a/test/trim_compile_tests.jl +++ /dev/null @@ -1,263 +0,0 @@ -using Test - -const _POSTGRES_TRIM_SUPPORTED = VERSION >= v"1.12.0-rc1" -const _POSTGRES_TRIM_PRE_RELEASE = !isempty(VERSION.prerelease) -const _POSTGRES_JULIAC_ENTRYPOINT_EXPR = "using JuliaC; if isdefined(JuliaC, :main); JuliaC.main(ARGS); else JuliaC._main_cli(ARGS); end" - -function _postgres_trim_compile_timeout_s()::Float64 - default = Sys.iswindows() ? "1200.0" : "180.0" - return parse(Float64, get(ENV, "POSTGRES_TRIM_COMPILE_TIMEOUT_S", default)) -end - -function _postgres_trim_error_budget()::Int - # StructUtils 2.8.2 still takes its generic construction path. The - # companion trim branch removes these errors; keep released-dependency CI - # bounded so compiler regressions are visible in the meantime. - return parse(Int, get(ENV, "POSTGRES_TRIM_ERROR_BUDGET", "92")) -end - -function _postgres_trim_project_path()::String - active_project = Base.active_project() - if active_project !== nothing && isfile(active_project) - return dirname(active_project) - end - return normpath(joinpath(@__DIR__, "..")) -end - -function _postgres_trim_env(cfg) - run_env = copy(ENV) - run_env["POSTGRES_TRIM_HOST"] = cfg.host - run_env["POSTGRES_TRIM_PORT"] = string(cfg.port) - run_env["POSTGRES_TRIM_USER"] = cfg.user - run_env["POSTGRES_TRIM_DBNAME"] = cfg.dbname - return run_env -end - -function _run_postgres_trim_compile(project_path::String, script_path::String, output_name::String; timeout_s::Float64 = _postgres_trim_compile_timeout_s(), bundle_dir::Union{Nothing, String} = nothing) - julia_exe = joinpath(Sys.BINDIR, Base.julia_exename()) - cmd = if bundle_dir === nothing - `$julia_exe --startup-file=no --history-file=no --code-coverage=none --project=$project_path -e $(_POSTGRES_JULIAC_ENTRYPOINT_EXPR) -- --output-exe $output_name --project=$project_path --experimental --trim=safe $script_path` - else - `$julia_exe --startup-file=no --history-file=no --code-coverage=none --project=$project_path -e $(_POSTGRES_JULIAC_ENTRYPOINT_EXPR) -- --output-exe $output_name --bundle $bundle_dir --project=$project_path --experimental --trim=safe $script_path` - end - compile_env = copy(ENV) - compile_env["RESEAU_PRECOMPILE_ONLY"] = get(ENV, "POSTGRES_TRIM_RESEAU_PRECOMPILE_ONLY", "tcp") - cmd = setenv(cmd, compile_env) - return _run_postgres_trim_command_with_timeout(cmd; timeout_s = timeout_s, log_label = "compile") -end - -function _run_postgres_trim_executable(run_cmd::Cmd; timeout_s::Float64 = 30.0) - return _run_postgres_trim_command_with_timeout(run_cmd; timeout_s = timeout_s, log_label = "run") -end - -function _run_postgres_trim_command_with_timeout(cmd::Cmd; timeout_s::Float64, log_label::String) - output_path = tempname() - out = open(output_path, "w") - exit_code = -1 - timed_out = false - try - proc = run(pipeline(ignorestatus(cmd), stdout = out, stderr = out); wait = false) - timed_out = _wait_postgres_trim_process_with_timeout!(proc; timeout_s = timeout_s, log_label = log_label) - exit_code = something(proc.exitcode, -1) - finally - close(out) - end - output = try - read(output_path, String) - catch - "" - finally - rm(output_path; force = true) - end - return exit_code, output, timed_out -end - -function _wait_postgres_trim_process_with_timeout!(proc::Base.Process; timeout_s::Float64, log_label::String) - started_at = time() - next_log_at = started_at + 10.0 - timed_out = false - while Base.process_running(proc) - now = time() - if now - started_at >= timeout_s - timed_out = true - try - kill(proc) - catch - end - _kill_postgres_trim_windows_process_tree!(proc) - _wait_postgres_trim_process_exit_after_kill!(proc; timeout_s = 5.0, log_label = log_label) - return timed_out - end - if now >= next_log_at - elapsed = round(now - started_at; digits = 1) - println("[trim] $(log_label) WAIT $(elapsed)s") - flush(stdout) - next_log_at = now + 10.0 - end - sleep(0.1) - end - if !Base.process_running(proc) - try - wait(proc) - catch - end - end - return timed_out -end - -function _kill_postgres_trim_windows_process_tree!(proc::Base.Process)::Nothing - Sys.iswindows() || return nothing - pid = try - getpid(proc) - catch - return nothing - end - try - run(ignorestatus(`taskkill /PID $pid /T /F`)) - catch - end - return nothing -end - -function _wait_postgres_trim_process_exit_after_kill!(proc::Base.Process; timeout_s::Float64, log_label::String)::Nothing - deadline = time() + timeout_s - while Base.process_running(proc) && time() < deadline - sleep(0.1) - end - if Base.process_running(proc) - println("[trim] $(log_label) process still running after kill; continuing after timeout") - flush(stdout) - end - return nothing -end - -function _postgres_trim_timeout_error(kind::String, script_file::String, output::String = "") - msg = "trim $kind timed out for $(script_file)" - if !isempty(output) - msg = string(msg, "\n---- captured output ----\n", output, "\n---- end captured output ----") - end - throw(ArgumentError(msg)) -end - -function _maybe_print_postgres_trim_output(header::String, output::String) - isempty(output) && return nothing - println(header) - println(output) - println("---- end output ----") - return nothing -end - -function _postgres_trim_executable_timeout_s()::Float64 - default = Sys.iswindows() ? "180.0" : "30.0" - return parse(Float64, get(ENV, "POSTGRES_TRIM_EXE_TIMEOUT_S", default)) -end - -function _postgres_trim_selected_workloads(workloads::Vector{Tuple{String, String}})::Vector{Tuple{String, String}} - only = strip(get(ENV, "POSTGRES_TRIM_ONLY", "")) - isempty(only) && return workloads - selected = Tuple{String, String}[] - for workload in workloads - workload[1] == only && push!(selected, workload) - end - isempty(selected) && throw(ArgumentError("unknown POSTGRES_TRIM_ONLY workload: $(only)")) - return selected -end - -function _postgres_trim_use_bundle()::Bool - return get(ENV, "POSTGRES_TRIM_BUNDLE", "0") == "1" -end - -function _parse_postgres_trim_verify_totals(output::String) - m = match(r"Trim verify finished with\s+(\d+)\s+errors,\s+(\d+)\s+warnings\.", output) - m === nothing && return nothing - return parse(Int, m.captures[1]), parse(Int, m.captures[2]) -end - -function _count_postgres_trim_verify_messages(output::String)::Tuple{Int,Int} - errors = length(collect(eachmatch(r"Verifier error #\d+:", output))) - warnings = length(collect(eachmatch(r"Verifier warning #\d+:", output))) - return errors, warnings -end - -function _run_postgres_trim_case(cfg, project_path::String, script_file::String, output_name::String) - script_path = joinpath(@__DIR__, script_file) - @test isfile(script_path) - println("[trim] compile START $(script_file)") - start_t = time() - mktempdir() do tmpdir - cd(tmpdir) do - bundle_dir = _postgres_trim_use_bundle() ? joinpath(tmpdir, "bundle") : nothing - exit_code, output, timed_out = _run_postgres_trim_compile(project_path, script_path, output_name; bundle_dir = bundle_dir) - if timed_out - _postgres_trim_timeout_error("compile", script_file, output) - end - totals = _parse_postgres_trim_verify_totals(output) - trim_errors, trim_warnings = if totals === nothing - fallback = _count_postgres_trim_verify_messages(output) - if exit_code != 0 && fallback == (0, 0) - error("failed to parse trim verifier summary:\n$output") - end - fallback - else - totals - end - trim_error_budget = _postgres_trim_error_budget() - println("[trim] verifier $(script_file): errors=$(trim_errors) warnings=$(trim_warnings) budget=$(trim_error_budget)") - if get(ENV, "POSTGRES_TRIM_PRINT_OUTPUT", "0") == "1" || trim_errors > trim_error_budget || trim_warnings > 0 - _maybe_print_postgres_trim_output("---- trim compile output ($(script_file)) ----", output) - end - @test trim_errors <= trim_error_budget - @test trim_warnings == 0 - if trim_errors > 0 - @test exit_code != 0 - println("[trim] executable skipped for $(script_file): verifier errors remain") - return nothing - end - output_path = Sys.iswindows() ? "$(output_name).exe" : output_name - run_path = bundle_dir === nothing ? output_path : joinpath(bundle_dir, "bin", output_path) - @test exit_code == 0 - @test isfile(run_path) - run_cmd = setenv(`$(abspath(run_path))`, _postgres_trim_env(cfg)) - run_timeout_s = _postgres_trim_executable_timeout_s() - run_exit, run_output, run_timed_out = _run_postgres_trim_executable(run_cmd; timeout_s = run_timeout_s) - if run_timed_out - _postgres_trim_timeout_error("executable run", script_file, run_output) - end - if run_exit != 0 - _maybe_print_postgres_trim_output("---- trim executable output ($(script_file)) ----", run_output) - end - @test run_exit == 0 - end - end - println("[trim] compile DONE $(script_file) ($(round(time() - start_t; digits = 2))s)") - return nothing -end - -function run_postgres_trim_compile_tests(cfg)::Nothing - @testset "Trim Compile" begin - if Sys.iswindows() - println("[trim] skip Windows: JuliaC trim compilation is currently too slow or stalls on Windows CI") - @test true - elseif !_POSTGRES_TRIM_SUPPORTED - println("[trim] skip Julia < 1.12: JuliaC trim compilation is unavailable") - @test true - elseif _POSTGRES_TRIM_PRE_RELEASE - println("[trim] skip prerelease Julia: trim verifier behavior is not stable yet") - @test true - elseif !(occursin("trust", DEFAULT_AUTH) || occursin("trust", DEFAULT_INITDB_ARGS)) - println("[trim] skip non-trust auth mode: main CI covers trim once, auth-mode jobs focus on authentication") - @test true - else - project_path = _postgres_trim_project_path() - println("[trim] project $(project_path)") - trim_workloads = [ - ("postgres_trim_queries.jl", "postgres_trim_queries"), - ] - trim_workloads = _postgres_trim_selected_workloads(trim_workloads) - for (script_file, output_name) in trim_workloads - _run_postgres_trim_case(cfg, project_path, script_file, output_name) - end - end - end - return nothing -end From 119ab47052fb89ac89cb45baea11380d5cc530af Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 6 Aug 2026 11:51:10 -0600 Subject: [PATCH 21/23] test: cover connection-level wire logging --- test/runtests.jl | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/runtests.jl b/test/runtests.jl index 885cde7..b20cd84 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1557,6 +1557,18 @@ end conn5 = DBInterface.connect(Postgres.Connection, params_reconnect; reconnect=false) @test !conn5.reconnect DBInterface.close!(conn5) + + # A connection-level debug setting applies to the normal + # prepare/describe/execute path without a per-call flag. + debug_conn = DBInterface.connect(Postgres.Connection, + cfg.host, cfg.user, cfg.password; dbname=cfg.dbname, + port=cfg.port) + debug_conn.debug = true + debug_result = @test_logs (:info, r"sending message") match_mode=:any begin + DBInterface.execute(debug_conn, "SELECT 1 AS n") + end + @test only(debug_result).n == 1 + DBInterface.close!(debug_conn) end @testset "SSL Modes" begin From 70486441be2be067c3001f68eab2efcbabc6b276 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 6 Aug 2026 12:05:40 -0600 Subject: [PATCH 22/23] docs: disclose assisted 1.0 development --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 24021ba..e70fe22 100644 --- a/README.md +++ b/README.md @@ -226,3 +226,10 @@ DBInterface.close!(pool) ## Errors and cancellation `Postgres.Error` represents server errors and includes SQLSTATE codes; `Postgres.PostgresInterfaceError` covers client-side failures. Use `Postgres.cancel_query!(conn)` to send a CancelRequest to the server. + +## Development disclosure + +The 1.0 release preparation used Claude Code and OpenAI Codex for implementation +assistance and adversarial review. Maintainer decisions, source history, review +discussion, and validation results are recorded in +[pull request #5](https://github.com/JuliaDatabases/Postgres.jl/pull/5). From 49823c316d118d3f9eb6166cda1127b130d631c5 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 6 Aug 2026 19:01:34 -0600 Subject: [PATCH 23/23] fix(ci): restore Linux release validation Copy the generated pg_hba.conf into container-owned storage so Linux bind-mount permissions cannot block TLS startup. Require Reseau 1.3.5 for the renewed precompile certificate and keep the exact lower-bound job aligned. --- .github/workflows/CI.yml | 2 +- Project.toml | 2 +- test/runtests.jl | 7 ++++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 03f92ee..0b4ecfd 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -93,7 +93,7 @@ jobs: Pkg.PackageSpec(name="JSON", version="1.0.0"), Pkg.PackageSpec(name="MD5", version="0.2.0"), Pkg.PackageSpec(name="Parsers", version="2.5.4"), - Pkg.PackageSpec(name="Reseau", version="1.1.1"), + Pkg.PackageSpec(name="Reseau", version="1.3.5"), Pkg.PackageSpec(name="SASLAuth", version="1.0.0"), Pkg.PackageSpec(name="StructUtils", version="2.3.0"), Pkg.PackageSpec(name="Tables", version="1.0.0"), diff --git a/Project.toml b/Project.toml index 41257e1..dce5a9c 100644 --- a/Project.toml +++ b/Project.toml @@ -27,7 +27,7 @@ JSON = "1" MD5 = "0.2" Parsers = "2.5.4" Random = "1.10" -Reseau = "1.1.1" +Reseau = "1.3.5" SASLAuth = "1" Sockets = "1.10" StructUtils = "2.3" diff --git a/test/runtests.jl b/test/runtests.jl index b20cd84..7f05410 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -161,10 +161,11 @@ mkdir -p "\$certdir" cp /certs/server.crt "\$certdir/server.crt" cp /certs/server.key "\$certdir/server.key" cp /certs/root.crt "\$certdir/root.crt" -chown postgres:postgres "\$certdir/server.crt" "\$certdir/server.key" "\$certdir/root.crt" -chmod 0644 "\$certdir/server.crt" "\$certdir/root.crt" +cp /certs/pg_hba.conf "\$certdir/pg_hba.conf" +chown postgres:postgres "\$certdir/server.crt" "\$certdir/server.key" "\$certdir/root.crt" "\$certdir/pg_hba.conf" +chmod 0644 "\$certdir/server.crt" "\$certdir/root.crt" "\$certdir/pg_hba.conf" chmod 0600 "\$certdir/server.key" -exec docker-entrypoint.sh postgres -c ssl=on -c ssl_cert_file="\$certdir/server.crt" -c ssl_key_file="\$certdir/server.key" -c ssl_ca_file="\$certdir/root.crt" -c hba_file=/certs/pg_hba.conf +exec docker-entrypoint.sh postgres -c ssl=on -c ssl_cert_file="\$certdir/server.crt" -c ssl_key_file="\$certdir/server.key" -c ssl_ca_file="\$certdir/root.crt" -c hba_file="\$certdir/pg_hba.conf" """ return ["sh", "-c", setup_script] end