diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index c4eeb0b..0b4ecfd 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -70,6 +70,60 @@ jobs: - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 + lower-bounds: + name: Lower dependency bounds - Julia 1.10 - ubuntu-latest + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v6 + - uses: julia-actions/setup-julia@v3 + with: + version: '1.10' + arch: x64 + - uses: julia-actions/cache@v3 + - name: Test declared dependency floors + run: | + using Pkg + mktempdir() do env + Pkg.activate(env) + Pkg.develop(path=ENV["GITHUB_WORKSPACE"]) + Pkg.add([ + Pkg.PackageSpec(name="ConcurrentUtilities", version="2.1.0"), + Pkg.PackageSpec(name="DBInterface", version="2.5.0"), + Pkg.PackageSpec(name="JSON", version="1.0.0"), + Pkg.PackageSpec(name="MD5", version="0.2.0"), + Pkg.PackageSpec(name="Parsers", version="2.5.4"), + Pkg.PackageSpec(name="Reseau", version="1.3.5"), + Pkg.PackageSpec(name="SASLAuth", version="1.0.0"), + Pkg.PackageSpec(name="StructUtils", version="2.3.0"), + Pkg.PackageSpec(name="Tables", version="1.0.0"), + Pkg.PackageSpec(name="URIs", version="1.0.0"), + ]) + Pkg.test("Postgres") + end + shell: julia --color=yes {0} + + postgres-versions: + name: PostgreSQL ${{ matrix.postgres }} - Julia 1 - ubuntu-latest + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + # PostgreSQL 16 is covered by the main OS and Julia-version matrix. + postgres: ['14', '15', '17', '18'] + env: + POSTGRES_IMAGE: postgres:${{ matrix.postgres }} + steps: + - uses: actions/checkout@v6 + - uses: julia-actions/setup-julia@v3 + with: + version: '1' + arch: x64 + - uses: julia-actions/cache@v3 + - uses: julia-actions/julia-buildpkg@v1 + - uses: julia-actions/julia-runtest@v1 + docs: name: Documentation runs-on: ubuntu-latest diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..4aa305a --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Jacob Quinn and contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/LICENSE.md b/LICENSE.md deleted file mode 100644 index 295bef4..0000000 --- a/LICENSE.md +++ /dev/null @@ -1,22 +0,0 @@ -The Example.jl package is licensed under the MIT "Expat" License: - -> Copyright (c) 2022: Jacob Quinn -> -> Permission is hereby granted, free of charge, to any person obtaining -> a copy of this software and associated documentation files (the -> "Software"), to deal in the Software without restriction, including -> without limitation the rights to use, copy, modify, merge, publish, -> distribute, sublicense, and/or sell copies of the Software, and to -> permit persons to whom the Software is furnished to do so, subject to -> the following conditions: -> -> The above copyright notice and this permission notice shall be -> included in all copies or substantial portions of the Software. -> -> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -> EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -> MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -> IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -> CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -> TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -> SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Project.toml b/Project.toml index 165d80d..dce5a9c 100644 --- a/Project.toml +++ b/Project.toml @@ -8,7 +8,6 @@ ConcurrentUtilities = "f0e56b4a-5159-44fe-b623-3e5288b988bb" DBInterface = "a10d1c49-ce27-4219-8d33-6db1a4562965" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" MD5 = "6ac74813-4b46-53a4-afec-0b5dc9d7885c" Parsers = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" @@ -19,24 +18,30 @@ URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] -ConcurrentUtilities = "2" +Aqua = "0.8" +ConcurrentUtilities = "2.1" DBInterface = "2.5" +Dates = "1.10" Harbor = "1" JSON = "1" MD5 = "0.2" -Parsers = "0.3, 1, 2" -Reseau = "1.1" +Parsers = "2.5.4" +Random = "1.10" +Reseau = "1.3.5" SASLAuth = "1" -StructUtils = "2" +Sockets = "1.10" +StructUtils = "2.3" Tables = "1" +Test = "1.10" URIs = "1" +UUIDs = "1.10" julia = "1.10" [extras] +Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" Harbor = "af79dbb9-1a80-47ad-8928-192a4af69376" -JuliaC = "acedd4c2-ced6-4a15-accc-2607eb759ba2" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Harbor", "JuliaC", "Sockets", "Test"] +test = ["Aqua", "Harbor", "Sockets", "Test"] diff --git a/README.md b/README.md index 91f0f23..e70fe22 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,10 @@ # Postgres.jl -Postgres.jl is a PostgreSQL client that implements the v3 wire protocol with `DBInterface` and `Tables` integration. +[![CI](https://github.com/JuliaDatabases/Postgres.jl/actions/workflows/CI.yml/badge.svg)](https://github.com/JuliaDatabases/Postgres.jl/actions/workflows/CI.yml) +[![docs](https://img.shields.io/badge/docs-dev-blue.svg)](https://JuliaDatabases.github.io/Postgres.jl/dev/) +[![codecov](https://codecov.io/gh/JuliaDatabases/Postgres.jl/branch/main/graph/badge.svg)](https://codecov.io/gh/JuliaDatabases/Postgres.jl) + +Postgres.jl is a PostgreSQL client written in Julia that implements the v3 wire protocol with `DBInterface` and `Tables` integration. ## Installation @@ -12,10 +16,10 @@ Pkg.add("Postgres") ## Quick start ```julia -using Postgres, DBInterface, Tables +using Postgres DBInterface.connect(Postgres.Connection, "host=127.0.0.1;port=5432;user=postgres;password=postgres;dbname=postgres") do conn - rows = Tables.rowtable(DBInterface.execute(conn, "SELECT 1 AS a")) - @show rows[1].a + row = only(DBInterface.execute(conn, "SELECT 1 AS a")) + @show row.a end ``` @@ -32,11 +36,15 @@ Connection options support: - libpq-style keyword strings such as `host=127.0.0.1 port=5432 user=postgres dbname=postgres`. - PostgreSQL URIs such as `postgresql://postgres:postgres@127.0.0.1:5432/postgres`. - Environment defaults: `PGHOST`, `PGPORT`, `PGUSER`, `PGPASSWORD`, `PGDATABASE`, `PGAPPNAME`, `PGCONNECT_TIMEOUT`, and TLS-related `PGSSL*` variables. -- `sslmode` values: `disable`, `prefer`, `require`, `verify-full` (only `verify-full` enforces certificate verification). -- TLS files: `sslrootcert`, `sslcert`, `sslkey`, `sslcapath`. +- `sslmode` values: `disable`, `prefer` (the default), `require`, `verify-full`. Only `verify-full` verifies the server's certificate; `require` encrypts without authenticating the server, and the default `prefer` falls back to an unencrypted connection if the server declines TLS. Use `verify-full` with `sslrootcert` when the connection needs to be authenticated. +- TLS files: `sslrootcert`, `sslcert`, `sslkey`, and `sslcapath` (`sslcapath` is a fallback CA bundle or directory, used only when `sslrootcert` is unset and ignored otherwise). `sslservername` overrides the TLS server name when connecting to a pre-resolved address; under `verify-full` it is also the name the certificate is verified against, so it must name the server you intend to authenticate. - `connect_timeout` (seconds) and `statement_timeout` (milliseconds). - `application_name` and `statement_cache_maxsize`. +See the [1.0 support policy](https://JuliaDatabases.github.io/Postgres.jl/dev/support/) +for tested Julia and PostgreSQL versions, TLS limits, and transaction-pooler +requirements. + You can also use `ConnectionParams`: ```julia @@ -51,9 +59,9 @@ DBInterface.close!(conn) ```julia using Postgres, DBInterface, Tables conn = DBInterface.connect(Postgres.Connection, "host=127.0.0.1;user=postgres;password=postgres;dbname=postgres") -rows = Tables.rowtable(DBInterface.execute(conn, "SELECT $1::int AS val", (42,))) +rows = Tables.rowtable(DBInterface.execute(conn, raw"SELECT $1::int AS val", (42,))) @show rows[1].val -stmt = DBInterface.prepare(conn, "SELECT $1::int AS val") +stmt = DBInterface.prepare(conn, raw"SELECT $1::int AS val") rows = Tables.rowtable(DBInterface.execute(stmt, (7,))) DBInterface.close!(stmt) DBInterface.close!(conn) @@ -87,7 +95,7 @@ StructUtils.@tags struct ProfileSummary createdAt::DateTime &(postgres=(name=:created_at,),) end -profile = DBInterface.execute(conn, """ +profile = DBInterface.execute(conn, raw""" SELECT profile_id, first_name, last_name, created_at FROM profiles WHERE profile_id = $1 @@ -103,7 +111,8 @@ profiles = DBInterface.execute(conn, """ `Postgres.command_tag(result)` and `Postgres.rows_affected(result)` expose PostgreSQL command completion metadata. -Statement caching is LRU-based. Set `statement_cache_maxsize=0` to disable caching. +Explicit named prepared statements use an LRU backend cache. Caller handles are +independent. Set `statement_cache_maxsize=0` to disable this cache. ```julia using Postgres, DBInterface @@ -183,15 +192,22 @@ DBInterface.close!(conn) ``` `Numeric` values are returned as `Postgres.Numeric`, `interval` values as `Dates.Period` or `Dates.CompoundPeriod`, and range types as `Postgres.PostgresRange{T}`. +Custom enum, composite, and range registration controls result decoding. Those +custom Julia values are not accepted as direct query parameters in 1.0; bind a +PostgreSQL text representation with an explicit SQL cast instead. -## Query logging +## Query logging and driver styles + +Driver behavior — query logging, server notices, asynchronous notifications — is customized by defining a driver "style": subtype `Postgres.AbstractPostgresStyle`, overload the behavior hooks for it, and pass an instance via the `style` connection keyword. ```julia using Postgres, DBInterface -conn = DBInterface.connect(Postgres.Connection, "host=127.0.0.1;user=postgres;password=postgres;dbname=postgres") -Postgres.set_query_logger!(conn) do event, info - @show event info.success info.duration_ns -end + +struct LoggingStyle <: Postgres.AbstractPostgresStyle end +Postgres.query_logging_enabled(::LoggingStyle) = true +Postgres.query_logger(::LoggingStyle, event::Symbol, info::NamedTuple) = @info "query" event info.success info.duration_ns + +conn = DBInterface.connect(Postgres.Connection, "host=127.0.0.1;user=postgres;password=postgres;dbname=postgres"; style=LoggingStyle()) DBInterface.execute(conn, "SELECT 1") DBInterface.close!(conn) ``` @@ -210,3 +226,10 @@ DBInterface.close!(pool) ## Errors and cancellation `Postgres.Error` represents server errors and includes SQLSTATE codes; `Postgres.PostgresInterfaceError` covers client-side failures. Use `Postgres.cancel_query!(conn)` to send a CancelRequest to the server. + +## Development disclosure + +The 1.0 release preparation used Claude Code and OpenAI Codex for implementation +assistance and adversarial review. Maintainer decisions, source history, review +discussion, and validation results are recorded in +[pull request #5](https://github.com/JuliaDatabases/Postgres.jl/pull/5). diff --git a/docs/make.jl b/docs/make.jl index a70d638..9278acf 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -6,9 +6,10 @@ makedocs( pages = [ "Home" => "index.md", "Manual" => "manual.md", + "Support Policy" => "support.md", ], ) if get(ENV, "POSTGRES_DOCS_DEPLOY", "false") == "true" - deploydocs(repo = "github.com/quinnj/Postgres.jl.git", push_preview = true) + deploydocs(repo = "github.com/JuliaDatabases/Postgres.jl.git", push_preview = true) end diff --git a/docs/src/index.md b/docs/src/index.md index de09dc5..fa84b34 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -3,6 +3,7 @@ Postgres.jl is a PostgreSQL client that speaks the v3 wire protocol with `DBInterface` and `Tables` integration. See the [Manual](@ref) for a guided walk through connections, queries, prepared statements, transactions, cancellation, notifications, and type translation. +See the [1.0 Support Policy](@ref) for tested versions and explicit limits. ## Installation @@ -17,11 +18,14 @@ Postgres.jl accepts DSN strings or PostgreSQL URIs and supports: - libpq-style keyword strings such as `host=127.0.0.1 port=5432 user=postgres dbname=postgres`. - Environment defaults from `PGHOST`, `PGPORT`, `PGUSER`, `PGPASSWORD`, `PGDATABASE`, `PGAPPNAME`, `PGCONNECT_TIMEOUT`, and TLS-related `PGSSL*` variables. -- `sslmode` values: `disable`, `prefer`, `require`, `verify-full` (only `verify-full` verifies certificates). -- TLS files: `sslrootcert`, `sslcert`, `sslkey`, `sslcapath`. +- `sslmode` values: `disable`, `prefer` (the default), `require`, `verify-full`. Only `verify-full` verifies the server's certificate; `require` encrypts without authenticating the server, and the default `prefer` falls back to an unencrypted connection if the server declines TLS. Use `verify-full` with `sslrootcert` when the connection needs to be authenticated. +- TLS files: `sslrootcert`, `sslcert`, `sslkey`, and `sslcapath` (`sslcapath` is a fallback CA bundle or directory, used only when `sslrootcert` is unset and ignored otherwise). `sslservername` overrides the TLS server name when connecting to a pre-resolved address; under `verify-full` it is also the name the certificate is verified against, so it must name the server you intend to authenticate. - `connect_timeout` (seconds), `statement_timeout` (milliseconds). - `application_name` and `statement_cache_maxsize`. +Options that request unsupported security or server-selection behavior are +rejected. They are not silently ignored. + ```julia using Postgres, DBInterface conn = DBInterface.connect(Postgres.Connection, "postgresql://postgres:postgres@127.0.0.1:5432/postgres?sslmode=disable") @@ -33,7 +37,7 @@ DBInterface.close!(conn) ```julia using Postgres, DBInterface, Tables conn = DBInterface.connect(Postgres.Connection, "host=127.0.0.1;user=postgres;password=postgres;dbname=postgres") -rows = Tables.rowtable(DBInterface.execute(conn, "SELECT $1::int AS val", (42,))) +rows = Tables.rowtable(DBInterface.execute(conn, raw"SELECT $1::int AS val", (42,))) @show rows[1].val DBInterface.close!(conn) ``` @@ -65,7 +69,7 @@ StructUtils.@tags struct ProfileSummary createdAt::DateTime &(postgres=(name=:created_at,),) end -profile = DBInterface.execute(conn, """ +profile = DBInterface.execute(conn, raw""" SELECT profile_id, first_name, last_name, created_at FROM profiles WHERE profile_id = $1 @@ -79,12 +83,12 @@ profiles = DBInterface.execute(conn, """ """, (), Vector{ProfileSummary}) ``` -Prepared statements are cached with LRU eviction; disable caching via `statement_cache_maxsize=0`. +Explicit named prepared statements use an LRU backend cache; disable it via `statement_cache_maxsize=0`. ```julia using Postgres, DBInterface, Tables conn = DBInterface.connect(Postgres.Connection, "host=127.0.0.1;user=postgres;password=postgres;dbname=postgres"; statement_cache_maxsize=5) -stmt = DBInterface.prepare(conn, "SELECT $1::int AS val") +stmt = DBInterface.prepare(conn, raw"SELECT $1::int AS val") rows = Tables.rowtable(DBInterface.execute(stmt, (7,))) DBInterface.close!(stmt) DBInterface.close!(conn) @@ -162,12 +166,16 @@ DBInterface.close!(conn) ## Query logging +Query logging (and other driver behavior) is customized with a driver style; see the [Manual](@ref) for details. + ```julia using Postgres, DBInterface -conn = DBInterface.connect(Postgres.Connection, "host=127.0.0.1;user=postgres;password=postgres;dbname=postgres") -Postgres.set_query_logger!(conn) do event, info - @show event info.success info.duration_ns -end + +struct LoggingStyle <: Postgres.AbstractPostgresStyle end +Postgres.query_logging_enabled(::LoggingStyle) = true +Postgres.query_logger(::LoggingStyle, event::Symbol, info::NamedTuple) = @info "query" event info.success info.duration_ns + +conn = DBInterface.connect(Postgres.Connection, "host=127.0.0.1;user=postgres;password=postgres;dbname=postgres"; style=LoggingStyle()) DBInterface.execute(conn, "SELECT 1") DBInterface.close!(conn) ``` @@ -187,6 +195,17 @@ DBInterface.close!(pool) `Postgres.Error` includes SQLSTATE information. Use `Postgres.cancel_query!(conn)` to cancel a running query. +## Reference + ```@autodocs Modules = [Postgres] ``` + +```@docs +Postgres.Error +Postgres.Notification +Postgres.Numeric +Postgres.PostgresRange +Postgres.ConnectionParams +Postgres.parse_dsn +``` diff --git a/docs/src/manual.md b/docs/src/manual.md index a0d010f..fe6baf5 100644 --- a/docs/src/manual.md +++ b/docs/src/manual.md @@ -92,7 +92,7 @@ StructUtils.@tags struct ProfileSummary createdAt::DateTime &(postgres=(name=:created_at,),) end -profile = DBInterface.execute(conn, """ +profile = DBInterface.execute(conn, raw""" SELECT profile_id, first_name, last_name, created_at FROM profiles WHERE profile_id = $1 @@ -117,24 +117,42 @@ that style, and pass an instance with the `style` connection keyword. The default `Postgres.PostgresStyle` keeps query logging disabled and reports server notices through Julia's logger. +```julia +struct LoggingStyle <: Postgres.AbstractPostgresStyle end +Postgres.query_logging_enabled(::LoggingStyle) = true +Postgres.query_logger(::LoggingStyle, event::Symbol, info::NamedTuple) = @info "query" event info.success info.duration_ns +Postgres.notice_callback(::LoggingStyle, notice) = @info "notice" notice +Postgres.notification_callback(::LoggingStyle, notification) = @info "notification" notification + +conn = DBInterface.connect(Postgres.Connection, "host=127.0.0.1 user=postgres dbname=postgres"; style=LoggingStyle()) +``` + +`query_logger`'s `info` includes the SQL and the bound parameter values, so a logger that writes them out records whatever sensitive data those queries carry. Redact or omit `info.params` when the log destination is less trusted than the database itself. + ```@docs Postgres.AbstractPostgresStyle Postgres.PostgresStyle +Postgres.query_logging_enabled +Postgres.query_logger +Postgres.notice_callback +Postgres.notification_callback ``` ## Parameters And Prepared Statements -Use PostgreSQL placeholders (`$1`, `$2`, ...) and pass a tuple or other iterable of parameter values. +Use PostgreSQL placeholders (`$1`, `$2`, ...) and pass a tuple or other iterable of parameter values. Note the use of `raw"..."` strings so that `$1` is not treated as Julia string interpolation. ```julia -rows = Tables.rowtable(DBInterface.execute(conn, "SELECT $1::int + $2::int AS total", (20, 22))) +rows = Tables.rowtable(DBInterface.execute(conn, raw"SELECT $1::int + $2::int AS total", (20, 22))) @show only(rows).total ``` -Prepared statements can be created explicitly. Postgres.jl also caches prepared statements internally with LRU eviction; set `statement_cache_maxsize=0` to disable caching. +Prepared statements can be created explicitly. Postgres.jl caches their named +backend statements with LRU eviction while returning an independent handle to +each caller. Set `statement_cache_maxsize=0` to disable this cache. ```julia -stmt = DBInterface.prepare(conn, "SELECT $1::text AS value") +stmt = DBInterface.prepare(conn, raw"SELECT $1::text AS value") rows = Tables.rowtable(DBInterface.execute(stmt, ("prepared",))) DBInterface.close!(stmt) ``` @@ -143,7 +161,7 @@ Bulk inserts can use `DBInterface.executemany`. ```julia DBInterface.execute(conn, "CREATE TEMP TABLE demo_many (id int, name text)") -stmt = DBInterface.prepare(conn, "INSERT INTO demo_many VALUES ($1, $2)") +stmt = DBInterface.prepare(conn, raw"INSERT INTO demo_many VALUES ($1, $2)") DBInterface.executemany(stmt, ([1, 2, 3], ["a", "b", "c"])) DBInterface.close!(stmt) ``` @@ -243,3 +261,33 @@ row = only(Tables.rowtable(DBInterface.execute(conn, "SELECT 'happy'::mood AS mo ``` Registering composite and range types follows the same pattern. + +Registration controls result decoding. Direct parameter binding for registered +enum, composite, and range Julia values is not part of the 1.0 interface. Bind +their PostgreSQL text representation and add an explicit SQL cast when needed. + +### Session Formats + +Text decoding assumes the server renders dates and timestamps in the `ISO` +`DateStyle` output format and intervals in the `postgres` `IntervalStyle`. The +driver checks the server-reported settings at connect time and issues a `SET` +for any that differ, preserving the configured date field order (`MDY`/`DMY`/ +`YMD`) since it decides how ambiguous input literals like `'01/02/2020'` are +read. The alignment is re-applied on reconnect, but not if the session is +changed afterwards: running `SET DateStyle = ...` or `SET IntervalStyle = ...` +mid-session breaks decoding — intervals and unparseable dates raise errors +rather than silently returning wrong values. + +Transaction-mode poolers cannot preserve session settings between logical +connections. See the [1.0 Support Policy](@ref) before using this mode. + +### Values Without A Julia Representation + +A few PostgreSQL values have no faithful Julia equivalent and raise +`Postgres.PostgresInterfaceError` when decoded rather than silently returning +a wrong value: `infinity`/`-infinity` dates and timestamps, dates in the BC +era, and `numeric` `NaN`/`infinity`. `"char"` columns (the 1-byte internal +catalog type) decode to `Char`, including the zero byte (`'\0'`) and high-bit +bytes; note that a `'\0'` read from such a column cannot be bound back as a +text parameter, because PostgreSQL rejects NUL bytes in text — write it with +an explicit cast such as `0::"char"` instead. diff --git a/docs/src/support.md b/docs/src/support.md new file mode 100644 index 0000000..94b6b43 --- /dev/null +++ b/docs/src/support.md @@ -0,0 +1,58 @@ +# 1.0 Support Policy + +Postgres.jl 1.0 supports Julia 1.10 and later. The release test matrix covers +PostgreSQL 14 through 18 on TCP connections. Unix-domain sockets are not +supported. + +## TLS + +`sslmode=verify-full` verifies the certificate chain and server name. Use a DNS +name as `host`, or set `sslservername` to the DNS name when dialing a resolved +address. IP-address subject-alternative-name matching on a TLS 1.2-only server +is not supported in 1.0. + +Client certificates require both `sslcert` and `sslkey`. Postgres.jl 1.0 limits +client-certificate connections to TLS 1.2 because the current Reseau 1.x mixed +TLS 1.2/1.3 client path does not send the certificate reliably. Server-only TLS +connections can negotiate TLS 1.2 or TLS 1.3. + +`connect_timeout` bounds the TCP connection and TLS handshake. It does not +bound PostgreSQL authentication or query execution. Use `statement_timeout` +for query execution on a direct or session-pooled connection. + +Keep a manual transaction or streaming cursor on the task that created it. +Do not run unrelated operations on that connection until the scope ends. Use +`ConnectionPool` to give concurrent tasks separate connections. + +## Transaction Poolers + +Connection-form `DBInterface.execute(conn, sql, params)` is safe through a +transaction-mode PgBouncer endpoint. Postgres.jl sends each unnamed extended +query as one dependent protocol segment. Explicit named prepared statements +require PgBouncer prepared-statement tracking, such as +`max_prepared_statements > 0`. + +A transaction pooler does not preserve session state between logical clients. +Do not use connection-level `statement_timeout`, `set_statement_timeout!`, +`LISTEN`, temporary tables, session advisory locks, or arbitrary `SET` state in +transaction mode. Configure PostgreSQL or the pooler defaults with +`DateStyle=ISO` and `IntervalStyle=postgres`; Postgres.jl needs these text +formats for correct decoding. Use direct connections or session pooling when +the application needs session state. + +`set_statement_timeout!` is rejected while a transaction is open. This keeps +the durable reconnect setting consistent with PostgreSQL's transactional `SET` +semantics. + +## Types + +Built-in scalar types, byte arrays, and one-dimensional PostgreSQL arrays can +be bound as parameters. Custom enum, composite, and range registration is a +result-decoding feature in 1.0. Bind a text representation with an explicit SQL +cast when writing those custom values. Multidimensional Julia arrays are not a +supported parameter form in 1.0. + +## Native Compilation + +JuliaC `--trim` compilation is not supported in Postgres.jl 1.0. Normal Julia +package precompilation is supported and is part of CI. diff --git a/src/Postgres.jl b/src/Postgres.jl index 0c1944d..cbba5ba 100644 --- a/src/Postgres.jl +++ b/src/Postgres.jl @@ -4,12 +4,21 @@ using DBInterface, Dates, UUIDs, Parsers, Tables, StructUtils, JSON, ConcurrentU export DBInterface -# For non-api errors that happen in Postgres.jl -struct PostgresInterfaceError +""" + Postgres.PostgresInterfaceError <: Exception + +A client-side error raised by Postgres.jl itself (closed connections, +parameter-count mismatches, unsupported features, ...), as opposed to +[`Postgres.Error`](@ref Postgres.API.Error), which represents an error +reported by the server. +""" +struct PostgresInterfaceError <: Exception msg::String end Base.showerror(io::IO, e::PostgresInterfaceError) = print(io, e.msg) +@noinline _reject_nul(what::String) = throw(PostgresInterfaceError("$what cannot contain a NUL byte")) + include("api/API.jl") using .API include("connection_string.jl") @@ -18,6 +27,63 @@ using .ConnectionString const Pools = ConcurrentUtilities.Pools const ReseauConn = Union{Reseau.TCP.Conn, Reseau.TLS.Conn} +# Observability must not change whether a database operation succeeds. A +# callback failure is reported, but never replaces the query result or error. +function query_log_safely(style, event::Symbol, info::NamedTuple) + try + API.query_logger(style, event, info) + catch err + @warn "Postgres query logger failed" event exception=(err, catch_backtrace()) + end + return nothing +end + +""" + Postgres.Connection + +A single connection to a PostgreSQL server, created via +`DBInterface.connect(Postgres.Connection, ...)`: + + DBInterface.connect(Postgres.Connection, host, user, password; dbname, port=5432, kwargs...) + DBInterface.connect(Postgres.Connection, dsn::String; kwargs...) + DBInterface.connect(Postgres.Connection, params::ConnectionParams; kwargs...) + +`dsn` may be a libpq-style keyword string (`"host=127.0.0.1 user=postgres dbname=postgres"`) +or a PostgreSQL URI (`"postgresql://user:pass@host:5432/dbname?sslmode=require"`). + +Supported keyword arguments. All are also available as DSN/URI options except +`style`, which is Julia-only: + +- `dbname`, `port`, `application_name` +- `connect_timeout` (seconds), `statement_timeout` (milliseconds) +- `sslmode` (`"disable"`, `"prefer"` (default), `"require"`, `"verify-full"`), + `sslrootcert`, `sslcert`, `sslkey`, `sslcapath`, and `sslservername`. + Only `verify-full` verifies the server's certificate; `require` encrypts + without authenticating the server, and the default `prefer` falls back to an + unencrypted connection if the server declines TLS. `sslcapath` is a + fallback CA bundle or directory used only when `sslrootcert` is unset (it is + ignored otherwise). + `sslservername` overrides the TLS server + name when the host is a pre-resolved address — note that under + `verify-full` this is also the name the certificate is verified against, + so it must name the server you intend to authenticate. +- `statement_cache_maxsize`: LRU backend cache size for explicit named prepared + statements (default 100; `0` disables) +- `reconnect`: automatically reconnect and re-prepare statements if the + connection is found dead (default `false`; never reconnects mid-transaction) +- `style`: a custom [`AbstractPostgresStyle`](@ref Postgres.API.AbstractPostgresStyle) + for query logging / notice / notification behavior +- `debug`: log wire protocol messages. Authentication messages are redacted, + but bind parameter values are not — treat a debug log as sensitive as the + data the connection carries. + +Individual operations are serialized on an internal lock. Keep each manual +transaction and streaming cursor on one task and do not run unrelated work on +that connection until the scope ends. Use a [`ConnectionPool`](@ref) for +concurrent work. `cancel_query!` is the intentional cross-task exception. +Close with `DBInterface.close!(conn)` or `close(conn)`; the do-block form +`DBInterface.connect(f, Postgres.Connection, ...)` closes automatically. +""" mutable struct Connection{T, S <: API.AbstractPostgresStyle} <: DBInterface.Connection const lock::ReentrantLock socket::ReseauConn @@ -52,7 +118,16 @@ mutable struct Connection{T, S <: API.AbstractPostgresStyle} <: DBInterface.Conn const style::S in_transaction::Bool # track transaction state transaction_depth::Int # track nested transactions (SAVEPOINTs) + transaction_savepoints::Vector{String} # driver-owned names, outermost first generation::Int # increment on reconnect to invalidate statements + # the server's own ReadyForQuery transaction status: unlike in_transaction + # it also sees a transaction opened by raw SQL (`execute(conn, "BEGIN")`) + server_in_transaction::Bool + # whether the outermost driver-managed transaction level issued the BEGIN: + # when start_transaction found a raw-SQL transaction already open, the base + # level is a savepoint inside the caller's transaction, and the final + # commit/rollback must not COMMIT/ROLLBACK the caller's work + owns_base_transaction::Bool function Connection(; host::AbstractString="", user::AbstractString="", password::Union{AbstractString, Nothing}=nothing, dbname::AbstractString="", port::Integer=5432, debug::Bool=false, reconnect::Bool=false, application_name::Union{AbstractString, Nothing}=nothing, connect_timeout::Union{Integer, Nothing}=nothing, sslmode::Union{AbstractString, Nothing}=nothing, sslrootcert::Union{AbstractString, Nothing}=nothing, sslcert::Union{AbstractString, Nothing}=nothing, sslkey::Union{AbstractString, Nothing}=nothing, sslcapath::Union{AbstractString, Nothing}=nothing, sslservername::Union{AbstractString, Nothing}=nothing, statement_timeout::Union{Integer, Nothing}=nothing, statement_cache_maxsize::Integer=100, style::API.AbstractPostgresStyle=PostgresStyle()) host = String(host) @@ -69,12 +144,18 @@ mutable struct Connection{T, S <: API.AbstractPostgresStyle} <: DBInterface.Conn sslcapath_val = sslcapath === nothing ? nothing : String(sslcapath) statement_timeout_val = statement_timeout === nothing ? nothing : Int(statement_timeout) sslservername_val = sslservername === nothing ? nothing : String(sslservername) + occursin('\0', host) && _reject_nul("host") + occursin('\0', user) && _reject_nul("user") + occursin('\0', dbname) && _reject_nul("dbname") + password !== nothing && occursin('\0', password) && _reject_nul("password") + app_name !== nothing && occursin('\0', app_name) && _reject_nul("application_name") + sslservername_val !== nothing && occursin('\0', sslservername_val) && _reject_nul("sslservername") + xor(sslcert_val === nothing, sslkey_val === nothing) && + throw(PostgresInterfaceError("sslcert and sslkey must be provided together")) maxsize = max(0, Int(statement_cache_maxsize)) - #TODO: if values have spaces, need to single-quote them - # also need to escape single quotes/backslahes then with backslashes socket, pid, skey, server_params = API.connect(host, port, dbname, user, password, debug, app_name, timeout, sslmode_val, sslrootcert_val, sslcert_val, sslkey_val, sslcapath_val, sslservername_val, statement_timeout_val) registry = Dict(API.DEFAULT_TYPE_REGISTRY) - return new{Statement{typeof(style)}, typeof(style)}(ReentrantLock(), socket, host, user, password, dbname, port, app_name, timeout, sslmode_val, sslrootcert_val, sslcert_val, sslkey_val, sslcapath_val, sslservername_val, statement_timeout_val, pid, skey, Dict{String, Statement{typeof(style)}}(), maxsize, 0, server_params, registry, false, reconnect, debug, style, false, 0, 1) + return new{Statement{typeof(style)}, typeof(style)}(ReentrantLock(), socket, host, user, password, dbname, port, app_name, timeout, sslmode_val, sslrootcert_val, sslcert_val, sslkey_val, sslcapath_val, sslservername_val, statement_timeout_val, pid, skey, Dict{String, Statement{typeof(style)}}(), maxsize, 0, server_params, registry, false, reconnect, debug, style, false, 0, String[], 1, false, true) end end @@ -117,30 +198,55 @@ function evict_lru_statement!(conn::Connection) end oldest_sql === nothing && return stmt = pop!(conn.statements, oldest_sql) - !stmt.closed && API.close_statement(conn.socket, stmt.name, conn.debug) + if !stmt.closed + API.close_statement(conn.socket, stmt.name, conn.debug, conn.server_parameters) + stmt.closed = true + end return end +""" + Postgres.get_cached_statements(conn) -> Dict{String, Statement} + +Return a copy of the connection's prepared-statement cache, keyed by SQL text. +""" function get_cached_statements(conn::Connection) - @lock conn.lock copy(conn.statements) + @lock conn.lock Dict(sql => statement_handle(stmt) for (sql, stmt) in conn.statements) end +""" + Postgres.clear_statement_cache!(conn) + +Close all server-side prepared statements in the connection's cache and empty it. +""" function clear_statement_cache!(conn::Connection) @lock conn.lock begin for (sql, stmt) in conn.statements - !stmt.closed && API.close_statement(conn.socket, stmt.name, conn.debug) + if !stmt.closed + API.close_statement(conn.socket, stmt.name, conn.debug, conn.server_parameters) + stmt.closed = true + end end empty!(conn.statements) end return conn end +""" + Postgres.set_statement_cache_maxsize!(conn, maxsize) + +Set the maximum number of prepared statements the connection caches (LRU +eviction). `0` disables caching and closes all currently cached statements. +""" function set_statement_cache_maxsize!(conn::Connection, maxsize::Integer) @lock conn.lock begin conn.statement_cache_maxsize = max(0, Int(maxsize)) if conn.statement_cache_maxsize == 0 for (sql, stmt) in conn.statements - !stmt.closed && API.close_statement(conn.socket, stmt.name, conn.debug) + if !stmt.closed + API.close_statement(conn.socket, stmt.name, conn.debug, conn.server_parameters) + stmt.closed = true + end end empty!(conn.statements) return conn @@ -153,43 +259,113 @@ function set_statement_cache_maxsize!(conn::Connection, maxsize::Integer) return conn end +""" + Postgres.get_server_parameter(conn, name) -> Union{String, Nothing} + +Return the server-reported value of runtime parameter `name` (e.g. +`"server_version"`, `"TimeZone"`), or `nothing` if the server has not reported it. +""" get_server_parameter(conn::Connection, param::String) = @lock conn.lock get(conn.server_parameters, param, nothing) +""" + Postgres.get_server_parameters(conn) -> Dict{String, String} + +Return a copy of all runtime parameters the server has reported on this connection. +""" get_server_parameters(conn::Connection) = @lock conn.lock copy(conn.server_parameters) # NOTE: runtime callback setters are gone — customize behavior by passing a custom # AbstractPostgresStyle to Connection(; style=...) and overloading the style-first # interface methods (query_logger / notice_callback / notification_callback). +""" + Postgres.get_statement_timeout(conn) -> Union{Int, Nothing} + +Return the statement timeout (milliseconds) configured on the connection, or +`nothing` if none was set. +""" function get_statement_timeout(conn::Connection) return @lock conn.lock conn.statement_timeout end +""" + Postgres.set_statement_timeout!(conn, timeout) + +Set the server `statement_timeout` for the connection, in milliseconds. +`nothing` or `0` disables the timeout. This setting is session state and cannot +be changed while a transaction is open. An explicit disable is retained across +automatic reconnects as `0`. +""" function set_statement_timeout!(conn::Connection, timeout::Union{Integer, Nothing}) timeout_val = timeout === nothing ? 0 : max(0, Int(timeout)) - DBInterface.execute(conn, "SET statement_timeout = $timeout_val") - @lock conn.lock conn.statement_timeout = timeout === nothing ? nothing : timeout_val + @lock conn.lock begin + checkconn(conn) + (conn.in_transaction || conn.server_in_transaction) && + throw(PostgresInterfaceError("statement_timeout cannot be changed while a transaction is open")) + DBInterface.execute(conn, "SET statement_timeout = $timeout_val") + conn.statement_timeout = timeout_val + end return conn end +""" + Postgres.escape_identifier(name) -> String + +Quote a string for use as a SQL identifier (double-quoted, embedded quotes +doubled). Throws if `name` contains a NUL byte. +""" function escape_identifier(name::AbstractString) + occursin('\0', name) && _reject_nul("identifier") return string("\"", replace(name, "\"" => "\"\""), "\"") end +""" + Postgres.escape_literal(val) -> String + +Quote a string for use as a SQL literal (single-quoted, embedded quotes +doubled). Throws if `val` contains a NUL byte. + +Prefer query parameters (`\$1`, `\$2`, ...) over literal interpolation +whenever possible — parameters are never parsed as SQL. Inputs containing a +backslash use PostgreSQL's explicit escape-string syntax, with backslashes and +quotes escaped so the result is independent of `standard_conforming_strings`. +""" function escape_literal(val::AbstractString) - return string("'", replace(val, "'" => "''"), "'") + occursin('\0', val) && _reject_nul("literal") + escaped = replace(val, "'" => "''") + occursin('\\', escaped) || return string("'", escaped, "'") + return string("E'", replace(escaped, "\\" => "\\\\"), "'") end +""" + Postgres.listen!(conn, channel) + +Execute `LISTEN channel` so the connection receives notifications for +`channel`. Use [`wait_for_notification`](@ref Postgres.wait_for_notification) +to block until one arrives. +""" function listen!(conn::Connection, channel::AbstractString) DBInterface.execute(conn, "LISTEN $(escape_identifier(channel))") return conn end +""" + Postgres.unlisten!(conn, channel) + +Execute `UNLISTEN channel` to stop receiving notifications for `channel`. +""" function unlisten!(conn::Connection, channel::AbstractString) DBInterface.execute(conn, "UNLISTEN $(escape_identifier(channel))") return conn end +""" + Postgres.notify!(conn, channel, payload=nothing) + +Execute `NOTIFY channel` (with optional `payload`), delivering a +[`Notification`](@ref Postgres.API.Notification) to all connections listening +on `channel`. +""" function notify!(conn::Connection, channel::AbstractString, payload::Union{AbstractString, Nothing}=nothing) channel_ident = escape_identifier(channel) sql = payload === nothing ? "NOTIFY $channel_ident" : "NOTIFY $channel_ident, $(escape_literal(payload))" @@ -199,7 +375,7 @@ end function update_server_parameters!(conn::Connection, buf::Vector{UInt8}) i = 1 - while i < length(buf) + GC.@preserve buf while i < length(buf) j = findnext(isequal(UInt8(0)), buf, i) j === nothing && break key = unsafe_string(pointer(buf, i), j - i) @@ -229,6 +405,36 @@ end const NOTIFICATION_POLL_INTERVAL_NS = Int64(100_000_000) +# A read deadline surfaces directly as DeadlineExceededError on a plain TCP +# connection, but the TLS layer wraps transport failures, so over TLS the same +# expiry arrives as a TLSError carrying it as the cause. +function _is_read_deadline_error(err) + err isa Reseau.IOPoll.DeadlineExceededError && return true + err isa Reseau.TLS.TLSError && return err.cause isa Reseau.IOPoll.DeadlineExceededError + return false +end + +""" + Postgres.wait_for_notification(conn; timeout=nothing) -> Union{Notification, Nothing} + +Block until a `NOTIFY` message arrives on the connection (see +[`listen!`](@ref Postgres.listen!)) and return it as a +[`Notification`](@ref Postgres.API.Notification). With a `timeout` (seconds), +return `nothing` if no message begins arriving in that window; once a message +starts, it is always read to completion so the connection is never left parked +mid-message. The connection lock is held while waiting, so use a dedicated +connection for listening — that is also the only way to receive every +notification, since notifications that arrive while the connection is busy +with a query are delivered to +[`notification_callback`](@ref Postgres.API.notification_callback) only during +the phases of a query that read result data. + +Over TLS the poll interval bounds a read on the underlying transport rather +than on the TLS record layer, so a record that arrives split across a poll +boundary cannot be resumed. That is detected on the following poll and closes +the connection with an error rather than returning corrupt data; a blocking +wait (no `timeout`) is not affected. +""" function wait_for_notification(conn::Connection; timeout::Union{Real, Nothing}=nothing) start_time = time() @lock conn.lock begin @@ -239,36 +445,91 @@ function wait_for_notification(conn::Connection; timeout::Union{Real, Nothing}=n else remaining_s = timeout - (time() - start_time) remaining_s <= 0 && return nothing - Int64(time_ns()) + min(NOTIFICATION_POLL_INTERVAL_NS, round(Int64, remaining_s * 1_000_000_000)) + # clamp before converting: an Inf or very large timeout would + # overflow the nanosecond conversion + remaining_ns = remaining_s >= 10.0 ? NOTIFICATION_POLL_INTERVAL_NS : round(Int64, remaining_s * 1_000_000_000) + Int64(time_ns()) + min(NOTIFICATION_POLL_INTERVAL_NS, remaining_ns) end + # The deadline covers only the first byte: if it expires there, + # nothing of a message has been consumed and polling again is safe. + # Once a byte arrives the rest of the message is read without a + # deadline, so a message straddling the poll boundary can never + # leave the stream parked mid-message. _set_read_deadline!(conn.socket, deadline_ns) - try - mt, len = API.readheader(conn.socket, conn.debug) + mt = try + read(conn.socket, UInt8) + catch err + if !_is_read_deadline_error(err) + # the stream position is unknowable, so the connection + # must never be reused (see the TLS caveat in the + # docstring: a deadline that expires partway through a TLS + # record surfaces here on the following poll) + close(conn.socket) + rethrow() + end + nothing + finally + # the deadline must be cleared on every path, including a + # rethrow: an expired deadline left set on the socket makes + # every later read on this connection fail + isopen(conn.socket) && _clear_read_deadline!(conn.socket) + end + # nothing arrived within this poll interval; nothing of a message + # has been consumed, so it is safe to loop and re-check the timeout + mt === nothing && continue + # A byte of a message has been consumed, so from here any failure + # leaves the stream at an unknowable position: close the connection + # rather than hand back one that still looks healthy. No deadline is + # in effect, so the message is read to completion. + # Read the message off the socket. Only the reading is guarded: + # once a message is fully consumed the stream is back at a clean + # boundary, so user callbacks and server errors are surfaced + # without destroying the connection. + message = try + len = ntoh(read(conn.socket, Int32)) - 4 + (len < 0 || len > API.MAX_MESSAGE_LEN) && + throw(API.Error("invalid message length $len from server; connection protocol state is corrupted")) + conn.debug && @info "readheader: $(Char(mt)), $len" if mt == UInt8('A') - notification = API.notificationResponse(len, conn.socket) - API.notification_callback(conn.style, notification) - return notification + API.notificationResponse(len, conn.socket) elseif mt == UInt8('N') - notice = API.noticeResponse(len, conn.socket) - API.notice_callback(conn.style, notice) + API.noticeResponse(len, conn.socket) elseif mt == UInt8('S') - buf = read(conn.socket, len) - update_server_parameters!(conn, buf) + update_server_parameters!(conn, read(conn.socket, len)) + nothing elseif mt == UInt8('E') - err = API.errorResponse(len, conn.socket, conn.debug) - throw(err) + API.errorResponse(len, conn.socket, conn.debug) else API.skipbytes!(conn.socket, len) + nothing end - catch err - err isa Reseau.IOPoll.DeadlineExceededError || rethrow() - finally - _clear_read_deadline!(conn.socket) + catch + close(conn.socket) + rethrow() + end + if message isa API.Notification + API.notification_callback(conn.style, message) + return message + elseif message isa API.Error + # FATAL and PANIC both terminate the session; close now so the + # next use reports the error rather than a bare EOF from a + # socket the server has already dropped + (message.severity == "FATAL" || message.severity == "PANIC") && close(conn.socket) + throw(message) + elseif message !== nothing + API.notice_callback(conn.style, message) end end end end +""" + Postgres.copy_from(conn, sql, data) + +Execute a `COPY ... FROM STDIN` statement, streaming `data` (an `IO`, string, +or byte vector) to the server. Supports all COPY formats, including +`(FORMAT BINARY)`. Returns `conn`. +""" function copy_from(conn::Connection, sql::AbstractString, data::IO; debug::Bool=false) log_enabled = API.query_logging_enabled(conn.style) start_ns = log_enabled ? time_ns() : 0 @@ -278,9 +539,9 @@ function copy_from(conn::Connection, sql::AbstractString, data::IO; debug::Bool= checkconn(conn) API.copy_in(conn.style, conn.socket, sql_str, data, debug || conn.debug) end - log_enabled && API.query_logger(conn.style, :copy_from, (sql=sql_str, duration_ns=time_ns() - start_ns, success=true)) + log_enabled && query_log_safely(conn.style, :copy_from, (sql=sql_str, duration_ns=time_ns() - start_ns, success=true)) catch err - log_enabled && API.query_logger(conn.style, :copy_from, (sql=sql_str, duration_ns=time_ns() - start_ns, success=false, error=err)) + log_enabled && query_log_safely(conn.style, :copy_from, (sql=sql_str, duration_ns=time_ns() - start_ns, success=false, error=err)) rethrow() end return conn @@ -291,6 +552,13 @@ function copy_from(conn::Connection, sql::AbstractString, data; debug::Bool=fals return copy_from(conn, sql, buffer; debug=debug) end +""" + Postgres.copy_to(conn, sql, [dest::IO]) + +Execute a `COPY ... TO STDOUT` statement. With a `dest` IO, the copy stream is +written to it and `dest` is returned; without one, the raw bytes are returned +as a `Vector{UInt8}`. +""" function copy_to(conn::Connection, sql::AbstractString, dest::IO; debug::Bool=false) log_enabled = API.query_logging_enabled(conn.style) start_ns = log_enabled ? time_ns() : 0 @@ -300,9 +568,9 @@ function copy_to(conn::Connection, sql::AbstractString, dest::IO; debug::Bool=fa checkconn(conn) API.copy_out(conn.style, conn.socket, sql_str, dest, debug || conn.debug) end - log_enabled && API.query_logger(conn.style, :copy_to, (sql=sql_str, duration_ns=time_ns() - start_ns, success=true)) + log_enabled && query_log_safely(conn.style, :copy_to, (sql=sql_str, duration_ns=time_ns() - start_ns, success=true)) catch err - log_enabled && API.query_logger(conn.style, :copy_to, (sql=sql_str, duration_ns=time_ns() - start_ns, success=false, error=err)) + log_enabled && query_log_safely(conn.style, :copy_to, (sql=sql_str, duration_ns=time_ns() - start_ns, success=false, error=err)) rethrow() end return dest @@ -314,32 +582,69 @@ function copy_to(conn::Connection, sql::AbstractString; debug::Bool=false) return take!(buffer) end +""" + Postgres.register_type!(conn, oid, julia_type; parser=nothing) + +Register a mapping from PostgreSQL type `oid` to `julia_type` in the +connection's type registry. `parser` is a `(val::String, registry) -> value` +function that converts the wire text. If `parser` is omitted, `julia_type` +must be `String`. See also [`register_enum!`](@ref Postgres.register_enum!), +[`register_composite!`](@ref Postgres.register_composite!), and +[`register_range!`](@ref Postgres.register_range!). +""" function register_type!(conn::Connection, oid::Integer, julia_type::Type; parser::Union{Function, Nothing}=nothing) + parser === nothing && julia_type !== String && + throw(ArgumentError("parser is required when julia_type is not String")) @lock conn.lock API.register_type!(conn.type_registry, oid, julia_type; parser=parser) return conn end function lookup_type_oid(conn::Connection, name::AbstractString, schema::AbstractString) rows = Tables.rowtable(DBInterface.execute(conn, """ - SELECT t.oid + SELECT t.oid, t.typarray FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace WHERE t.typname = \$1 AND n.nspname = \$2 """, (name, schema))) isempty(rows) && throw(PostgresInterfaceError("type $(schema).$(name) not found")) - return Int(rows[1].oid) + return Int(rows[1].oid), Int(rows[1].typarray) end +# Register the type's array OID alongside it, so `ARRAY[...]::name[]` values +# decode as arrays of the registered type instead of the raw literal string. +function register_array_type!(conn::Connection, arrayoid::Int, oid::Int, @nospecialize(julia_type::Type)) + arrayoid == 0 && return conn + register_type!(conn, arrayoid, Vector{julia_type}; + parser=(val, registry) -> API.parse_array_by_oid(val, oid, registry)) + return conn +end + +""" + Postgres.register_enum!(conn, name; schema="public", julia_type=Symbol) + +Look up the enum type `schema.name` on the server and register it so values +are returned as `julia_type` (by default `Symbol`). The supported types are +`Symbol` and `String`. +""" function register_enum!(conn::Connection, name::AbstractString; schema::AbstractString="public", julia_type::Type=Symbol) - oid = lookup_type_oid(conn, name, schema) + (julia_type === Symbol || julia_type === String) || + throw(ArgumentError("julia_type for register_enum! must be Symbol or String")) + oid, arrayoid = lookup_type_oid(conn, name, schema) parser = julia_type === Symbol ? (val, registry) -> Symbol(val) : nothing register_type!(conn, oid, julia_type; parser=parser) + register_array_type!(conn, arrayoid, oid, julia_type) return conn end +""" + Postgres.register_composite!(conn, name; schema="public") + +Look up the composite type `schema.name` on the server and register it so +values are returned as `NamedTuple`s with the composite's field names. +""" function register_composite!(conn::Connection, name::AbstractString; schema::AbstractString="public") rows = Tables.rowtable(DBInterface.execute(conn, """ - SELECT t.oid, a.attname, a.atttypid + SELECT t.oid, t.typarray, a.attname, a.atttypid FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace JOIN pg_class c ON c.oid = t.typrelid @@ -349,6 +654,7 @@ function register_composite!(conn::Connection, name::AbstractString; schema::Abs """, (name, schema))) isempty(rows) && throw(PostgresInterfaceError("composite type $(schema).$(name) not found")) oid = Int(rows[1].oid) + arrayoid = Int(rows[1].typarray) field_names = [Symbol(row.attname) for row in rows] field_oids = Int[row.atttypid for row in rows] tuple_type = NamedTuple{Tuple(field_names)} @@ -364,12 +670,25 @@ function register_composite!(conn::Connection, name::AbstractString; schema::Abs return tuple_type(Tuple(values)) end register_type!(conn, oid, tuple_type; parser=parser) + register_array_type!(conn, arrayoid, oid, tuple_type) return conn end +""" + Postgres.register_range!(conn, name; schema="public") + +Look up the range type `schema.name` on the server and register it so values +are returned as [`PostgresRange`](@ref Postgres.API.PostgresRange) of the +range's element type. + +The element type is captured when the range is registered, so register it +first: a range over a custom enum or composite must come after the +corresponding [`register_enum!`](@ref Postgres.register_enum!) / +[`register_composite!`](@ref Postgres.register_composite!) call. +""" function register_range!(conn::Connection, name::AbstractString; schema::AbstractString="public") rows = Tables.rowtable(DBInterface.execute(conn, """ - SELECT t.oid, r.rngsubtype + SELECT t.oid, t.typarray, r.rngsubtype FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace JOIN pg_range r ON r.rngtypid = t.oid @@ -377,19 +696,53 @@ function register_range!(conn::Connection, name::AbstractString; schema::Abstrac """, (name, schema))) isempty(rows) && throw(PostgresInterfaceError("range type $(schema).$(name) not found")) oid = Int(rows[1].oid) + arrayoid = Int(rows[1].typarray) subtype_oid = Int(rows[1].rngsubtype) subtype_type = API.type_info(conn.type_registry, subtype_oid).julia_type - parser = (val, registry) -> API.parse_range(val, subtype_oid, registry) + # bind the element type here rather than rediscovering it per value: the + # builtin parse_range only knows a fixed set of element types, so a range + # over anything else would register successfully and then fail on every value + parser = (val, registry) -> API.parse_range_of(subtype_type, val, subtype_oid, registry) register_type!(conn, oid, PostgresRange{subtype_type}; parser=parser) + register_array_type!(conn, arrayoid, oid, PostgresRange{subtype_type}) return conn end +# If the connection actually negotiated TLS, the cancel connection must +# require it too, even under the permissive default — otherwise a server +# answering 'N' to the cancel connection's SSLRequest downgrades the cancel +# key to cleartext. +function cancel_sslmode(socket_is_tls::Bool, sslmode::Union{String, Nothing}) + socket_is_tls || return sslmode + (sslmode === nothing || lowercase(sslmode) == "prefer") && return "require" + return sslmode +end + +""" + Postgres.cancel_query!(conn) + +Send a PostgreSQL CancelRequest for the query currently running on `conn` +(over a separate, short-lived connection, so it works while `conn` is busy). +The cancelled query fails with a [`Postgres.Error`](@ref Postgres.API.Error) +with SQLSTATE `57014`. + +The cancel connection uses the same TLS settings as `conn`, since the cancel +key it carries is a credential: if `conn` itself is on TLS, the cancel +connection requires TLS too. Throws a `PostgresInterfaceError` if the cancel +request could not be delivered (rather than failing silently, which would +leave the query running). +""" function cancel_query!(conn::Connection) host = conn.host port = conn.port pid = conn.pid skey = conn.skey debug = conn.debug + # Checked outside the lock deliberately: cancel_query! is called precisely + # when another task holds it running the query being cancelled, so a + # trylock-guarded check would be skipped in the case that matters. Reading + # the socket field unlocked matches how host/pid/skey are read below. + sslmode = cancel_sslmode(conn.socket isa Reseau.TLS.Conn, conn.sslmode) if trylock(conn.lock) try !isopen(conn.socket) && throw(PostgresInterfaceError("cannot cancel query: connection not open")) @@ -399,7 +752,8 @@ function cancel_query!(conn::Connection) unlock(conn.lock) end end - API.cancel_request(host, port, pid, skey, debug) + ok = API.cancel_request(host, port, pid, skey, debug, sslmode, conn.sslrootcert, conn.sslcert, conn.sslkey, conn.sslcapath, conn.sslservername, conn.connect_timeout) + ok || throw(PostgresInterfaceError("failed to deliver the cancel request to $(host):$(port)")) return conn end @@ -415,6 +769,10 @@ function checkconn(conn::Connection) empty!(conn.statements) conn.in_transaction = false conn.transaction_depth = 0 + empty!(conn.transaction_savepoints) + # the server-side transaction died with the old socket; a stale flag + # would trigger a spurious ROLLBACK on the fresh session + conn.server_in_transaction = false conn.generation += 1 conn.server_parameters = server_params @warn "postgres connection was closed; reconnected" @@ -427,15 +785,13 @@ function DBInterface.connect(::Type{Connection}, host::AbstractString, user::Abs Connection(host=host, user=user, password=passwd, dbname=dbname, port=port, debug=debug, reconnect=reconnect, application_name=application_name, connect_timeout=connect_timeout, sslmode=sslmode, sslrootcert=sslrootcert, sslcert=sslcert, sslkey=sslkey, sslcapath=sslcapath, sslservername=sslservername, statement_timeout=statement_timeout, statement_cache_maxsize=statement_cache_maxsize, style=style) end -function DBInterface.connect(::Type{Connection}, dsn::String; debug::Bool=false, reconnect::Bool=false, statement_cache_maxsize::Union{Integer, Nothing}=nothing) - params = parse_dsn(dsn) - actual_maxsize = isnothing(statement_cache_maxsize) ? params.statement_cache_maxsize : statement_cache_maxsize - Connection(host=params.host, user=params.user, password=params.password, dbname=params.dbname, port=params.port, debug=debug, reconnect=reconnect, application_name=params.application_name, connect_timeout=params.connect_timeout, sslmode=params.sslmode, sslrootcert=params.sslrootcert, sslcert=params.sslcert, sslkey=params.sslkey, sslcapath=params.sslcapath, statement_timeout=params.statement_timeout, statement_cache_maxsize=actual_maxsize) +function DBInterface.connect(::Type{Connection}, dsn::String; debug::Union{Bool, Nothing}=nothing, reconnect::Union{Bool, Nothing}=nothing, statement_cache_maxsize::Union{Integer, Nothing}=nothing, style::API.AbstractPostgresStyle=PostgresStyle()) + return DBInterface.connect(Connection, parse_dsn(dsn); debug=debug, reconnect=reconnect, statement_cache_maxsize=statement_cache_maxsize, style=style) end -function DBInterface.connect(::Type{Connection}, params::ConnectionParams; debug::Bool=false, reconnect::Bool=false, statement_cache_maxsize::Union{Integer, Nothing}=nothing) +function DBInterface.connect(::Type{Connection}, params::ConnectionParams; debug::Union{Bool, Nothing}=nothing, reconnect::Union{Bool, Nothing}=nothing, statement_cache_maxsize::Union{Integer, Nothing}=nothing, style::API.AbstractPostgresStyle=PostgresStyle()) actual_maxsize = isnothing(statement_cache_maxsize) ? params.statement_cache_maxsize : statement_cache_maxsize - Connection(host=params.host, user=params.user, password=params.password, dbname=params.dbname, port=params.port, debug=debug, reconnect=reconnect, application_name=params.application_name, connect_timeout=params.connect_timeout, sslmode=params.sslmode, sslrootcert=params.sslrootcert, sslcert=params.sslcert, sslkey=params.sslkey, sslcapath=params.sslcapath, statement_timeout=params.statement_timeout, statement_cache_maxsize=actual_maxsize) + Connection(host=params.host, user=params.user, password=params.password, dbname=params.dbname, port=params.port, debug=something(debug, params.debug), reconnect=something(reconnect, params.reconnect), application_name=params.application_name, connect_timeout=params.connect_timeout, sslmode=params.sslmode, sslrootcert=params.sslrootcert, sslcert=params.sslcert, sslkey=params.sslkey, sslcapath=params.sslcapath, sslservername=params.sslservername, statement_timeout=params.statement_timeout, statement_cache_maxsize=actual_maxsize, style=style) end function DBInterface.connect(f::Function, ::Type{Connection}, args...; kwargs...) @@ -458,31 +814,48 @@ function DBInterface.close!(conn::Connection) end Base.close(conn::Connection) = DBInterface.close!(conn) +""" + Postgres.ConnectionPool + +A pool of [`Connection`](@ref Postgres.Connection)s, created lazily up to +`limit` and reused across [`acquire`](@ref Postgres.acquire)/[`release`](@ref +Postgres.release) cycles. Locally closed connections are replaced. A peer close +that the local socket has not observed can surface on the borrower's first +operation; an ambiguous failed operation is never retried automatically. + + ConnectionPool(Postgres.Connection, host, user, password; limit=10, kwargs...) + ConnectionPool(dsn::String; limit=10, kwargs...) + ConnectionPool(params::ConnectionParams; limit=10, kwargs...) + ConnectionPool(connector::Function; limit=10) + +Prefer [`with_connection`](@ref Postgres.with_connection) over manual +acquire/release. Close all pooled connections with `DBInterface.close!(pool)`. +""" struct ConnectionPool pool::Pools.Pool connector::Function + closed::Threads.Atomic{Bool} + lifecycle_lock::ReentrantLock end function ConnectionPool(connector::Function; limit::Integer=10) pool = Pools.Pool{Connection}(max(1, Int(limit))) - return ConnectionPool(pool, connector) + return ConnectionPool(pool, connector, Threads.Atomic{Bool}(false), ReentrantLock()) end -function ConnectionPool(::Type{Connection}, host::AbstractString, user::AbstractString, passwd::Union{AbstractString, Nothing}; dbname::AbstractString="", port::Integer=5432, debug::Bool=false, reconnect::Bool=false, application_name::Union{AbstractString, Nothing}=nothing, connect_timeout::Union{Integer, Nothing}=nothing, sslmode::Union{AbstractString, Nothing}=nothing, sslrootcert::Union{AbstractString, Nothing}=nothing, sslcert::Union{AbstractString, Nothing}=nothing, sslkey::Union{AbstractString, Nothing}=nothing, sslcapath::Union{AbstractString, Nothing}=nothing, statement_timeout::Union{Integer, Nothing}=nothing, statement_cache_maxsize::Integer=100, limit::Integer=10, style::API.AbstractPostgresStyle=PostgresStyle()) - connector = () -> DBInterface.connect(Connection, host, user, passwd; dbname=dbname, port=port, debug=debug, reconnect=reconnect, application_name=application_name, connect_timeout=connect_timeout, sslmode=sslmode, sslrootcert=sslrootcert, sslcert=sslcert, sslkey=sslkey, sslcapath=sslcapath, statement_timeout=statement_timeout, statement_cache_maxsize=statement_cache_maxsize, style=style) +Base.isopen(pool::ConnectionPool) = !pool.closed[] + +function ConnectionPool(::Type{Connection}, host::AbstractString, user::AbstractString, passwd::Union{AbstractString, Nothing}; dbname::AbstractString="", port::Integer=5432, debug::Bool=false, reconnect::Bool=false, application_name::Union{AbstractString, Nothing}=nothing, connect_timeout::Union{Integer, Nothing}=nothing, sslmode::Union{AbstractString, Nothing}=nothing, sslrootcert::Union{AbstractString, Nothing}=nothing, sslcert::Union{AbstractString, Nothing}=nothing, sslkey::Union{AbstractString, Nothing}=nothing, sslcapath::Union{AbstractString, Nothing}=nothing, sslservername::Union{AbstractString, Nothing}=nothing, statement_timeout::Union{Integer, Nothing}=nothing, statement_cache_maxsize::Integer=100, limit::Integer=10, style::API.AbstractPostgresStyle=PostgresStyle()) + connector = () -> DBInterface.connect(Connection, host, user, passwd; dbname=dbname, port=port, debug=debug, reconnect=reconnect, application_name=application_name, connect_timeout=connect_timeout, sslmode=sslmode, sslrootcert=sslrootcert, sslcert=sslcert, sslkey=sslkey, sslcapath=sslcapath, sslservername=sslservername, statement_timeout=statement_timeout, statement_cache_maxsize=statement_cache_maxsize, style=style) return ConnectionPool(connector; limit=limit) end -function ConnectionPool(dsn::String; debug::Bool=false, reconnect::Bool=false, statement_cache_maxsize::Union{Integer, Nothing}=nothing, limit::Integer=10) - params = parse_dsn(dsn) - actual_maxsize = isnothing(statement_cache_maxsize) ? params.statement_cache_maxsize : statement_cache_maxsize - connector = () -> DBInterface.connect(Connection, params; debug=debug, reconnect=reconnect, statement_cache_maxsize=actual_maxsize) - return ConnectionPool(connector; limit=limit) +function ConnectionPool(dsn::String; debug::Union{Bool, Nothing}=nothing, reconnect::Union{Bool, Nothing}=nothing, statement_cache_maxsize::Union{Integer, Nothing}=nothing, limit::Integer=10, style::API.AbstractPostgresStyle=PostgresStyle()) + return ConnectionPool(parse_dsn(dsn); debug=debug, reconnect=reconnect, statement_cache_maxsize=statement_cache_maxsize, limit=limit, style=style) end -function ConnectionPool(params::ConnectionParams; debug::Bool=false, reconnect::Bool=false, statement_cache_maxsize::Union{Integer, Nothing}=nothing, limit::Integer=10) - actual_maxsize = isnothing(statement_cache_maxsize) ? params.statement_cache_maxsize : statement_cache_maxsize - connector = () -> DBInterface.connect(Connection, params; debug=debug, reconnect=reconnect, statement_cache_maxsize=actual_maxsize) +function ConnectionPool(params::ConnectionParams; debug::Union{Bool, Nothing}=nothing, reconnect::Union{Bool, Nothing}=nothing, statement_cache_maxsize::Union{Integer, Nothing}=nothing, limit::Integer=10, style::API.AbstractPostgresStyle=PostgresStyle()) + connector = () -> DBInterface.connect(Connection, params; debug=debug, reconnect=reconnect, statement_cache_maxsize=statement_cache_maxsize, style=style) return ConnectionPool(connector; limit=limit) end @@ -491,20 +864,95 @@ function pool_isvalid(conn::Connection) return valid end +""" + Postgres.acquire(pool; forcenew=false) -> Connection + +Take a connection from the pool, creating one if none is available (blocking +if the pool is at its limit). Return it with [`release`](@ref Postgres.release). +""" function acquire(pool::ConnectionPool; forcenew::Bool=false) + pool.closed[] && throw(PostgresInterfaceError("connection pool is closed")) conn = Pools.acquire(pool.connector, pool.pool; forcenew=forcenew, isvalid=pool_isvalid) + closed = @lock pool.lifecycle_lock pool.closed[] + if closed + try + DBInterface.close!(conn) + finally + Pools.release(pool.pool) + end + throw(PostgresInterfaceError("connection pool is closed")) + end return conn end +# A connection going back into the pool must not carry a transaction with it: +# the next borrower's `start_transaction` would issue a SAVEPOINT instead of +# BEGIN, and their commit would only decrement the depth — their writes would +# be silently discarded when the connection is later reset. Roll it back; if +# that can't be done, drop the connection instead of handing it on. +function reset_pooled_connection!(conn::Connection) + # the client flag misses a transaction opened by raw SQL, so trust the + # server's ReadyForQuery status too + (in_transaction(conn) || (@lock conn.lock conn.server_in_transaction)) || return true + try + @lock conn.lock begin + checkconn(conn) + try + execute_simple(conn, "ROLLBACK") + finally + conn.in_transaction = false + conn.transaction_depth = 0 + empty!(conn.transaction_savepoints) + end + end + return true + catch + try + DBInterface.close!(conn) + catch + # already unusable; nothing more to do + end + return false + end +end + +""" + Postgres.release(pool, conn) + +Return a connection previously taken with [`acquire`](@ref Postgres.acquire) +to the pool. A connection still inside a transaction is rolled back first, so +the next borrower starts from a clean session; if it can't be rolled back it +is closed rather than reused. +""" function release(pool::ConnectionPool, conn::Connection) - if pool_isvalid(conn) - Pools.release(pool.pool, conn) - else + reusable = !pool.closed[] && pool_isvalid(conn) && reset_pooled_connection!(conn) + if reusable + returned = @lock pool.lifecycle_lock begin + if pool.closed[] + false + else + Pools.release(pool.pool, conn) + true + end + end + returned && return pool + end + try + DBInterface.close!(conn) + catch + # already unusable; the permit still has to be returned + finally Pools.release(pool.pool) end return pool end +""" + Postgres.with_connection(f, pool; forcenew=false) + +Acquire a connection from the pool, call `f(conn)`, and release the connection +back to the pool afterwards. Returns `f`'s result. +""" function with_connection(f::Function, pool::ConnectionPool; forcenew::Bool=false) conn = acquire(pool; forcenew=forcenew) try @@ -515,15 +963,18 @@ function with_connection(f::Function, pool::ConnectionPool; forcenew::Bool=false end function DBInterface.close!(pool::ConnectionPool) - Base.@lock pool.pool.lock begin - for conn in pool.pool.values - try - DBInterface.close!(conn) - catch - # ignore close errors for pooled connections + Base.@lock pool.lifecycle_lock begin + pool.closed[] = true + Base.@lock pool.pool.lock begin + for conn in pool.pool.values + try + DBInterface.close!(conn) + catch + # ignore close errors for pooled connections + end end + empty!(pool.pool.values) end - empty!(pool.pool.values) end return pool end @@ -538,65 +989,211 @@ include("execute.jl") # mid-sequence and drop the unnamed statement ("unnamed prepared statement # does not exist"). Also one network round trip instead of three. Callers # must hold conn.lock. -function execute_simple(conn::Connection, sql::String) - API.exec(conn.style, conn.socket, sql, conn.debug) +function execute_simple(conn::Connection, sql::String; expected_tag::Union{Nothing, String}=nothing) + # capture the ReadyForQuery status through a Ref so a failed statement + # (a COMMIT hitting a deferred constraint) still refreshes the tracking: + # the server ended the transaction either way, and a stale flag draws a + # spurious ROLLBACK on the next pool release + status_ref = Ref{UInt8}(UInt8('I')) + command_tag_ref = Ref{Union{Nothing, String}}(nothing) + try + API.exec(conn.style, conn.socket, sql, conn.debug, status_ref, + command_tag_ref, conn.server_parameters) + finally + conn.server_in_transaction = API.in_transaction_status(status_ref[]) + end + if expected_tag !== nothing && command_tag_ref[] != expected_tag + actual = something(command_tag_ref[], "no command tag") + throw(PostgresInterfaceError("expected $expected_tag but PostgreSQL completed $actual")) + end return conn end +function new_transaction_savepoint!() + return string("postgres_jl_", replace(string(UUIDs.uuid4()), "-" => "")) +end + +""" + Postgres.start_transaction(conn) + +Begin a transaction (`BEGIN`). If a transaction is already open, create a +savepoint instead, so transactions nest. Pair with [`commit`](@ref +Postgres.commit) or [`rollback`](@ref Postgres.rollback); prefer +[`transaction`](@ref Postgres.transaction) or +[`@transaction`](@ref Postgres.@transaction) for automatic handling. +""" function start_transaction(conn::Connection) @lock conn.lock begin checkconn(conn) if !conn.in_transaction - execute_simple(conn, "BEGIN") + if conn.server_in_transaction + # a transaction opened with raw SQL (execute(conn, "BEGIN")) + # belongs to the caller: nest inside it with a savepoint, as + # for driver-owned nesting, so our commit can't commit — and + # our rollback can't destroy — their work + savepoint = new_transaction_savepoint!() + execute_simple(conn, "SAVEPOINT $savepoint"; expected_tag="SAVEPOINT") + push!(conn.transaction_savepoints, savepoint) + conn.owns_base_transaction = false + else + execute_simple(conn, "BEGIN"; expected_tag="BEGIN") + conn.owns_base_transaction = true + end conn.in_transaction = true conn.transaction_depth = 1 else # Start a SAVEPOINT for nested transactions - savepoint = "sp_$(conn.transaction_depth)" - execute_simple(conn, "SAVEPOINT $savepoint") + savepoint = new_transaction_savepoint!() + execute_simple(conn, "SAVEPOINT $savepoint"; expected_tag="SAVEPOINT") + push!(conn.transaction_savepoints, savepoint) conn.transaction_depth += 1 end end return conn end +""" + Postgres.in_transaction(conn) -> Bool + +Whether the connection currently has an open transaction. +""" in_transaction(conn::Connection) = @lock conn.lock conn.in_transaction +# Forget a transaction whose session is already gone. Nothing can be sent to +# end it, and leaving the flags set makes checkconn refuse to reconnect. +function clear_transaction_state!(conn::Connection) + @lock conn.lock begin + conn.in_transaction = false + conn.transaction_depth = 0 + empty!(conn.transaction_savepoints) + conn.server_in_transaction = false + end + return +end + +""" + Postgres.commit(conn) + +Commit the current transaction (or release one level of transaction nesting). +""" function commit(conn::Connection) @lock conn.lock begin - checkconn(conn) !conn.in_transaction && throw(PostgresInterfaceError("no transaction in progress")) - if conn.transaction_depth == 1 - execute_simple(conn, "COMMIT") + # a dead socket took the transaction with it: clear the bookkeeping + # before reporting, or checkconn will refuse to reconnect forever + # ("reconnect disabled during transaction") + if !isopen(conn.socket) conn.in_transaction = false conn.transaction_depth = 0 + empty!(conn.transaction_savepoints) + disconnected() + end + checkconn(conn) + if conn.transaction_depth == 1 + # COMMIT ends the transaction server-side whether it succeeds or + # fails (and a dead connection ends it too), so the client's + # transaction state must be cleared either way — leaving it set + # would block reconnects and make the next cursor skip its BEGIN + try + if conn.owns_base_transaction + execute_simple(conn, "COMMIT"; expected_tag="COMMIT") + else + # the base transaction is the caller's raw-SQL one: keep + # this level's work pending inside it and leave it open + savepoint = only(conn.transaction_savepoints) + execute_simple(conn, "RELEASE SAVEPOINT $savepoint"; expected_tag="RELEASE") + end + finally + conn.in_transaction = false + conn.transaction_depth = 0 + empty!(conn.transaction_savepoints) + end else # Release SAVEPOINT for nested transaction + savepoint = last(conn.transaction_savepoints) + execute_simple(conn, "RELEASE SAVEPOINT $savepoint"; expected_tag="RELEASE") + pop!(conn.transaction_savepoints) conn.transaction_depth -= 1 - # Don't need to release SAVEPOINT explicitly, just commit will handle it end end return conn end +""" + Postgres.rollback(conn) + +Roll back the current transaction (or, in a nested transaction, roll back to +the enclosing savepoint). +""" function rollback(conn::Connection) @lock conn.lock begin - checkconn(conn) !conn.in_transaction && throw(PostgresInterfaceError("no transaction in progress")) - if conn.transaction_depth == 1 - execute_simple(conn, "ROLLBACK") + # as in commit: a dead session already ended the transaction + if !isopen(conn.socket) conn.in_transaction = false conn.transaction_depth = 0 + empty!(conn.transaction_savepoints) + disconnected() + end + checkconn(conn) + if conn.transaction_depth == 1 + # as in commit: the transaction is over server-side regardless of + # how ROLLBACK fares, so don't leave client state describing it + try + if conn.owns_base_transaction + execute_simple(conn, "ROLLBACK"; expected_tag="ROLLBACK") + else + # undo only this level; a plain ROLLBACK would destroy the + # caller's raw-SQL transaction along with it + savepoint = only(conn.transaction_savepoints) + execute_simple(conn, "ROLLBACK TO SAVEPOINT $savepoint"; expected_tag="ROLLBACK") + execute_simple(conn, "RELEASE SAVEPOINT $savepoint"; expected_tag="RELEASE") + end + finally + conn.in_transaction = false + conn.transaction_depth = 0 + empty!(conn.transaction_savepoints) + end else # Rollback to SAVEPOINT for nested transaction + savepoint = last(conn.transaction_savepoints) + execute_simple(conn, "ROLLBACK TO SAVEPOINT $savepoint"; expected_tag="ROLLBACK") + execute_simple(conn, "RELEASE SAVEPOINT $savepoint"; expected_tag="RELEASE") + pop!(conn.transaction_savepoints) conn.transaction_depth -= 1 - savepoint = "sp_$(conn.transaction_depth)" - execute_simple(conn, "ROLLBACK TO SAVEPOINT $savepoint") end end return conn end +# Roll back after the body (or the COMMIT) failed. Only acts if a transaction +# is still open — a failed COMMIT has already ended it — and never lets its own +# failure replace the original error, which is what the caller needs to see. +function rollback_for_failed_transaction!(conn::Connection) + in_transaction(conn) || return + try + rollback(conn) + catch + # the connection is already failing; the original error is the useful one + end + return +end + +""" + Postgres.transaction(f, conn) + +Run `f(conn)` inside a transaction: committed if `f` returns normally, rolled +back if it throws. Nested calls use savepoints. Returns `f`'s result. + +A transaction already opened with raw SQL (`execute(conn, "BEGIN")`) is +treated as the enclosing level: the block nests inside it with a savepoint +and leaves it open, so the caller's own `COMMIT`/`ROLLBACK` stays in control +of their transaction. + + Postgres.transaction(conn) do conn + DBInterface.execute(conn, "INSERT INTO t VALUES (1)") + end +""" function transaction(f::F, conn::Connection) where {F} start_transaction(conn) try @@ -604,7 +1201,7 @@ function transaction(f::F, conn::Connection) where {F} commit(conn) return result catch - rollback(conn) + rollback_for_failed_transaction!(conn) rethrow() end end @@ -616,29 +1213,66 @@ function DBInterface.transaction(f::F, conn::Connection) where {F} commit(conn) return result catch - rollback(conn) + rollback_for_failed_transaction!(conn) rethrow() end end +struct TransactionReturn{T} <: Exception + value::T +end + +function rewrite_transaction_returns(expr) + expr isa Expr || return expr + if expr.head === :return + value = isempty(expr.args) ? nothing : rewrite_transaction_returns(only(expr.args)) + marker = GlobalRef(@__MODULE__, :TransactionReturn) + return Expr(:call, GlobalRef(Core, :throw), Expr(:call, marker, value)) + elseif expr.head === :function || expr.head === :(->) || expr.head === :quote + # A return in a nested function belongs to that function, not to the + # scope that contains this transaction macro. + return expr + end + return Expr(expr.head, map(rewrite_transaction_returns, expr.args)...) +end + +""" + Postgres.@transaction conn expr + +Run `expr` inside a transaction: committed if it completes, rolled back if it +throws. Evaluates to `expr`'s value. +""" macro transaction(conn, expr) + body = rewrite_transaction_returns(expr) quote + # bind once: the connection expression may have side effects + # (`@transaction acquire(pool) ...` would otherwise take a different + # connection for the BEGIN, the COMMIT and the ROLLBACK) + local c = $(esc(conn)) local success = false - start_transaction($(esc(conn))) + start_transaction(c) try - result = $(esc(expr)) - commit($(esc(conn))) + local result + try + result = $(esc(body)) + catch err + if err isa TransactionReturn + commit(c) + success = true + return err.value + end + rethrow() + end + commit(c) success = true result catch - !success && rollback($(esc(conn))) + !success && rollback_for_failed_transaction!(c) rethrow() end end end -# escape(conn::Connection, s::AbstractString) = API.escape(conn.pg, s) - struct Describe resultset::Any end @@ -671,6 +1305,12 @@ function Base.show(io::IO, desc::Describe) end end +""" + Postgres.describe(conn, table; schema="public") + +Return a printable summary of a table's columns: name, type, nullability, +default, primary-key flag, and foreign-key reference. +""" function describe(conn::Connection, table::AbstractString; schema::String="public") Describe(DBInterface.execute(conn, """ WITH column_info AS ( @@ -696,13 +1336,29 @@ function describe(conn::Connection, table::AbstractString; schema::String="publi FROM information_schema.columns c LEFT JOIN - information_schema.key_column_usage kcu ON c.table_name = kcu.table_name AND c.column_name = kcu.column_name + information_schema.key_column_usage kcu ON + c.table_catalog = kcu.table_catalog AND + c.table_schema = kcu.table_schema AND + c.table_name = kcu.table_name AND + c.column_name = kcu.column_name LEFT JOIN - information_schema.table_constraints tc ON kcu.constraint_name = tc.constraint_name + information_schema.table_constraints tc ON + kcu.constraint_catalog = tc.constraint_catalog AND + kcu.constraint_schema = tc.constraint_schema AND + kcu.constraint_name = tc.constraint_name AND + kcu.table_schema = tc.table_schema AND + kcu.table_name = tc.table_name LEFT JOIN - information_schema.referential_constraints rc ON tc.constraint_name = rc.constraint_name + information_schema.referential_constraints rc ON + tc.constraint_catalog = rc.constraint_catalog AND + tc.constraint_schema = rc.constraint_schema AND + tc.constraint_name = rc.constraint_name LEFT JOIN - information_schema.key_column_usage fk ON rc.unique_constraint_name = fk.constraint_name AND fk.table_schema = c.table_schema + information_schema.key_column_usage fk ON + rc.unique_constraint_catalog = fk.constraint_catalog AND + rc.unique_constraint_schema = fk.constraint_schema AND + rc.unique_constraint_name = fk.constraint_name AND + kcu.position_in_unique_constraint = fk.ordinal_position WHERE c.table_name = \$1 AND c.table_schema = \$2 ) @@ -721,4 +1377,20 @@ function describe(conn::Connection, table::AbstractString; schema::String="publi """, (table, schema))) end +# the supported API surface (`public` requires Julia 1.11+; the names are +# parsed from a string so the file still loads on 1.10) +@static if VERSION >= v"1.11" + eval(Meta.parse( + "public Connection, ConnectionPool, ConnectionParams, PostgresInterfaceError, " * + "Error, Notification, Numeric, PostgresRange, AbstractPostgresStyle, PostgresStyle, " * + "query_logging_enabled, query_logger, notice_callback, notification_callback, parse_dsn, " * + "transaction, @transaction, start_transaction, commit, rollback, in_transaction, " * + "cursor, copy_from, copy_to, listen!, unlisten!, notify!, wait_for_notification, " * + "register_type!, register_enum!, register_composite!, register_range!, " * + "command_tag, rows_affected, cancel_query!, escape_identifier, escape_literal, " * + "get_cached_statements, clear_statement_cache!, set_statement_cache_maxsize!, " * + "get_server_parameter, get_server_parameters, get_statement_timeout, set_statement_timeout!, " * + "acquire, release, with_connection, describe")) +end + end diff --git a/src/api/API.jl b/src/api/API.jl index 9a07b5d..fb62870 100644 --- a/src/api/API.jl +++ b/src/api/API.jl @@ -1,13 +1,26 @@ module API -using UUIDs, Dates, Reseau, SASLAuth, MD5, Parsers, StructUtils, Logging, JSON, Random +using UUIDs, Dates, Reseau, SASLAuth, MD5, Parsers, StructUtils, JSON, Random import ..PostgresInterfaceError -export PostgresStyle, AbstractPostgresStyle, query_logging_enabled, query_logger, notice_callback, notification_callback, Error, Notification, Numeric, PostgresRange +export PostgresStyle, AbstractPostgresStyle, query_logging_enabled, query_logger, notice_callback, notification_callback, Error, Notification, Numeric, PostgresRange, cancel_request const ReseauConn = Union{Reseau.TCP.Conn, Reseau.TLS.Conn} const SKIP_BUFFER_SIZE = 8192 +""" + Postgres.Error <: Exception + +A PostgreSQL server error (an `ErrorResponse` message). Carries the fields the +server reported: `severity` (non-localized when the server supplies it, so it +can be compared against `"FATAL"`, `"ERROR"`, ... regardless of the server's +`lc_messages`), `code` (the SQLSTATE, e.g. `"23505"`), `message`, +and optional context such as `detail`, `hint`, `position`, `schema`, `table`, +`column`, and `constraint`. A small number of protocol-level failures detected +client-side (unsupported authentication methods, protocol desync) also use +this type, with an empty `code`; other client-side failures throw +`Postgres.PostgresInterfaceError`. +""" struct Error <: Exception severity::String code::String @@ -28,6 +41,14 @@ struct Error <: Exception routine::Union{String, Nothing} end +""" + Postgres.Notification + +An asynchronous `NOTIFY` message received from the server, with the notifying +backend's `pid`, the `channel` name, and the `payload` string (empty when the +notification had no payload). See `Postgres.listen!` and +`Postgres.wait_for_notification`. +""" struct Notification pid::Int32 channel::String @@ -51,33 +72,29 @@ function Base.showerror(io::IO, e::Error) return end -# error code => (name, should_be_shown) -const ERROR_CODE = Dict{Char, Tuple{String, Bool}}( - 'S' => ("Severity", true), - 'V' => ("Severity", false), - 'C' => ("Code", false), - 'M' => ("Message", true), - 'D' => ("Detail", true), - 'H' => ("Hint", true), - 'P' => ("Position", false), - 'p' => ("Internal Position", false), - 'q' => ("Internal Query", false), - 'W' => ("Where", true), - 's' => ("Schema Name", true), - 't' => ("Table Name", true), - 'c' => ("Column Name", true), - 'd' => ("Data Type Name", true), - 'n' => ("Constraint Name", true), - 'F' => ("File", false), - 'L' => ("Line", false), - 'R' => ("Routine", false), -) +# Read a NUL-terminated string from `buf` starting at `pos`, bounded by the +# buffer's actual length: `read` returns a short buffer at EOF, and a hostile +# or failing server can send an unterminated or truncated field, so an +# unbounded scan (plain `unsafe_string(pointer(buf, pos))`) would read past +# the allocation. Returns (string, next_pos). +function cstring_at(buf::Vector{UInt8}, pos::Int) + n = length(buf) + pos > n && return "", n + 1 + stop = findnext(isequal(UInt8(0)), buf, pos) + if stop === nothing + return GC.@preserve(buf, unsafe_string(pointer(buf, pos), n - pos + 1)), n + 1 + end + return GC.@preserve(buf, unsafe_string(pointer(buf, pos), stop - pos)), stop + 1 +end function errorResponse(len, socket, debug) buf = read(socket, len) # parse error fields i = 1 severity = "" + # 'V' is the non-localized severity (PostgreSQL 9.6+); 'S' is translated + # per the server's lc_messages, so it can't be compared against literals + severity_nonlocalized = "" code = "" message = "" detail = nothing @@ -94,13 +111,16 @@ function errorResponse(len, socket, debug) file = nothing line = nothing routine = nothing - while i < len + while i <= length(buf) ccode = Char(buf[i]) + # the field list is terminated by a zero byte + ccode == '\0' && break i += 1 - val = unsafe_string(pointer(buf, i)) - i += sizeof(val) + 1 + val, i = cstring_at(buf, i) if ccode == 'S' severity = val + elseif ccode == 'V' + severity_nonlocalized = val elseif ccode == 'C' code = val elseif ccode == 'M' @@ -135,6 +155,9 @@ function errorResponse(len, socket, debug) routine = val end end + # prefer the non-localized severity so callers can compare it to "FATAL" + # and friends regardless of the server's locale + isempty(severity_nonlocalized) || (severity = severity_nonlocalized) err = Error(severity, code, message, detail, hint, position, internal_position, internal_query, where, schema, table, column, datatype, constraint, file, line, routine) debug && @error err return err @@ -144,30 +167,47 @@ function noticeResponse(len, socket) buf = read(socket, len) i = 1 notice = Dict{String, String}() - while i < length(buf) + while i <= length(buf) ccode = Char(buf[i]) + ccode == '\0' && break i += 1 - val = unsafe_string(pointer(buf, i)) - i += sizeof(val) + 1 + val, i = cstring_at(buf, i) notice[string(ccode)] = val end return notice end function notificationResponse(len, socket) + # the body is at least the 4-byte pid; a shorter one would make the + # channel/payload read consume the next message + len < 4 && throw(Error("truncated NotificationResponse from server")) pid = ntoh(read(socket, Int32)) buf = read(socket, len - 4) i = 1 channel = "" payload = "" if !isempty(buf) - channel = unsafe_string(pointer(buf, i)) - i += sizeof(channel) + 1 - i <= length(buf) && (payload = unsafe_string(pointer(buf, i))) + channel, i = cstring_at(buf, i) + i <= length(buf) && ((payload, i) = cstring_at(buf, i)) end return Notification(pid, channel, payload) end +function parameterStatus!(parameters::Dict{String, String}, len, socket) + buf = read(socket, len) + length(buf) == len || close_and_throw(socket, Error("truncated ParameterStatus message from server")) + first_nul = findfirst(isequal(UInt8(0)), buf) + first_nul === nothing && close_and_throw(socket, Error("invalid ParameterStatus message from server")) + second_nul = findnext(isequal(UInt8(0)), buf, first_nul + 1) + (second_nul === length(buf) && first_nul > 1) || + close_and_throw(socket, Error("invalid ParameterStatus message from server")) + key = GC.@preserve buf unsafe_string(pointer(buf), first_nul - 1) + value_start = first_nul + 1 + value = GC.@preserve buf unsafe_string(pointer(buf, value_start), second_nul - value_start) + parameters[key] = value + return nothing +end + include("types.jl") struct Params @@ -185,11 +225,14 @@ _msgsizeof_parts(parts::Tuple) = msgsizeof(first(parts)) + _msgsizeof_parts(Base writepart(io, x) = write(io, x) function writepart(io, x::String) + occursin('\0', x) && throw(Postgres.PostgresInterfaceError("PostgreSQL protocol strings cannot contain a NUL byte")) write(io, x) write(io, UInt8(0)) end writepart(io, x::Integer) = write(io, hton(x)) function writepart(io, x::Tuple{String, String}) + (occursin('\0', x[1]) || occursin('\0', x[2])) && + throw(Postgres.PostgresInterfaceError("PostgreSQL startup parameters cannot contain a NUL byte")) write(io, x[1]) write(io, UInt8(0)) write(io, x[2]) @@ -256,18 +299,20 @@ function writestartupmessage( application_name::Union{Nothing, String}, statement_timeout::Union{Nothing, Int}, )::Nothing - timeout_options = statement_timeout === nothing ? nothing : string("-c statement_timeout=", statement_timeout) - len = 8 + msgsizeof(("user", user)) + msgsizeof(("database", dbname)) + 1 + # statement_timeout is applied with a SET after connect rather than through + # the startup `options` parameter: poolers (pgbouncer) reject unknown + # startup options outright, so sending it here fails the whole connection. + len = 8 + msgsizeof(("user", user)) + msgsizeof(("database", dbname)) + + msgsizeof(("client_encoding", "UTF8")) + 1 application_name !== nothing && (len += msgsizeof(("application_name", application_name))) - timeout_options !== nothing && (len += msgsizeof(("options", timeout_options))) debug && @info "sending startup message" buf = IOBuffer(Vector{UInt8}(undef, len); write=true) write(buf, hton(Int32(len))) write(buf, hton(Int32(196608))) _write_startup_param(buf, "user", user) _write_startup_param(buf, "database", dbname) + _write_startup_param(buf, "client_encoding", "UTF8") application_name !== nothing && _write_startup_param(buf, "application_name", application_name) - timeout_options !== nothing && _write_startup_param(buf, "options", timeout_options) write(buf, UInt8(0)) write(socket, take!(buf)) flush(socket) @@ -298,16 +343,39 @@ function skipbytes!(io::IO, n::Integer) buf = Vector{UInt8}(undef, min(SKIP_BUFFER_SIZE, remaining)) while remaining > 0 nb = min(length(buf), remaining) - readbytes!(io, buf, nb) - remaining -= nb + nr = readbytes!(io, buf, nb) + nr == nb || throw(EOFError()) + remaining -= nr end return nothing end -function readheader(socket, debug=false) +# PostgreSQL's own protocol maximum (PQ_LARGE_MESSAGE_LIMIT): no valid message +# body exceeds 1 GiB. The length is server-supplied and the transport allocates +# it up front, so bound it here — the single point every message passes through +# — rather than letting a bogus header commit gigabytes. This is reachable +# before authentication (an ErrorResponse to the SSLRequest), so it must not +# depend on a trusted peer. +const MAX_MESSAGE_LEN = Int32(1) << 30 +# Authentication and startup messages are small. A separate bound prevents an +# unauthenticated peer from forcing a process-sized allocation before the +# connection is established. +const MAX_PREAUTH_MESSAGE_LEN = Int32(1) << 20 + +# A bogus length means the stream is desynchronized, so the socket must be +# closed before throwing — callers such as describeprepared treat a surviving +# `Error` as "the stream is clean, at ReadyForQuery" and would otherwise keep +# using a connection whose position is unknowable. +@noinline function _bad_message_length(socket, len) + close(socket) + throw(Error("invalid message length $len from server; connection protocol state is corrupted")) +end + +function readheader(socket, debug=false, max_message_len::Int32=MAX_MESSAGE_LEN) mt = read(socket, UInt8) len = ntoh(read(socket, Int32)) - 4 debug && @info "readheader: $(Char(mt)), $len" + (len < 0 || len > max_message_len) && _bad_message_length(socket, len) return mt, len end @@ -344,7 +412,8 @@ end # ErrorResponse is read fully, the stream drained through ReadyForQuery (the # connection stays usable), and thrown as a Postgres.Error. Any other message # type means the stream is desynchronized: close the connection and throw. -function read_expected(socket, debug, expected::Char...) +function read_expected(socket, debug, expected::Char...; + server_parameters::Union{Nothing, Dict{String, String}}=nothing) while true mt, len = readheader(socket, debug) if any(c -> mt == UInt8(c), expected) @@ -353,7 +422,10 @@ function read_expected(socket, debug, expected::Char...) err = errorResponse(len, socket, debug) drain_to_ready!(socket, debug) throw(err) - elseif mt == UInt8('S') || mt == UInt8('N') || mt == UInt8('A') + elseif mt == UInt8('S') + server_parameters === nothing ? skipbytes!(socket, len) : + parameterStatus!(server_parameters, len, socket) + elseif mt == UInt8('N') || mt == UInt8('A') skipbytes!(socket, len) else close_and_throw(socket, Error("unexpected message type '$(Char(mt))' from server; connection protocol state is corrupted")) @@ -368,7 +440,9 @@ function expect_auth_message(socket, debug, mt, len) end # wait for code, then ready -function waitfor(socket, debug::Bool, codes::Vararg{Char, N}) where {N} +function waitfor(socket, debug::Bool, codes::Vararg{Char, N}; + max_message_len::Int32=MAX_MESSAGE_LEN, + server_parameters::Union{Nothing, Dict{String, String}}=nothing) where {N} error = false error_msg = nothing found = _sum_codes(codes) @@ -377,7 +451,7 @@ function waitfor(socket, debug::Bool, codes::Vararg{Char, N}) where {N} debug && @info "waitfor: $codes" try while true - mt, len = readheader(socket, debug) + mt, len = readheader(socket, debug, max_message_len) if mt == UInt8('E') # error error = true @@ -387,20 +461,12 @@ function waitfor(socket, debug::Bool, codes::Vararg{Char, N}) where {N} skipbytes!(socket, len) break elseif mt == UInt8('S') - # parameter status - buf = read(socket, len) - i = 1 - while i < len - j = findnext(isequal(UInt8(0)), buf, i) - j === nothing && break - key = unsafe_string(pointer(buf, i), j - i) - i = j + 1 - j = findnext(isequal(UInt8(0)), buf, i) - j === nothing && break - val = unsafe_string(pointer(buf, i), j - i) - server_params[key] = val - i = j + 1 - end + # ParameterStatus is part of the connection's public state. + # During startup, collect it in the return value. During later + # waits, update the connection-owned dictionary supplied by + # the caller. + target = server_parameters === nothing ? server_params : server_parameters + parameterStatus!(target, len, socket) elseif _contains_code(mt, codes) # found found -= mt @@ -412,6 +478,12 @@ function waitfor(socket, debug::Bool, codes::Vararg{Char, N}) where {N} skipbytes!(socket, len) end found == 0 && break + else + # any other message (notices, notifications, ...): discard the + # body. Without this the body is read as the next header and + # the stream desynchronizes — e.g. a NOTIFY delivered on a + # connection that is also running queries. + skipbytes!(socket, len) end end catch @@ -428,6 +500,14 @@ function waitfor(socket, debug::Bool, codes::Vararg{Char, N}) where {N} return pid, skey, server_params end +# password material must never reach the debug log: password-bearing messages +# are written with debug=false and a redacted line is logged instead +function write_password_message(socket, debug::Bool, password::String) + debug && @info "sending message: p, (password redacted)" + writemessage(socket, false, 'p', password) + return +end + function authRequest(debug, len, socket, user, password, client::Union{Nothing, SASLAuth.SCRAMSHA256Client}=nothing) auth_code = ntoh(read(socket, Int32)) debug && @info "auth code: $auth_code" @@ -440,8 +520,8 @@ function authRequest(debug, len, socket, user, password, client::Union{Nothing, throw(Error("kerberos v5 authentication not supported")) elseif auth_code == 3 # send cleartext password message - writemessage(socket, debug, 'p', password) - mt, len = readheader(socket, debug) + write_password_message(socket, debug, password) + mt, len = readheader(socket, debug, MAX_PREAUTH_MESSAGE_LEN) if mt == UInt8('E') # error close_and_throw_error_response(socket, len, debug) @@ -464,8 +544,8 @@ function authRequest(debug, len, socket, user, password, client::Union{Nothing, # Calculate the MD5 password pass = string("md5", bytes2hex(md5(vcat(Vector{UInt8}(bytes2hex(md5(string(password, user)))), salt)))) # Send password message - writemessage(socket, debug, 'p', pass) - mt, len = readheader(socket, debug) + write_password_message(socket, debug, pass) + mt, len = readheader(socket, debug, MAX_PREAUTH_MESSAGE_LEN) if mt == UInt8('E') # error close_and_throw_error_response(socket, len, debug) @@ -503,16 +583,21 @@ function authRequest(debug, len, socket, user, password, client::Union{Nothing, client = SASLAuth.SCRAMSHA256Client(user, password) msg, _ = SASLAuth.step!(client, nothing) bytes = Vector{UInt8}(msg) - writemessage(socket, debug, 'p', "SCRAM-SHA-256", Int32(length(bytes)), bytes) - mt, len = readheader(socket, debug) + # SASL messages carry the client nonce and (in the client-final message) + # the client proof, from which the password is brute-forcible offline: + # write them with debug=false and log a redacted line instead + debug && @info "sending message: p, (SASL initial response redacted)" + writemessage(socket, false, 'p', "SCRAM-SHA-256", Int32(length(bytes)), bytes) + mt, len = readheader(socket, debug, MAX_PREAUTH_MESSAGE_LEN) expect_auth_message(socket, debug, mt, len) return authRequest(debug, len, socket, user, password, client) elseif auth_code == 11 # SASL Challenge challenge = String(read(socket, len - 4)) msg, _ = SASLAuth.step!(client, challenge) - writemessage(socket, debug, 'p', Vector{UInt8}(msg)) - mt, len = readheader(socket, debug) + debug && @info "sending message: p, (SASL response redacted)" + writemessage(socket, false, 'p', Vector{UInt8}(msg)) + mt, len = readheader(socket, debug, MAX_PREAUTH_MESSAGE_LEN) expect_auth_message(socket, debug, mt, len) return authRequest(debug, len, socket, user, password, client) elseif auth_code == 12 @@ -520,7 +605,7 @@ function authRequest(debug, len, socket, user, password, client::Union{Nothing, final_msg = String(read(socket, len - 4)) _, done = SASLAuth.step!(client, final_msg) done || close_and_throw(socket, Error("SASL authentication did not complete")) - mt, len = readheader(socket, debug) + mt, len = readheader(socket, debug, MAX_PREAUTH_MESSAGE_LEN) expect_auth_message(socket, debug, mt, len) auth_code = ntoh(read(socket, Int32)) auth_code == 0 || close_and_throw(socket, Error("SASL authentication failed: $auth_code")) @@ -553,8 +638,17 @@ end connectsocket(host::AbstractString, port::Integer; connect_timeout::Union{Int, Nothing}=nothing) = connectsocket(host, port, connect_timeout) +# `host:port` for Reseau, with IPv6 literals bracketed (`[::1]:5432`) so the +# address parser doesn't reject the extra colons +function hostport_address(host::AbstractString, port::Integer) + startswith(host, '/') && throw(PostgresInterfaceError("unix socket connections are not supported; provide a TCP host")) + h = String(host) + (startswith(h, '[') && endswith(h, ']')) && return string(h, ":", Int(port)) + return occursin(':', h) ? string("[", h, "]:", Int(port)) : string(h, ":", Int(port)) +end + function connectsocket(host::AbstractString, port::Integer, @nospecialize(connect_timeout::Union{Int, Nothing})) - address = string(host, ":", Int(port)) + address = hostport_address(host, port) return if connect_timeout === nothing Reseau.TCP.connect(address) else @@ -592,9 +686,12 @@ function tlsupgrade(socket::Reseau.TCP.Conn, @nospecialize(connect_timeout::Unio nothing, nothing, ca_file, nothing, String[], UInt16[], handshake_timeout_ns, Reseau.TLS.TLS1_2_VERSION, nothing, false) else + # Reseau 1.3.x does not send a configured client certificate on its + # mixed TLS 1.2/1.3 client path. Its TLS 1.2 path does. Keep mTLS + # functional and fail-closed until the dependency fixes that path. Reseau.TLS.Config(sni, verify_peer, verify_peer, Reseau.TLS.ClientAuthMode.NoClientCert, ssl_cert::String, ssl_key::String, ca_file, nothing, String[], UInt16[], - handshake_timeout_ns, Reseau.TLS.TLS1_2_VERSION, nothing, false) + handshake_timeout_ns, Reseau.TLS.TLS1_2_VERSION, Reseau.TLS.TLS1_2_VERSION, false) end tls_conn = Reseau.TLS.client(socket, config) try @@ -625,6 +722,11 @@ function connect(host::String, port::Integer, dbname::String, user::String, @nos sslservername_v = sslservername::Union{String, Nothing} statement_timeout_v = statement_timeout::Union{Int, Nothing} socket = connectsocket(host, port, connect_timeout_v) + # Any failure from here on must close the socket: nothing else holds a + # reference to it, and the transport has no finalizer, so an escaping + # exception would leak the descriptor for the life of the process — + # a pool or reconnect loop against a flapping server would hit EMFILE. + try sslmode_str = sslmode_v === nothing ? "prefer" : lowercase(String(sslmode_v)) sslmode_str == "disable" || sslmode_str == "prefer" || sslmode_str == "require" || sslmode_str == "verify-full" || throw(Error("invalid sslmode: $sslmode_str")) if sslmode_str != "disable" @@ -640,8 +742,11 @@ function connect(host::String, port::Integer, dbname::String, user::String, @nos elseif mt == UInt8('N') (sslmode_str == "require" || sslmode_str == "verify-full") && throw(Error("server does not support SSL")) elseif mt == UInt8('E') - # server may answer SSLRequest with a full ErrorResponse + # server may answer SSLRequest with a full ErrorResponse. This is + # pre-TLS and pre-auth, so bound the length like readheader does + # before handing it to the allocating read. len = ntoh(read(socket, Int32)) - 4 + (len < 0 || len > MAX_PREAUTH_MESSAGE_LEN) && close_and_throw(socket, Error("invalid message length $len from server")) close_and_throw_error_response(socket, len, debug) else close_and_throw(socket, Error("unexpected response to SSLRequest: $(Char(mt))")) @@ -654,7 +759,7 @@ function connect(host::String, port::Integer, dbname::String, user::String, @nos writestartupmessage(socket::Reseau.TLS.Conn, debug, user, dbname, application_name_v, statement_timeout_v) end # read initial response - mt, len = readheader(socket, debug) + mt, len = readheader(socket, debug, MAX_PREAUTH_MESSAGE_LEN) if mt == UInt8('E') # error close_and_throw_error_response(socket, len, debug) @@ -664,55 +769,131 @@ function connect(host::String, port::Integer, dbname::String, user::String, @nos # server version too old close_and_throw(socket, Error("server version too old")) end - pid, skey, server_params = waitfor(socket, debug, 'K', 'Z') + pid, skey, server_params = waitfor(socket, debug, 'K', 'Z'; max_message_len=MAX_PREAUTH_MESSAGE_LEN) + uppercase(replace(get(server_params, "client_encoding", ""), "-" => "")) == "UTF8" || + close_and_throw(socket, Error("server did not confirm UTF8 client_encoding")) + # socket-union isa split so the call resolves under --trim, as above + if socket isa Reseau.TCP.Conn + align_session_formats!(socket::Reseau.TCP.Conn, server_params, debug, statement_timeout_v) + else + align_session_formats!(socket::Reseau.TLS.Conn, server_params, debug, statement_timeout_v) + end return socket, pid, skey, server_params + catch + close(socket) + rethrow() + end +end + +# The text-format parsers only understand ISO dates and postgres-style +# intervals; against any other setting values decode into silently wrong dates +# or fail with an error that points nowhere near the cause. The server reports +# both in its startup ParameterStatus, so correct them only when they actually +# differ: a default server pays nothing, and no extra startup parameters are +# sent (poolers such as pgbouncer reject `options` unless it is allowlisted). +# The field-order half of DateStyle ("MDY"/"DMY"/"YMD") decides how ambiguous +# *input* like '01/02/2020' is read; only the output half has to be ISO. Keep +# whatever order the server was configured with so correcting the output format +# doesn't silently change what the user's literals mean. +function date_order(datestyle::AbstractString) + for part in split(datestyle, ',') + order = uppercase(strip(part)) + (order == "MDY" || order == "DMY" || order == "YMD") && return order + end + return "MDY" +end + +function align_session_formats!(socket, server_params::Dict{String, String}, debug::Bool, @nospecialize(statement_timeout::Union{Int, Nothing})=nothing) + if statement_timeout !== nothing + exec(PostgresStyle(), socket, string("SET statement_timeout = ", statement_timeout::Int), + debug, nothing, nothing, server_params) + end + # A parameter the server didn't report (a pooler may not forward it) must + # be treated as unknown, i.e. corrected — assuming it is already right is + # how intervals silently decode to zero. + datestyle = get(server_params, "DateStyle", "") + if isempty(datestyle) + # unreported (a pooler may not forward ParameterStatus): setting just + # the format half preserves whatever field order is configured + # server-side, which we can't see — naming an order here would flip a + # DMY session to MDY + exec(PostgresStyle(), socket, "SET DateStyle = 'ISO'", debug, + nothing, nothing, server_params) + server_params["DateStyle"] = "ISO" + elseif !startswith(datestyle, "ISO") + wanted = string("ISO, ", date_order(datestyle)) + exec(PostgresStyle(), socket, string("SET DateStyle = '", wanted, "'"), + debug, nothing, nothing, server_params) + server_params["DateStyle"] = wanted + end + if get(server_params, "IntervalStyle", "") != "postgres" + exec(PostgresStyle(), socket, "SET IntervalStyle = 'postgres'", debug, + nothing, nothing, server_params) + server_params["IntervalStyle"] = "postgres" + end + return end -function prepare(socket, sql::String, debug::Bool; name::Union{Nothing, String}=nothing) +function prepare(socket, sql::String, debug::Bool, + server_parameters::Union{Nothing, Dict{String, String}}=nothing; + name::Union{Nothing, String}=nothing) stmtname = name === nothing ? randstring(Random.RandomDevice(), 36) : String(name) writemessages(socket, debug, ('P', stmtname, sql, Int16(0)), ('S',)) - waitfor(socket, debug, '1', 'Z') + waitfor(socket, debug, '1', 'Z'; server_parameters=server_parameters) return stmtname end _symbol(ptr, len) = ccall(:jl_symbol_n, Ref{Symbol}, (Ptr{UInt8}, Int), ptr, len) -function describeprepared(socket, name::String, debug::Bool) +function describeprepared(socket, name::String, debug::Bool, + server_parameters::Union{Nothing, Dict{String, String}}=nothing) writemessages(socket, debug, ('D', UInt8('S'), name), ('S',)) + nparams, cols, types = readprepareddescription(socket, debug, server_parameters) + waitfor(socket, debug, 'Z'; server_parameters=server_parameters) + return nparams, cols, types +end + +function readprepareddescription(socket, debug::Bool, + server_parameters::Union{Nothing, Dict{String, String}}=nothing) nparams = 0 ncols = 0 cols = Symbol[] types = Int[] try - mt, len = read_expected(socket, debug, 't') + mt, len = read_expected(socket, debug, 't'; server_parameters=server_parameters) + len >= 2 || close_and_throw(socket, Error("truncated ParameterDescription message from server")) nparams = Int(ntoh(read(socket, Int16))) + nparams >= 0 || close_and_throw(socket, Error("invalid ParameterDescription count from server")) + len == 2 + 4nparams || close_and_throw(socket, Error("invalid ParameterDescription length from server")) skipbytes!(socket, len - 2) - mt, len = read_expected(socket, debug, 'T', 'n') + mt, len = read_expected(socket, debug, 'T', 'n'; server_parameters=server_parameters) if mt == UInt8('n') # no data - waitfor(socket, debug, 'Z') return nparams, cols, types end + len >= 2 || close_and_throw(socket, Error("truncated RowDescription message from server")) ncols = Int(ntoh(read(socket, Int16))) + ncols >= 0 || close_and_throw(socket, Error("invalid RowDescription column count from server")) buf = read(socket, len - 2) + length(buf) == len - 2 || close_and_throw(socket, Error("truncated RowDescription message from server")) i = 1 - while i < len - 2 - ptr = pointer(buf, i) - plen = Int(@ccall strlen(ptr::Ptr{Cvoid})::Csize_t) - name = _symbol(ptr, plen) - i += plen + 1 - i += 4 # skip table oid - i += 2 # skip column number - typeId = Int(ntoh(unsafe_load(Ptr{Int32}(pointer(buf, i))))) + # each field: name (cstring), table oid (4), column number (2), + # type oid (4), type length (2), type modifier (4), format code (2). + # All offsets are bounds-checked against the buffer actually received: + # a short read or a malformed RowDescription must not read past it. + GC.@preserve buf while i <= length(buf) + stop = findnext(isequal(UInt8(0)), buf, i) + stop === nothing && close_and_throw(socket, Error("truncated RowDescription message from server")) + name = _symbol(pointer(buf, i), stop - i) + i = stop + 1 + i + 17 <= length(buf) || close_and_throw(socket, Error("truncated RowDescription message from server")) + typeId = Int(ntoh(unsafe_load(Ptr{Int32}(pointer(buf, i + 6))))) + i += 18 push!(types, typeId) - i += 4 - i += 2 # skip type length - # typeModifier = Int(ntoh(unsafe_load(Ptr{Int32}(pointer(buf, i))))) - i += 4 - i += 2 # skip format code push!(cols, name) end - waitfor(socket, debug, 'Z') + length(cols) == ncols || close_and_throw(socket, Error("RowDescription column count does not match its fields")) + i == length(buf) + 1 || close_and_throw(socket, Error("RowDescription message has trailing bytes")) return nparams, cols, types catch err # a deliberately-thrown Error leaves the stream at ReadyForQuery (or @@ -739,40 +920,80 @@ end function StructUtils.applyeach(::AbstractPostgresStyle, f, dr::DataRow) buf = dr.buf + nbuf = length(buf) GC.@preserve buf begin + nbuf >= 2 || throw(Error("truncated DataRow message from server")) ncols = Int(ntoh(unsafe_load(Ptr{Int16}(pointer(buf))))) + # the protocol mandates one value per described column; anything else + # (including a negative count, which is signed on the wire) would leave + # the caller's row partly unfilled — an UndefRefError downstream + (ncols == length(dr.names) && ncols == length(dr.typeIds)) || + throw(Error("DataRow column count does not match the row description")) pos = 3 for i = 1:ncols + # column lengths come off the wire: validate each against the + # message actually received before reading the value + pos + 3 <= nbuf || throw(Error("truncated DataRow message from server")) len = Int(ntoh(unsafe_load(Ptr{Int32}(pointer(buf, pos))))) pos += 4 if len == -1 # null f(dr.names[i], nothing) else + (len >= 0 && pos + len - 1 <= nbuf) || throw(Error("truncated DataRow message from server")) str = unsafe_string(pointer(buf, pos), len) pos += len @inbounds applycast(f, dr.names[i], dr.typeIds[i], str, dr.type_registry) end end + pos == nbuf + 1 || throw(Error("DataRow message has trailing bytes")) end return end +# Resolve the intersection with StructUtils.applyeach(f, ::StructStyle, x). +# A StructStyle is not a row callback, so this argument order is invalid. +function StructUtils.applyeach(style::AbstractPostgresStyle, + callback::StructUtils.StructStyle, + dr::DataRow) + throw(MethodError(StructUtils.applyeach, (style, callback, dr))) +end + struct Exec{S <: AbstractPostgresStyle} style::S socket::ReseauConn names::Vector{Symbol} typeIds::Vector{Int} type_registry::Dict{Int, TypeInfo} + server_parameters::Dict{String, String} debug::Bool command_tag::Base.RefValue{Union{Nothing, String}} rows_affected::Base.RefValue{Union{Nothing, Int}} + # ReadyForQuery's transaction status: 'I' idle, 'T' in a transaction, + # 'E' in a failed transaction. The server's own view, which is the only + # thing that knows about a transaction opened by raw SQL. + tx_status::Base.RefValue{UInt8} +end + +# Read ReadyForQuery's required one-byte transaction status. +function read_ready_status(socket, len) + len == 1 || close_and_throw(socket, Error("invalid ReadyForQuery message length from server")) + status = read(socket, UInt8) + status in (UInt8('I'), UInt8('T'), UInt8('E')) || + close_and_throw(socket, Error("invalid ReadyForQuery transaction status from server")) + return status end +in_transaction_status(status::UInt8) = status == UInt8('T') || status == UInt8('E') + function commandComplete(len, socket) buf = read(socket, len) - isempty(buf) && return "" - return unsafe_string(pointer(buf)) + length(buf) == len || throw(Error("truncated CommandComplete message from server")) + isempty(buf) && throw(Error("empty CommandComplete message from server")) + findfirst(isequal(UInt8(0)), buf) == length(buf) || + throw(Error("invalid CommandComplete message from server")) + tag, _ = cstring_at(buf, 1) + return tag end function rows_affected_from_command_tag(tag::String) @@ -789,6 +1010,8 @@ function StructUtils.applyeach(::AbstractPostgresStyle, f, e::Exec) nrows = 0 server_error = nothing consumer_error = nothing + copy_in_statement = false + copy_out_statement = false try while true mt, len = readheader(e.socket, e.debug) @@ -796,7 +1019,7 @@ function StructUtils.applyeach(::AbstractPostgresStyle, f, e::Exec) # error; keep reading until ready-for-query, thrown below server_error = errorResponse(len, e.socket, e.debug) elseif mt == UInt8('Z') - skipbytes!(e.socket, len) + e.tx_status[] = read_ready_status(e.socket, len) break elseif mt == UInt8('D') nrows += 1 @@ -818,8 +1041,26 @@ function StructUtils.applyeach(::AbstractPostgresStyle, f, e::Exec) tag = commandComplete(len, e.socket) e.command_tag[] = tag e.rows_affected[] = rows_affected_from_command_tag(tag) - elseif mt == UInt8('T') || mt == UInt8('n') || mt == UInt8('I') || mt == UInt8('S') - # row description / no data / empty query response / parameter status + elseif mt == UInt8('S') + parameterStatus!(e.server_parameters, len, e.socket) + elseif mt == UInt8('T') || mt == UInt8('n') || mt == UInt8('I') + # row description / no data / empty query response + skipbytes!(e.socket, len) + elseif mt == UInt8('G') + # CopyInResponse: the statement was a COPY ... FROM STDIN, which + # execute doesn't support. Abort the copy with CopyFail so the + # server returns to ready and the connection stays usable; a + # clear client error is thrown below. A fresh Sync must follow: + # the one sent with Bind/Execute was ignored during copy-in + # mode, and the aborted extended-query sequence only reaches + # ReadyForQuery once a Sync arrives after the error. + skipbytes!(e.socket, len) + copy_in_statement = true + writemessages(e.socket, e.debug, ('f', "COPY FROM STDIN is not supported via execute"), ('S',)) + elseif mt == UInt8('H') || mt == UInt8('d') || mt == UInt8('c') + # CopyOutResponse/CopyData/CopyDone: drain the copy-out stream + # through ReadyForQuery; a clear client error is thrown below + mt == UInt8('H') && (copy_out_statement = true) skipbytes!(e.socket, len) elseif mt == UInt8('A') # notification response @@ -843,6 +1084,16 @@ function StructUtils.applyeach(::AbstractPostgresStyle, f, e::Exec) server_error === nothing || throw(server_error) rethrow() end + # COPY misuse throws a clear client error; the stream was drained above, + # so the connection stays usable. For copy-in the server error is just the + # CopyFail artifact, so the client error wins; for copy-out a server error + # is a genuine mid-stream failure (e.g. inside COPY (SELECT ...) TO + # STDOUT) and is more informative than the misuse error. + copy_in_statement && throw(PostgresInterfaceError("COPY ... FROM STDIN is not supported via execute; use Postgres.copy_from")) + if copy_out_statement + server_error === nothing || throw(server_error) + throw(PostgresInterfaceError("COPY ... TO STDOUT is not supported via execute; use Postgres.copy_to")) + end # server errors take precedence; otherwise surface a consumer error that # aborted materialization (the stream was still drained above) server_error === nothing || throw(server_error) @@ -850,19 +1101,62 @@ function StructUtils.applyeach(::AbstractPostgresStyle, f, e::Exec) return end -function exec(style::S, socket::ReseauConn, stmtname::String, params::Vector{Union{String, Missing}}, names, typeIds, type_registry::Dict{Int, TypeInfo}, debug::Bool, rowlimit::Int=0) where {S <: AbstractPostgresStyle} +function StructUtils.applyeach(style::AbstractPostgresStyle, + callback::StructUtils.StructStyle, + e::Exec) + throw(MethodError(StructUtils.applyeach, (style, callback, e))) +end + +function exec(style::S, socket::ReseauConn, stmtname::String, + params::Vector{Union{String, Missing}}, names, typeIds, + type_registry::Dict{Int, TypeInfo}, debug::Bool, rowlimit::Int=0, + server_parameters::Dict{String, String}=Dict{String, String}()) where {S <: AbstractPostgresStyle} #TODO: support binary format: here and in applycast npformats = Int16(0) # all params use text format nparams = Int16(length(params)) # bind, then execute, then sync writemessages(socket, debug, ('B', "", stmtname, npformats, nparams, Params(params), Int16(0)), ('E', "", Int32(rowlimit)), ('S',)) - waitfor(socket, debug, '2') - return Exec{S}(style, socket, names, typeIds, type_registry, debug, Ref{Union{Nothing, String}}(nothing), Ref{Union{Nothing, Int}}(nothing)) + waitfor(socket, debug, '2'; server_parameters=server_parameters) + return Exec{S}(style, socket, names, typeIds, type_registry, + server_parameters, debug, Ref{Union{Nothing, String}}(nothing), + Ref{Union{Nothing, Int}}(nothing), Ref{UInt8}(UInt8('I'))) +end + +# Execute an unnamed statement as one extended-query segment. There is one +# Sync, after Parse/Describe/Bind/Execute. A transaction-mode pooler therefore +# cannot return the backend between dependent protocol messages and replace the +# unnamed statement with another client's statement. +function exec_unnamed(style::S, socket::ReseauConn, sql::String, + params::Vector{Union{String, Missing}}, + type_registry::Dict{Int, TypeInfo}, debug::Bool, + rowlimit::Int=0, + server_parameters::Dict{String, String}=Dict{String, String}()) where {S <: AbstractPostgresStyle} + npformats = Int16(0) + nparams = Int16(length(params)) + writemessages(socket, debug, + ('P', "", sql, Int16(0)), + ('D', UInt8('S'), ""), + ('B', "", "", npformats, nparams, Params(params), Int16(0)), + ('E', "", Int32(rowlimit)), + ('S',)) + mt, len = read_expected(socket, debug, '1'; server_parameters=server_parameters) + skipbytes!(socket, len) + _, names, typeIds = readprepareddescription(socket, debug, server_parameters) + mt, len = read_expected(socket, debug, '2'; server_parameters=server_parameters) + skipbytes!(socket, len) + return Exec{S}(style, socket, names, typeIds, type_registry, server_parameters, debug, + Ref{Union{Nothing, String}}(nothing), + Ref{Union{Nothing, Int}}(nothing), + Ref{UInt8}(UInt8('I'))) end -function exec(style::S, socket::ReseauConn, query::String, debug::Bool) where {S <: AbstractPostgresStyle} +function exec(style::S, socket::ReseauConn, query::String, debug::Bool, + tx_status_ref::Union{Nothing, Base.RefValue{UInt8}}=nothing, + command_tag_ref::Union{Nothing, Base.RefValue{Union{Nothing, String}}}=nothing, + server_parameters::Union{Nothing, Dict{String, String}}=nothing) where {S <: AbstractPostgresStyle} writemessages(socket, debug, ('Q', query)) server_error = nothing + tx_status = UInt8('I') try while true mt, len = readheader(socket, debug) @@ -871,15 +1165,25 @@ function exec(style::S, socket::ReseauConn, query::String, debug::Bool) where {S # server error so the connection remains reusable. server_error = errorResponse(len, socket, debug) elseif mt == UInt8('Z') - skipbytes!(socket, len) + tx_status = read_ready_status(socket, len) + # publish through the Ref as well: a failed statement (a + # COMMIT hitting a deferred constraint) throws below, but its + # ReadyForQuery status is authoritative and the caller's + # transaction tracking must not go stale + tx_status_ref === nothing || (tx_status_ref[] = tx_status) break elseif mt == UInt8('N') notice_callback(style, noticeResponse(len, socket)) elseif mt == UInt8('A') notification_callback(style, notificationResponse(len, socket)) - elseif mt == UInt8('C') || mt == UInt8('T') || mt == UInt8('D') || - mt == UInt8('I') || mt == UInt8('S') - # CommandComplete and any incidental simple-query result data. + elseif mt == UInt8('C') + tag = commandComplete(len, socket) + command_tag_ref === nothing || (command_tag_ref[] = tag) + elseif mt == UInt8('S') + server_parameters === nothing ? skipbytes!(socket, len) : + parameterStatus!(server_parameters, len, socket) + elseif mt == UInt8('T') || mt == UInt8('D') || mt == UInt8('I') + # Incidental simple-query result data. skipbytes!(socket, len) else close_and_throw(socket, Error("unexpected message type '$(Char(mt))' from server; connection protocol state is corrupted")) @@ -891,7 +1195,7 @@ function exec(style::S, socket::ReseauConn, query::String, debug::Bool) where {S rethrow() end server_error === nothing || throw(server_error) - return + return tx_status end exec(socket::ReseauConn, query::String, debug::Bool) = exec(PostgresStyle(), socket, query, debug) @@ -899,120 +1203,265 @@ exec(socket::ReseauConn, query::String, debug::Bool) = exec(PostgresStyle(), soc function copy_in(style::S, socket, query::String, source::IO, debug::Bool) where {S <: AbstractPostgresStyle} writemessage(socket, debug, 'Q', query) error_msg = nothing - while true - mt, len = readheader(socket, debug) - if mt == UInt8('G') - skipbytes!(socket, len) - break - elseif mt == UInt8('E') - error_msg = errorResponse(len, socket, debug) - elseif mt == UInt8('N') - notice = noticeResponse(len, socket) - notice_callback(style, notice) - elseif mt == UInt8('A') - notification = notificationResponse(len, socket) - notification_callback(style, notification) - else - skipbytes!(socket, len) + copy_started = false + extra_statement = false + saw_noncopy_response = false + try + while true + mt, len = readheader(socket, debug) + if mt == UInt8('G') + skipbytes!(socket, len) + copy_started = true + extra_statement |= saw_noncopy_response + break + elseif mt == UInt8('E') + error_msg = errorResponse(len, socket, debug) + elseif mt == UInt8('Z') + # ReadyForQuery without CopyInResponse: the statement errored or + # wasn't a COPY ... FROM STDIN; the stream is back at ready + skipbytes!(socket, len) + break + elseif mt == UInt8('N') + notice = noticeResponse(len, socket) + notice_callback(style, notice) + elseif mt == UInt8('A') + notification = notificationResponse(len, socket) + notification_callback(style, notification) + else + # Any command response before CopyInResponse belongs to a + # separate statement in the simple-query string. Continue to + # the COPY so the server cannot be left waiting for input, but + # reject the call after the stream returns to ReadyForQuery. + mt in (UInt8('C'), UInt8('T'), UInt8('D'), UInt8('I'), + UInt8('n'), UInt8('H'), UInt8('d'), UInt8('c')) && + (saw_noncopy_response = true) + skipbytes!(socket, len) + end end + catch + # bailed mid-stream: the position is unknowable, never reuse the socket + close(socket) + # if the server reported an error before the connection died, surface + # it over the raw IO error — it explains what actually went wrong + error_msg === nothing || throw(error_msg) + rethrow() end error_msg === nothing || throw(error_msg) - buf = Vector{UInt8}(undef, 16384) - while !eof(source) - n = readbytes!(source, buf, length(buf)) - n == 0 && break - writemessage(socket, debug, 'd', view(buf, 1:n)) + copy_started || throw(PostgresInterfaceError("statement did not initiate COPY ... FROM STDIN")) + try + buf = Vector{UInt8}(undef, 16384) + while !eof(source) + n = readbytes!(source, buf, length(buf)) + n == 0 && break + writemessage(socket, debug, 'd', view(buf, 1:n)) + end + writemessage(socket, debug, 'c') + catch + # the user's data source failed mid-copy: abort the copy so the + # connection returns to ready, then rethrow the source error + try + writemessage(socket, debug, 'f', "client-side data source failed") + drain_to_ready!(socket, debug) + catch + close(socket) + end + rethrow() end - writemessage(socket, debug, 'c') error_msg = nothing - while true - mt, len = readheader(socket, debug) - if mt == UInt8('E') - error_msg = errorResponse(len, socket, debug) - elseif mt == UInt8('C') - skipbytes!(socket, len) - elseif mt == UInt8('N') - notice = noticeResponse(len, socket) - notice_callback(style, notice) - elseif mt == UInt8('A') - notification = notificationResponse(len, socket) - notification_callback(style, notification) - elseif mt == UInt8('Z') - skipbytes!(socket, len) - break - else - skipbytes!(socket, len) + second_copy = false + copy_completed = false + try + while true + mt, len = readheader(socket, debug) + if mt == UInt8('E') + copy_completed && (extra_statement = true) + error_msg = errorResponse(len, socket, debug) + elseif mt == UInt8('C') + commandComplete(len, socket) + if copy_completed + extra_statement = true + else + copy_completed = true + end + elseif mt == UInt8('G') + # a second CopyInResponse (multi-statement query string): the + # server is waiting for more copy data, so abort with CopyFail + # instead of deadlocking; a clear error is thrown below + skipbytes!(socket, len) + second_copy = true + extra_statement = true + writemessage(socket, debug, 'f', "copy_from supports a single COPY FROM STDIN statement") + elseif mt == UInt8('N') + notice = noticeResponse(len, socket) + notice_callback(style, notice) + elseif mt == UInt8('A') + notification = notificationResponse(len, socket) + notification_callback(style, notification) + elseif mt == UInt8('Z') + read_ready_status(socket, len) + break + else + mt in (UInt8('T'), UInt8('D'), UInt8('I'), UInt8('n'), + UInt8('H'), UInt8('d'), UInt8('c')) && + (extra_statement = true) + skipbytes!(socket, len) + end end + catch + # bailed mid-stream: the position is unknowable, never reuse the socket + close(socket) + error_msg === nothing || throw(error_msg) + rethrow() end + (extra_statement || second_copy) && + throw(PostgresInterfaceError("copy_from supports a single COPY ... FROM STDIN statement per call")) error_msg === nothing || throw(error_msg) + copy_completed || throw(PostgresInterfaceError("COPY ... FROM STDIN did not complete")) return end function copy_out(style::S, socket, query::String, dest::IO, debug::Bool) where {S <: AbstractPostgresStyle} writemessage(socket, debug, 'Q', query) error_msg = nothing - while true - mt, len = readheader(socket, debug) - if mt == UInt8('H') - skipbytes!(socket, len) - elseif mt == UInt8('d') - write(dest, read(socket, len)) - elseif mt == UInt8('c') - skipbytes!(socket, len) - elseif mt == UInt8('C') - skipbytes!(socket, len) - elseif mt == UInt8('E') - error_msg = errorResponse(len, socket, debug) - elseif mt == UInt8('N') - notice = noticeResponse(len, socket) - notice_callback(style, notice) - elseif mt == UInt8('A') - notification = notificationResponse(len, socket) - notification_callback(style, notification) - elseif mt == UInt8('Z') - skipbytes!(socket, len) - break - else - skipbytes!(socket, len) + copy_started = false + copy_completed = false + extra_statement = false + saw_noncopy_response = false + wrong_direction = false + try + while true + mt, len = readheader(socket, debug) + if mt == UInt8('H') + skipbytes!(socket, len) + (copy_started || copy_completed || saw_noncopy_response) && + (extra_statement = true) + copy_started = true + elseif mt == UInt8('d') + write(dest, read(socket, len)) + elseif mt == UInt8('c') + skipbytes!(socket, len) + elseif mt == UInt8('C') + commandComplete(len, socket) + if copy_started && !copy_completed + copy_completed = true + elseif copy_completed + extra_statement = true + else + saw_noncopy_response = true + end + elseif mt == UInt8('G') + # CopyInResponse: the statement was COPY ... FROM STDIN. The server + # is now waiting on us for data, so abort the copy with CopyFail to + # return the stream to ready instead of deadlocking. + skipbytes!(socket, len) + if copy_started || copy_completed + extra_statement = true + else + wrong_direction = true + end + writemessage(socket, debug, 'f', "COPY FROM STDIN is not supported via copy_to") + elseif mt == UInt8('E') + copy_completed && (extra_statement = true) + error_msg = errorResponse(len, socket, debug) + elseif mt == UInt8('N') + notice = noticeResponse(len, socket) + notice_callback(style, notice) + elseif mt == UInt8('A') + notification = notificationResponse(len, socket) + notification_callback(style, notification) + elseif mt == UInt8('Z') + read_ready_status(socket, len) + break + else + if mt in (UInt8('T'), UInt8('D'), UInt8('I'), UInt8('n')) + if copy_started + extra_statement = true + else + saw_noncopy_response = true + end + end + skipbytes!(socket, len) + end end + catch + # bailed mid-stream (socket failure, or the user's dest IO threw): + # the position is unknowable, never reuse the socket + close(socket) + error_msg === nothing || throw(error_msg) + rethrow() end + (extra_statement || (copy_started && saw_noncopy_response)) && + throw(PostgresInterfaceError("copy_to supports a single COPY ... TO STDOUT statement per call")) + wrong_direction && throw(PostgresInterfaceError("statement initiated COPY ... FROM STDIN; use Postgres.copy_from")) error_msg === nothing || throw(error_msg) + copy_started || throw(PostgresInterfaceError("statement did not initiate COPY ... TO STDOUT")) + copy_completed || throw(PostgresInterfaceError("COPY ... TO STDOUT did not complete")) return dest end -function close_statement(socket, name::String, debug::Bool) +function close_statement(socket, name::String, debug::Bool, + server_parameters::Union{Nothing, Dict{String, String}}=nothing) writemessages(socket, debug, ('C', UInt8('S'), name), ('S',)) - waitfor(socket, debug, '3', 'Z') + waitfor(socket, debug, '3', 'Z'; server_parameters=server_parameters) return end -function cancel_request(host::String, port::Int, pid::Int32, skey::Int32, debug::Bool=false) - socket = connectsocket(host, port) +# The cancel key is a credential: anyone holding it can cancel that backend's +# queries for the life of the connection, so the CancelRequest goes over TLS +# whenever the connection it cancels uses TLS. +function cancel_request(host::String, port::Int, pid::Int32, skey::Int32, debug::Bool=false, + @nospecialize(sslmode::Union{String, Nothing}=nothing), + @nospecialize(sslrootcert::Union{String, Nothing}=nothing), + @nospecialize(sslcert::Union{String, Nothing}=nothing), + @nospecialize(sslkey::Union{String, Nothing}=nothing), + @nospecialize(sslcapath::Union{String, Nothing}=nothing), + @nospecialize(sslservername::Union{String, Nothing}=nothing), + @nospecialize(connect_timeout::Union{Int, Nothing}=nothing)) + sslmode_v = sslmode::Union{String, Nothing} + connect_timeout_v = connect_timeout::Union{Int, Nothing} + sslmode_str = sslmode_v === nothing ? "prefer" : lowercase(String(sslmode_v)) + tls_required = sslmode_str == "require" || sslmode_str == "verify-full" + socket = connectsocket(host, port, connect_timeout_v) + refused_cleartext = false + sent = false try - buf = IOBuffer(Vector{UInt8}(undef, 16); write=true) - write(buf, hton(Int32(16))) - write(buf, hton(Int32(80877102))) # CancelRequest code - write(buf, hton(pid)) - write(buf, hton(skey)) - write(socket, take!(buf)) - flush(socket) - return true + send_key = true + if sslmode_str != "disable" + writemessage(socket, debug, '\0', Int32(80877103)) + mt = read(socket, UInt8) + if mt == UInt8('S') + socket = tlsupgrade(socket, connect_timeout_v, + sslservername isa String ? sslservername::String : host, + sslmode_str == "verify-full", + sslcert::Union{String, Nothing}, sslkey::Union{String, Nothing}, + sslrootcert::Union{String, Nothing}, sslcapath::Union{String, Nothing}) + elseif tls_required + # never send the cancel key in the clear when TLS was required + refused_cleartext = true + send_key = false + end + end + if send_key + buf = IOBuffer(Vector{UInt8}(undef, 16); write=true) + write(buf, hton(Int32(16))) + write(buf, hton(Int32(80877102))) # CancelRequest code + write(buf, hton(pid)) + write(buf, hton(skey)) + write(socket, take!(buf)) + flush(socket) + sent = true + end catch - return false + sent = false finally close(socket) end + # refusing to send is a hard failure the caller must hear about, not a + # silent no-op: the query they asked to cancel is still running + refused_cleartext && throw(PostgresInterfaceError("server refused TLS on the cancel connection; not sending the cancel key in cleartext under sslmode=$sslmode_str")) + return sent end -export cancel_request - -# function escape(conn::PGconn, s::AbstractString) -# str = C.PQescapeLiteral(ptr, s, sizeof(s)) -# escaped = unsafe_string(str) -# C.PQfreemem(str) -# return escaped -# end - include("../array_parsing.jl") using .ArrayParsing diff --git a/src/api/types.jl b/src/api/types.jl index a2cc1c1..d41ab7a 100644 --- a/src/api/types.jl +++ b/src/api/types.jl @@ -12,6 +12,11 @@ behavior interface on it: Postgres.notice_callback(::MyStyle, notice) = ... Postgres.notification_callback(::MyStyle, notification) = ... +`query_logger`'s `info` carries the SQL and the bound parameter values, so a +logger that writes them out will record whatever sensitive data those queries +carry — redact or omit `info.params` if the log is not as trusted as the +database. + Custom styles inherit the default row-materialization traits (lift/structlike/...), which dispatch on `AbstractPostgresStyle`, and are used as the StructUtils style when materializing query results — so `StructUtils.lift` overloads on a custom style apply @@ -24,13 +29,55 @@ abstract type AbstractPostgresStyle <: StructUtils.StructStyle end struct PostgresStyle <: AbstractPostgresStyle end # behavior interface (style-first; overload on your own style) + +""" + Postgres.query_logging_enabled(style) -> Bool + +Whether [`query_logger`](@ref Postgres.API.query_logger) should be called for +queries on connections using `style`. `false` by default, which also skips the +timing work the logger would need. +""" query_logging_enabled(::AbstractPostgresStyle) = false + +""" + Postgres.query_logger(style, event::Symbol, info::NamedTuple) + +Called after each query when +[`query_logging_enabled`](@ref Postgres.API.query_logging_enabled) is true for +`style`. `event` is `:execute`, `:copy_from`, or `:copy_to`; `info` carries +`sql`, `duration_ns`, `success`, the bound `params` (for `:execute`), and +`error` when the query failed. + +`info.params` holds the query's parameter values, so a logger that records +them will record whatever sensitive data those queries carry. +""" query_logger(::AbstractPostgresStyle, event::Symbol, info::NamedTuple) = nothing + +""" + Postgres.notice_callback(style, notice) + +Called for each `NoticeResponse` the server sends. `notice` is a `Dict` of the +raw notice fields, keyed by their single-character protocol codes (`"M"` is +the message, `"S"` the severity). Emits the message as a `@warn` by default. +""" function notice_callback(::AbstractPostgresStyle, notice) msg = get(notice, "M", "") !isempty(msg) && @warn msg return nothing end + +""" + Postgres.notification_callback(style, notification::Notification) + +Called for each asynchronous `NOTIFY` ([`Notification`](@ref +Postgres.API.Notification)) received while reading query results. Does nothing +by default. + +A connection only observes notifications while it is reading from the server, +so a connection that is idle or busy in another phase of a query may not see +one. Use [`wait_for_notification`](@ref Postgres.wait_for_notification) on a +dedicated connection to receive every notification on a channel. +""" notification_callback(::AbstractPostgresStyle, notification) = nothing StructUtils.fieldtagkey(::AbstractPostgresStyle) = :postgres @@ -40,6 +87,17 @@ StructUtils.lift(::AbstractPostgresStyle, ::Type{T}, x::T) where {T<:JSON.LazyVa StructUtils.lift(::AbstractPostgresStyle, ::Type{T}, x::T, tags) where {T<:JSON.LazyValue} = x, nothing +""" + Postgres.Numeric + +Exact decimal representation of a PostgreSQL `numeric`/`decimal` value: +`coeff * 10^-scale`, where `coeff` is a `BigInt` and `scale` the number of +digits after the decimal point. Preserves the value and scale exactly (no +floating-point rounding). `print`/`string` produce the decimal text form. + +The PostgreSQL special values `NaN`, `Infinity`, and `-Infinity` cannot be +represented and throw an error when encountered. +""" struct Numeric coeff::BigInt scale::Int @@ -48,12 +106,28 @@ StructUtils.structlike(::AbstractPostgresStyle, ::Type{Numeric}) = false Base.:(==)(a::Numeric, b::Numeric) = a.coeff == b.coeff && a.scale == b.scale +""" + Postgres.PostgresRange{T} + +A PostgreSQL range value (`int4range`, `numrange`, `tstzrange`, ...). `lower` +and `upper` are the bounds (`missing` when unbounded), `lower_inclusive` and +`upper_inclusive` indicate whether each bound is inclusive, and `empty` is +`true` for the empty range. +""" struct PostgresRange{T} lower::Union{T, Missing} upper::Union{T, Missing} lower_inclusive::Bool upper_inclusive::Bool empty::Bool + + function PostgresRange{T}(lower, upper, lower_inclusive::Bool, + upper_inclusive::Bool, empty::Bool) where {T} + converted_lower = ismissing(lower) ? missing : convert(T, lower) + converted_upper = ismissing(upper) ? missing : convert(T, upper) + return new{T}(converted_lower, converted_upper, lower_inclusive, + upper_inclusive, empty) + end end # ── CastFn: trim-safe type-erased value caster (the Reseau TaskFn pattern) ── @@ -149,6 +223,7 @@ function _populate_default_type_registry!() 1560 => TypeInfo(Bool, nothing), 1000 => TypeInfo(Vector{Bool}, nothing), 1001 => TypeInfo(Vector{Vector{UInt8}}, (val, registry) -> parse_array_by_oid(val, 17, registry)), + 1002 => TypeInfo(Vector{Char}, (val, registry) -> parse_array_by_oid(val, 18, registry)), 1005 => TypeInfo(Vector{Int16}, nothing), 1007 => TypeInfo(Vector{Int32}, nothing), 1016 => TypeInfo(Vector{Int64}, nothing), @@ -190,8 +265,6 @@ function register_type!(registry::Dict{Int, TypeInfo}, oid::Integer, julia_type: return registry end -const DATETIME_OPTIONS = Parsers.Options(dateformat=dateformat"yyyy-mm-dd HH:MM:SS.s") - @inline function tzoffset_seconds(offset::AbstractString) isempty(offset) && return 0 sign = offset[1] == '-' ? -1 : 1 @@ -199,7 +272,10 @@ const DATETIME_OPTIONS = Parsers.Options(dateformat=dateformat"yyyy-mm-dd HH:MM: isempty(digits) && return 0 hours = parse(Int, digits[1:2]) mins = length(digits) >= 4 ? parse(Int, digits[3:4]) : 0 - return sign * (hours * 3600 + mins * 60) + # pre-standardization (LMT-era) timestamps in named zones carry a seconds + # field ("+05:21:10"); dropping it silently shifts the decoded value + secs = length(digits) >= 6 ? parse(Int, digits[5:6]) : 0 + return sign * (hours * 3600 + mins * 60 + secs) end # ── hand-rolled postgres text-format date/time parsing ────────────────────── @@ -212,13 +288,33 @@ end @inline _pg_digit(b::UInt8)::Int = Int(b - UInt8('0')) @inline _pg_isdigit(b::UInt8)::Bool = UInt8('0') <= b <= UInt8('9') -@inline function _pg_date_at(c, o::Int)::Date - y = _pg_digit(c[o]) * 1000 + _pg_digit(c[o+1]) * 100 + _pg_digit(c[o+2]) * 10 + _pg_digit(c[o+3]) - m = _pg_digit(c[o+5]) * 10 + _pg_digit(c[o+6]) - d = _pg_digit(c[o+8]) * 10 + _pg_digit(c[o+9]) - return Date(y, m, d) +# The year is normally 4 digits but PostgreSQL emits more beyond year 9999, so +# scan it rather than assuming a fixed width. Returns the date and the offset +# just past it. The digit-count bound and separator checks matter: an ISO year +# is zero-padded to at least 4 digits, so accepting fewer would silently +# mis-decode other DateStyle renderings ("03-04-2020" is Postgres-style for +# 2020-03-04, not year 3), and an unbounded scan would overflow Int on +# adversarial input. +@inline function _pg_date_at_end(c, o::Int) + y = 0 + i = o + n = length(c) + while i <= n && _pg_isdigit(c[i]) + y = y * 10 + _pg_digit(c[i]) + i += 1 + end + ndigits = i - o + (4 <= ndigits <= 9) || throw(ArgumentError("invalid postgres date")) + (i + 5 <= n && c[i] == UInt8('-') && c[i+3] == UInt8('-') && + _pg_isdigit(c[i+1]) && _pg_isdigit(c[i+2]) && _pg_isdigit(c[i+4]) && _pg_isdigit(c[i+5])) || + throw(ArgumentError("invalid postgres date")) + m = _pg_digit(c[i+1]) * 10 + _pg_digit(c[i+2]) + d = _pg_digit(c[i+4]) * 10 + _pg_digit(c[i+5]) + return Date(y, m, d), i + 6 end +@inline _pg_date_at(c, o::Int)::Date = first(_pg_date_at_end(c, o)) + @inline function _pg_hms_at(c, o::Int) h = _pg_digit(c[o]) * 10 + _pg_digit(c[o+1]) mi = _pg_digit(c[o+3]) * 10 + _pg_digit(c[o+4]) @@ -239,8 +335,31 @@ end return h, mi, se, ms end +# `"char"` output: byte 0 renders as an empty string. High bytes render as +# backslash-octal on current PostgreSQL releases, while PostgreSQL 14 can send +# the raw byte. A backslash byte itself renders as a lone "\\", so only the +# exact 4-byte escape shapes are decoded. +function pg_parse_char(s::String) + isempty(s) && return '\0' + c = codeunits(s) + # Indexing a String that contains one raw high byte produces Julia's + # invalid-UTF8 Char sentinel. The PostgreSQL type is one byte, so decode + # that byte value directly. + length(c) == 1 && return Char(c[1]) + if length(c) == 4 && c[1] == UInt8('\\') && + UInt8('0') <= c[2] <= UInt8('3') && UInt8('0') <= c[3] <= UInt8('7') && UInt8('0') <= c[4] <= UInt8('7') + return Char((_pg_digit(c[2]) << 6) | (_pg_digit(c[3]) << 3) | _pg_digit(c[4])) + end + if length(c) == 4 && c[1] == UInt8('\\') && + (c[2] == UInt8('x') || c[2] == UInt8('X')) + return Char((hexnibble(c[3]) << 4) | hexnibble(c[4])) + end + return s[1] +end + function pg_parse_date(s::AbstractString)::Date c = codeunits(s) + _check_temporal_special(s, "date") length(c) >= 10 || throw(ArgumentError("invalid postgres date")) return _pg_date_at(c, 1) end @@ -249,14 +368,29 @@ function pg_parse_time(s::AbstractString)::Time c = codeunits(s) length(c) >= 8 || throw(ArgumentError("invalid postgres time")) h, mi, se, ms = _pg_hms_at(c, 1) + # postgres permits '24:00:00' as a time value; Julia's Time does not + h == 24 && throw(PostgresInterfaceError("postgres time value \"$s\" cannot be represented as a Julia Time")) return Time(h, mi, se, ms) end +@noinline _reject_temporal_special(s::AbstractString, what::String) = + throw(PostgresInterfaceError("postgres $what value \"$s\" cannot be represented as a Julia $(what == "date" ? "Date" : "DateTime")")) + +@inline function _check_temporal_special(s::AbstractString, what::String) + (s == "infinity" || s == "-infinity") && _reject_temporal_special(s, what) + # BC years are a different numbering than Julia's (proleptic, no year 0), + # so decoding them as AD would silently produce the wrong date + endswith(s, " BC") && throw(PostgresInterfaceError("postgres BC $what value \"$s\" is not supported")) + return +end + function pg_parse_datetime(s::AbstractString)::DateTime c = codeunits(s) + _check_temporal_special(s, "timestamp") length(c) >= 19 || throw(ArgumentError("invalid postgres timestamp")) - d = _pg_date_at(c, 1) - h, mi, se, ms = _pg_hms_at(c, 12) + # the time starts one space past the date, whose year may be wider than 4 + d, after_date = _pg_date_at_end(c, 1) + h, mi, se, ms = _pg_hms_at(c, after_date + 1) return DateTime(Dates.year(d), Dates.month(d), Dates.day(d), h, mi, se, ms) end @@ -272,6 +406,10 @@ end @inline function parse_timestamptz(val::String) lastindex(val) == 0 && throw(ArgumentError("invalid postgres timestamptz")) + # timestamptz output puts " BC" after the zone offset, so the check inside + # pg_parse_datetime (which sees only the offset-stripped prefix) can't + # catch it; check the full value here + _check_temporal_special(val, "timestamp") if val[end] == 'Z' ts = SubString(val, 1, prevind(val, lastindex(val))) return pg_parse_datetime(ts) @@ -315,6 +453,9 @@ Base.show(io::IO, num::Numeric) = print(io, numeric_string(num)) function parse_numeric(val::String) stripped = strip(val) stripped == "" && return Numeric(BigInt(0), 0) + lowered = lowercase(stripped) + (lowered == "nan" || lowered == "infinity" || lowered == "-infinity" || lowered == "+infinity") && + throw(PostgresInterfaceError("postgres numeric special value \"$stripped\" cannot be represented as Postgres.Numeric")) sign = 1 if stripped[1] == '-' sign = -1 @@ -325,7 +466,16 @@ function parse_numeric(val::String) exp_index = findfirst(c -> c == 'e' || c == 'E', stripped) exp_val = 0 if exp_index !== nothing - exp_val = parse(Int, stripped[exp_index + 1:end]) + # postgres numeric tops out at 16383 digits either side of the point; + # bound the exponent so a bogus value can't drive an enormous BigInt + # scaling below (tryparse so an oversized exponent reports the same + # error as an out-of-range one, rather than an OverflowError) + parsed_exp = tryparse(Int, stripped[exp_index + 1:end]) + # compared without abs: abs(typemin(Int)) wraps back to itself and + # would slip past the bound + (parsed_exp === nothing || parsed_exp < -100_000 || parsed_exp > 100_000) && + throw(PostgresInterfaceError("postgres numeric exponent out of range: $stripped")) + exp_val = parsed_exp stripped = stripped[1:exp_index - 1] end parts = split(stripped, '.'; limit=2) @@ -368,20 +518,34 @@ function parse_interval_time(token::AbstractString) return periods end +@noinline _reject_interval(val::String) = + throw(PostgresInterfaceError("unrecognized postgres interval \"$val\"; this driver requires IntervalStyle=postgres")) + function parse_interval(val::String) tokens = split(strip(val)) periods = Dates.Period[] + # whether any token was understood; distinguishes a real zero interval + # ("00:00:00") from text in a style this parser can't read + recognized = false i = 1 while i <= length(tokens) token = tokens[i] if occursin(':', token) + # a postgres time component is exactly h:m:s; anything else with a + # colon comes from a different IntervalStyle + count(isequal(':'), token) == 2 && (recognized = true) append!(periods, parse_interval_time(token)) i += 1 continue end i == length(tokens) && break - amount = parse(Int, token) + amount = tryparse(Int, token) + # a non-numeric token ("@", sql_standard year-month like "1-2") means a + # different IntervalStyle; fail with the actionable error, not a raw + # integer-parse failure + amount === nothing && _reject_interval(val) unit = lowercase(tokens[i + 1]) + nbefore = length(periods) if startswith(unit, "year") push!(periods, Dates.Year(amount)) elseif startswith(unit, "mon") @@ -395,9 +559,19 @@ function parse_interval(val::String) elseif startswith(unit, "sec") push!(periods, Dates.Second(amount)) end + # every understood unit pushes a period, so growth means this token + # pair was genuinely matched + length(periods) > nbefore && (recognized = true) i += 2 end - isempty(periods) && return Dates.Millisecond(0) + if isempty(periods) + # a genuine zero interval renders as "00:00:00"; anything else that + # yielded no periods is a format this parser doesn't understand (a + # mid-session `SET IntervalStyle` to sql_standard or iso_8601), and + # silently returning zero would be wrong data + recognized || _reject_interval(val) + return Dates.Millisecond(0) + end length(periods) == 1 && return only(periods) # n=0 construction skips CompoundPeriod's canonicalize loop (whose Period + # merge is dynamic dispatch under --trim); pg interval text is already @@ -431,9 +605,34 @@ function split_range_values(val::String) return left, right end +# A range bound is quoted whenever it contains whitespace, a comma, a quote, a +# backslash or a bracket — which every timestamp bound does. The quotes and +# their backslash escapes have to come off before the element parser sees it. +function unquote_range_bound(token::String) + cu = codeunits(token) + (length(cu) >= 2 && cu[1] == UInt8('"') && cu[end] == UInt8('"')) || return token + out = IOBuffer() + i = 2 + last = length(cu) - 1 + while i <= last + c = cu[i] + if c == UInt8('\\') && i < last + i += 1 + write(out, cu[i]) + elseif c == UInt8('"') && i < last && cu[i + 1] == UInt8('"') + write(out, UInt8('"')) + i += 1 + else + write(out, c) + end + i += 1 + end + return String(take!(out)) +end + function parse_range_value(token::String, typeId::Int, registry::Dict{Int, TypeInfo}) token == "" && return missing - return parse_value(typeId, token, registry) + return parse_value(typeId, unquote_range_bound(token), registry) end # construct over the standard range element types explicitly: PostgresRange{T} @@ -460,13 +659,32 @@ function parse_range(val::String, typeId::Int, registry::Dict{Int, TypeInfo}) lowercase(val) == "empty" && return _range_typed(T, missing, missing, false, false, true) lower_inclusive = val[1] == '[' upper_inclusive = val[end] == ']' - inner = val[2:end - 1] + # the brackets are ASCII but the bounds may not be: slice by character + # index, or a bound ending in a multibyte character throws StringIndexError + inner = val[2:prevind(val, lastindex(val))] left, right = split_range_values(inner) lower = parse_range_value(left, typeId, registry) upper = parse_range_value(right, typeId, registry) return _range_typed(T, lower, upper, lower_inclusive, upper_inclusive, false) end +# Range parsing for an element type discovered at registration time +# (`register_range!`). The builtin registry keeps using `parse_range`'s fixed +# dispatch, which is what stays resolvable under `--trim`; this path is only +# reachable once a user registers a range type at runtime. +function parse_range_of(::Type{T}, val::String, typeId::Int, registry::Dict{Int, TypeInfo}) where {T} + lowercase(val) == "empty" && return PostgresRange{T}(missing, missing, false, false, true) + lower_inclusive = val[1] == '[' + upper_inclusive = val[end] == ']' + # character slicing, as in parse_range: bounds may end in multibyte text + left, right = split_range_values(val[2:prevind(val, lastindex(val))]) + lower = parse_range_value(left, typeId, registry) + upper = parse_range_value(right, typeId, registry) + l = lower === missing ? missing : convert(T, lower) + u = upper === missing ? missing : convert(T, upper) + return PostgresRange{T}(l, u, lower_inclusive, upper_inclusive, false) +end + parse_array_scalar(typeId::Int, registry::Dict{Int, TypeInfo}, value::Missing) = missing parse_array_scalar(typeId::Int, registry::Dict{Int, TypeInfo}, value::AbstractString) = parse_value(typeId, String(value), registry) # explicit two-level walk: the self-recursive AbstractVector method is an @@ -632,7 +850,7 @@ function parse_value(typeId::Int, val::String, registry::Dict{Int, TypeInfo}) end return val == "t" elseif T == Char - return val[1] + return pg_parse_char(val) elseif T == DateTime if typeId == 1184 return parse_timestamptz(val) @@ -704,7 +922,7 @@ end StructUtils.lift(::AbstractPostgresStyle, ::Type{Int8}, s::String) = Parsers.parse(Int8, s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Bool}, s::String) = (s == "t" || s == "1"), nothing -StructUtils.lift(::AbstractPostgresStyle, ::Type{Char}, s::String) = s[1], nothing +StructUtils.lift(::AbstractPostgresStyle, ::Type{Char}, s::String) = pg_parse_char(s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Int16}, s::String) = Parsers.parse(Int16, s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Int32}, s::String) = Parsers.parse(Int32, s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Int64}, s::String) = Parsers.parse(Int64, s), nothing @@ -731,6 +949,7 @@ StructUtils.lift(::AbstractPostgresStyle, ::Type{Vector{Time}}, s::String) = par StructUtils.lift(::AbstractPostgresStyle, ::Type{Vector{DateTime}}, s::String) = parse_array(s, DateTime), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Vector{UUID}}, s::String) = parse_array(s, UUID), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Vector{Numeric}}, s::String) = parse_array(s, Numeric), nothing +StructUtils.lift(::AbstractPostgresStyle, ::Type{Vector{Char}}, s::String) = parse_array(s, Char), nothing # For array-typed fields the generic `make` takes its arraylike branch (applyeach # over the source) before consulting lifts — but our source is the wire STRING, diff --git a/src/array_parsing.jl b/src/array_parsing.jl index d7b5d67..ac39195 100644 --- a/src/array_parsing.jl +++ b/src/array_parsing.jl @@ -1,7 +1,7 @@ module ArrayParsing using Parsers, Dates, UUIDs -import ..pg_parse_date, ..pg_parse_time, ..pg_parse_datetime_any, ..parse_numeric, ..Numeric +import ..pg_parse_date, ..pg_parse_time, ..pg_parse_datetime_any, ..parse_numeric, ..Numeric, ..pg_parse_char const BRACKET_OPEN = UInt8('[') const BRACKET_CLOSE = UInt8(']') @@ -50,7 +50,12 @@ function parse_unquoted(code::Base.CodeUnits{UInt8, String}, pos::Ref{Int}) buf = UInt8[] while pos[] <= length(code) c = code[pos[]] - if c == COMMA || c == BRACE_CLOSE || c == BRACKET_CLOSE + # ']' is NOT a terminator: postgres only quotes elements containing + # '"', '\\', '{', '}', ',' or whitespace, so an unquoted ']' (file + # paths, "x[1]", embedded JSON) is ordinary element data. Treating it + # as a terminator silently truncated the element and dropped every + # element after it. + if c == COMMA || c == BRACE_CLOSE break elseif c == BACKSLASH pos[] += 1 @@ -90,6 +95,7 @@ function parse_scalar(token::String, inner_type::Type{T}, quoted::Bool) where {T inner_type === DateTime && return pg_parse_datetime_any(token) inner_type === UUID && return UUID(token) inner_type === Numeric && return parse_numeric(token) + inner_type === Char && return pg_parse_char(token) return token end @@ -101,6 +107,9 @@ function coerce_array(values::Vector{Any}, inner_type::Type{T}) where {T} value isa AbstractVector && (has_nested = true; break) value === missing && (has_missing = true) end + # nested arrays stay Vector{Any}: narrowing to the runtime element type + # would need a runtime-typed Vector construction, which `juliac --trim` + # cannot resolve. The values themselves are fully parsed either way. has_nested && return values if has_missing dest = Union{inner_type, Missing}[] @@ -127,7 +136,7 @@ end function parse_array_value(code::Base.CodeUnits{UInt8, String}, pos::Ref{Int}, inner_type::Type{T}) where {T} c = code[pos[]] - if c == BRACE_OPEN || c == BRACKET_OPEN + if c == BRACE_OPEN return parse_array_inner(code, pos, inner_type) elseif c == QUOTE token = parse_quoted(code, pos) @@ -145,7 +154,7 @@ function parse_array_inner(code::Base.CodeUnits{UInt8, String}, pos::Ref{Int}, i skip_ws(code, pos) pos[] > length(code) && break c = code[pos[]] - if c == BRACE_CLOSE || c == BRACKET_CLOSE + if c == BRACE_CLOSE pos[] += 1 break end @@ -159,6 +168,34 @@ function parse_array_inner(code::Base.CodeUnits{UInt8, String}, pos::Ref{Int}, i return coerce_array(values, inner_type) end +const EQUALS = UInt8('=') + +# PostgreSQL prefixes an array literal with explicit dimensions whenever a +# lower bound isn't 1: "[0:2]={a,b,c}", "[1:2][1:2]={{1,2},{3,4}}". Skip the +# prefix so the rest parses as an ordinary array literal — otherwise the '[' +# is taken for an array-open bracket and the real elements are lost. +function skip_dimension_prefix!(code::Base.CodeUnits{UInt8, String}, pos::Ref{Int}) + start = pos[] + n = length(code) + while pos[] <= n && code[pos[]] == BRACKET_OPEN + close_idx = pos[] + while close_idx <= n && code[close_idx] != BRACKET_CLOSE + close_idx += 1 + end + if close_idx > n + pos[] = start + return + end + pos[] = close_idx + 1 + end + if pos[] > start && pos[] <= n && code[pos[]] == EQUALS + pos[] += 1 + return + end + pos[] = start + return +end + function parse_array(str::String, inner_type::Type{T}) where {T} code = codeunits(str) pos = Ref{Int}(1) @@ -166,22 +203,19 @@ function parse_array(str::String, inner_type::Type{T}) where {T} if pos[] > length(code) return inner_type[] end + skip_dimension_prefix!(code, pos) + skip_ws(code, pos) + if pos[] > length(code) + return inner_type[] + end c = code[pos[]] - if c == BRACE_OPEN || c == BRACKET_OPEN + if c == BRACE_OPEN return parse_array_inner(code, pos, inner_type) end value = parse_scalar(str, inner_type, true) return coerce_array(Any[value], inner_type) end -function parse_to_julia_array(str::String, julia_type::Type) - try - return parse_array(str, julia_type) - catch - return str - end -end - -export parse_array, parse_to_julia_array +export parse_array end diff --git a/src/connection_string.jl b/src/connection_string.jl index f636e23..18f487d 100644 --- a/src/connection_string.jl +++ b/src/connection_string.jl @@ -2,6 +2,21 @@ module ConnectionString using URIs +""" + Postgres.ConnectionParams(; host="localhost", port=5432, user="", password=nothing, + dbname="", kwargs...) + +Structured connection options, an alternative to DSN strings: + + params = Postgres.ConnectionParams(host="127.0.0.1", user="postgres", dbname="postgres") + conn = DBInterface.connect(Postgres.Connection, params) + +Also produced by `Postgres.parse_dsn`. Supported keyword +arguments mirror the connection keywords: `application_name`, +`connect_timeout`, `sslmode`, `sslrootcert`, `sslcert`, `sslkey`, `sslcapath`, +`sslservername`, `statement_timeout`, `statement_cache_maxsize`, `debug`, and +`reconnect`. +""" struct ConnectionParams host::String port::Int @@ -15,22 +30,33 @@ struct ConnectionParams sslcert::Union{String, Nothing} sslkey::Union{String, Nothing} sslcapath::Union{String, Nothing} + sslservername::Union{String, Nothing} statement_timeout::Union{Int, Nothing} statement_cache_maxsize::Int debug::Bool reconnect::Bool end -function ConnectionParams(; host::String="localhost", port::Int=5432, user::String="", password::Union{String, Nothing}=nothing, dbname::String="", application_name::Union{String, Nothing}=nothing, connect_timeout::Union{Int, Nothing}=nothing, sslmode::Union{String, Nothing}=nothing, sslrootcert::Union{String, Nothing}=nothing, sslcert::Union{String, Nothing}=nothing, sslkey::Union{String, Nothing}=nothing, sslcapath::Union{String, Nothing}=nothing, statement_timeout::Union{Int, Nothing}=nothing, statement_cache_maxsize::Int=100, debug::Bool=false, reconnect::Bool=false) - return ConnectionParams(host, port, user, password, dbname, application_name, connect_timeout, sslmode, sslrootcert, sslcert, sslkey, sslcapath, statement_timeout, statement_cache_maxsize, debug, reconnect) +function ConnectionParams(; host::String="localhost", port::Int=5432, user::String="", password::Union{String, Nothing}=nothing, dbname::String="", application_name::Union{String, Nothing}=nothing, connect_timeout::Union{Int, Nothing}=nothing, sslmode::Union{String, Nothing}=nothing, sslrootcert::Union{String, Nothing}=nothing, sslcert::Union{String, Nothing}=nothing, sslkey::Union{String, Nothing}=nothing, sslcapath::Union{String, Nothing}=nothing, sslservername::Union{String, Nothing}=nothing, statement_timeout::Union{Int, Nothing}=nothing, statement_cache_maxsize::Int=100, debug::Bool=false, reconnect::Bool=false) + return ConnectionParams(host, port, user, password, dbname, application_name, connect_timeout, sslmode, sslrootcert, sslcert, sslkey, sslcapath, sslservername, statement_timeout, statement_cache_maxsize, debug, reconnect) end +function Base.show(io::IO, params::ConnectionParams) + password = params.password === nothing ? "nothing" : "***" + print(io, "Postgres.ConnectionParams(host=", repr(params.host), + ", port=", params.port, ", user=", repr(params.user), + ", password=", password, ", dbname=", repr(params.dbname), ")") +end +Base.show(io::IO, ::MIME"text/plain", params::ConnectionParams) = show(io, params) + default_user() = get(ENV, "PGUSER", get(ENV, "USER", get(ENV, "USERNAME", ""))) -function parse_optional_int(value::Union{String, Nothing}) +function parse_optional_int(value::Union{String, Nothing}, key::String="") value === nothing && return nothing isempty(value) && return nothing - return parse(Int, value) + parsed = tryparse(Int, value) + parsed === nothing && throw(ArgumentError("invalid value \"$value\" for connection parameter \"$key\"; expected an integer")) + return parsed end function connection_defaults() @@ -61,7 +87,82 @@ function apply_env_defaults!(values::Dict{String, String}) return values end +const KNOWN_PARAMS = Set([ + "host", "port", "user", "password", "dbname", "application_name", + "connect_timeout", "sslmode", "sslrootcert", "sslcert", "sslkey", + "sslcapath", "sslservername", "statement_timeout", + "statement_cache_maxsize", "debug", "reconnect", +]) + +# libpq keywords this driver doesn't implement. They are accepted and ignored +# rather than rejected: managed-PostgreSQL providers routinely include them in +# the connection URI they hand users, and failing on a DSN that names a real +# libpq option would be worse than not honoring it. +const IGNORED_PARAMS = Set([ + "channel_binding", "target_session_attrs", "options", "gssencmode", + "gsslib", "krbsrvname", "sslnegotiation", "sslcompression", "sslcrl", + "sslcrldir", "sslpassword", "requiressl", "requirepeer", "hostaddr", + "client_encoding", "passfile", "service", "fallback_application_name", + "keepalives", "keepalives_idle", "keepalives_interval", "keepalives_count", + "tcp_user_timeout", "load_balance_hosts", "replication", +]) + +function parse_bool_param(value::Union{String, Nothing}, default::Bool, key::String) + value === nothing && return default + # an empty value means "unset", as it does for the integer parameters + isempty(value) && return default + lowered = lowercase(value) + lowered in ("1", "on", "true", "yes") && return true + lowered in ("0", "off", "false", "no") && return false + throw(ArgumentError("invalid value \"$value\" for connection parameter \"$key\"; expected a boolean (on/off, true/false, yes/no, 1/0)")) +end + +# Ignored keywords that change security or connection-selection behavior when +# set: silently dropping "channel_binding=require" or a CRL file would leave +# the caller believing a protection is in place. The values listed are the +# no-op defaults for each keyword; any other value is rejected. +const SECURITY_SENSITIVE_IGNORED = Dict( + "channel_binding" => ("", "prefer", "disable"), + "target_session_attrs" => ("", "any"), + "options" => ("",), + "gssencmode" => ("", "prefer", "disable"), + "sslnegotiation" => ("", "postgres"), + "sslcompression" => ("", "0"), + "sslcrl" => ("",), + "sslcrldir" => ("",), + "sslpassword" => ("",), + "requiressl" => ("", "0"), + "requirepeer" => ("",), + "hostaddr" => ("",), + "client_encoding" => ("", "UTF8", "UTF-8", "utf8", "utf-8"), + "passfile" => ("",), + "service" => ("",), + "load_balance_hosts" => ("", "disable"), + "replication" => ("", "0", "false", "off"), +) + +function check_ignored_param(key::String, value::String) + inert = get(SECURITY_SENSITIVE_IGNORED, key, nothing) + inert === nothing && return + value in inert && return + throw(ArgumentError("connection parameter \"$key=$value\" is not supported by Postgres.jl and cannot be safely ignored")) +end + +# An unrecognized key is almost always a typo, and silently dropping it is +# dangerous: "ssl_mode=verify-full" would leave sslmode unset and fall back to +# an unverified connection while the caller believes otherwise. libpq errors +# on unknown keywords for the same reason. +function check_known_params(values::Dict{String, String}) + for (key, value) in values + (key in KNOWN_PARAMS || key in IGNORED_PARAMS) || + throw(ArgumentError("unrecognized connection parameter \"$key\"; recognized parameters are $(join(sort!(collect(KNOWN_PARAMS)), ", "))")) + key in IGNORED_PARAMS && check_ignored_param(key, value) + end + return values +end + function params_from_values(values::Dict{String, String}) + check_known_params(values) merged = connection_defaults() merge!(merged, values) apply_env_defaults!(merged) @@ -70,19 +171,27 @@ function params_from_values(values::Dict{String, String}) return ConnectionParams( ; host=get(merged, "host", "localhost"), - port=parse(Int, get(merged, "port", "5432")), + # empty values (an unset PGPORT expanded into the environment) fall + # back to the default rather than failing to parse + port=something(parse_optional_int(get(merged, "port", nothing), "port"), 5432), user=user, password=get(merged, "password", nothing), dbname=dbname, application_name=get(merged, "application_name", nothing), - connect_timeout=parse_optional_int(get(merged, "connect_timeout", nothing)), + connect_timeout=parse_optional_int(get(merged, "connect_timeout", nothing), "connect_timeout"), + # deliberately NOT empty-tolerant, matching libpq: an unexpanded + # ${PGSSLMODE} that was meant to be verify-full must fail loudly + # rather than fall back to the unauthenticated default sslmode=haskey(merged, "sslmode") ? lowercase(merged["sslmode"]) : nothing, sslrootcert=get(merged, "sslrootcert", nothing), sslcert=get(merged, "sslcert", nothing), sslkey=get(merged, "sslkey", nothing), sslcapath=get(merged, "sslcapath", nothing), - statement_timeout=parse_optional_int(get(merged, "statement_timeout", nothing)), - statement_cache_maxsize=parse(Int, get(merged, "statement_cache_maxsize", "100")), + sslservername=get(merged, "sslservername", nothing), + statement_timeout=parse_optional_int(get(merged, "statement_timeout", nothing), "statement_timeout"), + statement_cache_maxsize=something(parse_optional_int(get(merged, "statement_cache_maxsize", nothing), "statement_cache_maxsize"), 100), + debug=parse_bool_param(get(merged, "debug", nothing), false, "debug"), + reconnect=parse_bool_param(get(merged, "reconnect", nothing), false, "reconnect"), ) end @@ -94,55 +203,80 @@ function parse_keyword_dsn(dsn::String) i = nextind(dsn, i) end i > lastindex(dsn) && break + key_start = i - while i <= lastindex(dsn) && dsn[i] != '=' + while i <= lastindex(dsn) && dsn[i] != '=' && + !isspace(dsn[i]) && dsn[i] != ';' i = nextind(dsn, i) end - i > lastindex(dsn) && break - key = lowercase(strip(dsn[key_start:prevind(dsn, i)])) + key_end = prevind(dsn, i) + key = key_end < key_start ? "" : lowercase(String(dsn[key_start:key_end])) + isempty(key) && throw(ArgumentError("empty connection parameter name")) + while i <= lastindex(dsn) && isspace(dsn[i]) + i = nextind(dsn, i) + end + (i <= lastindex(dsn) && dsn[i] == '=') || + throw(ArgumentError("connection parameter \"$key\" is missing '='")) i = nextind(dsn, i) while i <= lastindex(dsn) && isspace(dsn[i]) i = nextind(dsn, i) end + buf = IOBuffer() if i <= lastindex(dsn) && dsn[i] == '\'' i = nextind(dsn, i) + closed_quote = false while i <= lastindex(dsn) c = dsn[i] if c == '\\' i = nextind(dsn, i) - if i <= lastindex(dsn) - write(buf, dsn[i]) - i = nextind(dsn, i) - end + i <= lastindex(dsn) || + throw(ArgumentError("dangling escape in value for connection parameter \"$key\"")) + write(buf, dsn[i]) + i = nextind(dsn, i) elseif c == '\'' i = nextind(dsn, i) + closed_quote = true break else write(buf, c) i = nextind(dsn, i) end end + closed_quote || + throw(ArgumentError("unterminated quoted value for connection parameter \"$key\"")) + if i <= lastindex(dsn) && !isspace(dsn[i]) && dsn[i] != ';' + throw(ArgumentError("unexpected text after quoted value for connection parameter \"$key\"")) + end else while i <= lastindex(dsn) && !isspace(dsn[i]) && dsn[i] != ';' c = dsn[i] if c == '\\' i = nextind(dsn, i) - if i <= lastindex(dsn) - write(buf, dsn[i]) - i = nextind(dsn, i) - end + i <= lastindex(dsn) || + throw(ArgumentError("dangling escape in value for connection parameter \"$key\"")) + write(buf, dsn[i]) + i = nextind(dsn, i) else write(buf, c) i = nextind(dsn, i) end end end - !isempty(key) && (values[key] = String(take!(buf))) + values[key] = String(take!(buf)) end return values end +""" + Postgres.parse_dsn(dsn) -> ConnectionParams + +Parse a libpq-style keyword string (`"host=127.0.0.1 user=postgres"`) or a +PostgreSQL URI (`"postgresql://user:pass@host:5432/dbname"`) into +`ConnectionParams`. Unset options fall back +to the `PGHOST`, `PGPORT`, `PGUSER`, `PGPASSWORD`, `PGDATABASE`, `PGAPPNAME`, +`PGCONNECT_TIMEOUT`, and `PGSSL*` environment variables, then to defaults. +""" function parse_dsn(dsn::String) lowered = lowercase(dsn) (startswith(lowered, "postgres://") || startswith(lowered, "postgresql://")) && return parse_uri(dsn) @@ -178,8 +312,12 @@ function parse_uri(uri::String) query = String(parsed.query) if !isempty(query) params = URIs.queryparams(query) - for key in ("host", "port", "user", "password", "dbname", "application_name", "connect_timeout", "sslmode", "sslrootcert", "sslcert", "sslkey", "sslcapath", "statement_timeout", "statement_cache_maxsize") - haskey(params, key) && (values[key] = params[key]) + for (key, value) in params + (key in KNOWN_PARAMS || key in IGNORED_PARAMS) || + throw(ArgumentError("unrecognized connection parameter \"$key\" in URI; recognized parameters are $(join(sort!(collect(KNOWN_PARAMS)), ", "))")) + key in IGNORED_PARAMS && check_ignored_param(key, value) + # keys we accept but don't implement must not reach params_from_values + key in KNOWN_PARAMS && (values[key] = value) end end return params_from_values(values) diff --git a/src/execute.jl b/src/execute.jl index 93f9715..41aa995 100644 --- a/src/execute.jl +++ b/src/execute.jl @@ -16,7 +16,21 @@ end Base.size(r::Result) = (length(r.rows),) Base.getindex(r::Result, i::Integer) = r.rows[i] + +""" + Postgres.command_tag(result) -> Union{String, Nothing} + +The PostgreSQL command completion tag for the executed statement, e.g. +`"SELECT 5"`, `"INSERT 0 2"`, or `"UPDATE 3"`. +""" command_tag(r::Result) = r.command_tag + +""" + Postgres.rows_affected(result) -> Union{Int, Nothing} + +The number of rows the statement affected (parsed from the command tag), or +`nothing` when the statement doesn't report one. +""" rows_affected(r::Result) = r.rows_affected getdata(r::ResultRow) = getfield(r, :data) @@ -36,8 +50,6 @@ Tables.getcolumn(r::ResultRow, nm::Symbol) = Tables.getcolumn(r, getlookup(r)[nm Tables.schema(r::Result) = Tables.Schema(r.names, r.types) -# DBInterface.lastrowid(result::Result) = API.lastrowid(result.result) - function DBInterface.close!(::Result) return end @@ -76,6 +88,8 @@ mutable struct Cursor{S <: API.AbstractPostgresStyle} index::Int done::Bool rowcount::Int + portal_closed::Bool + owned_statement::Union{Nothing, Statement{S}} owns_transaction::Bool end @@ -92,34 +106,52 @@ function checkstmt(stmt::Statement) checkconn(stmt.conn) stmt.closed && throw(PostgresInterfaceError("statement has been closed")) if stmt.cached - if stmt.generation != stmt.conn.generation || !haskey(stmt.conn.statements, stmt.sql) - # if the connection was reset, we need to re-prepare the statement - stmt.name = API.prepare(stmt.conn.socket, stmt.sql, stmt.conn.debug) - stmt.conn.statements[stmt.sql] = stmt + cached = get(stmt.conn.statements, stmt.sql, nothing) + if stmt.generation != stmt.conn.generation || cached === nothing || + cached.name != stmt.name || cached.generation != stmt.conn.generation + # The cache entry was evicted, cleared, replaced, or lost on + # reconnect. Keep this caller handle valid as an independent + # server statement; do not silently repopulate or overrun the + # current cache policy. + stmt.name = API.prepare(stmt.conn.socket, stmt.sql, stmt.conn.debug, + stmt.conn.server_parameters) stmt.generation = stmt.conn.generation + stmt.cached = false + else + touch_statement!(stmt.conn, cached) + stmt.last_used = cached.last_used end - touch_statement!(stmt.conn, stmt) elseif stmt.generation != stmt.conn.generation - stmt.name = API.prepare(stmt.conn.socket, stmt.sql, stmt.conn.debug) + stmt.name = API.prepare(stmt.conn.socket, stmt.sql, stmt.conn.debug, + stmt.conn.server_parameters) stmt.generation = stmt.conn.generation end - !stmt.cached && touch_statement!(stmt.conn, stmt) + stmt.cached || touch_statement!(stmt.conn, stmt) return end +function statement_handle(stmt::Statement) + params = Union{String, Missing}[missing for _ = 1:stmt.nparams] + return Statement{_style_type(stmt.conn)}( + stmt.conn, stmt.name, stmt.sql, stmt.nfields, stmt.names, stmt.typeIds, + stmt.nparams, params, false, true, stmt.generation, stmt.last_used) +end + function DBInterface.prepare(conn::Connection, sql::AbstractString; debug::Bool=false) sql_str = String(sql) + actual_debug = debug || conn.debug @lock conn.lock begin checkconn(conn) # check if we've already prepared this sql before if haskey(conn.statements, sql_str) - stmt = conn.statements[sql_str] - touch_statement!(conn, stmt) - return stmt + cached = conn.statements[sql_str] + touch_statement!(conn, cached) + return statement_handle(cached) end if conn.statement_cache_maxsize == 0 - name = API.prepare(conn.socket, sql_str, debug) - nparams, names, types = API.describeprepared(conn.socket, name, debug) + name = API.prepare(conn.socket, sql_str, actual_debug, conn.server_parameters) + nparams, names, types = API.describeprepared( + conn.socket, name, actual_debug, conn.server_parameters) params = Union{String, Missing}[missing for _ = 1:nparams] last_used = next_statement_clock!(conn) return Statement{_style_type(conn)}(conn, name, sql_str, length(names), names, types, nparams, params, false, false, conn.generation, last_used) @@ -129,13 +161,14 @@ function DBInterface.prepare(conn::Connection, sql::AbstractString; debug::Bool= evict_lru_statement!(conn) end # new statement to prepare - name = API.prepare(conn.socket, sql_str, debug) - nparams, names, types = API.describeprepared(conn.socket, name, debug) + name = API.prepare(conn.socket, sql_str, actual_debug, conn.server_parameters) + nparams, names, types = API.describeprepared( + conn.socket, name, actual_debug, conn.server_parameters) params = Union{String, Missing}[missing for _ = 1:nparams] last_used = next_statement_clock!(conn) - stmt = Statement{_style_type(conn)}(conn, name, sql_str, length(names), names, types, nparams, params, false, true, conn.generation, last_used) - conn.statements[sql_str] = stmt - return stmt + cached = Statement{_style_type(conn)}(conn, name, sql_str, length(names), names, types, nparams, params, false, true, conn.generation, last_used) + conn.statements[sql_str] = cached + return statement_handle(cached) end end @@ -143,28 +176,85 @@ function DBInterface.close!(stmt::Statement) @lock stmt.conn.lock begin stmt.closed && return if !isopen(stmt.conn.socket) - stmt.cached && haskey(stmt.conn.statements, stmt.sql) && delete!(stmt.conn.statements, stmt.sql) stmt.closed = true return end - stmt.cached && haskey(stmt.conn.statements, stmt.sql) && stmt.conn.statements[stmt.sql] === stmt && delete!(stmt.conn.statements, stmt.sql) - API.close_statement(stmt.conn.socket, stmt.name, stmt.conn.debug) + # Cached backend statements belong to the connection cache, not to a + # caller handle. Closing one handle must not invalidate independent + # handles or remove the cache entry. + stmt.cached || API.close_statement(stmt.conn.socket, stmt.name, stmt.conn.debug, + stmt.conn.server_parameters) stmt.closed = true end return end +# Finish the transaction a cursor opened for itself. If the connection died, +# clear the bookkeeping directly rather than trying to COMMIT: leaving +# in_transaction set would make checkconn refuse to reconnect forever, and the +# server-side transaction is already gone with the session. +function finish_cursor_transaction!(conn::Connection) + if !isopen(conn) + clear_transaction_state!(conn) + return + end + in_transaction(conn) && commit(conn) + return +end + +# same, for the failure path: roll back rather than commit +function abort_cursor_transaction!(conn::Connection) + if !isopen(conn) + clear_transaction_state!(conn) + return + end + in_transaction(conn) && rollback(conn) + return +end + function DBInterface.close!(cursor::Cursor) owns_transaction = cursor.owns_transaction - @lock cursor.conn.lock begin - if !cursor.done - API.writemessages(cursor.conn.socket, cursor.conn.debug, ('C', UInt8('P'), cursor.portal), ('S',)) - API.waitfor(cursor.conn.socket, cursor.conn.debug, '3', 'Z') + owned_statement = cursor.owned_statement + cursor.owned_statement = nothing + closed_cleanly = false + try + @lock cursor.conn.lock begin + if !cursor.portal_closed && isopen(cursor.conn.socket) + API.writemessages(cursor.conn.socket, cursor.conn.debug, ('C', UInt8('P'), cursor.portal), ('S',)) + API.waitfor(cursor.conn.socket, cursor.conn.debug, '3', 'Z'; + server_parameters=cursor.conn.server_parameters) + cursor.portal_closed = true + end + cursor.done = true + empty!(cursor.buffer) + owned_statement === nothing || DBInterface.close!(owned_statement) + end + closed_cleanly = true + finally + # take responsibility exactly once: closing an already-closed cursor + # must not commit whatever transaction the caller has open now + cursor.owns_transaction = false + if owns_transaction + if closed_cleanly + # a COMMIT failure here means the caller's writes did not land, + # so it must propagate rather than be swallowed + finish_cursor_transaction!(cursor.conn) + else + try + abort_cursor_transaction!(cursor.conn) + catch + # already unwinding; don't mask the original error + end + end + end + if !closed_cleanly && owned_statement !== nothing && !owned_statement.closed + try + DBInterface.close!(owned_statement) + catch + # Preserve the portal-close error already in flight. + end end - cursor.done = true - empty!(cursor.buffer) end - owns_transaction && in_transaction(cursor.conn) && commit(cursor.conn) return end Base.close(cursor::Cursor) = DBInterface.close!(cursor) @@ -231,6 +321,15 @@ function build_params(params, nparams::Int, sql::AbstractString) return dest end +function build_unchecked_params(params) + dest = Union{String, Missing}[] + params === nothing && return dest + for param in params + push!(dest, _param(param)) + end + return dest +end + mutable struct RowClosure data::Vector{Any} types::Vector{Type} @@ -246,8 +345,13 @@ end @inbounds f.types[f.i] = Union{f.types[f.i], Missing} @inbounds f.data[f.i] = missing else - if v isa AbstractVector{>:Missing} - @inbounds f.types[f.i] = typeof(v) + # the OID-derived column type is a default; widen the schema whenever + # the parsed value doesn't fit it (nullable or nested array elements, + # values from a custom parser), so Tables.schema stays truthful. + # Widening must be monotone — replacing would let a later row narrow + # the schema back to a type earlier rows don't satisfy. + @inbounds if !(v isa f.types[f.i]) + @inbounds f.types[f.i] = Union{f.types[f.i], typeof(v)} end @inbounds f.data[f.i] = v end @@ -273,6 +377,8 @@ function read_portal_batch!(cursor::Cursor) rows = ResultRow[] error_msg = nothing consumer_error = nothing + copy_in_statement = false + copy_out_statement = false done = false try while true @@ -299,16 +405,32 @@ function read_portal_batch!(cursor::Cursor) elseif mt == UInt8('C') API.skipbytes!(conn.socket, len) done = true + elseif mt == UInt8('G') + # CopyInResponse: a COPY ... FROM STDIN statement was used with + # a cursor. Abort the copy with CopyFail so the stream returns + # to ready instead of deadlocking; a clear error is thrown + # below. A fresh Sync must follow: the one sent with + # Bind/Execute was ignored during copy-in mode. + API.skipbytes!(conn.socket, len) + copy_in_statement = true + API.writemessages(conn.socket, conn.debug, ('f', "COPY FROM STDIN is not supported via cursor"), ('S',)) + elseif mt == UInt8('H') || mt == UInt8('d') || mt == UInt8('c') + # CopyOutResponse/CopyData/CopyDone: drain the copy-out stream + mt == UInt8('H') && (copy_out_statement = true) + API.skipbytes!(conn.socket, len) elseif mt == UInt8('N') notice = API.noticeResponse(len, conn.socket) API.notice_callback(conn.style, notice) elseif mt == UInt8('A') notification = API.notificationResponse(len, conn.socket) API.notification_callback(conn.style, notification) + elseif mt == UInt8('S') + API.parameterStatus!(conn.server_parameters, len, conn.socket) elseif mt == UInt8('E') error_msg = API.errorResponse(len, conn.socket, conn.debug) elseif mt == UInt8('Z') - API.skipbytes!(conn.socket, len) + status = API.read_ready_status(conn.socket, len) + conn.server_in_transaction = API.in_transaction_status(status) break else API.skipbytes!(conn.socket, len) @@ -320,6 +442,18 @@ function read_portal_batch!(cursor::Cursor) error_msg === nothing || throw(error_msg) rethrow() end + # same precedence as the execute path: for copy-in the server error is + # just the CopyFail artifact; for copy-out a server error is a genuine + # mid-stream failure and wins over the misuse error + if copy_in_statement + cursor.done = true + throw(PostgresInterfaceError("COPY ... FROM STDIN is not supported via cursor; use Postgres.copy_from")) + end + if copy_out_statement + cursor.done = true + error_msg === nothing || throw(error_msg) + throw(PostgresInterfaceError("COPY ... TO STDOUT is not supported via cursor; use Postgres.copy_to")) + end error_msg === nothing || throw(error_msg) consumer_error === nothing || throw(consumer_error) cursor.buffer = rows @@ -349,8 +483,9 @@ function Base.iterate(cursor::Cursor, state=nothing) return row, nothing end -function DBInterface.execute(stmt::Statement, params=nothing, ::Type{T}=Any; debug::Bool=false, binary::Bool=false) where {T} +function DBInterface.execute(stmt::Statement, params=nothing, ::Type{T}=Any; debug::Bool=false) where {T} style = stmt.conn.style + actual_debug = debug || stmt.conn.debug log_enabled = API.query_logging_enabled(style) start_ns = log_enabled ? time_ns() : 0 result = nothing @@ -358,22 +493,40 @@ function DBInterface.execute(stmt::Statement, params=nothing, ::Type{T}=Any; deb @lock stmt.conn.lock begin # check that connection/statement are ok checkstmt(stmt) - bind_params!(stmt.params, params, stmt.sql) - # isa-split the socket union with per-branch typeasserts (identical calls in - # both branches get tail-merged back into one dynamic call by the optimizer), - # so the exec call resolves statically under `juliac --trim` - socket = stmt.conn.socket - e = if socket isa Reseau.TCP.Conn - API.exec(style, socket::Reseau.TCP.Conn, stmt.name, stmt.params, stmt.names, stmt.typeIds, stmt.conn.type_registry, debug, 0) - else - API.exec(style, socket::Reseau.TLS.Conn, stmt.name, stmt.params, stmt.names, stmt.typeIds, stmt.conn.type_registry, debug, 0) + e = try + bind_params!(stmt.params, params, stmt.sql) + # isa-split the socket union with per-branch typeasserts + # (identical calls in both branches get tail-merged back into + # one dynamic call by the optimizer). + socket = stmt.conn.socket + e = if socket isa Reseau.TCP.Conn + API.exec(style, socket::Reseau.TCP.Conn, stmt.name, stmt.params, + stmt.names, stmt.typeIds, stmt.conn.type_registry, + actual_debug, 0, stmt.conn.server_parameters) + else + API.exec(style, socket::Reseau.TLS.Conn, stmt.name, stmt.params, + stmt.names, stmt.typeIds, stmt.conn.type_registry, + actual_debug, 0, stmt.conn.server_parameters) + end + e + finally + # Bound strings can contain credentials or personal data. The + # vector must be cleared even if local parameter validation or + # the server Bind fails. + fill!(stmt.params, missing) + end + # in a finally: a failed statement still drained to ReadyForQuery + # and its status is authoritative. + try + result = T === Any ? makeresult(e) : StructUtils.arraylike(T) ? StructUtils.make(T, e, style) : only(StructUtils.make(Vector{T}, e, style)) + finally + stmt.conn.server_in_transaction = API.in_transaction_status(e.tx_status[]) end - result = T === Any ? makeresult(e) : StructUtils.arraylike(T) ? StructUtils.make(T, e, style) : only(StructUtils.make(Vector{T}, e, style)) end - log_enabled && API.query_logger(style, :execute, (sql=stmt.sql, params=params, duration_ns=time_ns() - start_ns, success=true)) + log_enabled && query_log_safely(style, :execute, (sql=stmt.sql, params=params, duration_ns=time_ns() - start_ns, success=true)) return result catch err - log_enabled && API.query_logger(style, :execute, (sql=stmt.sql, params=params, duration_ns=time_ns() - start_ns, success=false, error=err)) + log_enabled && query_log_safely(style, :execute, (sql=stmt.sql, params=params, duration_ns=time_ns() - start_ns, success=false, error=err)) rethrow() end end @@ -381,50 +534,142 @@ end function DBInterface.execute(conn::Connection, sql::AbstractString, params=nothing, ::Type{T}=Any; debug::Bool=false) where {T} sql_str = String(sql) style = conn.style + actual_debug = debug || conn.debug log_enabled = API.query_logging_enabled(style) start_ns = log_enabled ? time_ns() : 0 result = nothing try @lock conn.lock begin checkconn(conn) - stmtname = API.prepare(conn.socket, sql_str, debug; name="") - nparams, names, types = API.describeprepared(conn.socket, stmtname, debug) - params_vec = build_params(params, nparams, sql_str) + params_vec = build_unchecked_params(params) # see the statement-execute method: socket union isa-split for --trim socket = conn.socket e = if socket isa Reseau.TCP.Conn - API.exec(style, socket::Reseau.TCP.Conn, stmtname, params_vec, names, types, conn.type_registry, debug, 0) + API.exec_unnamed(style, socket::Reseau.TCP.Conn, sql_str, params_vec, + conn.type_registry, actual_debug, 0, + conn.server_parameters) else - API.exec(style, socket::Reseau.TLS.Conn, stmtname, params_vec, names, types, conn.type_registry, debug, 0) + API.exec_unnamed(style, socket::Reseau.TLS.Conn, sql_str, params_vec, + conn.type_registry, actual_debug, 0, + conn.server_parameters) + end + # in a finally, as in the statement-execute method above + try + result = T === Any ? makeresult(e) : StructUtils.arraylike(T) ? StructUtils.make(T, e, style) : only(StructUtils.make(Vector{T}, e, style)) + finally + conn.server_in_transaction = API.in_transaction_status(e.tx_status[]) end - result = T === Any ? makeresult(e) : StructUtils.arraylike(T) ? StructUtils.make(T, e, style) : only(StructUtils.make(Vector{T}, e, style)) end - log_enabled && API.query_logger(style, :execute, (sql=sql_str, params=params, duration_ns=time_ns() - start_ns, success=true)) + log_enabled && query_log_safely(style, :execute, (sql=sql_str, params=params, duration_ns=time_ns() - start_ns, success=true)) return result catch err - log_enabled && API.query_logger(style, :execute, (sql=sql_str, params=params, duration_ns=time_ns() - start_ns, success=false, error=err)) + log_enabled && query_log_safely(style, :execute, (sql=sql_str, params=params, duration_ns=time_ns() - start_ns, success=false, error=err)) rethrow() end end -function cursor(stmt::Statement, params=nothing; fetchsize::Integer=1000, owns_transaction::Bool=false) +# DBInterface's generic connection overload prepares a statement without +# closing it. That leaks named server statements when this driver's cache is +# disabled. Keep the handle lifetime explicit for both bulk fallbacks. +function DBInterface.executemany(conn::Connection, sql::AbstractString, params) + stmt = DBInterface.prepare(conn, sql) + try + return DBInterface.executemany(stmt, params) + finally + DBInterface.close!(stmt) + end +end + +function DBInterface.executemultiple(conn::Connection, sql::AbstractString, params) + stmt = DBInterface.prepare(conn, sql) + try + return DBInterface.executemultiple(stmt, params) + finally + DBInterface.close!(stmt) + end +end + +function cursor(stmt::Statement, params=nothing; fetchsize::Integer=1000, + owns_transaction::Bool=false, owns_statement::Bool=false) conn = stmt.conn - @lock conn.lock begin - checkstmt(stmt) - bind_params!(stmt.params, params, stmt.sql) - portal = string(UUIDs.uuid4()) - types = Type[API.juliatype(x -> x, i, conn.type_registry) for i in stmt.typeIds] - lookup = Dict(x => i for (i, x) in enumerate(stmt.names)) - cursor = Cursor{_style_type(conn)}(conn, portal, stmt.names, stmt.typeIds, types, lookup, max(1, Int(fetchsize)), ResultRow[], 1, false, 0, owns_transaction) - API.writemessages(conn.socket, conn.debug, ('B', portal, stmt.name, Int16(0), Int16(length(stmt.params)), API.Params(stmt.params), Int16(0)), ('E', portal, Int32(cursor.fetchsize)), ('S',)) - read_portal_batch!(cursor) - return cursor + started_here = false + if !owns_transaction + already_in_tx = @lock conn.lock (conn.in_transaction || conn.server_in_transaction) + if !already_in_tx + start_transaction(conn) + owns_transaction = true + started_here = true + end + end + try + @lock conn.lock begin + checkstmt(stmt) + try + bind_params!(stmt.params, params, stmt.sql) + portal = string(UUIDs.uuid4()) + types = Type[API.juliatype(x -> x, i, conn.type_registry) for i in stmt.typeIds] + lookup = Dict(x => i for (i, x) in enumerate(stmt.names)) + owned_stmt = owns_statement ? stmt : nothing + cursor = Cursor{_style_type(conn)}(conn, portal, stmt.names, + stmt.typeIds, types, lookup, max(1, Int(fetchsize)), + ResultRow[], 1, false, 0, false, owned_stmt, + owns_transaction) + API.writemessages(conn.socket, conn.debug, ('B', portal, stmt.name, Int16(0), Int16(length(stmt.params)), API.Params(stmt.params), Int16(0)), ('E', portal, Int32(cursor.fetchsize)), ('S',)) + read_portal_batch!(cursor) + return cursor + finally + fill!(stmt.params, missing) + end + end + catch + if started_here + try + abort_cursor_transaction!(conn) + catch + end + end + rethrow() end end +""" + Postgres.cursor(conn, sql, params=nothing; fetchsize=1000) -> Cursor + Postgres.cursor(stmt, params=nothing; fetchsize=1000) -> Cursor + +Execute a query and stream its result rows in batches of `fetchsize` instead +of materializing them all at once. The returned cursor iterates rows; close it +with `DBInterface.close!(cursor)`. A cursor requires a transaction: one is +started (and committed on close) if the connection isn't already in one. +""" function cursor(conn::Connection, sql::AbstractString, params=nothing; fetchsize::Integer=1000, debug::Bool=false) owns_transaction = false - in_transaction(conn) || (start_transaction(conn); owns_transaction = true) - stmt = DBInterface.prepare(conn, sql; debug=debug) - return cursor(stmt, params; fetchsize=fetchsize, owns_transaction=owns_transaction) + stmt = nothing + # a transaction opened with raw SQL counts as "already in one": the server + # status sees it even though the client flag doesn't, and owning it here + # would mean committing the caller's transaction on cursor close + already_in_tx = @lock conn.lock (conn.in_transaction || conn.server_in_transaction) + already_in_tx || (start_transaction(conn); owns_transaction = true) + try + stmt = DBInterface.prepare(conn, sql; debug=debug) + return cursor(stmt, params; fetchsize=fetchsize, owns_transaction=owns_transaction, + owns_statement=true) + catch + if stmt !== nothing && !stmt.closed + try + DBInterface.close!(stmt) + catch + end + end + # Don't leave the transaction we started dangling on a failed cursor; + # if the connection died, clear the state directly (a ROLLBACK can't be + # delivered, and leaving it set would block reconnect forever) + if owns_transaction + try + abort_cursor_transaction!(conn) + catch + # already unwinding; don't mask the original error + end + end + rethrow() + end end diff --git a/test/postgres_trim_queries.jl b/test/postgres_trim_queries.jl deleted file mode 100644 index 560502a..0000000 --- a/test/postgres_trim_queries.jl +++ /dev/null @@ -1,176 +0,0 @@ -using Dates -using DBInterface -using Postgres -using StructUtils -using UUIDs - -struct TrimId - profile_id::Int32 -end - -struct TrimCount - count::Int32 -end - -struct TrimName - display_name::String -end - -StructUtils.@tags struct TrimProfile - profileId::Int32 &(postgres=(name=:profile_id,),) - displayName::String &(postgres=(name=:display_name,),) - createdAt::DateTime &(postgres=(name=:created_at,),) - active::Bool - score::Union{Missing, Int32} - uid::UUID - flags::Vector{Int32} -end - -function _postgres_trim_connect() - host = get(ENV, "POSTGRES_TRIM_HOST", "127.0.0.1") - port = parse(Int, get(ENV, "POSTGRES_TRIM_PORT", "5432")) - user = get(ENV, "POSTGRES_TRIM_USER", "postgres") - dbname = get(ENV, "POSTGRES_TRIM_DBNAME", "postgres") - return DBInterface.connect( - Postgres.Connection, - host, - user, - nothing; - dbname=dbname, - port=port, - sslmode="disable", - connect_timeout=2, - application_name="postgres_trim", - statement_cache_maxsize=4, - ) -end - -function _assert_trim_profile(profile::TrimProfile, id::Int32, name::String)::Nothing - profile.profileId == id || error("unexpected profile id") - profile.displayName == name || error("unexpected profile name") - profile.active || error("expected active profile") - profile.uid == UUID("12345678-1234-5678-1234-567812345678") || id != 1 || error("unexpected UUID") - !isempty(profile.flags) || error("expected non-empty flags array") - return nothing -end - -function run_postgres_trim_queries()::Nothing - conn = _postgres_trim_connect() - try - DBInterface.execute(conn, """ - CREATE TEMP TABLE trim_compile_profiles ( - profile_id integer PRIMARY KEY, - display_name text NOT NULL, - created_at timestamp NOT NULL, - active boolean NOT NULL, - score integer, - uid uuid NOT NULL, - flags integer[] NOT NULL - ) - """) - - insert_sql = raw""" - INSERT INTO trim_compile_profiles ( - profile_id, - display_name, - created_at, - active, - score, - uid, - flags - ) VALUES ( - $1, - $2, - $3, - $4, - $5, - $6, - $7::integer[] - ) - RETURNING profile_id - """ - - first_id = DBInterface.execute( - conn, - insert_sql, - ( - Int32(1), - "Ada", - DateTime(2024, 1, 2, 3, 4, 5), - true, - Int32(99), - UUID("12345678-1234-5678-1234-567812345678"), - Int32[1, 2, 3], - ), - TrimId, - ) - first_id.profile_id == 1 || error("unexpected inserted id") - - DBInterface.transaction(conn) do - DBInterface.execute( - conn, - insert_sql, - ( - Int32(2), - "Grace", - DateTime(2024, 1, 3, 4, 5, 6), - true, - missing, - UUID("87654321-4321-8765-4321-876543218765"), - Int32[4, 5], - ), - ) - end - - stmt = DBInterface.prepare(conn, raw""" - SELECT profile_id, display_name, created_at, active, score, uid, flags - FROM trim_compile_profiles - WHERE profile_id = $1 - """) - try - profile = DBInterface.execute(stmt, (Int32(1),), TrimProfile) - _assert_trim_profile(profile, Int32(1), "Ada") - finally - DBInterface.close!(stmt) - end - - profiles = DBInterface.execute(conn, """ - SELECT profile_id, display_name, created_at, active, score, uid, flags - FROM trim_compile_profiles - ORDER BY profile_id - """, (), Vector{TrimProfile}) - length(profiles) == 2 || error("expected two profiles") - _assert_trim_profile(profiles[2], Int32(2), "Grace") - - count_row = DBInterface.execute( - conn, - "SELECT count(*)::integer AS count FROM trim_compile_profiles", - (), - TrimCount, - ) - count_row.count == 2 || error("unexpected typed count") - - name_row = DBInterface.execute( - conn, - "SELECT display_name FROM trim_compile_profiles WHERE profile_id = 1", - (), - TrimName, - ) - name_row.display_name == "Ada" || error("unexpected typed name") - - update_result = DBInterface.execute(conn, "UPDATE trim_compile_profiles SET score = coalesce(score, 0) + 1") - Postgres.rows_affected(update_result) == 2 || error("unexpected rows affected") - occursin("UPDATE", Postgres.command_tag(update_result)) || error("unexpected command tag") - finally - DBInterface.close!(conn) - end - return nothing -end - -function @main(args::Vector{String})::Cint - _ = args - run_postgres_trim_queries() - return 0 -end - -Base.Experimental.entrypoint(main, (Vector{String},)) diff --git a/test/runtests.jl b/test/runtests.jl index 76609c5..7f05410 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,4 +1,5 @@ using Test +using Aqua using Dates using UUIDs using DBInterface @@ -19,6 +20,14 @@ Postgres.API.query_logging_enabled(::LoggingStyle) = true Postgres.API.query_logger(::LoggingStyle, event::Symbol, info::NamedTuple) = (push!(LOGGED_EVENTS, (event=event, info=info)); nothing) Postgres.API.notice_callback(::LoggingStyle, notice) = (NOTICE_SEEN[] = true; nothing) +const FAILING_LOGGER_CALLS = Ref(0) +struct FailingLoggerStyle <: Postgres.API.AbstractPostgresStyle end +Postgres.API.query_logging_enabled(::FailingLoggerStyle) = true +function Postgres.API.query_logger(::FailingLoggerStyle, event::Symbol, info::NamedTuple) + FAILING_LOGGER_CALLS[] += 1 + error("logger failed") +end + # Integration tests for Postgres.jl protocol and API behavior. const JSONType = typeof(JSON.lazy("{}")) @@ -61,6 +70,14 @@ struct Int8Row s::String end +# user-IO failure injection for the COPY hardening tests +struct ThrowingSource <: IO end +Base.eof(::ThrowingSource) = false +Base.readbytes!(::ThrowingSource, ::Vector{UInt8}, n) = error("source failed") + +struct FailingDest <: IO end +Base.write(::FailingDest, ::Vector{UInt8}) = error("dest failed") + struct PgConfig host::String port::Int @@ -144,10 +161,11 @@ mkdir -p "\$certdir" cp /certs/server.crt "\$certdir/server.crt" cp /certs/server.key "\$certdir/server.key" cp /certs/root.crt "\$certdir/root.crt" -chown postgres:postgres "\$certdir/server.crt" "\$certdir/server.key" "\$certdir/root.crt" -chmod 0644 "\$certdir/server.crt" "\$certdir/root.crt" +cp /certs/pg_hba.conf "\$certdir/pg_hba.conf" +chown postgres:postgres "\$certdir/server.crt" "\$certdir/server.key" "\$certdir/root.crt" "\$certdir/pg_hba.conf" +chmod 0644 "\$certdir/server.crt" "\$certdir/root.crt" "\$certdir/pg_hba.conf" chmod 0600 "\$certdir/server.key" -exec docker-entrypoint.sh postgres -c ssl=on -c ssl_cert_file="\$certdir/server.crt" -c ssl_key_file="\$certdir/server.key" -c ssl_ca_file="\$certdir/root.crt" +exec docker-entrypoint.sh postgres -c ssl=on -c ssl_cert_file="\$certdir/server.crt" -c ssl_key_file="\$certdir/server.key" -c ssl_ca_file="\$certdir/root.crt" -c hba_file="\$certdir/pg_hba.conf" """ return ["sh", "-c", setup_script] end @@ -161,6 +179,11 @@ function generate_ssl_material(dir::AbstractString) server_csr = joinpath(dir, "server.csr") server_cert = joinpath(dir, "server.crt") server_config = joinpath(dir, "server-openssl.cnf") + client_key = joinpath(dir, "client.key") + client_csr = joinpath(dir, "client.csr") + client_cert = joinpath(dir, "client.crt") + client_config = joinpath(dir, "client-openssl.cnf") + hba_file = joinpath(dir, "pg_hba.conf") open(server_config, "w") do io write(io, """ @@ -183,13 +206,44 @@ IP.1 = 127.0.0.1 """) end + open(client_config, "w") do io + write(io, """ +[req] +distinguished_name = req_distinguished_name +prompt = no +req_extensions = v3_req + +[req_distinguished_name] +CN = postgres_mtls + +[v3_req] +basicConstraints = CA:FALSE +keyUsage = digitalSignature, keyEncipherment +extendedKeyUsage = clientAuth +""") + end + + open(hba_file, "w") do io + write(io, """ +local all all trust +hostssl all postgres_mtls 0.0.0.0/0 trust clientcert=verify-full +hostssl all postgres_mtls ::/0 trust clientcert=verify-full +host all all 0.0.0.0/0 trust +host all all ::/0 trust +""") + end + run_openssl("req", "-x509", "-newkey", "rsa:2048", "-days", "1", "-nodes", "-keyout", root_key, "-out", root_cert, "-subj", "/CN=Postgres.jl Test Root CA") run_openssl("req", "-x509", "-newkey", "rsa:2048", "-days", "1", "-nodes", "-keyout", wrong_root_key, "-out", wrong_root_cert, "-subj", "/CN=Postgres.jl Wrong Root CA") run_openssl("req", "-new", "-newkey", "rsa:2048", "-nodes", "-keyout", server_key, "-out", server_csr, "-config", server_config) run_openssl("x509", "-req", "-in", server_csr, "-CA", root_cert, "-CAkey", root_key, "-CAcreateserial", "-out", server_cert, "-days", "1", "-sha256", "-extensions", "v3_req", "-extfile", server_config) + run_openssl("req", "-new", "-newkey", "rsa:2048", "-nodes", "-keyout", client_key, "-out", client_csr, "-config", client_config) + run_openssl("x509", "-req", "-in", client_csr, "-CA", root_cert, "-CAkey", root_key, "-CAcreateserial", "-out", client_cert, "-days", "1", "-sha256", "-extensions", "v3_req", "-extfile", client_config) chmod(server_key, 0o600) + chmod(client_key, 0o600) - return (rootcert=root_cert, wrongrootcert=wrong_root_cert, certdir=dir) + return (rootcert=root_cert, wrongrootcert=wrong_root_cert, + clientcert=client_cert, clientkey=client_key, certdir=dir) end function with_ssl_postgres(f::Function) @@ -417,8 +471,30 @@ function random_array_string(rng::AbstractRNG) end @testset "Postgres" begin + Aqua.test_all(Postgres) + @testset "Export Surface" begin - @test Set(names(Postgres)) == Set([:DBInterface, :Postgres]) + exported = Set([:DBInterface, :Postgres]) + @static if VERSION >= v"1.11" + # names() includes `public` declarations on Julia 1.11+ + public_names = Set([ + :Connection, :ConnectionPool, :ConnectionParams, :PostgresInterfaceError, + :Error, :Notification, :Numeric, :PostgresRange, :AbstractPostgresStyle, :PostgresStyle, + :query_logging_enabled, :query_logger, :notice_callback, :notification_callback, :parse_dsn, + :transaction, Symbol("@transaction"), :start_transaction, :commit, :rollback, :in_transaction, + :cursor, :copy_from, :copy_to, :listen!, :unlisten!, :notify!, :wait_for_notification, + :register_type!, :register_enum!, :register_composite!, :register_range!, + :command_tag, :rows_affected, :cancel_query!, :escape_identifier, :escape_literal, + :get_cached_statements, :clear_statement_cache!, :set_statement_cache_maxsize!, + :get_server_parameter, :get_server_parameters, :get_statement_timeout, :set_statement_timeout!, + :acquire, :release, :with_connection, :describe, + ]) + @test Set(names(Postgres)) == union(exported, public_names) + else + @test Set(names(Postgres)) == exported + end + @test Postgres.PostgresInterfaceError <: Exception + @test Postgres.Error <: Exception end @testset "Connection String Parsing" begin @@ -470,6 +546,89 @@ end @test query_host_params.port == 5436 @test query_host_params.dbname == "postgres" + sni_params = Postgres.parse_dsn("host=203.0.113.7 sslmode=require sslservername=db.example.com") + @test sni_params.sslservername == "db.example.com" + sni_uri_params = Postgres.parse_dsn("postgresql://postgres@203.0.113.7/postgres?sslservername=db.example.com") + @test sni_uri_params.sslservername == "db.example.com" + + # IPv6 hosts must be bracketed for the transport address parser; + # unix socket paths are rejected with a clear error + @test Postgres.API.hostport_address("::1", 5432) == "[::1]:5432" + @test Postgres.API.hostport_address("127.0.0.1", 5432) == "127.0.0.1:5432" + @test Postgres.API.hostport_address("db.example.com", 6432) == "db.example.com:6432" + @test_throws Postgres.PostgresInterfaceError Postgres.API.hostport_address("/var/run/postgresql", 5432) + + # debug/reconnect are accepted from a DSN and actually applied + flag_params = Postgres.parse_dsn("host=h debug=true reconnect=on") + @test flag_params.debug + @test flag_params.reconnect + @test !Postgres.parse_dsn("host=h").debug + @test !Postgres.parse_dsn("host=h").reconnect + + # an unrecognized parameter is a typo, not something to silently drop: + # "ssl_mode=verify-full" would otherwise leave sslmode unset and + # quietly fall back to an unverified connection + @test_throws ArgumentError Postgres.parse_dsn("host=h ssl_mode=verify-full") + @test_throws ArgumentError Postgres.parse_dsn("postgresql://u@h/db?ssl_mode=require") + + # A libpq option that requests behavior this driver cannot enforce is + # rejected. Warning and connecting would falsely report a security or + # routing guarantee to the caller. + @test_throws ArgumentError Postgres.parse_dsn("postgresql://u:p@h/db?sslmode=require&channel_binding=require") + @test_throws ArgumentError Postgres.parse_dsn("postgresql://u@h/db?target_session_attrs=read-write") + @test_throws ArgumentError Postgres.parse_dsn("host=h options=-csearch_path=x") + @test_throws ArgumentError Postgres.parse_dsn("host=h sslcrl=/tmp/crl.pem") + @test_throws ArgumentError Postgres.parse_dsn("host=h requiressl=1") + @test_throws ArgumentError Postgres.parse_dsn("host=h gssencmode=require") + @test_throws ArgumentError Postgres.parse_dsn("host=h requirepeer=postgres") + @test_throws ArgumentError Postgres.parse_dsn("host=h hostaddr=203.0.113.1") + @test_throws ArgumentError Postgres.parse_dsn("host=h client_encoding=LATIN1") + # the no-op defaults for those keywords stay silent, as do keywords + # with no security consequence + @test_logs Postgres.parse_dsn("host=h channel_binding=prefer target_session_attrs=any requiressl=0") + @test_logs Postgres.parse_dsn("host=h keepalives=1 client_encoding=UTF8") + @test_logs Postgres.parse_dsn("postgresql://u@h/db?channel_binding=disable") + + # Displaying structured connection options must never reveal a secret. + shown = repr(Postgres.ConnectionParams(host="h", user="u", password="top-secret", dbname="d")) + plain_shown = repr(MIME"text/plain"(), Postgres.ConnectionParams( + host="h", user="u", password="top-secret", dbname="d")) + @test !occursin("top-secret", shown) + @test !occursin("top-secret", plain_shown) + @test occursin("password=***", shown) + @test occursin("password=***", plain_shown) + + # Malformed keyword DSNs must never degrade to a usable partial + # configuration. In particular, a discarded security option could + # change which endpoint or transport is selected. + @test_throws ArgumentError Postgres.parse_dsn("host=h broken") + @test_throws ArgumentError Postgres.parse_dsn("host='unterminated") + @test_throws ArgumentError Postgres.parse_dsn("host=abc\\") + @test_throws ArgumentError Postgres.parse_dsn("host='h'trailing") + + # invalid values for a recognized parameter are reported against that + # parameter rather than silently defaulting + @test_throws ArgumentError Postgres.parse_dsn("host=h reconnect=ture") + @test_throws ArgumentError Postgres.parse_dsn("host=h port=abc") + + # an empty value (an unset PGPORT expanded by a process manager) falls + # back to the default instead of failing to parse + withenv("PGPORT" => "") do + @test Postgres.parse_dsn("host=h").port == 5432 + @test Postgres.parse_dsn(nothing).port == 5432 + end + # ... and the same for the boolean and TLS-mode parameters (quoted so + # the empty value can't swallow the next key: a bare "reconnect=" takes + # the following token as its value, as libpq does) + @test !Postgres.parse_dsn("host=h reconnect='' debug=''").reconnect + @test !Postgres.parse_dsn("host=h reconnect='' debug=''").debug + # sslmode is deliberately NOT empty-tolerant (libpq rejects it too): + # an unexpanded ${PGSSLMODE} meant to be verify-full must fail loudly + # rather than fall back to the unauthenticated default + withenv("PGSSLMODE" => "") do + @test Postgres.parse_dsn("host=h").sslmode == "" + end + withenv( "PGHOST" => "envhost", "PGPORT" => "5544", @@ -502,19 +661,202 @@ end @test string(Postgres.API.parse_numeric("-0.00120")) == "-0.00120" @test string(Postgres.API.parse_numeric("1.23e3")) == "1230" @test Postgres.API.parse_numeric("+42") == Postgres.Numeric(BigInt(42), 0) + # numeric special values can't be represented and must fail clearly + @test_throws Postgres.PostgresInterfaceError Postgres.API.parse_numeric("NaN") + @test_throws Postgres.PostgresInterfaceError Postgres.API.parse_numeric("Infinity") + @test_throws Postgres.PostgresInterfaceError Postgres.API.parse_numeric("-Infinity") + # an absurd exponent must be rejected, not turned into a huge BigInt + @test_throws Postgres.PostgresInterfaceError Postgres.API.parse_numeric("1e999999999999") + @test_throws Postgres.PostgresInterfaceError Postgres.API.parse_numeric("1e99999999999999999999999999") + # abs(typemin(Int)) wraps to itself, so the bound must not use abs + @test_throws Postgres.PostgresInterfaceError Postgres.API.parse_numeric("1e-9223372036854775808") + # ordinary scientific notation still round-trips + @test string(Postgres.API.parse_numeric("1.5e2")) == "150" + @test string(Postgres.API.parse_numeric("1.5e-2")) == "0.015" + + # message-field parsing is bounded by the buffer actually received: a + # truncated or unterminated field must not read past the allocation + @test Postgres.API.cstring_at(UInt8[], 1) == ("", 1) + @test Postgres.API.cstring_at(UInt8['a', 'b', 0x00], 1) == ("ab", 4) + @test Postgres.API.cstring_at(UInt8['a', 'b'], 1) == ("ab", 3) + @test Postgres.API.cstring_at(UInt8['a', 0x00, 'c', 0x00], 3) == ("c", 5) + @test Postgres.API.cstring_at(UInt8['a', 0x00], 5) == ("", 3) + @test_throws EOFError Postgres.API.skipbytes!(IOBuffer(UInt8[0x01]), 2) + @test Postgres.API.read_ready_status(IOBuffer(UInt8['T']), 1) == UInt8('T') + @test_throws Postgres.API.Error Postgres.API.read_ready_status(IOBuffer(UInt8[]), 0) + @test Postgres.API.commandComplete(7, IOBuffer(UInt8[codeunits("SELECT\0")...])) == "SELECT" + @test_throws Postgres.API.Error Postgres.API.commandComplete(0, IOBuffer()) + @test_throws Postgres.API.Error Postgres.API.commandComplete(4, IOBuffer(UInt8['O', 'K', 0x00, 0x00])) + let socket = IOBuffer() + write(socket, UInt8('D')) + write(socket, hton(Postgres.API.MAX_PREAUTH_MESSAGE_LEN + Int32(5))) + seekstart(socket) + @test_throws Postgres.API.Error Postgres.API.readheader( + socket, false, Postgres.API.MAX_PREAUTH_MESSAGE_LEN) + end + + # a malformed DataRow must fail with a clear protocol error rather + # than reading past the buffer or leaving the row partly unfilled + let nms = Symbol[:a, :b], tids = Int[23, 23], + mk = b -> Postgres.API.DataRow(b, nms, tids, registry), + consume = row -> StructUtils.applyeach(Postgres.API.PostgresStyle(), (k, v) -> nothing, row) + # ncols is signed on the wire: -1 must not pass an upper-bound check + @test_throws Postgres.API.Error consume(mk(UInt8[0xff, 0xff])) + # the count must equal the described column count exactly: too few + # would leave the caller's row partly unfilled + @test_throws Postgres.API.Error consume(mk(UInt8[0x00, 0x09])) + @test_throws Postgres.API.Error consume(mk(UInt8[0x00, 0x01, 0x00, 0x00, 0x00, 0x01, UInt8('7')])) + # truncated header, and a column length running past the body + @test_throws Postgres.API.Error consume(mk(UInt8[0x00])) + @test_throws Postgres.API.Error consume(mk(UInt8[0x00, 0x01, 0x00, 0x00, 0x00, 0x7f])) + # a well-formed row still parses, including a NULL column + vals = Any[] + consume_ok = Postgres.API.DataRow( + UInt8[0x00, 0x02, 0x00, 0x00, 0x00, 0x01, UInt8('5'), 0xff, 0xff, 0xff, 0xff], + nms, tids, registry) + StructUtils.applyeach(Postgres.API.PostgresStyle(), (k, v) -> push!(vals, v), consume_ok) + @test vals == Any[Int32(5), nothing] + end + + # a cancel connection must never downgrade to cleartext when the + # connection being cancelled actually negotiated TLS + @test Postgres.cancel_sslmode(true, nothing) == "require" + @test Postgres.cancel_sslmode(true, "prefer") == "require" + @test Postgres.cancel_sslmode(true, "PREFER") == "require" + @test Postgres.cancel_sslmode(true, "verify-full") == "verify-full" + @test Postgres.cancel_sslmode(true, "require") == "require" + # an explicitly plaintext connection is left alone, as are non-TLS ones + @test Postgres.cancel_sslmode(true, "disable") == "disable" + @test Postgres.cancel_sslmode(false, nothing) === nothing + @test Postgres.cancel_sslmode(false, "prefer") == "prefer" + + # escaping helpers reject embedded NULs rather than emitting SQL the + # server would truncate mid-statement + @test Postgres.escape_identifier("a\"b") == "\"a\"\"b\"" + @test Postgres.escape_literal("a'b") == "'a''b'" + @test Postgres.escape_literal("a\\b") == "E'a\\\\b'" + @test_throws Postgres.PostgresInterfaceError Postgres.escape_identifier("a\0b") + @test_throws Postgres.PostgresInterfaceError Postgres.escape_literal("a\0b") + @test_throws Postgres.PostgresInterfaceError Postgres.Connection(host="127.0.0.1", port=1, user="u\0x") + + # severity must come from the non-localized 'V' field when the server + # sends it: 'S' is translated, so comparing it to "FATAL" would depend + # on the server's lc_messages + let socket = IOBuffer(Vector{UInt8}(vcat( + UInt8('S'), Vector{UInt8}("SCHWERWIEGEND"), 0x00, + UInt8('V'), Vector{UInt8}("FATAL"), 0x00, + UInt8('C'), Vector{UInt8}("57P01"), 0x00, + UInt8('M'), Vector{UInt8}("terminating connection"), 0x00, + 0x00))) + err = Postgres.API.errorResponse(bytesavailable(socket), socket, false) + @test err.severity == "FATAL" + @test err.code == "57P01" + end + # without 'V' the localized 'S' is still reported + let socket = IOBuffer(Vector{UInt8}(vcat( + UInt8('S'), Vector{UInt8}("ERROR"), 0x00, + UInt8('C'), Vector{UInt8}("42601"), 0x00, + 0x00))) + err = Postgres.API.errorResponse(bytesavailable(socket), socket, false) + @test err.severity == "ERROR" + end @test Postgres.API.parse_value(1184, "2024-02-13 05:28:17+02", registry) == DateTime(2024, 2, 13, 3, 28, 17) @test Postgres.API.parse_value(1184, "2024-02-13 05:28:17+02:30", registry) == DateTime(2024, 2, 13, 2, 58, 17) @test Postgres.API.parse_value(1184, "2024-02-13 05:28:17Z", registry) == DateTime(2024, 2, 13, 5, 28, 17) + # LMT-era offsets in named zones carry a seconds field; dropping it + # silently shifted the decoded value + @test Postgres.API.parse_value(1184, "1880-01-01 05:21:10+05:21:10", registry) == DateTime(1880, 1, 1, 0, 0, 0) + @test Postgres.API.parse_value(1184, "1879-12-31 18:38:50-05:21:10", registry) == DateTime(1880, 1, 1, 0, 0, 0) @test Postgres.API.parse_interval("1 year 2 mons 3 days 04:05:06.789") == Dates.CompoundPeriod(Dates.Year(1), Dates.Month(2), Dates.Day(3), Dates.Hour(4), Dates.Minute(5), Dates.Second(6), Dates.Millisecond(789)) @test Postgres.API.parse_interval("-04:05:06.789") == Dates.CompoundPeriod(Dates.Hour(-4), Dates.Minute(-5), Dates.Second(-6), Dates.Millisecond(-789)) + # a genuine zero interval renders as "00:00:00" in the postgres style + @test Postgres.API.parse_interval("00:00:00") == Dates.Millisecond(0) + # text in an IntervalStyle this parser can't read (a mid-session SET + # to sql_standard or iso_8601) must fail loudly, not silently decode + # to a zero interval + for foreign in ("+1 +2:00:00", "1 2:00:00", "P1DT2H", "PT0S", "@ 1 day 2 hours", "1-2") + @test_throws Postgres.PostgresInterfaceError Postgres.API.parse_interval(foreign) + end + + # the field-order half of DateStyle survives the correction to ISO + @test Postgres.API.date_order("German, DMY") == "DMY" + @test Postgres.API.date_order("SQL, MDY") == "MDY" + @test Postgres.API.date_order("Postgres, YMD") == "YMD" + @test Postgres.API.date_order("ISO, DMY") == "DMY" + # unreported or unrecognized styles fall back to the postgres default + @test Postgres.API.date_order("") == "MDY" + @test Postgres.API.date_order("German") == "MDY" @test Postgres.API.parse_value(17, raw"\xDEADBEEF", registry) == UInt8[0xde, 0xad, 0xbe, 0xef] @test Postgres.API.decode_bytea(raw"\141\\") == UInt8['a', '\\'] @test_throws ArgumentError Postgres.API.decode_bytea(raw"\xabc") @test_throws ArgumentError Postgres.API.decode_bytea(raw"\xzz") + # timestamp range bounds are quoted on the wire; without unquoting them + # every tsrange/tstzrange value fails to decode + ts_range = Postgres.API.parse_range("[\"2020-01-01 00:00:00\",\"2020-01-02 00:00:00\")", 1114, registry) + @test ts_range.lower == DateTime(2020, 1, 1) + @test ts_range.upper == DateTime(2020, 1, 2) + @test ts_range.lower_inclusive + @test !ts_range.upper_inclusive + @test Postgres.API.unquote_range_bound("\"a\\\"b\"") == "a\"b" + @test Postgres.API.unquote_range_bound("plain") == "plain" + + # the "char" type renders its zero value as an empty string + @test Postgres.API.parse_value(18, "", registry) == '\0' + @test Postgres.API.parse_value(18, "Z", registry) == 'Z' + # ... and high-bit bytes as backslash-octal escapes + @test Postgres.API.parse_value(18, "\\200", registry) == Char(0x80) + @test Postgres.API.parse_value(18, "\\377", registry) == Char(0xff) + @test Postgres.API.parse_value(18, "\\x80", registry) == Char(0x80) + @test Postgres.API.parse_value(18, "\\xFF", registry) == Char(0xff) + # a backslash byte renders as a lone backslash, not an escape + @test Postgres.API.parse_value(18, "\\", registry) == '\\' + @test Postgres.API.pg_parse_char("\\310") == Char(0xc8) + # "char"[] (oid 1002) decodes elements, escapes included; on the wire + # the escape's backslash is itself array-quoted as "\\200" + @test isequal(Postgres.API.parse_value(1002, "{a,\"\\\\200\",NULL}", registry), Any['a', Char(0x80), missing]) + @test Postgres.API.ArrayParsing.parse_array("{a,b}", Char) == ['a', 'b'] + + # infinite timestamps/dates can't be represented and must say so + @test_throws Postgres.PostgresInterfaceError Postgres.API.pg_parse_datetime("infinity") + @test_throws Postgres.PostgresInterfaceError Postgres.API.pg_parse_datetime("-infinity") + @test_throws Postgres.PostgresInterfaceError Postgres.API.pg_parse_date("infinity") + # BC dates are a different year numbering than Julia's (no year zero); + # decoding them as AD would be silent corruption + @test_throws Postgres.PostgresInterfaceError Postgres.API.pg_parse_date("0044-03-15 BC") + @test_throws Postgres.PostgresInterfaceError Postgres.API.pg_parse_datetime("0044-03-15 12:00:00 BC") + # timestamptz puts " BC" after the zone offset, past where the + # offset-stripped datetime parse can see it + @test_throws Postgres.PostgresInterfaceError Postgres.API.parse_timestamptz("0044-03-15 12:00:00+00 BC") + # years beyond 9999 widen the year field rather than misparsing + @test Postgres.API.pg_parse_date("10000-01-01") == Date(10000, 1, 1) + @test Postgres.API.pg_parse_datetime("294276-12-31 23:59:59") == DateTime(294276, 12, 31, 23, 59, 59) + @test Postgres.API.parse_value(1184, "10000-01-02 03:04:05+02", registry) == DateTime(10000, 1, 2, 1, 4, 5) + # ... but only genuine ISO renderings: an ISO year is zero-padded to + # at least 4 digits, so "03-04-2020" (Postgres-style for 2020-03-04 + # after a mid-session SET DateStyle) must throw, not decode as year 3 + @test_throws ArgumentError Postgres.API.pg_parse_date("03-04-2020") + @test_throws ArgumentError Postgres.API.pg_parse_date("123-01-01") + # adversarial input: bounded digits (no Int overflow), checked layout + @test_throws ArgumentError Postgres.API.pg_parse_date("99999999999999999999-01-01") + @test_throws ArgumentError Postgres.API.pg_parse_date("12345678-9") + @test_throws ArgumentError Postgres.API.pg_parse_datetime("999999999- 03:04:05 ") + + # postgres permits time '24:00:00'; Julia's Time does not + @test_throws Postgres.PostgresInterfaceError Postgres.API.pg_parse_time("24:00:00") + @test Postgres.API.pg_parse_time("23:59:59.999") == Time(23, 59, 59, 999) + + # range bounds may end in multibyte text; byte-index slicing threw + # StringIndexError on every such value + uni_range = Postgres.API.parse_range_of(String, "[α,ω)", 25, registry) + @test uni_range.lower == "α" + @test uni_range.upper == "ω" + @test uni_range.lower_inclusive + @test !uni_range.upper_inclusive + range = Postgres.API.parse_range("[1,5)", 23, registry) @test range == Postgres.PostgresRange{Int32}(1, 5, true, false, false) unbounded = Postgres.API.parse_range("(,5]", 23, registry) @@ -530,6 +872,22 @@ end fields = Postgres.API.parse_composite_fields("(\"a,b\",,\"a\\\"b\",\"c\\\\d\",plain)") @test isequal(fields, Union{String, Missing}["a,b", missing, "a\"b", "c\\d", "plain"]) + # postgres prefixes the literal with explicit dimensions whenever a + # lower bound isn't 1; without handling it the elements are silently + # dropped (text) or the parse throws (numeric) + @test Postgres.API.parse_array_by_oid("[0:2]={x,y,z}", 25, registry) == ["x", "y", "z"] + @test Postgres.API.parse_array_by_oid("[0:2]={1,2,3}", 23, registry) == [1, 2, 3] + @test Postgres.API.parse_value(1009, "[0:1]={a,b}", registry) == ["a", "b"] + @test Postgres.API.ArrayParsing.parse_array("[1:2][1:2]={{1,2},{3,4}}", Int64) == [[1, 2], [3, 4]] + # ']' is ordinary element data: postgres doesn't quote it, so treating + # it as a terminator silently truncated the element and dropped every + # element after it + @test Postgres.API.parse_array_by_oid("{a]b,x[1],plain}", 25, registry) == ["a]b", "x[1]", "plain"] + @test Postgres.API.parse_array_by_oid("{]}", 25, registry) == ["]"] + @test Postgres.API.parse_array_by_oid("{/var/log/x[1].txt,b}", 25, registry) == ["/var/log/x[1].txt", "b"] + + @test Postgres.API.ArrayParsing.parse_array("{1,2}", Int64) isa Vector{Int64} + rng = MersenneTwister(0x5097) for _ in 1:200 values = [random_array_string(rng) for _ in 1:rand(rng, 0:8)] @@ -558,8 +916,6 @@ end end end - include("trim_compile_tests.jl") - if !docker_available() @info "Docker not available; skipping Postgres integration tests." @test true @@ -672,7 +1028,8 @@ end DBInterface.execute(connp, "SELECT pg_terminate_backend($(conn_victim.pid))") err = fetch(victim_task) @test err isa Postgres.API.Error - @test occursin("terminat", err.message) + # 57P01: admin_shutdown — the code, not the localized message + @test err.code == "57P01" @test !isopen(conn_victim.socket) DBInterface.close!(conn_victim) @@ -682,6 +1039,34 @@ end stmt = DBInterface.prepare(conn, raw"SELECT $1::int AS val") res = Tables.rowtable(DBInterface.execute(stmt, (1,))) @test res[1].val == 1 + @test all(ismissing, stmt.params) + + failing_stmt = DBInterface.prepare(conn, raw"SELECT 10 / $1::int AS val") + @test_throws Postgres.API.Error DBInterface.execute(failing_stmt, (0,)) + @test all(ismissing, failing_stmt.params) + mismatch_stmt = DBInterface.prepare(conn, + raw"SELECT $1::text AS a, $2::text AS b") + @test_throws Postgres.PostgresInterfaceError DBInterface.execute( + mismatch_stmt, ("must-not-remain",)) + @test all(ismissing, mismatch_stmt.params) + + # Each prepare call returns an independent caller handle, + # even when both handles share one cache-owned server + # statement. Closing one must not close the other. + held = DBInterface.prepare(conn, "SELECT 42 AS val") + alias = DBInterface.prepare(conn, "SELECT 42 AS val") + @test held !== alias + DBInterface.close!(alias) + @test only(DBInterface.execute(held)).val == 42 + function_value = DBInterface.execute(conn, "SELECT 42 AS val", nothing) do result + only(result).val + end + @test function_value == 42 + @test only(DBInterface.execute(held)).val == 42 + + DBInterface.close!(held) + DBInterface.close!(mismatch_stmt) + DBInterface.close!(failing_stmt) DBInterface.close!(stmt) @test_throws Postgres.PostgresInterfaceError DBInterface.execute(stmt, (1,)) end @@ -728,10 +1113,29 @@ end @test ismissing(row.nullable_text) @test row.int_array == expected[23] @test eltype(row.int_array) == Int32 + # a column mixing null-free and null-bearing arrays must report + # a schema type every row satisfies, whatever the row order + for order in ("'{1,2}'::int[]), ('{1,NULL}'::int[]), ('{3,4}'::int[]", + "'{1,NULL}'::int[]), ('{1,2}'::int[]), ('{3,4}'::int[]") + mixed = DBInterface.execute(conn, "SELECT a FROM (VALUES ($order)) t(a)") + schema_type = Tables.schema(mixed).types[1] + @test all(row -> Tables.getcolumn(row, 1) isa schema_type, mixed) + @test length(Tables.columntable(mixed).a) == 3 + end + # a text array element containing ']' must survive the round trip + bracket_param = ["a]b", "x[1]", "]", "plain"] + bracket_row = only(Tables.rowtable(DBInterface.execute(conn, raw"SELECT $1::text[] AS arr", (bracket_param,)))) + @test bracket_row.arr == bracket_param array_row = only(Tables.rowtable(DBInterface.execute(conn, "SELECT '{1,NULL,3}'::int[] AS arr"))) @test isequal(array_row.arr, Union{Missing, Int32}[1, missing, 3]) nested_row = only(Tables.rowtable(DBInterface.execute(conn, "SELECT '{{1,2},{3,4}}'::int[] AS arr"))) @test nested_row.arr == [Int32[1, 2], Int32[3, 4]] + # arrays whose lower bound isn't 1 come back with an explicit + # dimension prefix; the elements must survive it + lb_row = only(Tables.rowtable(DBInterface.execute(conn, "SELECT array_fill(7, ARRAY[3], ARRAY[0]) AS arr"))) + @test lb_row.arr == [7, 7, 7] + lb_text_row = only(Tables.rowtable(DBInterface.execute(conn, "SELECT array_fill('x'::text, ARRAY[3], ARRAY[0]) AS arr"))) + @test lb_text_row.arr == ["x", "x", "x"] typed = DBInterface.execute(conn, raw"SELECT * FROM types_test WHERE id = $1", (id,), TypeRow) @test typed isa TypeRow @test typed.uuid_col == expected[12] @@ -779,6 +1183,89 @@ end @test row.span.upper == 5 @test row.span.lower_inclusive @test !row.span.upper_inclusive + + # a range over an element type outside the builtin set must + # decode after registration, not throw on every value + DBInterface.execute(conn, "DROP TYPE IF EXISTS textrange CASCADE") + DBInterface.execute(conn, "CREATE TYPE textrange AS RANGE (subtype = text)") + Postgres.register_range!(conn, "textrange"; schema="public") + trow = only(Tables.rowtable(DBInterface.execute(conn, "SELECT textrange('a','z') AS r, textrange('x','y','[]') AS s, 'empty'::textrange AS e"))) + @test trow.r isa Postgres.PostgresRange{String} + @test trow.r.lower == "a" + @test trow.r.upper == "z" + @test trow.r.lower_inclusive + @test !trow.r.upper_inclusive + @test trow.s.lower == "x" + @test trow.s.upper_inclusive + @test trow.e.empty + # bounds ending in multibyte text arrive unquoted + uni_row = only(Tables.rowtable(DBInterface.execute(conn, "SELECT textrange('α','ω') AS r"))) + @test uni_row.r.lower == "α" + @test uni_row.r.upper == "ω" + + # arrays of registered custom types decode as arrays, not + # raw literal strings + enumarr = only(Tables.rowtable(DBInterface.execute(conn, "SELECT ARRAY['sad','happy']::mood[] AS a, ARRAY['ok',NULL]::mood[] AS b"))) + @test isequal(collect(enumarr.a), Any[:sad, :happy]) + @test isequal(collect(enumarr.b), Any[:ok, missing]) + comparr = only(Tables.rowtable(DBInterface.execute(conn, "SELECT ARRAY[ROW('A',1)::address, ROW('B',2)::address] AS a"))) + @test comparr.a[1] == (street="A", number=1) + @test comparr.a[2] == (street="B", number=2) + rangearr = only(Tables.rowtable(DBInterface.execute(conn, "SELECT ARRAY['[1,3)'::int4range, '[5,7)'::int4range] AS a"))) + @test rangearr.a[1].lower == 1 + @test rangearr.a[2].upper == 7 + textrangearr = only(Tables.rowtable(DBInterface.execute(conn, "SELECT ARRAY[textrange('a','c'), textrange('α','ω')] AS a"))) + @test textrangearr.a[1].upper == "c" + @test textrangearr.a[2].lower == "α" + + # Registry metadata must describe values the parser can + # actually produce, including empty and all-NULL results. + @test_throws ArgumentError Postgres.register_type!(conn, 900_000, Int) + @test_throws ArgumentError Postgres.register_enum!(conn, "mood"; julia_type=Int) + Postgres.register_enum!(conn, "mood"; julia_type=String) + empty_enum = DBInterface.execute(conn, "SELECT mood FROM custom_types WHERE false") + @test Tables.schema(empty_enum).types[1] === String + null_enum = DBInterface.execute(conn, "SELECT NULL::mood AS mood") + @test Tables.schema(null_enum).types[1] == Union{Missing, String} + @test ismissing(only(null_enum).mood) + string_enum = only(DBInterface.execute(conn, "SELECT 'happy'::mood AS mood")) + @test string_enum.mood == "happy" + DBInterface.execute(conn, "DROP TYPE textrange CASCADE") + end + + @testset "Describe Schema Isolation" begin + DBInterface.execute(conn, "DROP SCHEMA IF EXISTS postgres_describe_a CASCADE") + DBInterface.execute(conn, "DROP SCHEMA IF EXISTS postgres_describe_b CASCADE") + DBInterface.execute(conn, "CREATE SCHEMA postgres_describe_a") + DBInterface.execute(conn, "CREATE SCHEMA postgres_describe_b") + try + DBInterface.execute(conn, "CREATE TABLE postgres_describe_a.parent_a (id int PRIMARY KEY)") + DBInterface.execute(conn, "CREATE TABLE postgres_describe_b.parent_b (id int PRIMARY KEY)") + DBInterface.execute(conn, """ + CREATE TABLE postgres_describe_a.child ( + id int PRIMARY KEY, + parent_id int, + CONSTRAINT shared_fk FOREIGN KEY (parent_id) + REFERENCES postgres_describe_a.parent_a(id) + ) + """) + DBInterface.execute(conn, """ + CREATE TABLE postgres_describe_b.child ( + id int PRIMARY KEY, + parent_id int, + CONSTRAINT shared_fk FOREIGN KEY (parent_id) + REFERENCES postgres_describe_b.parent_b(id) + ) + """) + description = Postgres.describe(conn, "child"; schema="postgres_describe_a") + rows = Tables.rowtable(description.resultset) + @test length(rows) == 2 + parent_row = only(filter(row -> row.column_name == "parent_id", rows)) + @test parent_row.foreign_key_reference == "parent_a.id" + finally + DBInterface.execute(conn, "DROP SCHEMA IF EXISTS postgres_describe_a CASCADE") + DBInterface.execute(conn, "DROP SCHEMA IF EXISTS postgres_describe_b CASCADE") + end end @testset "Transactions" begin DBInterface.execute(conn, "DROP TABLE IF EXISTS trans_test") @@ -800,6 +1287,162 @@ end @test_throws Postgres.PostgresInterfaceError Postgres.commit(conn) @test_throws Postgres.PostgresInterfaceError Postgres.rollback(conn) + # A COMMIT that fails server-side (deferred constraint) must + # surface the server's error with its SQLSTATE — a retry + # loop keys on that — and must not leave the transaction + # open on the client after the server has ended it. + DBInterface.execute(conn, "DROP TABLE IF EXISTS deferred_child") + DBInterface.execute(conn, "DROP TABLE IF EXISTS deferred_parent") + DBInterface.execute(conn, "CREATE TABLE deferred_parent (id int PRIMARY KEY)") + DBInterface.execute(conn, """ + CREATE TABLE deferred_child ( + id int, + parent_id int REFERENCES deferred_parent(id) DEFERRABLE INITIALLY DEFERRED + ) + """) + for wrapper in (:plain, :helper, :macro) + err = try + if wrapper === :plain + Postgres.start_transaction(conn) + DBInterface.execute(conn, "INSERT INTO deferred_child VALUES (1, 999)") + Postgres.commit(conn) + elseif wrapper === :helper + Postgres.transaction(conn) do tx + DBInterface.execute(tx, "INSERT INTO deferred_child VALUES (1, 999)") + end + else + Postgres.@transaction conn begin + DBInterface.execute(conn, "INSERT INTO deferred_child VALUES (1, 999)") + end + end + nothing + catch e + e + end + @test err isa Postgres.API.Error + @test err.code == "23503" + @test !Postgres.in_transaction(conn) + # the failed COMMIT's own ReadyForQuery says the + # transaction is over; the tracked server status must + # not stay stale-true from the preceding INSERT + @test !(@lock conn.lock conn.server_in_transaction) + @test isempty(Tables.rowtable(DBInterface.execute(conn, "SELECT * FROM deferred_child"))) + end + + # PostgreSQL changes COMMIT to ROLLBACK when a statement + # error was caught inside the body. The helper must not + # return the body value as if the write committed. + DBInterface.execute(conn, "DROP TABLE IF EXISTS caught_error_tx") + DBInterface.execute(conn, "CREATE TABLE caught_error_tx (id int)") + for wrapper in (:plain, :helper, :macro) + err = try + if wrapper === :plain + Postgres.start_transaction(conn) + DBInterface.execute(conn, "INSERT INTO caught_error_tx VALUES (1)") + try + DBInterface.execute(conn, "SELECT 1/0") + catch + end + Postgres.commit(conn) + elseif wrapper === :helper + Postgres.transaction(conn) do tx + DBInterface.execute(tx, "INSERT INTO caught_error_tx VALUES (1)") + try + DBInterface.execute(tx, "SELECT 1/0") + catch + end + :body_value + end + else + Postgres.@transaction conn begin + DBInterface.execute(conn, "INSERT INTO caught_error_tx VALUES (1)") + try + DBInterface.execute(conn, "SELECT 1/0") + catch + end + :body_value + end + end + nothing + catch e + e + end + @test err isa Postgres.PostgresInterfaceError + @test occursin("completed ROLLBACK", sprint(showerror, err)) + @test !Postgres.in_transaction(conn) + @test isempty(Tables.rowtable(DBInterface.execute(conn, "SELECT * FROM caught_error_tx"))) + end + + # a transaction opened with raw SQL belongs to the caller: + # driver helpers must nest inside it (savepoints), never + # commit it, and never destroy it on their rollback + DBInterface.execute(conn, "DROP TABLE IF EXISTS rawtx_test") + DBInterface.execute(conn, "CREATE TABLE rawtx_test (id int)") + DBInterface.execute(conn, "BEGIN") + DBInterface.execute(conn, "INSERT INTO rawtx_test VALUES (1)") + Postgres.transaction(conn) do tx + DBInterface.execute(tx, "INSERT INTO rawtx_test VALUES (2)") + end + # a failing block rolls back only its own level, leaving + # the caller's transaction alive and usable + raw_err = try + Postgres.transaction(conn) do tx + DBInterface.execute(tx, "INSERT INTO rawtx_test VALUES (3)") + DBInterface.execute(tx, "SELECT 1/0") + end + nothing + catch e + e + end + @test raw_err isa Postgres.API.Error + # a cursor sees the open transaction and must not own + # (and so commit) it on close — nor start any driver-level + # nesting of its own + raw_cur = Postgres.cursor(conn, "SELECT id FROM rawtx_test ORDER BY id"; fetchsize=1) + @test !Postgres.in_transaction(conn) + @test [row.id for row in raw_cur] == [1, 2] + DBInterface.close!(raw_cur) + @test @lock conn.lock conn.server_in_transaction + # the caller's ROLLBACK is still in control of all of it + DBInterface.execute(conn, "ROLLBACK") + @test isempty(Tables.rowtable(DBInterface.execute(conn, "SELECT id FROM rawtx_test"))) + + # Driver savepoints must not shadow a caller savepoint with + # the same name. A failed helper must also release its own + # savepoint after rolling back to it. + DBInterface.execute(conn, "BEGIN") + DBInterface.execute(conn, "SAVEPOINT sp_0") + DBInterface.execute(conn, "INSERT INTO rawtx_test VALUES (10)") + @test_throws ErrorException Postgres.transaction(conn) do tx + DBInterface.execute(tx, "INSERT INTO rawtx_test VALUES (20)") + error("fail nested work") + end + DBInterface.execute(conn, "ROLLBACK TO SAVEPOINT sp_0") + @test isempty(Tables.rowtable(DBInterface.execute(conn, "SELECT id FROM rawtx_test"))) + DBInterface.execute(conn, "ROLLBACK") + # ... and after a raw COMMIT, the driver-level work sticks + DBInterface.execute(conn, "BEGIN") + Postgres.transaction(conn) do tx + DBInterface.execute(tx, "INSERT INTO rawtx_test VALUES (4)") + end + DBInterface.execute(conn, "COMMIT") + @test only(Tables.rowtable(DBInterface.execute(conn, "SELECT id FROM rawtx_test"))).id == 4 + DBInterface.execute(conn, "DROP TABLE rawtx_test") + + # COMMIT/ROLLBACK end the transaction server-side even when + # they fail, so client state must not be left behind + fail_conn = DBInterface.connect(Postgres.Connection, cfg.host, cfg.user, cfg.password; dbname=cfg.dbname, port=cfg.port, reconnect=true) + Postgres.start_transaction(fail_conn) + close(fail_conn.socket) + try + Postgres.commit(fail_conn) + catch + # the connection is gone; the COMMIT cannot be delivered + end + @test !Postgres.in_transaction(fail_conn) + @test Tables.rowtable(DBInterface.execute(fail_conn, "SELECT 1 AS a"))[1].a == 1 + DBInterface.close!(fail_conn) + Postgres.start_transaction(conn) @test_throws Postgres.API.Error DBInterface.execute(conn, "INVALID SQL") Postgres.rollback(conn) @@ -820,6 +1463,14 @@ end DBInterface.execute(conn, "CREATE TABLE macro_test (id SERIAL PRIMARY KEY, value INTEGER)") DBInterface.execute(conn, "INSERT INTO macro_test (value) VALUES (1)") + # the macro must evaluate its connection expression once + conn_evals = Ref(0) + eval_conn = () -> (conn_evals[] += 1; conn) + Postgres.@transaction eval_conn() begin + DBInterface.execute(conn, "SELECT 1") + end + @test conn_evals[] == 1 + result = Postgres.@transaction conn begin DBInterface.execute(conn, "INSERT INTO macro_test (value) VALUES (2)") length(Tables.rowtable(DBInterface.execute(conn, "SELECT * FROM macro_test"))) @@ -831,6 +1482,18 @@ end DBInterface.execute(conn, "INVALID SQL") end @test length(Tables.rowtable(DBInterface.execute(conn, "SELECT * FROM macro_test"))) == 2 + + # An early return must commit before it leaves the caller. + early_return = function(c) + Postgres.@transaction c begin + DBInterface.execute(c, "INSERT INTO macro_test (value) VALUES (4)") + return :early + end + return :late + end + @test early_return(conn) === :early + @test !Postgres.in_transaction(conn) + @test only(Tables.rowtable(DBInterface.execute(conn, "SELECT count(*)::int AS n FROM macro_test"))).n == 3 end @testset "Nested Transactions" begin @@ -877,6 +1540,36 @@ end conn3 = DBInterface.connect(Postgres.Connection, params_with_app) @test isopen(conn3) DBInterface.close!(conn3) + + # ConnectionParams debug/reconnect fields are honored (and + # overridable via keyword arguments) + params_reconnect = Postgres.ConnectionParams( + host=cfg.host, + port=cfg.port, + user=cfg.user, + password=cfg.password, + dbname=cfg.dbname, + reconnect=true + ) + conn4 = DBInterface.connect(Postgres.Connection, params_reconnect) + @test conn4.reconnect + @test !conn4.debug + DBInterface.close!(conn4) + conn5 = DBInterface.connect(Postgres.Connection, params_reconnect; reconnect=false) + @test !conn5.reconnect + DBInterface.close!(conn5) + + # A connection-level debug setting applies to the normal + # prepare/describe/execute path without a per-call flag. + debug_conn = DBInterface.connect(Postgres.Connection, + cfg.host, cfg.user, cfg.password; dbname=cfg.dbname, + port=cfg.port) + debug_conn.debug = true + debug_result = @test_logs (:info, r"sending message") match_mode=:any begin + DBInterface.execute(debug_conn, "SELECT 1 AS n") + end + @test only(debug_result).n == 1 + DBInterface.close!(debug_conn) end @testset "SSL Modes" begin @@ -918,6 +1611,48 @@ end Postgres.clear_statement_cache!(conn_cache) @test length(Postgres.get_cached_statements(conn_cache)) == 0 DBInterface.close!(conn_cache) + + # Retained handles survive eviction and cache disablement, + # but private re-prepare must never exceed or repopulate + # the configured cache. + retained_conn = DBInterface.connect(Postgres.Connection, + cfg.host, cfg.user, cfg.password; dbname=cfg.dbname, + port=cfg.port, statement_cache_maxsize=1) + retained_a = DBInterface.prepare(retained_conn, "SELECT 101 AS n") + retained_b = DBInterface.prepare(retained_conn, "SELECT 102 AS n") + @test length(Postgres.get_cached_statements(retained_conn)) == 1 + @test only(DBInterface.execute(retained_a)).n == 101 + retained_cache = Postgres.get_cached_statements(retained_conn) + @test length(retained_cache) == 1 + @test haskey(retained_cache, "SELECT 102 AS n") + replacement_a = DBInterface.prepare(retained_conn, "SELECT 101 AS n") + @test only(DBInterface.execute(retained_a)).n == 101 + @test length(Postgres.get_cached_statements(retained_conn)) == 1 + Postgres.set_statement_cache_maxsize!(retained_conn, 0) + @test only(DBInterface.execute(retained_b)).n == 102 + @test isempty(Postgres.get_cached_statements(retained_conn)) + for retained in (retained_a, retained_b, replacement_a) + DBInterface.close!(retained) + end + DBInterface.close!(retained_conn) + + # Connection-form bulk helpers own and close the private + # statements they create when caching is disabled. + bulk_conn = DBInterface.connect(Postgres.Connection, + cfg.host, cfg.user, cfg.password; dbname=cfg.dbname, + port=cfg.port, statement_cache_maxsize=0) + DBInterface.execute(bulk_conn, "CREATE TEMP TABLE cache_bulk_test (id int)") + for i in 1:3 + DBInterface.executemany(bulk_conn, + raw"INSERT INTO cache_bulk_test VALUES ($1)", ([i],)) + resultsets = DBInterface.executemultiple( + bulk_conn, raw"SELECT $1::int AS n", (i,)) + @test only(only(resultsets)).n == i + prepared_count = only(DBInterface.execute(bulk_conn, + "SELECT count(*)::int AS n FROM pg_prepared_statements")).n + @test prepared_count == 0 + end + DBInterface.close!(bulk_conn) end @testset "Do-Block Helpers" begin @@ -987,8 +1722,66 @@ end return rows[1].a end @test pooled_result == 1 + + # A connection returned to the pool mid-transaction must not + # hand that transaction to the next borrower: their BEGIN + # would become a SAVEPOINT and their commit would be lost. + Postgres.with_connection(pool) do pooled_conn + DBInterface.execute(pooled_conn, "DROP TABLE IF EXISTS pool_tx_test") + DBInterface.execute(pooled_conn, "CREATE TABLE pool_tx_test (id int)") + end + try + Postgres.with_connection(pool) do pooled_conn + Postgres.start_transaction(pooled_conn) + DBInterface.execute(pooled_conn, "INSERT INTO pool_tx_test VALUES (1)") + error("abandon the block mid-transaction") + end + catch + # the caller's error propagates; the pool must still be clean + end + Postgres.with_connection(pool) do pooled_conn + @test !Postgres.in_transaction(pooled_conn) + Postgres.transaction(pooled_conn) do tx + DBInterface.execute(tx, "INSERT INTO pool_tx_test VALUES (2)") + end + end + + # ... including a transaction opened by raw SQL, which the + # client-side flag never sees + try + Postgres.with_connection(pool) do pooled_conn + DBInterface.execute(pooled_conn, "BEGIN") + DBInterface.execute(pooled_conn, "INSERT INTO pool_tx_test VALUES (99)") + error("abandon a raw-SQL transaction") + end + catch + end + Postgres.with_connection(pool) do pooled_conn + rows = Tables.rowtable(DBInterface.execute(pooled_conn, "SELECT id FROM pool_tx_test ORDER BY id")) + @test [row.id for row in rows] == [2] + end + # the abandoned insert rolled back; the committed one landed + Postgres.with_connection(pool) do pooled_conn + ids = [row.id for row in Tables.rowtable(DBInterface.execute(pooled_conn, "SELECT id FROM pool_tx_test ORDER BY id"))] + @test ids == [2] + end DBInterface.close!(pool) + @test !isopen(pool) @test !isopen(conn_a) + @test_throws Postgres.PostgresInterfaceError Postgres.acquire(pool) + DBInterface.close!(pool) + + # Closing a pool is terminal even when a connection is + # still checked out. Its later release must close it, not + # add it back to the closed pool. + active_pool = Postgres.ConnectionPool(Postgres.Connection, cfg.host, cfg.user, cfg.password; + dbname=cfg.dbname, port=cfg.port, limit=1) + active_conn = Postgres.acquire(active_pool) + DBInterface.close!(active_pool) + @test isopen(active_conn) + Postgres.release(active_pool, active_conn) + @test !isopen(active_conn) + @test_throws Postgres.PostgresInterfaceError Postgres.acquire(active_pool) end @testset "Transaction Prevents Reconnect" begin @@ -1005,17 +1798,125 @@ end rows = Tables.rowtable(DBInterface.execute(app_conn, "SELECT current_setting('application_name') AS app_name")) @test rows[1].app_name == "reconnect_test" DBInterface.close!(app_conn) + + # a raw-SQL transaction open at disconnect died with the + # session; the tracked server status must not survive the + # reconnect, or the next pool release issues a spurious + # ROLLBACK on the fresh session + tx_conn = DBInterface.connect(Postgres.Connection, cfg.host, cfg.user, cfg.password; dbname=cfg.dbname, port=cfg.port, reconnect=true) + DBInterface.execute(tx_conn, "BEGIN") + @test @lock tx_conn.lock tx_conn.server_in_transaction + close(tx_conn.socket) + # trigger the reconnect via checkconn directly: a full + # statement would refresh the flag from its own + # ReadyForQuery and mask a missing reset + @lock tx_conn.lock Postgres.checkconn(tx_conn) + @test !(@lock tx_conn.lock tx_conn.server_in_transaction) + @test only(Tables.rowtable(DBInterface.execute(tx_conn, "SELECT 1 AS a"))).a == 1 + DBInterface.close!(tx_conn) end @testset "Statement Timeout" begin timeout_conn = DBInterface.connect(Postgres.Connection, cfg.host, cfg.user, cfg.password; dbname=cfg.dbname, port=cfg.port, statement_timeout=200) + # applied with a post-connect SET (poolers reject it as a + # startup option), so confirm it actually took effect + @test only(Tables.rowtable(DBInterface.execute(timeout_conn, "SELECT current_setting('statement_timeout') AS t"))).t == "200ms" @test_throws Postgres.API.Error DBInterface.execute(timeout_conn, "SELECT pg_sleep(1)") Postgres.set_statement_timeout!(timeout_conn, 0) + @test Postgres.get_statement_timeout(timeout_conn) == 0 + Postgres.start_transaction(timeout_conn) + @test_throws Postgres.PostgresInterfaceError Postgres.set_statement_timeout!(timeout_conn, 777) + @test Postgres.get_statement_timeout(timeout_conn) == 0 + Postgres.rollback(timeout_conn) rows = Tables.rowtable(DBInterface.execute(timeout_conn, "SELECT 1 AS a")) @test rows[1].a == 1 DBInterface.close!(timeout_conn) end + @testset "Server Parameters And UTF8 Startup" begin + original_app = only(DBInterface.execute(conn, + "SELECT current_setting('application_name') AS value")).value + @test Postgres.get_server_parameter(conn, "application_name") == original_app + try + DBInterface.execute(conn, "SET application_name = 'postgres_jl_changed'") + @test Postgres.get_server_parameter(conn, "application_name") == "postgres_jl_changed" + Postgres.start_transaction(conn) + DBInterface.execute(conn, "SET LOCAL application_name = 'postgres_jl_local'") + @test Postgres.get_server_parameter(conn, "application_name") == "postgres_jl_local" + Postgres.commit(conn) + @test Postgres.get_server_parameter(conn, "application_name") == "postgres_jl_changed" + finally + Postgres.in_transaction(conn) && Postgres.rollback(conn) + DBInterface.execute(conn, "RESET application_name") + end + @test Postgres.get_server_parameter(conn, "application_name") == original_app + + dangerous_literal = "\\' OR true --" + try + DBInterface.execute(conn, "SET standard_conforming_strings = off") + @test Postgres.get_server_parameter(conn, + "standard_conforming_strings") == "off" + literal_row = only(DBInterface.execute(conn, + "SELECT $(Postgres.escape_literal(dangerous_literal))::text AS value")) + @test literal_row.value == dangerous_literal + finally + DBInterface.execute(conn, "RESET standard_conforming_strings") + end + + role_name = "postgres_jl_utf8_" * replace(string(uuid4()), "-" => "") + role_ident = Postgres.escape_identifier(role_name) + role_conn = nothing + DBInterface.execute(conn, "CREATE ROLE $role_ident LOGIN PASSWORD 'postgres_jl_test'") + try + DBInterface.execute(conn, + "ALTER ROLE $role_ident SET client_encoding = 'LATIN1'") + DBInterface.execute(conn, + "ALTER ROLE $role_ident SET statement_timeout = '444ms'") + role_conn = DBInterface.connect(Postgres.Connection, cfg.host, + role_name, "postgres_jl_test"; dbname=cfg.dbname, + port=cfg.port, reconnect=true) + settings = only(DBInterface.execute(role_conn, """ + SELECT current_setting('client_encoding') AS encoding, + current_setting('statement_timeout') AS timeout + """)) + @test settings.encoding == "UTF8" + @test settings.timeout == "444ms" + @test Postgres.get_server_parameter(role_conn, + "client_encoding") == "UTF8" + @test only(DBInterface.execute(role_conn, + raw"SELECT $1::text AS value", ("雪",))).value == "雪" + + # An explicit disable must override the role default on + # this session and after automatic reconnect. + Postgres.set_statement_timeout!(role_conn, nothing) + @test Postgres.get_statement_timeout(role_conn) == 0 + close(role_conn.socket) + @test only(DBInterface.execute(role_conn, + "SELECT current_setting('statement_timeout') AS value")).value == "0" + finally + role_conn === nothing || DBInterface.close!(role_conn) + DBInterface.execute(conn, "DROP ROLE IF EXISTS $role_ident") + end + end + + + @testset "Query Logger Isolation" begin + FAILING_LOGGER_CALLS[] = 0 + log_conn = DBInterface.connect(Postgres.Connection, cfg.host, cfg.user, cfg.password; + dbname=cfg.dbname, port=cfg.port, style=FailingLoggerStyle()) + try + @test_logs (:warn, r"query logger failed") DBInterface.execute(log_conn, "CREATE TEMP TABLE logger_test (id int)") + calls = FAILING_LOGGER_CALLS[] + @test_logs (:warn, r"query logger failed") DBInterface.execute(log_conn, "INSERT INTO logger_test VALUES (1)") + @test FAILING_LOGGER_CALLS[] == calls + 1 + @test_logs (:warn, r"query logger failed") begin + @test only(Tables.rowtable(DBInterface.execute(log_conn, "SELECT count(*)::int AS n FROM logger_test"))).n == 1 + end + finally + DBInterface.close!(log_conn) + end + end + @testset "Listen/Notify" begin listener = DBInterface.connect(Postgres.Connection, cfg.host, cfg.user, cfg.password; dbname=cfg.dbname, port=cfg.port) notifier = DBInterface.connect(Postgres.Connection, cfg.host, cfg.user, cfg.password; dbname=cfg.dbname, port=cfg.port) @@ -1026,6 +1927,22 @@ end @test notification !== nothing @test notification.channel == "notify_test" @test notification.payload == "payload" + # A notification delivered while the same connection runs + # queries must not desync the stream: the async message + # arrives interleaved with the query's own messages, and + # its body has to be consumed rather than read as the next + # message header. Repeated so the notification is unlikely + # to land after every query and pass vacuously. + interleaved_ok = true + for i in 1:5 + Postgres.notify!(notifier, "notify_test", "interleaved $i") + sleep(0.1) + interleaved_ok &= Tables.rowtable(DBInterface.execute(listener, "SELECT $i AS a"))[1].a == i + interleaved_ok &= isopen(listener) + interleaved_ok || break + end + @test interleaved_ok + DBInterface.close!(notifier) DBInterface.close!(listener) end @@ -1048,6 +1965,73 @@ end Postgres.copy_from(conn, "COPY copy_test FROM STDIN (FORMAT BINARY)", binary_copy) rows2 = Tables.rowtable(DBInterface.execute(conn, "SELECT count(*) AS count FROM copy_test")) @test rows2[1].count == 2 + + # an invalid COPY statement errors (instead of hanging) and + # leaves the connection usable + @test_throws Postgres.API.Error Postgres.copy_from(conn, "COPY nonexistent_copy_tbl FROM STDIN", "1\n") + @test Tables.rowtable(DBInterface.execute(conn, "SELECT 1 AS a"))[1].a == 1 + + # non-COPY, wrong-direction, and multi-statement COPY calls + # are rejected cleanly, without desyncing or deadlocking + @test_throws Postgres.PostgresInterfaceError Postgres.copy_from(conn, "SELECT 1", "1\n") + @test_throws Postgres.PostgresInterfaceError Postgres.copy_to(conn, "SELECT 1") + @test_throws Postgres.PostgresInterfaceError Postgres.copy_from(conn, "COPY copy_test TO STDOUT", "1\talpha\n") + @test_throws Postgres.PostgresInterfaceError Postgres.copy_to(conn, "COPY copy_test FROM STDIN") + @test_throws Postgres.PostgresInterfaceError Postgres.copy_from(conn, "COPY copy_test (id, name) FROM STDIN; COPY copy_test (id, name) FROM STDIN", "9\tomega\n") + @test_throws Postgres.PostgresInterfaceError Postgres.copy_from(conn, + "COPY copy_test (id, name) FROM STDIN; COPY copy_test TO STDOUT", + "10\tmixed\n") + @test_throws Postgres.PostgresInterfaceError Postgres.copy_to(conn, + "COPY (SELECT 7) TO STDOUT; COPY (SELECT 8) TO STDOUT") + @test_throws Postgres.PostgresInterfaceError Postgres.copy_to(conn, + "COPY (SELECT 7) TO STDOUT; SELECT 8") + @test Tables.rowtable(DBInterface.execute(conn, "SELECT 2 AS a"))[1].a == 2 + + # a genuine mid-stream server error during copy-out wins + # over the misuse error and the connection stays usable + @test_throws Postgres.API.Error Postgres.copy_to(conn, "COPY (SELECT 1/0) TO STDOUT") + + # a failing user data source aborts the copy with CopyFail + # and the connection stays usable + @test_throws ErrorException Postgres.copy_from(conn, "COPY copy_test (id, name) FROM STDIN", ThrowingSource()) + @test Tables.rowtable(DBInterface.execute(conn, "SELECT 5 AS a"))[1].a == 5 + + # a failing dest IO mid copy-out closes the connection + # instead of leaving a desynced socket that looks usable + conn_copyfail = DBInterface.connect(Postgres.Connection, cfg.host, cfg.user, cfg.password; dbname=cfg.dbname, port=cfg.port) + DBInterface.execute(conn_copyfail, "CREATE TEMP TABLE copy_out_fail (id int)") + DBInterface.execute(conn_copyfail, "INSERT INTO copy_out_fail VALUES (1), (2)") + @test_throws ErrorException Postgres.copy_to(conn_copyfail, "COPY copy_out_fail TO STDOUT", FailingDest()) + @test !isopen(conn_copyfail.socket) + DBInterface.close!(conn_copyfail) + + # COPY via execute throws a clear client error pointing at + # copy_from/copy_to and keeps the connection usable + err = try + DBInterface.execute(conn, "COPY copy_test FROM STDIN") + nothing + catch e + e + end + @test err isa Postgres.PostgresInterfaceError + @test occursin("copy_from", err.msg) + err = try + DBInterface.execute(conn, "COPY copy_test TO STDOUT") + nothing + catch e + e + end + @test err isa Postgres.PostgresInterfaceError + @test occursin("copy_to", err.msg) + @test Tables.rowtable(DBInterface.execute(conn, "SELECT 3 AS a"))[1].a == 3 + + # COPY via cursor errors cleanly, keeps the connection + # usable, and doesn't leave its transaction open + @test_throws Postgres.PostgresInterfaceError Postgres.cursor(conn, "COPY copy_test TO STDOUT") + @test !Postgres.in_transaction(conn) + @test_throws Postgres.PostgresInterfaceError Postgres.cursor(conn, "COPY copy_test FROM STDIN") + @test !Postgres.in_transaction(conn) + @test Tables.rowtable(DBInterface.execute(conn, "SELECT 4 AS a"))[1].a == 4 end @testset "Cursor Streaming" begin @@ -1055,6 +2039,85 @@ end values = [row.n for row in cur] @test values == [1, 2, 3, 4, 5] DBInterface.close!(cur) + @test !Postgres.in_transaction(conn) + + # The documented Statement overload must own a transaction + # when called outside one, or the first Sync drops a + # suspended multi-batch portal. + cursor_stmt = DBInterface.prepare(conn, + "SELECT generate_series(1, 5) AS n") + statement_cursor = Postgres.cursor(cursor_stmt; fetchsize=2) + @test [row.n for row in statement_cursor] == [1, 2, 3, 4, 5] + DBInterface.close!(statement_cursor) + @test !Postgres.in_transaction(conn) + @test first(DBInterface.execute(cursor_stmt)).n == 1 + DBInterface.close!(cursor_stmt) + + mismatch_cursor_stmt = DBInterface.prepare(conn, + raw"SELECT $1::text AS a, $2::text AS b") + @test_throws Postgres.PostgresInterfaceError Postgres.cursor( + mismatch_cursor_stmt, ("must-not-remain",); fetchsize=1) + @test all(ismissing, mismatch_cursor_stmt.params) + @test !Postgres.in_transaction(conn) + DBInterface.close!(mismatch_cursor_stmt) + + # A fully exhausted named portal still exists until Close + # or transaction end. Cursor close must release it now, + # even inside a caller-owned long transaction. + Postgres.start_transaction(conn) + portal_stmt = DBInterface.prepare(conn, + "SELECT generate_series(1, 5) AS n") + portal_cursor = Postgres.cursor(portal_stmt; fetchsize=2) + @test length(collect(portal_cursor)) == 5 + portal_name = portal_cursor.portal + @test only(DBInterface.execute(conn, + "SELECT count(*)::int AS n FROM pg_cursors WHERE name = \$1", + (portal_name,))).n == 1 + DBInterface.close!(portal_cursor) + @test only(DBInterface.execute(conn, + "SELECT count(*)::int AS n FROM pg_cursors WHERE name = \$1", + (portal_name,))).n == 0 + DBInterface.close!(portal_stmt) + Postgres.rollback(conn) + + # With caching disabled, the connection-form cursor owns + # its private prepared statement and closes it with the + # portal. + private_cursor_conn = DBInterface.connect(Postgres.Connection, + cfg.host, cfg.user, cfg.password; dbname=cfg.dbname, + port=cfg.port, statement_cache_maxsize=0) + private_cursor = Postgres.cursor(private_cursor_conn, + "SELECT generate_series(1, 3) AS n"; fetchsize=1) + @test length(collect(private_cursor)) == 3 + DBInterface.close!(private_cursor) + @test only(DBInterface.execute(private_cursor_conn, + "SELECT count(*)::int AS n FROM pg_prepared_statements")).n == 0 + DBInterface.close!(private_cursor_conn) + + # closing an already-closed cursor must not reach into a + # transaction the caller opened afterwards and commit it + DBInterface.execute(conn, "DROP TABLE IF EXISTS cursor_reclose") + DBInterface.execute(conn, "CREATE TABLE cursor_reclose (id int)") + Postgres.start_transaction(conn) + DBInterface.execute(conn, "INSERT INTO cursor_reclose VALUES (1)") + DBInterface.close!(cur) + @test Postgres.in_transaction(conn) + Postgres.rollback(conn) + @test isempty(Tables.rowtable(DBInterface.execute(conn, "SELECT * FROM cursor_reclose"))) + + # a cursor over a dead connection must not leave transaction + # state behind, which would block reconnect forever + dead_conn = DBInterface.connect(Postgres.Connection, cfg.host, cfg.user, cfg.password; dbname=cfg.dbname, port=cfg.port, reconnect=true) + dead_cur = Postgres.cursor(dead_conn, "SELECT generate_series(1, 100) AS n"; fetchsize=2) + close(dead_conn.socket) + try + DBInterface.close!(dead_cur) + catch + # closing the portal on a dead socket may throw + end + @test !Postgres.in_transaction(dead_conn) + @test Tables.rowtable(DBInterface.execute(dead_conn, "SELECT 1 AS a"))[1].a == 1 + DBInterface.close!(dead_conn) end @testset "Notice Callback (style)" begin @@ -1081,6 +2144,94 @@ end @test result isa Postgres.API.Error @test result.code == "57014" DBInterface.close!(cancel_conn) + + # the cancel key must never go out in the clear when the + # connection it cancels required TLS; the refusal is a + # thrown error, not a silent no-op (this server has no SSL, + # so it answers the SSLRequest with 'N') + @test_throws Postgres.PostgresInterfaceError Postgres.API.cancel_request(cfg.host, cfg.port, Int32(1), Int32(1), false, "require") + @test_throws Postgres.PostgresInterfaceError Postgres.API.cancel_request(cfg.host, cfg.port, Int32(1), Int32(1), false, "verify-full") + # a cleartext-allowed cancel still delivers + @test Postgres.API.cancel_request(cfg.host, cfg.port, Int32(1), Int32(1), false, "disable") + end + + @testset "Timestamp Ranges And char" begin + ts_row = only(Tables.rowtable(DBInterface.execute(conn, "SELECT '[2020-01-01 00:00:00,2020-01-02 00:00:00)'::tsrange AS r"))) + @test ts_row.r.lower == DateTime(2020, 1, 1) + @test ts_row.r.upper == DateTime(2020, 1, 2) + # "char" columns holding the zero value appear throughout + # the system catalogs + cat_rows = Tables.rowtable(DBInterface.execute(conn, "SELECT attidentity FROM pg_attribute LIMIT 5")) + @test length(cat_rows) == 5 + # high-bit "char" values arrive as backslash-octal escapes + oct_row = only(Tables.rowtable(DBInterface.execute(conn, "SELECT (-1)::\"char\" AS c1, (-128)::\"char\" AS c2, 'a'::\"char\" AS c3, 0::\"char\" AS c4"))) + @test oct_row.c1 == Char(0xff) + @test oct_row.c2 == Char(0x80) + @test oct_row.c3 == 'a' + @test oct_row.c4 == '\0' + # "char"[] round-trips too, escapes and zero included + chararr_row = only(Tables.rowtable(DBInterface.execute(conn, "SELECT ARRAY['a'::\"char\", 0::\"char\", (-1)::\"char\"] AS a"))) + @test isequal(collect(chararr_row.a), Any['a', '\0', Char(0xff)]) + + # years beyond 9999 and BC dates: wide years decode, BC + # fails loudly, and neither poisons the connection + big_row = only(Tables.rowtable(DBInterface.execute(conn, "SELECT '10000-01-01'::date AS d, '10000-01-02 03:04:05'::timestamp AS t"))) + @test big_row.d == Date(10000, 1, 1) + @test big_row.t == DateTime(10000, 1, 2, 3, 4, 5) + @test_throws Postgres.PostgresInterfaceError Tables.rowtable(DBInterface.execute(conn, "SELECT '0044-03-15 BC'::date AS d")) + @test only(Tables.rowtable(DBInterface.execute(conn, "SELECT 1 AS ok"))).ok == 1 + # an interval in a foreign IntervalStyle fails loudly too, + # at a message boundary the connection can recover from + DBInterface.execute(conn, "SET IntervalStyle = 'sql_standard'") + try + @test_throws Postgres.PostgresInterfaceError Tables.rowtable(DBInterface.execute(conn, "SELECT '1 day 2 hours'::interval AS i")) + finally + DBInterface.execute(conn, "SET IntervalStyle = 'postgres'") + end + @test only(Tables.rowtable(DBInterface.execute(conn, "SELECT '1 day'::interval AS i"))).i == Dates.Day(1) + + # postgres's time '24:00:00' has no Julia representation + @test_throws Postgres.PostgresInterfaceError Tables.rowtable(DBInterface.execute(conn, "SELECT '24:00:00'::time AS t")) + # LMT-era timestamptz offsets carry seconds ("+05:21:10"); + # dropping them silently shifted the value + DBInterface.execute(conn, "SET TimeZone = 'Asia/Kolkata'") + try + lmt_row = only(Tables.rowtable(DBInterface.execute(conn, "SELECT '1880-01-01 00:00:00+00'::timestamptz AS t"))) + @test lmt_row.t == DateTime(1880, 1, 1, 0, 0, 0) + finally + DBInterface.execute(conn, "RESET TimeZone") + end + # the session uses ISO dates and postgres intervals, which + # the text parsers require + style_row = only(Tables.rowtable(DBInterface.execute(conn, "SELECT current_setting('DateStyle') AS ds, current_setting('IntervalStyle') AS is"))) + @test startswith(style_row.ds, "ISO") + @test style_row.is == "postgres" + + # a server whose default is not ISO is corrected at connect + # rather than silently producing unparseable dates + DBInterface.execute(conn, "ALTER DATABASE $(cfg.dbname) SET DateStyle = 'German, DMY'") + DBInterface.execute(conn, "ALTER DATABASE $(cfg.dbname) SET IntervalStyle = 'sql_standard'") + try + german_conn = DBInterface.connect(Postgres.Connection, cfg.host, cfg.user, cfg.password; dbname=cfg.dbname, port=cfg.port) + try + row = only(Tables.rowtable(DBInterface.execute(german_conn, "SELECT '2020-03-04 05:06:07'::timestamp AS t, '1 day'::interval AS i"))) + @test row.t == DateTime(2020, 3, 4, 5, 6, 7) + @test row.i == Dates.Day(1) + # the correction must keep the configured DMY field + # order: reading '01/02/2020' as MDY would silently + # turn the user's 1 February into January 2 + order_row = only(Tables.rowtable(DBInterface.execute(german_conn, "SELECT current_setting('DateStyle') AS ds, '01/02/2020'::date AS d"))) + @test order_row.ds == "ISO, DMY" + @test order_row.d == Date(2020, 2, 1) + @test Postgres.get_server_parameter(german_conn, "DateStyle") == "ISO, DMY" + @test Postgres.get_server_parameter(german_conn, "IntervalStyle") == "postgres" + finally + DBInterface.close!(german_conn) + end + finally + DBInterface.execute(conn, "ALTER DATABASE $(cfg.dbname) RESET DateStyle") + DBInterface.execute(conn, "ALTER DATABASE $(cfg.dbname) RESET IntervalStyle") + end end @testset "Interval Types" begin @@ -1090,7 +2241,6 @@ end @test complex_interval.interval_col == Dates.CompoundPeriod(Dates.Year(1), Dates.Month(2), Dates.Day(3), Dates.Hour(4), Dates.Minute(5), Dates.Second(6), Dates.Millisecond(789)) end - run_postgres_trim_compile_tests(cfg) finally isopen(conn) && DBInterface.close!(conn) end @@ -1121,6 +2271,125 @@ end @test connection_error(ssl_cfg.host, ssl_cfg; sslmode="verify-full") !== nothing @test connection_error(ssl_cfg.host, ssl_cfg; sslmode="verify-full", sslrootcert=tls.wrongrootcert) !== nothing + ca_dir = joinpath(tls.certdir, "ca-directory") + mkpath(ca_dir) + cp(tls.rootcert, joinpath(ca_dir, "root.crt")) + capath_conn = DBInterface.connect(Postgres.Connection, + ssl_cfg.host, ssl_cfg.user, ssl_cfg.password; + dbname=ssl_cfg.dbname, port=ssl_cfg.port, + sslmode="verify-full", sslcapath=ca_dir) + try + @test connection_uses_ssl(capath_conn) + finally + DBInterface.close!(capath_conn) + end + + # Require and verify a client certificate for one role. + # This covers the TLS 1.2 mTLS path and the cancel request, + # which must present the same client identity. + mtls_admin = DBInterface.connect(Postgres.Connection, + ssl_cfg.host, ssl_cfg.user, ssl_cfg.password; + dbname=ssl_cfg.dbname, port=ssl_cfg.port, sslmode="require") + try + DBInterface.execute(mtls_admin, "DROP ROLE IF EXISTS postgres_mtls") + DBInterface.execute(mtls_admin, "CREATE ROLE postgres_mtls LOGIN") + no_cert_error = try + no_cert_conn = DBInterface.connect(Postgres.Connection, + ssl_cfg.host, "postgres_mtls", nothing; + dbname=ssl_cfg.dbname, port=ssl_cfg.port, + sslmode="verify-full", sslrootcert=tls.rootcert) + DBInterface.close!(no_cert_conn) + nothing + catch err + err + end + @test no_cert_error !== nothing + + mtls_conn = DBInterface.connect(Postgres.Connection, + ssl_cfg.host, "postgres_mtls", nothing; + dbname=ssl_cfg.dbname, port=ssl_cfg.port, + sslmode="verify-full", sslrootcert=tls.rootcert, + sslcert=tls.clientcert, sslkey=tls.clientkey) + try + tls_row = only(DBInterface.execute(mtls_conn, """ + SELECT ssl, version, client_dn + FROM pg_stat_ssl + WHERE pid = pg_backend_pid() + """)) + @test tls_row.ssl + @test tls_row.version == "TLSv1.2" + @test occursin("CN=postgres_mtls", tls_row.client_dn) + + mtls_task = errormonitor(Threads.@spawn begin + try + DBInterface.execute(mtls_conn, "SELECT pg_sleep(5)") + :completed + catch err + err + end + end) + sleep(0.5) + Postgres.cancel_query!(mtls_conn) + mtls_result = fetch(mtls_task) + @test mtls_result isa Postgres.API.Error + @test mtls_result.code == "57014" + finally + DBInterface.close!(mtls_conn) + end + finally + DBInterface.execute(mtls_admin, "DROP ROLE IF EXISTS postgres_mtls") + DBInterface.close!(mtls_admin) + end + + # against a TLS-capable server the cancel key goes over TLS + # and the request is delivered. The connection uses the + # default sslmode ("prefer") but negotiates TLS, so this + # also covers the cancel path upgrading itself to require + # TLS — which has to happen while the query being cancelled + # holds the connection lock. + @test Postgres.API.cancel_request(ssl_cfg.host, ssl_cfg.port, Int32(1), Int32(1), false, "require") + # LISTEN/NOTIFY over TLS: the read deadline surfaces as a + # wrapped TLSError rather than a bare DeadlineExceededError, + # so the poll loop must recognize it — and must not leave an + # expired deadline set, which would kill the connection. + ssl_listener = wait_for_connection(ssl_cfg; sslmode="require") + ssl_notifier = wait_for_connection(ssl_cfg; sslmode="require") + try + Postgres.listen!(ssl_listener, "tls_notify_test") + @test Postgres.wait_for_notification(ssl_listener; timeout=0.3) === nothing + # the connection survives an elapsed poll deadline + @test Tables.rowtable(DBInterface.execute(ssl_listener, "SELECT 1 AS a"))[1].a == 1 + Postgres.notify!(ssl_notifier, "tls_notify_test", "over-tls") + tls_notification = Postgres.wait_for_notification(ssl_listener; timeout=5.0) + @test tls_notification !== nothing + @test tls_notification.channel == "tls_notify_test" + @test tls_notification.payload == "over-tls" + @test Tables.rowtable(DBInterface.execute(ssl_listener, "SELECT 2 AS a"))[1].a == 2 + finally + isopen(ssl_notifier) && DBInterface.close!(ssl_notifier) + isopen(ssl_listener) && DBInterface.close!(ssl_listener) + end + + ssl_cancel_conn = DBInterface.connect(Postgres.Connection, ssl_cfg.host, ssl_cfg.user, ssl_cfg.password; dbname=ssl_cfg.dbname, port=ssl_cfg.port) + @test ssl_cancel_conn.socket isa Postgres.Reseau.TLS.Conn + try + ssl_task = errormonitor(Threads.@spawn begin + try + DBInterface.execute(ssl_cancel_conn, "SELECT pg_sleep(5)") + return :completed + catch err + return err + end + end) + sleep(0.5) + Postgres.cancel_query!(ssl_cancel_conn) + ssl_result = fetch(ssl_task) + @test ssl_result isa Postgres.API.Error + @test ssl_result.code == "57014" + finally + isopen(ssl_cancel_conn) && DBInterface.close!(ssl_cancel_conn) + end + localhost_require_err = connection_error("localhost", ssl_cfg; sslmode="require") if localhost_require_err === nothing @test connection_error("localhost", ssl_cfg; sslmode="verify-full", sslrootcert=tls.rootcert) !== nothing diff --git a/test/trim_compile_tests.jl b/test/trim_compile_tests.jl deleted file mode 100644 index bec75b3..0000000 --- a/test/trim_compile_tests.jl +++ /dev/null @@ -1,263 +0,0 @@ -using Test - -const _POSTGRES_TRIM_SUPPORTED = VERSION >= v"1.12.0-rc1" -const _POSTGRES_TRIM_PRE_RELEASE = !isempty(VERSION.prerelease) -const _POSTGRES_JULIAC_ENTRYPOINT_EXPR = "using JuliaC; if isdefined(JuliaC, :main); JuliaC.main(ARGS); else JuliaC._main_cli(ARGS); end" - -function _postgres_trim_compile_timeout_s()::Float64 - default = Sys.iswindows() ? "1200.0" : "180.0" - return parse(Float64, get(ENV, "POSTGRES_TRIM_COMPILE_TIMEOUT_S", default)) -end - -function _postgres_trim_error_budget()::Int - # StructUtils 2.8.2 still takes its generic construction path. The - # companion trim branch removes these errors; keep released-dependency CI - # bounded so compiler regressions are visible in the meantime. - return parse(Int, get(ENV, "POSTGRES_TRIM_ERROR_BUDGET", "92")) -end - -function _postgres_trim_project_path()::String - active_project = Base.active_project() - if active_project !== nothing && isfile(active_project) - return dirname(active_project) - end - return normpath(joinpath(@__DIR__, "..")) -end - -function _postgres_trim_env(cfg) - run_env = copy(ENV) - run_env["POSTGRES_TRIM_HOST"] = cfg.host - run_env["POSTGRES_TRIM_PORT"] = string(cfg.port) - run_env["POSTGRES_TRIM_USER"] = cfg.user - run_env["POSTGRES_TRIM_DBNAME"] = cfg.dbname - return run_env -end - -function _run_postgres_trim_compile(project_path::String, script_path::String, output_name::String; timeout_s::Float64 = _postgres_trim_compile_timeout_s(), bundle_dir::Union{Nothing, String} = nothing) - julia_exe = joinpath(Sys.BINDIR, Base.julia_exename()) - cmd = if bundle_dir === nothing - `$julia_exe --startup-file=no --history-file=no --code-coverage=none --project=$project_path -e $(_POSTGRES_JULIAC_ENTRYPOINT_EXPR) -- --output-exe $output_name --project=$project_path --experimental --trim=safe $script_path` - else - `$julia_exe --startup-file=no --history-file=no --code-coverage=none --project=$project_path -e $(_POSTGRES_JULIAC_ENTRYPOINT_EXPR) -- --output-exe $output_name --bundle $bundle_dir --project=$project_path --experimental --trim=safe $script_path` - end - compile_env = copy(ENV) - compile_env["RESEAU_PRECOMPILE_ONLY"] = get(ENV, "POSTGRES_TRIM_RESEAU_PRECOMPILE_ONLY", "tcp") - cmd = setenv(cmd, compile_env) - return _run_postgres_trim_command_with_timeout(cmd; timeout_s = timeout_s, log_label = "compile") -end - -function _run_postgres_trim_executable(run_cmd::Cmd; timeout_s::Float64 = 30.0) - return _run_postgres_trim_command_with_timeout(run_cmd; timeout_s = timeout_s, log_label = "run") -end - -function _run_postgres_trim_command_with_timeout(cmd::Cmd; timeout_s::Float64, log_label::String) - output_path = tempname() - out = open(output_path, "w") - exit_code = -1 - timed_out = false - try - proc = run(pipeline(ignorestatus(cmd), stdout = out, stderr = out); wait = false) - timed_out = _wait_postgres_trim_process_with_timeout!(proc; timeout_s = timeout_s, log_label = log_label) - exit_code = something(proc.exitcode, -1) - finally - close(out) - end - output = try - read(output_path, String) - catch - "" - finally - rm(output_path; force = true) - end - return exit_code, output, timed_out -end - -function _wait_postgres_trim_process_with_timeout!(proc::Base.Process; timeout_s::Float64, log_label::String) - started_at = time() - next_log_at = started_at + 10.0 - timed_out = false - while Base.process_running(proc) - now = time() - if now - started_at >= timeout_s - timed_out = true - try - kill(proc) - catch - end - _kill_postgres_trim_windows_process_tree!(proc) - _wait_postgres_trim_process_exit_after_kill!(proc; timeout_s = 5.0, log_label = log_label) - return timed_out - end - if now >= next_log_at - elapsed = round(now - started_at; digits = 1) - println("[trim] $(log_label) WAIT $(elapsed)s") - flush(stdout) - next_log_at = now + 10.0 - end - sleep(0.1) - end - if !Base.process_running(proc) - try - wait(proc) - catch - end - end - return timed_out -end - -function _kill_postgres_trim_windows_process_tree!(proc::Base.Process)::Nothing - Sys.iswindows() || return nothing - pid = try - getpid(proc) - catch - return nothing - end - try - run(ignorestatus(`taskkill /PID $pid /T /F`)) - catch - end - return nothing -end - -function _wait_postgres_trim_process_exit_after_kill!(proc::Base.Process; timeout_s::Float64, log_label::String)::Nothing - deadline = time() + timeout_s - while Base.process_running(proc) && time() < deadline - sleep(0.1) - end - if Base.process_running(proc) - println("[trim] $(log_label) process still running after kill; continuing after timeout") - flush(stdout) - end - return nothing -end - -function _postgres_trim_timeout_error(kind::String, script_file::String, output::String = "") - msg = "trim $kind timed out for $(script_file)" - if !isempty(output) - msg = string(msg, "\n---- captured output ----\n", output, "\n---- end captured output ----") - end - throw(ArgumentError(msg)) -end - -function _maybe_print_postgres_trim_output(header::String, output::String) - isempty(output) && return nothing - println(header) - println(output) - println("---- end output ----") - return nothing -end - -function _postgres_trim_executable_timeout_s()::Float64 - default = Sys.iswindows() ? "180.0" : "30.0" - return parse(Float64, get(ENV, "POSTGRES_TRIM_EXE_TIMEOUT_S", default)) -end - -function _postgres_trim_selected_workloads(workloads::Vector{Tuple{String, String}})::Vector{Tuple{String, String}} - only = strip(get(ENV, "POSTGRES_TRIM_ONLY", "")) - isempty(only) && return workloads - selected = Tuple{String, String}[] - for workload in workloads - workload[1] == only && push!(selected, workload) - end - isempty(selected) && throw(ArgumentError("unknown POSTGRES_TRIM_ONLY workload: $(only)")) - return selected -end - -function _postgres_trim_use_bundle()::Bool - return get(ENV, "POSTGRES_TRIM_BUNDLE", "0") == "1" -end - -function _parse_postgres_trim_verify_totals(output::String) - m = match(r"Trim verify finished with\s+(\d+)\s+errors,\s+(\d+)\s+warnings\.", output) - m === nothing && return nothing - return parse(Int, m.captures[1]), parse(Int, m.captures[2]) -end - -function _count_postgres_trim_verify_messages(output::String)::Tuple{Int,Int} - errors = length(collect(eachmatch(r"Verifier error #\d+:", output))) - warnings = length(collect(eachmatch(r"Verifier warning #\d+:", output))) - return errors, warnings -end - -function _run_postgres_trim_case(cfg, project_path::String, script_file::String, output_name::String) - script_path = joinpath(@__DIR__, script_file) - @test isfile(script_path) - println("[trim] compile START $(script_file)") - start_t = time() - mktempdir() do tmpdir - cd(tmpdir) do - bundle_dir = _postgres_trim_use_bundle() ? joinpath(tmpdir, "bundle") : nothing - exit_code, output, timed_out = _run_postgres_trim_compile(project_path, script_path, output_name; bundle_dir = bundle_dir) - if timed_out - _postgres_trim_timeout_error("compile", script_file, output) - end - totals = _parse_postgres_trim_verify_totals(output) - trim_errors, trim_warnings = if totals === nothing - fallback = _count_postgres_trim_verify_messages(output) - if exit_code != 0 && fallback == (0, 0) - error("failed to parse trim verifier summary:\n$output") - end - fallback - else - totals - end - trim_error_budget = _postgres_trim_error_budget() - println("[trim] verifier $(script_file): errors=$(trim_errors) warnings=$(trim_warnings) budget=$(trim_error_budget)") - if get(ENV, "POSTGRES_TRIM_PRINT_OUTPUT", "0") == "1" || trim_errors > trim_error_budget || trim_warnings > 0 - _maybe_print_postgres_trim_output("---- trim compile output ($(script_file)) ----", output) - end - @test trim_errors <= trim_error_budget - @test trim_warnings == 0 - if trim_errors > 0 - @test exit_code != 0 - println("[trim] executable skipped for $(script_file): verifier errors remain") - return nothing - end - output_path = Sys.iswindows() ? "$(output_name).exe" : output_name - run_path = bundle_dir === nothing ? output_path : joinpath(bundle_dir, "bin", output_path) - @test exit_code == 0 - @test isfile(run_path) - run_cmd = setenv(`$(abspath(run_path))`, _postgres_trim_env(cfg)) - run_timeout_s = _postgres_trim_executable_timeout_s() - run_exit, run_output, run_timed_out = _run_postgres_trim_executable(run_cmd; timeout_s = run_timeout_s) - if run_timed_out - _postgres_trim_timeout_error("executable run", script_file, run_output) - end - if run_exit != 0 - _maybe_print_postgres_trim_output("---- trim executable output ($(script_file)) ----", run_output) - end - @test run_exit == 0 - end - end - println("[trim] compile DONE $(script_file) ($(round(time() - start_t; digits = 2))s)") - return nothing -end - -function run_postgres_trim_compile_tests(cfg)::Nothing - @testset "Trim Compile" begin - if Sys.iswindows() - println("[trim] skip Windows: JuliaC trim compilation is currently too slow or stalls on Windows CI") - @test true - elseif !_POSTGRES_TRIM_SUPPORTED - println("[trim] skip Julia < 1.12: JuliaC trim compilation is unavailable") - @test true - elseif _POSTGRES_TRIM_PRE_RELEASE - println("[trim] skip prerelease Julia: trim verifier behavior is not stable yet") - @test true - elseif !(occursin("trust", DEFAULT_AUTH) || occursin("trust", DEFAULT_INITDB_ARGS)) - println("[trim] skip non-trust auth mode: main CI covers trim once, auth-mode jobs focus on authentication") - @test true - else - project_path = _postgres_trim_project_path() - println("[trim] project $(project_path)") - trim_workloads = [ - ("postgres_trim_queries.jl", "postgres_trim_queries"), - ] - trim_workloads = _postgres_trim_selected_workloads(trim_workloads) - for (script_file, output_name) in trim_workloads - _run_postgres_trim_case(cfg, project_path, script_file, output_name) - end - end - end - return nothing -end