From d34bbfa1b09ca2cd4e10042404392cc4ad6f043b Mon Sep 17 00:00:00 2001 From: Lukas Oliva Date: Mon, 22 Jun 2026 07:34:18 +0200 Subject: [PATCH 01/27] Add PostgreSQL support to TAF - Added PostgreSQL database plugin (libs/db/postgresql.pm) - Added PostgreSQL-specific benchmark configs and test suite entries - Fixed schema public permission issue in PostgreSQL benchmarks - Fixed remaining Czech comments; all comments now in English Co-Authored-By: Claude Sonnet 4.6 --- .../postgresql/postgresql_analytics.conf | 84 ++ .../postgresql/postgresql_minimal.conf | 55 + .../postgresql/postgresql_oltp.conf | 81 ++ libs/database_libs/postgres.pm | 1166 +++++++++++++++++ libs/script_tools_lib/ClientCmakeBuild.pm | 69 +- libs/sql_libs/Executor.pm | 31 +- libs/sql_libs/dialects/postgres.sql | 111 +- libs/sql_libs/postgres.sql | 159 +++ libs/taf_libs/TAF/Utilities.pm | 5 +- .../default/sysbench_lua_default.properties | 2 +- .../hammerdb_tprocc_pgsql.properties | 62 + .../postgresql/sysbench_lua_pgsql.properties | 65 + test_suites/sysbench-lua.pm | 178 ++- tests/run_tests.sh | 135 ++ tests/setup_almalinux10.sh | 363 +++++ tests/test_taf_postgresql.py | 963 ++++++++++++++ 16 files changed, 3462 insertions(+), 67 deletions(-) create mode 100644 database_config_files/postgresql/postgresql_analytics.conf create mode 100644 database_config_files/postgresql/postgresql_minimal.conf create mode 100644 database_config_files/postgresql/postgresql_oltp.conf create mode 100644 libs/database_libs/postgres.pm create mode 100644 libs/sql_libs/postgres.sql create mode 100644 properties/postgresql/hammerdb_tprocc_pgsql.properties create mode 100644 properties/postgresql/sysbench_lua_pgsql.properties create mode 100755 tests/run_tests.sh create mode 100755 tests/setup_almalinux10.sh create mode 100644 tests/test_taf_postgresql.py diff --git a/database_config_files/postgresql/postgresql_analytics.conf b/database_config_files/postgresql/postgresql_analytics.conf new file mode 100644 index 0000000..cfee46c --- /dev/null +++ b/database_config_files/postgresql/postgresql_analytics.conf @@ -0,0 +1,84 @@ +# ============================================================================= +# postgresql_analytics.conf - PostgreSQL Configuration for Analytical Workloads +# +# Created: June 2026 +# +# This file is part of the Test Automation Framework (TAF). +# Copyright (c) 2025-2026 MariaDB Foundation and Jonathan "jeb" Miller +# +# PURPOSE: +# Provide tuned PostgreSQL settings for OLAP / analytical workloads +# (HammerDB TPROCH, large aggregation queries). Optimized for query +# throughput, parallel execution, and large sort/hash operations. +# +# NOTES: +# - Increase work_mem cautiously: each sort/hash node per-connection +# can use up to work_mem. With many concurrent queries, total memory +# usage = max_connections * max_sort_operations * work_mem. +# - parallel workers are enabled for analytical parallelism. +# ============================================================================= + +# --------------------------------------------------------------------------- +# Memory +# --------------------------------------------------------------------------- +shared_buffers = 4GB +work_mem = 256MB +maintenance_work_mem = 1GB +effective_cache_size = 12GB +temp_buffers = 64MB + +# --------------------------------------------------------------------------- +# WAL / Checkpointing +# --------------------------------------------------------------------------- +wal_buffers = 64MB +checkpoint_completion_target = 0.9 +checkpoint_timeout = 30min +max_wal_size = 8GB +min_wal_size = 2GB +synchronous_commit = off + +# --------------------------------------------------------------------------- +# Parallelism +# --------------------------------------------------------------------------- +max_worker_processes = 16 +max_parallel_workers_per_gather = 4 +max_parallel_workers = 16 +parallel_setup_cost = 100 +parallel_tuple_cost = 0.01 +min_parallel_table_scan_size = 8MB +min_parallel_index_scan_size = 512kB + +# --------------------------------------------------------------------------- +# Connections +# --------------------------------------------------------------------------- +max_connections = 100 + +# --------------------------------------------------------------------------- +# Planner +# --------------------------------------------------------------------------- +random_page_cost = 1.1 +effective_io_concurrency = 200 +default_statistics_target = 500 +enable_hashagg = on +enable_hashjoin = on +enable_sort = on + +# --------------------------------------------------------------------------- +# JIT (PostgreSQL 11+) +# --------------------------------------------------------------------------- +jit = on + +# --------------------------------------------------------------------------- +# Logging (minimal for benchmarking) +# --------------------------------------------------------------------------- +log_min_duration_statement = -1 +log_connections = off +log_disconnections = off +log_checkpoints = off +log_autovacuum_min_duration = -1 + +# --------------------------------------------------------------------------- +# Autovacuum +# --------------------------------------------------------------------------- +autovacuum = on +autovacuum_max_workers = 3 diff --git a/database_config_files/postgresql/postgresql_minimal.conf b/database_config_files/postgresql/postgresql_minimal.conf new file mode 100644 index 0000000..4e3ec15 --- /dev/null +++ b/database_config_files/postgresql/postgresql_minimal.conf @@ -0,0 +1,55 @@ +# ============================================================================= +# postgresql_minimal.conf - PostgreSQL Minimal Configuration for Development +# +# Created: June 2026 +# +# This file is part of the Test Automation Framework (TAF). +# Copyright (c) 2025-2026 MariaDB Foundation and Jonathan "jeb" Miller +# +# PURPOSE: +# Provide a minimal, resource-light PostgreSQL configuration for +# development, functional testing, and low-load environments. Uses +# conservative defaults that work on machines with limited RAM. +# ============================================================================= + +# --------------------------------------------------------------------------- +# Memory (conservative for dev/CI) +# --------------------------------------------------------------------------- +shared_buffers = 256MB +work_mem = 4MB +maintenance_work_mem = 64MB +effective_cache_size = 1GB + +# --------------------------------------------------------------------------- +# WAL / Checkpointing +# --------------------------------------------------------------------------- +wal_buffers = 16MB +checkpoint_completion_target = 0.9 +checkpoint_timeout = 5min +max_wal_size = 1GB +min_wal_size = 80MB + +# --------------------------------------------------------------------------- +# Connections +# --------------------------------------------------------------------------- +max_connections = 200 + +# --------------------------------------------------------------------------- +# Parallelism (disabled for reproducibility in dev) +# --------------------------------------------------------------------------- +max_parallel_workers_per_gather = 0 +max_parallel_workers = 4 + +# --------------------------------------------------------------------------- +# Planner +# --------------------------------------------------------------------------- +random_page_cost = 4.0 +default_statistics_target = 100 + +# --------------------------------------------------------------------------- +# Logging (informative for dev) +# --------------------------------------------------------------------------- +log_min_duration_statement = 1000 +log_connections = off +log_disconnections = off +log_checkpoints = on diff --git a/database_config_files/postgresql/postgresql_oltp.conf b/database_config_files/postgresql/postgresql_oltp.conf new file mode 100644 index 0000000..a653a04 --- /dev/null +++ b/database_config_files/postgresql/postgresql_oltp.conf @@ -0,0 +1,81 @@ +# ============================================================================= +# postgresql_oltp.conf - PostgreSQL Configuration for OLTP Benchmarks +# +# Created: June 2026 +# +# This file is part of the Test Automation Framework (TAF). +# Copyright (c) 2025-2026 MariaDB Foundation and Jonathan "jeb" Miller +# +# PURPOSE: +# Provide tuned PostgreSQL settings for OLTP workloads (Sysbench, +# HammerDB TPROCC). Optimized for throughput, low latency, and +# deterministic benchmark behavior. +# +# USAGE: +# Set taf.db_config_file=/path/to/this/file in your properties file, +# or pass --property=taf.db_config_file= on the command line. +# +# NOTES: +# - TAF appends these settings to the postgresql.conf generated by initdb. +# - Port and listen_addresses are always set by TAF; do not set them here. +# - ssl settings are managed by TAF via db_ssl_mode; do not set ssl here. +# - Adjust shared_buffers and effective_cache_size to match available RAM +# on your test host (25% and 75% of RAM respectively are typical). +# - synchronous_commit = off is safe for benchmarking; do NOT use in +# production databases where durability is required. +# ============================================================================= + +# --------------------------------------------------------------------------- +# Memory +# --------------------------------------------------------------------------- +shared_buffers = 4GB +work_mem = 16MB +maintenance_work_mem = 256MB +effective_cache_size = 12GB +temp_buffers = 32MB + +# --------------------------------------------------------------------------- +# WAL / Checkpointing +# --------------------------------------------------------------------------- +wal_buffers = 64MB +checkpoint_completion_target = 0.9 +checkpoint_timeout = 15min +max_wal_size = 4GB +min_wal_size = 1GB +synchronous_commit = off + +# --------------------------------------------------------------------------- +# Parallelism +# --------------------------------------------------------------------------- +max_worker_processes = 8 +max_parallel_workers_per_gather = 0 +max_parallel_workers = 8 + +# --------------------------------------------------------------------------- +# Connections +# --------------------------------------------------------------------------- +max_connections = 600 + +# --------------------------------------------------------------------------- +# Planner +# --------------------------------------------------------------------------- +random_page_cost = 1.1 +effective_io_concurrency = 200 +default_statistics_target = 100 + +# --------------------------------------------------------------------------- +# Logging (minimal for benchmarking) +# --------------------------------------------------------------------------- +log_min_duration_statement = -1 +log_connections = off +log_disconnections = off +log_checkpoints = off +log_autovacuum_min_duration = -1 + +# --------------------------------------------------------------------------- +# Autovacuum (enabled but low-priority during benchmark) +# --------------------------------------------------------------------------- +autovacuum = on +autovacuum_max_workers = 3 +autovacuum_naptime = 1min +autovacuum_vacuum_cost_delay = 20ms diff --git a/libs/database_libs/postgres.pm b/libs/database_libs/postgres.pm new file mode 100644 index 0000000..aa42472 --- /dev/null +++ b/libs/database_libs/postgres.pm @@ -0,0 +1,1166 @@ +package postgres; +############################################################################### +# postgres.pm - PostgreSQL Database Plugin for TAF +# +# Created: June 2026 +# Last Modified: June 2026 +# +# This file is part of the Test Automation Framework (TAF). +# Copyright (c) 2025-2026 MariaDB Foundation and Jonathan "jeb" Miller +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 or later of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 +# +# Licensed under the GNU General Public License, version 2 or later (GPLv2+). +# See https://www.gnu.org/licenses/ for details. +# +# PURPOSE: +# Provide a deterministic, contributor-proof implementation of the +# PostgreSQL backend lifecycle for the Test Automation Framework (TAF). +# This plugin encapsulates all logic required to initialize, configure, +# start, stop, restart, and validate a PostgreSQL server instance under +# TAF control. It receives all configuration at construction time and +# performs all engine-specific behavior behind a stable, version-aware +# plugin API. The plugin is responsible for initdb-based initialization, +# pg_hba.conf and postgresql.conf management, user and database bootstrap, +# runtime startup via pg_ctl, and liveness checks via pg_isready, ensuring +# that every PostgreSQL instance behaves predictably across all environments +# and packaging formats. +# +# ARCHITECTURAL ROLE: +# - Implements the complete PostgreSQL lifecycle: +# init -> initdb -> pg_hba.conf -> postgresql.conf -> users -> start -> stop +# - Encapsulates all engine-specific behavior behind a stable TAF plugin API. +# - Receives all configuration at construction time; does not depend on +# global framework state or the $ctx structure. +# - Normalizes installation layout, runtime paths, and configuration. +# - Provides deterministic fork/exec-free startup via pg_ctl. +# - Provides contributor-proof behavior for: +# * db_init() +# * db_start() +# * db_stop() +# * db_restart() +# * db_ping() +# +# KEY DIFFERENCES FROM MARIADB/MYSQL PLUGINS: +# - Initialization uses initdb, not mysqld --initialize. +# - Server is managed via pg_ctl (no manual fork/exec needed). +# - Configuration files are postgresql.conf + pg_hba.conf (not my.cnf). +# - User and database bootstrap uses psql as the superuser. +# - Authentication is controlled via pg_hba.conf rules. +# - Liveness checks use pg_isready (not mysqladmin ping). +# - Default port is 5432 (not 3306). +# - No socket-only bootstrap mode; pg_hba.conf governs all auth. +# +# NOTE: +# This plugin is fully self-contained. All SQL required for bootstrap, +# user creation, grants, and lifecycle validation is executed through the +# psql client binary. No external SQL libraries are used. +# +# CONTRACT: +# - Must be instantiated via ->new(%args) with all required DB configuration. +# - Must implement db_ping(), db_start(), db_stop(), and db_init() +# without requiring the framework context. +# - Must not modify global TAF state. +# - Must return OK/ERROR codes consistently. +############################################################################### +our $_me = "PostgreSQL"; + +################################################################################ +# Includes +################################################################################ +use strict; +use warnings; +use File::Spec; +use File::Path (); +use File::Basename (); +use Carp; +use POSIX qw(setsid); +use FindBin qw($Bin); +use lib "$Bin/../taf_libs"; +use TAF::Logging qw( + PrintError + PrintWarning + PrintVerbose + StageStart + StageEnd +); + +################################################################################ +# Constants +################################################################################ +use constant OK => 0; +use constant ERROR => 1; +use constant TRUE => 1; +use constant FALSE => 0; + +################################################################################ +# new +# +# PURPOSE: +# Construct and return a new PostgreSQL plugin object. The object captures +# all configuration, paths, binaries, SSL settings, and lifecycle state +# required for deterministic PostgreSQL behavior under TAF. +# +# BEHAVIOR: +# - Stores all constructor arguments directly into the plugin object. +# - Resolves postgres, pg_ctl, psql, initdb, and pg_isready binaries. +# - Validates that required binaries exist and are executable. +# - Sets the default port to 5432 when not supplied. +# +# NOTES: +# - PostgreSQL binaries may live under versioned paths such as +# /usr/pgsql-16/bin/ or /usr/lib/postgresql/16/bin/. The _find_binary() +# helper searches install_root/bin/ first, then common system paths. +# - PGPASSWORD is set in the environment at runtime to avoid interactive +# password prompts when running psql commands. +################################################################################ +sub new { + my ($class, %args) = @_; + + my $self = { + + # Instanced pid + db_pid => undef, + + # Install and data paths + install_root => $args{db_software_install_dir}, + data_dir => $args{db_data_dir}, + trans_logs_dir => $args{db_trans_logs_dir}, + + # Config (postgresql.conf path; pg_hba.conf is derived from data_dir) + config => $args{db_config_file}, + + # Binaries (resolved below) + postgres_bin => undef, + pg_ctl_bin => undef, + psql_bin => undef, + initdb_bin => undef, + pg_isready_bin => undef, + + # Error log + error_log => undef, + + # Connectivity + port => $args{db_port} // 5432, + + # SSL (TAF unified SSL contract) + ssl_mode => $args{db_ssl_mode}, + ssl_ca => $args{db_ssl_ca}, + ssl_cert => $args{db_ssl_cert}, + ssl_key => $args{db_ssl_key}, + + # Database and users + database => $args{database} // 'test', + db_user => $args{db_user} // 'pgsql_tester', + db_user_pass => $args{db_user_pass} // 'PostgresPass_@123', + db_user_permissions => $args{db_user_permissions} // 'ALL PRIVILEGES', + db_root_user => $args{db_root_user} // 'postgres', + db_root_pass => $args{db_root_pass} // 'PostgresPass_@123', + + # Locality and performance + cpus => $args{db_task_set}, + db_start_wait => $args{db_start_wait}, + db_stop_wait => $args{db_stop_wait}, + tmpdir => $args{tmp_dir}, + + # Extras + extra_args => $args{db_extra_args}, + + # State flags + initialized => FALSE, + users_created => FALSE, + + # Version metadata (populated during init) + pg_version => undef, + pg_version_num => undef, + }; + + bless $self, $class; + + # Validate tmpdir early — it is required throughout the lifecycle + unless ($self->{tmpdir} && -d $self->{tmpdir}) { + PrintError("$_me::new - tmpdir is missing or not a directory: " . + ($self->{tmpdir} // "")); + return undef; + } + + # Validate install_root + unless ($self->{install_root} && -d $self->{install_root}) { + PrintError("$_me::new - install_root is missing or not a directory: " . + ($self->{install_root} // "")); + return undef; + } + + # Resolve binaries + $self->{postgres_bin} = _find_binary($self->{install_root}, 'postgres'); + $self->{pg_ctl_bin} = _find_binary($self->{install_root}, 'pg_ctl'); + $self->{psql_bin} = _find_binary($self->{install_root}, 'psql'); + $self->{initdb_bin} = _find_binary($self->{install_root}, 'initdb'); + $self->{pg_isready_bin} = _find_binary($self->{install_root}, 'pg_isready'); + + # Validate required binaries + for my $b (qw(postgres_bin pg_ctl_bin psql_bin initdb_bin pg_isready_bin)) { + unless ($self->{$b} && -x $self->{$b}) { + PrintError("$_me::new - Required binary '$b' not found under " . + $self->{install_root}); + return undef; + } + } + + $self->{log_init} = File::Spec->catfile($self->{tmpdir}, "postgresql_initdb.log"); + $self->{log_start} = File::Spec->catfile($self->{tmpdir}, "postgresql_start.log"); + $self->{pidfile} = File::Spec->catfile($self->{tmpdir}, "postgresql_runtime.pid"); + + # When TAF runs as root, initdb and pg_ctl must run as the postgres OS user. + # Detect this at construction time so all lifecycle methods can wrap commands. + $self->{is_root} = ($> == 0) ? 1 : 0; + if ($self->{is_root}) { + my @pw = getpwnam('postgres'); + if (@pw) { + $self->{os_user} = 'postgres'; + $self->{os_uid} = $pw[2]; + $self->{os_gid} = $pw[3]; + PrintVerbose("$_me::new - running as root; cluster operations will use OS user 'postgres' (uid=$pw[2])"); + + # If data_dir or tmpdir are under /root (not accessible to postgres), + # redirect them to /tmp where the postgres user can traverse. + my $pg_base = "/tmp/taf_pg_$$"; + if ($self->{data_dir} && $self->{data_dir} =~ m{^/root(/|$)}) { + $self->{data_dir} = "$pg_base/data"; + PrintVerbose("$_me::new - data_dir relocated to $self->{data_dir} (root path inaccessible to postgres)"); + } + if ($self->{tmpdir} && $self->{tmpdir} =~ m{^/root(/|$)}) { + $self->{tmpdir} = "$pg_base/tmp"; + PrintVerbose("$_me::new - tmpdir relocated to $self->{tmpdir} (root path inaccessible to postgres)"); + # Ensure log paths are updated + $self->{log_init} = File::Spec->catfile($self->{tmpdir}, "postgresql_initdb.log"); + $self->{log_start} = File::Spec->catfile($self->{tmpdir}, "postgresql_start.log"); + $self->{pidfile} = File::Spec->catfile($self->{tmpdir}, "postgresql_runtime.pid"); + } + # Create and chown the pg_base dirs so postgres can write into them + if ($self->{data_dir} =~ m{^\Q$pg_base\E} || $self->{tmpdir} =~ m{^\Q$pg_base\E}) { + File::Path::make_path("$pg_base/data", "$pg_base/tmp") + or PrintWarning("$_me::new - could not pre-create $pg_base dirs"); + chown($pw[2], $pw[3], $pg_base, "$pg_base/data", "$pg_base/tmp"); + chmod(0700, $pg_base, "$pg_base/data", "$pg_base/tmp"); + } + } else { + PrintWarning("$_me::new - running as root but OS user 'postgres' not found; initdb may fail"); + $self->{os_user} = undef; + $self->{os_uid} = undef; + $self->{os_gid} = undef; + } + } + + return $self; +} + +################################################################################ +# db_init +# +# PURPOSE: +# Execute the full PostgreSQL initialization lifecycle. This routine +# prepares the datadir via initdb, writes postgresql.conf and pg_hba.conf, +# starts the server, creates the TAF tester user and database, then stops +# the server. The cluster is left in a clean, initialized state ready for +# db_start() by the framework. +# +# BEHAVIOR: +# 1. Validate binaries and tmpdir. +# 2. Prepare an empty datadir. +# 3. Run initdb to create the PostgreSQL cluster. +# 4. Apply postgresql.conf (user-supplied or built-in defaults). +# 5. Write pg_hba.conf to allow TCP and local connections. +# 6. Start the server. +# 7. Create the tester role and test database via psql. +# 8. Stop the server. +# +# NOTES: +# - pg_hba.conf always allows connections from 127.0.0.1 and ::1 using +# md5 authentication for the tester and postgres users. This is required +# for sysbench and other TCP-based benchmark clients. +# - The superuser password is set during initdb via --pwfile. +################################################################################ +sub db_init { + my ($self) = @_; + my $_init = StageStart("$_me -> Init Database ->"); + + # Validate binaries + return ERROR if $self->_db_validate_binaries() != OK; + + # Prepare empty data directory + return ERROR if $self->_db_prepare_data_dir() != OK; + + # Detect PostgreSQL version + $self->{pg_version} = $self->_detect_pg_version($self->{postgres_bin}); + unless ($self->{pg_version}) { + PrintError("$_init Failed to detect PostgreSQL version"); + return ERROR; + } + PrintVerbose("$_init Detected PostgreSQL version: $self->{pg_version}"); + + # Run initdb to create the cluster + return ERROR if $self->_db_run_initdb() != OK; + + # Apply postgresql.conf + return ERROR if $self->_db_apply_postgresql_conf() != OK; + + # Write pg_hba.conf + return ERROR if $self->_db_write_pg_hba_conf() != OK; + + # Start server for user bootstrap + return ERROR if $self->db_start() != OK; + + # Create tester role and test database + return ERROR if $self->_db_setup_users() != OK; + + # Stop server after bootstrap + return ERROR if $self->db_stop() != OK; + + $self->{initialized} = TRUE; + StageEnd($_init); + return OK; +} + +################################################################################ +# db_start +# +# PURPOSE: +# Start the PostgreSQL server using pg_ctl. Waits for the server to become +# ready using pg_isready before returning OK. +# +# BEHAVIOR: +# - Builds and executes: pg_ctl start -D {data_dir} -l {log} -w -t {timeout} +# - pg_ctl writes the server PID into {data_dir}/postmaster.pid. +# - Reads the PID from postmaster.pid after successful startup. +# - Waits via _wait_for_start() which polls pg_isready. +################################################################################ +sub db_start { + my ($self, $wait_seconds) = @_; + my $_st = StageStart("$_me -> Database Start ->"); + + my $pg_ctl = $self->{pg_ctl_bin}; + my $data_dir = $self->{data_dir}; + my $log = $self->{log_start}; + my $timeout = $wait_seconds // $self->{db_start_wait} // 90; + + unless ($pg_ctl && -x $pg_ctl) { + PrintError("$_st pg_ctl binary not executable: " . ($pg_ctl // "")); + return ERROR; + } + + unless ($data_dir && -d $data_dir) { + PrintError("$_st data_dir does not exist: " . ($data_dir // "")); + return ERROR; + } + + # Port is set in postgresql.conf by _db_apply_postgresql_conf(); no need to + # pass it via -o here. Passing "-o -p N" through _run_command (which uses + # shell string form of system()) would cause shell splitting issues. + my @cmd = ( + $self->_os_prefix(), + $pg_ctl, + 'start', + "-D", $data_dir, + "-l", $log, + "-w", + "-t", $timeout, + ); + + if ($self->{extra_args}) { + push @cmd, "-o", "\"$self->{extra_args}\""; + } + + PrintVerbose("$_st Running: @cmd"); + + my $rc = $self->_run_command(\@cmd, "start", undef); + if ($rc != 0) { + PrintError("$_st pg_ctl start failed (exit $rc), see $log"); + return ERROR; + } + + # Confirm readiness via pg_isready + if ($self->_wait_for_start($timeout) != OK) { + PrintError("$_st PostgreSQL did not become ready, see $log"); + return ERROR; + } + + # Read PID from postmaster.pid + my $pidfile = File::Spec->catfile($data_dir, "postmaster.pid"); + if (-f $pidfile) { + if (open(my $fh, '<', $pidfile)) { + my $pid = <$fh>; + close $fh; + chomp $pid; + if ($pid =~ /^\d+$/) { + $self->{db_pid} = $pid; + PrintVerbose("$_st PostgreSQL runtime PID: $pid"); + } + } + } + + StageEnd($_st); + return OK; +} + +################################################################################ +# db_stop +# +# PURPOSE: +# Stop the PostgreSQL server using pg_ctl stop -m fast. Waits for the +# server to exit before returning OK. +################################################################################ +sub db_stop { + my ($self, $wait_seconds) = @_; + my $_st = StageStart("$_me -> Database Stop ->"); + + my $pg_ctl = $self->{pg_ctl_bin}; + my $data_dir = $self->{data_dir}; + my $timeout = $wait_seconds // $self->{db_stop_wait} // 120; + + unless ($pg_ctl && -x $pg_ctl) { + PrintError("$_st pg_ctl binary not executable: " . ($pg_ctl // "")); + return ERROR; + } + + # Check whether the server is actually running + my $status_rc = $self->_pg_ctl_status(); + if ($status_rc != 0) { + PrintVerbose("$_st PostgreSQL is not running (pg_ctl status=$status_rc); nothing to stop"); + StageEnd($_st); + return OK; + } + + my @cmd = ( + $self->_os_prefix(), + $pg_ctl, + 'stop', + "-D", $data_dir, + "-m", "fast", + "-w", + "-t", $timeout, + ); + + PrintVerbose("$_st Running: @cmd"); + + my $rc = $self->_run_command(\@cmd, "stop", undef); + if ($rc != 0) { + PrintError("$_st pg_ctl stop failed (exit $rc)"); + return ERROR; + } + + $self->{db_pid} = undef; + + PrintVerbose("$_st PostgreSQL stopped"); + StageEnd($_st); + return OK; +} + +################################################################################ +# db_restart +################################################################################ +sub db_restart { + my ($self) = @_; + my $_st = StageStart("$_me -> Database Restart ->"); + + if ($self->db_stop() != OK) { + PrintError("$_st db_stop() failed during restart"); + return ERROR; + } + + if ($self->db_start() != OK) { + PrintError("$_st db_start() failed during restart"); + return ERROR; + } + + StageEnd($_st); + return OK; +} + +################################################################################ +# db_ping +# +# PURPOSE: +# Verify that the PostgreSQL server is responsive using pg_isready, then +# confirm SQL execution via a trivial SELECT 1. +################################################################################ +sub db_ping { + my ($self) = @_; + my $_st = StageStart("$_me -> Ping ->"); + + my $rc = $self->_db_execute_no_return_query("SELECT 1"); + if ($rc != OK) { + PrintError("$_st Ping failed"); + return ERROR; + } + + PrintVerbose("$_st Ping successful"); + StageEnd($_st); + return OK; +} + +################################################################################ +# db_pid +################################################################################ +sub db_pid { + my ($self) = @_; + + my $pid = $self->{db_pid}; + unless (defined $pid && $pid =~ /^\d+$/) { + PrintError("$_me::db_pid - PID not set or invalid"); + return undef; + } + return $pid; +} + +#=============================================================================== +# Internal Subs +#=============================================================================== + +################################################################################ +# _db_execute_no_return_query +# +# PURPOSE: +# Execute a SQL statement through psql as the root (postgres) superuser. +# Used for bootstrap SQL and liveness checks. Does not return result sets. +# +# BEHAVIOR: +# - Uses TCP connection to 127.0.0.1:{port} (pg_hba.conf must allow it). +# - Sets PGPASSWORD in the environment to avoid interactive prompts. +# - Appends -c to run the statement directly. +# - Returns OK on exit 0, ERROR otherwise. +################################################################################ +sub _db_execute_no_return_query { + my ($self, $sql, $as_root) = @_; + my $_tag = "$_me -> _db_execute_no_return_query ->"; + + my $psql = $self->{psql_bin}; + unless ($psql && -x $psql) { + PrintError("$_tag psql not executable: " . ($psql // "")); + return ERROR; + } + + my $user = $as_root ? $self->{db_root_user} : $self->{db_user}; + my $pass = $as_root ? $self->{db_root_pass} : $self->{db_user_pass}; + my $db = $as_root ? 'postgres' : $self->{database}; + + PrintVerbose("$_tag Executing: $sql"); + + # Set PGPASSWORD to avoid interactive prompt + local $ENV{PGPASSWORD} = $pass if $pass; + + my @cmd = ( + $psql, + "-h", "127.0.0.1", + "-p", $self->{port}, + "-U", $user, + "-d", $db, + "-c", $sql, + "-q", + "--no-psqlrc", + ); + + my $rc = system(@cmd); + if ($rc != 0) { + my $exit = $rc >> 8; + PrintError("$_tag Query failed (exit $exit): $sql"); + return ERROR; + } + + return OK; +} + +################################################################################ +# _db_execute_as_superuser +# +# PURPOSE: +# Execute a SQL statement as the postgres superuser against the postgres +# maintenance database. Used exclusively during bootstrap. +################################################################################ +sub _db_execute_as_superuser { + my ($self, $sql, $db) = @_; + $db //= 'postgres'; # default: maintenance database + my $_tag = "$_me -> _db_execute_as_superuser ->"; + + my $psql = $self->{psql_bin}; + unless ($psql && -x $psql) { + PrintError("$_tag psql not executable"); + return ERROR; + } + + PrintVerbose("$_tag Executing (db=$db): $sql"); + + local $ENV{PGPASSWORD} = $self->{db_root_pass} if $self->{db_root_pass}; + + my @cmd = ( + $psql, + "-h", "127.0.0.1", + "-p", $self->{port}, + "-U", $self->{db_root_user}, + "-d", $db, + "-c", $sql, + "-q", + "--no-psqlrc", + ); + + my $rc = system(@cmd); + if ($rc != 0) { + my $exit = $rc >> 8; + PrintError("$_tag Superuser query failed (exit $exit): $sql"); + return ERROR; + } + + return OK; +} + +################################################################################ +# _db_setup_users +# +# PURPOSE: +# Create the TAF tester role and test database during initialization. +# +# BEHAVIOR: +# - Sets the postgres superuser password. +# - Drops and recreates the tester role. +# - Drops and recreates the test database owned by tester. +# - Grants privileges on the test database to the tester. +# +# CONTRACT: +# - Must be called after db_start() has launched the server. +# - Operates via TCP connections to 127.0.0.1 (pg_hba.conf must allow it). +################################################################################ +sub _db_setup_users { + my ($self) = @_; + my $_st = StageStart("$_me -> Setup Users ->"); + + my $root = $self->{db_root_user}; + my $rootpass = $self->{db_root_pass}; + my $user = $self->{db_user}; + my $pass = $self->{db_user_pass}; + my $db = $self->{database}; + + # Set superuser password + if ($rootpass) { + my $sql = "ALTER USER \"$root\" WITH PASSWORD '$rootpass'"; + return ERROR if $self->_db_execute_as_superuser($sql) != OK; + PrintVerbose("$_st Superuser password set"); + } + + # Drop tester role if exists (clean re-init semantics) + { + my $sql = "DROP DATABASE IF EXISTS \"$db\""; + return ERROR if $self->_db_execute_as_superuser($sql) != OK; + + $sql = "DROP ROLE IF EXISTS \"$user\""; + return ERROR if $self->_db_execute_as_superuser($sql) != OK; + } + + # Create tester role with login and password + { + my $sql = "CREATE ROLE \"$user\" WITH LOGIN PASSWORD '$pass'"; + return ERROR if $self->_db_execute_as_superuser($sql) != OK; + PrintVerbose("$_st Tester role created: $user"); + } + + # Create test database owned by tester + { + my $sql = "CREATE DATABASE \"$db\" OWNER \"$user\""; + return ERROR if $self->_db_execute_as_superuser($sql) != OK; + PrintVerbose("$_st Test database created: $db"); + } + + # Grant all privileges on the database + { + my $sql = "GRANT ALL PRIVILEGES ON DATABASE \"$db\" TO \"$user\""; + return ERROR if $self->_db_execute_as_superuser($sql) != OK; + } + + # PG 15+: GRANT CREATE on the public schema — revoked from PUBLIC by default. + # Must connect to the test database (not postgres) to GRANT on its schema. + { + my $sql = "GRANT ALL ON SCHEMA public TO \"$user\""; + return ERROR if $self->_db_execute_as_superuser($sql, $db) != OK; + PrintVerbose("$_st GRANT public schema to $user (PG15+ requirement)"); + } + + $self->{users_created} = TRUE; + PrintVerbose("$_st Tester user setup complete"); + + StageEnd($_st); + return OK; +} + +################################################################################ +# _db_run_initdb +# +# PURPOSE: +# Initialize a new PostgreSQL cluster using initdb. +# +# BEHAVIOR: +# - Writes a temporary password file for the superuser. +# - Runs: initdb -D {data_dir} -U {db_root_user} -E UTF8 --pwfile= +# - Removes the password file after initdb completes. +################################################################################ +sub _db_run_initdb { + my ($self) = @_; + my $_tag = StageStart("$_me -> RunInitdb ->"); + + my $initdb = $self->{initdb_bin}; + my $data_dir = $self->{data_dir}; + my $root = $self->{db_root_user}; + my $rootpass = $self->{db_root_pass}; + my $log = $self->{log_init}; + + unless ($initdb && -x $initdb) { + PrintError("$_tag initdb not executable: " . ($initdb // "")); + return ERROR; + } + + unless (-d $data_dir) { + PrintError("$_tag data_dir does not exist: $data_dir"); + return ERROR; + } + + # Write superuser password to a temporary pwfile. + # When running as root, the initdb process runs as the postgres OS user and + # cannot access paths under /root. Use /tmp for the pwfile so it is always + # readable, and remove it immediately after initdb completes. + my $pwfile_dir = ($self->{is_root} && $self->{os_user}) ? '/tmp' : $self->{tmpdir}; + my $pwfile = File::Spec->catfile($pwfile_dir, "pg_pwfile_taf_$$.tmp"); + if (open(my $fh, '>', $pwfile)) { + print $fh $rootpass // ''; + close $fh; + # World-readable so the postgres OS user can read it; short-lived file + chmod(($self->{is_root} ? 0644 : 0600), $pwfile); + } else { + PrintError("$_tag Cannot write pwfile: $pwfile"); + return ERROR; + } + + my @cmd = ( + $self->_os_prefix(), + $initdb, + "-D", $data_dir, + "-U", $root, + "-E", "UTF8", + "--locale=C", + "--pwfile=$pwfile", + ); + + PrintVerbose("$_tag Running: @cmd"); + + my $rc = $self->_run_command(\@cmd, "initdb", $log); + unlink $pwfile; + + if ($rc != 0) { + PrintError("$_tag initdb failed (exit $rc), see $log"); + return ERROR; + } + + PrintVerbose("$_tag initdb completed"); + StageEnd($_tag); + return OK; +} + +################################################################################ +# _db_apply_postgresql_conf +# +# PURPOSE: +# Apply postgresql.conf settings. If the user supplied a config file via +# db_config_file, its contents are appended to (not replaced) the +# postgresql.conf created by initdb. This preserves initdb-generated +# defaults while layering TAF-specific tuning on top. +# +# BEHAVIOR: +# - Always writes the port setting to ensure the configured port is used. +# - If a user config file is supplied and readable, appends its contents. +# - If no user config is supplied, writes safe benchmark defaults. +################################################################################ +sub _db_apply_postgresql_conf { + my ($self) = @_; + my $_tag = "$_me -> _db_apply_postgresql_conf ->"; + + my $pg_conf = File::Spec->catfile($self->{data_dir}, "postgresql.conf"); + + unless (-w $pg_conf) { + PrintError("$_tag postgresql.conf not writable: $pg_conf"); + return ERROR; + } + + # Append TAF port setting unconditionally + if (open(my $fh, '>>', $pg_conf)) { + print $fh "\n# === TAF-managed settings ===\n"; + print $fh "port = $self->{port}\n"; + print $fh "listen_addresses = '*'\n"; + + # SSL settings + my $ssl_mode = lc($self->{ssl_mode} // 'off'); + if ($ssl_mode ne 'off') { + print $fh "ssl = on\n"; + print $fh "ssl_ca_file = '$self->{ssl_ca}'\n" if $self->{ssl_ca}; + print $fh "ssl_cert_file = '$self->{ssl_cert}'\n" if $self->{ssl_cert}; + print $fh "ssl_key_file = '$self->{ssl_key}'\n" if $self->{ssl_key}; + } else { + print $fh "ssl = off\n"; + } + + # If user supplied a config file, append its contents + if ($self->{config} && -r $self->{config}) { + print $fh "\n# === User-supplied TAF config ===\n"; + if (open(my $ufh, '<', $self->{config})) { + while (my $line = <$ufh>) { + # Skip port/listen/ssl — already written above + next if $line =~ /^\s*(port|listen_addresses|ssl)\s*=/i; + print $fh $line; + } + close $ufh; + PrintVerbose("$_tag Appended user config: $self->{config}"); + } + } else { + # Write safe benchmark defaults when no user config supplied + print $fh "\n# === TAF benchmark defaults ===\n"; + print $fh "shared_buffers = 256MB\n"; + print $fh "work_mem = 4MB\n"; + print $fh "maintenance_work_mem = 64MB\n"; + print $fh "effective_cache_size = 1GB\n"; + print $fh "checkpoint_completion_target = 0.9\n"; + print $fh "wal_buffers = 16MB\n"; + print $fh "max_connections = 500\n"; + print $fh "log_min_duration_statement = -1\n"; + print $fh "log_connections = off\n"; + print $fh "log_disconnections = off\n"; + } + + close $fh; + } else { + PrintError("$_tag Cannot open postgresql.conf for writing: $pg_conf"); + return ERROR; + } + + PrintVerbose("$_tag postgresql.conf configured"); + return OK; +} + +################################################################################ +# _db_write_pg_hba_conf +# +# PURPOSE: +# Write pg_hba.conf to allow TCP and local connections for both the +# postgres superuser and the TAF tester user. This is required because +# initdb generates a restrictive pg_hba.conf (peer/ident auth), which +# would block psql TCP connections used by TAF and benchmark clients. +# +# BEHAVIOR: +# - Writes a minimal, TAF-controlled pg_hba.conf. +# - Allows md5 authentication for all users from 127.0.0.1/32 and ::1/128. +# - Allows local (Unix socket) connections for the postgres user for +# pg_ctl and maintenance operations. +# - If ssl_mode is not 'off', adds hostssl rules in addition to host rules. +################################################################################ +sub _db_write_pg_hba_conf { + my ($self) = @_; + my $_tag = "$_me -> _db_write_pg_hba_conf ->"; + + my $hba = File::Spec->catfile($self->{data_dir}, "pg_hba.conf"); + + unless (open(my $fh, '>', $hba)) { + PrintError("$_tag Cannot write pg_hba.conf: $hba"); + return ERROR; + } else { + my $ssl_mode = lc($self->{ssl_mode} // 'off'); + my $auth = "md5"; + + print $fh "# TAF-managed pg_hba.conf\n"; + print $fh "# TYPE DATABASE USER ADDRESS METHOD\n"; + print $fh "\n"; + + # Local (Unix socket) — postgres superuser only, for pg_ctl and psql maintenance + print $fh "local all postgres trust\n"; + print $fh "local all all md5\n"; + print $fh "\n"; + + # TCP IPv4 and IPv6 — all users + print $fh "host all all 127.0.0.1/32 $auth\n"; + print $fh "host all all ::1/128 $auth\n"; + print $fh "\n"; + + # SSL connections when SSL is enabled + if ($ssl_mode ne 'off') { + print $fh "hostssl all all 0.0.0.0/0 $auth\n"; + } + + close $fh; + } + + PrintVerbose("$_tag pg_hba.conf written"); + return OK; +} + +################################################################################ +################################################################################ +# _os_prefix +# +# PURPOSE: +# Return the command prefix needed to run a command as the postgres OS user +# when TAF is executing as root. Returns an empty list when not root or +# when the postgres OS user could not be resolved. +# +# USAGE: +# my @cmd = ($self->_os_prefix(), $binary, @args); +################################################################################ +sub _os_prefix { + my ($self) = @_; + return () unless $self->{is_root} && $self->{os_user}; + return ('runuser', '-u', $self->{os_user}, '--'); +} + +# _db_prepare_data_dir +# +# PURPOSE: +# Ensure the data directory is empty and ready for initdb. Removes any +# existing content. Creates the directory fresh. When running as root, +# also chowns the directory to the postgres OS user so initdb can write it. +################################################################################ +sub _db_prepare_data_dir { + my ($self) = @_; + my $dir = $self->{data_dir}; + + if (-d $dir) { + PrintVerbose("$_me -> Removing existing data directory: $dir"); + File::Path::remove_tree($dir, {error => \my $err}); + if (@$err) { + PrintError("_db_prepare_data_dir: Failed to remove $dir"); + return ERROR; + } + } + + File::Path::make_path($dir) or do { + PrintError("_db_prepare_data_dir: Failed to create $dir"); + return ERROR; + }; + + # When running as root, hand ownership to the postgres OS user so initdb + # and pg_ctl can read and write the cluster directory. + if ($self->{is_root} && defined $self->{os_uid}) { + chown($self->{os_uid}, $self->{os_gid}, $dir) + or PrintWarning("_db_prepare_data_dir: chown $dir to $self->{os_user} failed: $!"); + # Also chown tmpdir so pg_ctl can write the startup log + chown($self->{os_uid}, $self->{os_gid}, $self->{tmpdir}) + or PrintWarning("_db_prepare_data_dir: chown tmpdir failed: $!"); + } + + return OK; +} + +################################################################################ +# _db_validate_binaries +# +# PURPOSE: +# Validate that all required PostgreSQL binaries resolved during new() +# exist and are executable. +################################################################################ +sub _db_validate_binaries { + my ($self) = @_; + my $_tag = "$_me -> _db_validate_binaries ->"; + + for my $b (qw(postgres_bin pg_ctl_bin psql_bin initdb_bin pg_isready_bin)) { + unless ($self->{$b} && -x $self->{$b}) { + PrintError("$_tag Binary '$b' not found or not executable: " . + ($self->{$b} // "")); + return ERROR; + } + } + + PrintVerbose("$_tag All required binaries validated"); + return OK; +} + +################################################################################ +# _detect_pg_version +# +# PURPOSE: +# Detect the PostgreSQL server version. Runs: postgres --version +# Returns the version string (e.g. "16.3") or undef on failure. +################################################################################ +sub _detect_pg_version { + my ($self, $binary) = @_; + + return undef unless defined $binary && -x $binary; + + my $output = `"$binary" --version 2>&1`; + return undef unless defined $output && length $output; + + # Expected: "postgres (PostgreSQL) 16.3" + my ($version) = $output =~ /PostgreSQL\)\s+(\d+\.\d+(?:\.\d+)?)/; + return undef unless $version; + + # Extract numeric major version (e.g. 16 from 16.3) + my ($major) = $version =~ /^(\d+)/; + $self->{pg_version_num} = $major; + + $self->{server_version_raw} = $output; + $self->{server_version_norm} = $version; + + PrintVerbose("$_me::_detect_pg_version: $version"); + return $version; +} + +################################################################################ +# _wait_for_start +# +# PURPOSE: +# Poll pg_isready until the server is accepting connections or timeout +# is reached. +# +# BEHAVIOR: +# - Calls pg_isready -h 127.0.0.1 -p {port} in a loop. +# - Polls at 1-second intervals up to $timeout seconds. +# - Returns OK when pg_isready exits 0; ERROR on timeout. +################################################################################ +sub _wait_for_start { + my ($self, $timeout) = @_; + my $_tag = "$_me -> _wait_for_start ->"; + + $timeout //= $self->{db_start_wait} // 90; + + my $pg_isready = $self->{pg_isready_bin}; + unless ($pg_isready && -x $pg_isready) { + PrintError("$_tag pg_isready not executable"); + return ERROR; + } + + PrintVerbose("$_tag Waiting up to $timeout seconds for PostgreSQL readiness..."); + + for my $i (1 .. $timeout) { + my @cmd = ( + $pg_isready, + "-h", "127.0.0.1", + "-p", $self->{port}, + "-q", + ); + + my $rc = system(@cmd); + if ($rc == 0) { + PrintVerbose("$_tag PostgreSQL is ready (attempt $i)"); + return OK; + } + + sleep 1; + } + + PrintError("$_tag PostgreSQL did not become ready within $timeout seconds"); + return ERROR; +} + +################################################################################ +# _pg_ctl_status +# +# PURPOSE: +# Run pg_ctl status -D {data_dir} and return the exit code. +# Exit 0 means the server is running; non-zero means it is not. +################################################################################ +sub _pg_ctl_status { + my ($self) = @_; + + my $pg_ctl = $self->{pg_ctl_bin}; + my $data_dir = $self->{data_dir}; + + return 1 unless $pg_ctl && -x $pg_ctl && $data_dir && -d $data_dir; + + # pg_ctl status: -q is not valid for all pg_ctl versions (e.g. PG 11). + # Suppress output by redirecting through the shell instead. + my $cmd = join(' ', ($self->_os_prefix()), $pg_ctl, 'status', "-D", $data_dir, '>/dev/null 2>&1'); + my $rc = system($cmd); + return ($rc == 0) ? 0 : 1; +} + +################################################################################ +# _find_binary +# +# PURPOSE: +# Locate a PostgreSQL binary under the install root or standard system +# paths. Searches in deterministic order: +# /bin/ +# /sbin/ +# / +# +# NOTES: +# - PostgreSQL binaries may also exist under versioned system paths such as +# /usr/pgsql-16/bin/ or /usr/lib/postgresql/16/bin/. The install_root +# passed by TAF::DatabaseSoftwareInstalls should already point to the +# correct versioned prefix; this routine only searches under that root. +################################################################################ +sub _find_binary { + my ($base, $binary) = @_; + + return undef unless defined $base && length $base; + return undef unless defined $binary && length $binary; + + my @paths = ( + File::Spec->catfile($base, "bin", $binary), + File::Spec->catfile($base, "sbin", $binary), + File::Spec->catfile($base, $binary), + ); + + for my $p (@paths) { + return $p if -e $p && -x $p; + } + + return undef; +} + +################################################################################ +# _run_command +# +# PURPOSE: +# Execute a system command from an array reference. Optionally redirects +# stdout/stderr to a logfile. Returns the normalized exit code. +################################################################################ +sub _run_command { + my ($self, $cmd_ref, $tag, $logfile) = @_; + my $_tag = "$_me::_run_command($tag): "; + + my $cmd_str = join(' ', @$cmd_ref); + + if ($logfile) { + if (open(my $fh, '>>', $logfile)) { + print $fh "=== _run_command [$tag] ===\n"; + print $fh "$cmd_str\n"; + close $fh; + } + $cmd_str .= " >> \"$logfile\" 2>&1"; + } + + PrintVerbose("$_tag $cmd_str"); + + my $rc = system($cmd_str); + + if ($rc == -1) { + PrintError("$_tag Failed to execute: $!"); + return 1; + } + + my $exit = $rc >> 8; + + if ($exit != 0) { + PrintError("$_tag Exit code $exit"); + } + + return $exit; +} + +############################################################################# +# Module terminator +############################################################################# +1; diff --git a/libs/script_tools_lib/ClientCmakeBuild.pm b/libs/script_tools_lib/ClientCmakeBuild.pm index 3028713..94b1c0e 100644 --- a/libs/script_tools_lib/ClientCmakeBuild.pm +++ b/libs/script_tools_lib/ClientCmakeBuild.pm @@ -410,9 +410,7 @@ sub SetLibAndInclude { return _SetLibAndInclude_MySQLFamily($installDir); } elsif ($maker eq 'postgres' || $maker eq 'postgresql') { - # TO BE ADDED - DebugPrint("ERROR: PostgreSQL client builds are not supported by this module"); - return ERROR; + return _SetLibAndInclude_PostgreSQL($installDir); } elsif ($maker eq 'oracle') { # TO BE ADDED @@ -753,9 +751,74 @@ sub _DetectMakerFromInstallDir { return $maker if $path =~ m{/\Q$maker\E[^/]*}i; } + # Fallback: probe for pg_config to detect system-package PostgreSQL (e.g. /usr) + my $pg_config = File::Spec->catfile($installDir, 'bin', 'pg_config'); + return 'postgres' if -x $pg_config; + return undef; } +#------------------------------------------------------------------------------- +# Subroutine: _SetLibAndInclude_PostgreSQL +# +# PURPOSE: +# Resolve include and library directories for PostgreSQL client builds +# using pg_config. Sets $ENV{INC} and $ENV{LIB} for use by cmake. +# +# PARAMETERS: +# $installDir - Root of the PostgreSQL installation. +# +# RETURNS: +# OK - INC and LIB resolved via pg_config. +# ERROR - pg_config not found or returned invalid paths. +#------------------------------------------------------------------------------- +sub _SetLibAndInclude_PostgreSQL { + my ($installDir) = @_; + + DebugPrint("SetLibAndInclude - PostgreSQL family"); + + # Locate pg_config: first under installDir, then system-wide + my $pgConfig; + for my $candidate ( + File::Spec->catfile($installDir, 'bin', 'pg_config'), + '/usr/bin/pg_config', + ) { + if (-x $candidate) { + $pgConfig = $candidate; + last; + } + } + + unless ($pgConfig) { + DebugPrint("ERROR: pg_config not found under $installDir/bin or /usr/bin"); + return ERROR; + } + + DebugPrint("pg_config = $pgConfig"); + + my $includeDir = `$pgConfig --includedir 2>/dev/null`; + my $libDir = `$pgConfig --libdir 2>/dev/null`; + chomp($includeDir); + chomp($libDir); + + unless ($includeDir && -d $includeDir) { + DebugPrint("ERROR: pg_config --includedir returned invalid directory: '$includeDir'"); + return ERROR; + } + unless ($libDir && -d $libDir) { + DebugPrint("ERROR: pg_config --libdir returned invalid directory: '$libDir'"); + return ERROR; + } + + $ENV{INC} = $includeDir; + $ENV{LIB} = $libDir; + + DebugPrint("PostgreSQL INC = $includeDir"); + DebugPrint("PostgreSQL LIB = $libDir"); + + return OK; +} + ############################################################################# # Module terminator ############################################################################# diff --git a/libs/sql_libs/Executor.pm b/libs/sql_libs/Executor.pm index 61e7007..7c894b9 100644 --- a/libs/sql_libs/Executor.pm +++ b/libs/sql_libs/Executor.pm @@ -19,7 +19,7 @@ package sql_libs::Executor; # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 # # Licensed under the GNU General Public License, version 2 or later (GPLv2+). # See https://www.gnu.org/licenses/ for details. @@ -391,8 +391,11 @@ sub DbCreateDatabase { my $db = $ctx->{options}{database} or croak "DbCreateDatabase: ctx->{options}{database} is undefined"; + my $user = $ctx->{options}{db_user}; + my $sql = _LoadDialect($ctx, "create_database"); $sql =~ s/\{db\}/$db/g; + $sql =~ s/\{user\}/$user/g if defined $user; return DbExecuteNoReturnQuery($sql, $ctx); } @@ -570,6 +573,30 @@ sub _BuildCommand { croak "_BuildCommand requires db_port or db_socket in options" unless defined $connection; + my $maker = _NormalizeMaker($ctx->{taf_var}{db_maker} // ''); + + # PostgreSQL uses psql syntax which differs from MySQL-family clients + if ($maker eq 'postgres') { + my $cmd = ''; + $cmd .= "PGPASSWORD='$pass' " if defined $pass; + $cmd .= "$client -U $user"; + # Always use TCP loopback — pg_hba.conf allows 127.0.0.1; db_socket is MySQL-style + my $pg_port = $opt->{db_port} // 5432; + $cmd .= " -h 127.0.0.1 -p $pg_port"; + # Connect to maintenance database for DDL; -q suppresses notices + $cmd .= " -d postgres -q"; + # SSL for postgres + if ($opt->{ssl_enabled}) { + $cmd .= " --set=sslmode=require"; + $cmd .= " --set=sslrootcert=$opt->{ssl_ca}" if $opt->{ssl_ca}; + $cmd .= " --set=sslcert=$opt->{ssl_cert}" if $opt->{ssl_cert}; + $cmd .= " --set=sslkey=$opt->{ssl_key}" if $opt->{ssl_key}; + } + $cmd .= " $extra" if $extra; + $cmd .= " -c \"$sql\""; + return $cmd; + } + my $cmd = "$client -u $user"; if (defined $pass) { @@ -585,7 +612,7 @@ sub _BuildCommand { # SSL options (normalized earlier in TAF) if ($opt->{ssl_enabled}) { - if ($maker eq 'mariadb' || $maker eq 'mysql') { + if ($maker eq 'mariadb' || $maker eq 'mysql') { $cmd .= " --ssl-ca=$opt->{ssl_ca}" if $opt->{ssl_ca}; $cmd .= " --ssl-cert=$opt->{ssl_cert}" if $opt->{ssl_cert}; $cmd .= " --ssl-key=$opt->{ssl_key}" if $opt->{ssl_key}; diff --git a/libs/sql_libs/dialects/postgres.sql b/libs/sql_libs/dialects/postgres.sql index 4908aea..213804f 100644 --- a/libs/sql_libs/dialects/postgres.sql +++ b/libs/sql_libs/dialects/postgres.sql @@ -46,7 +46,114 @@ FROM pg_database; SELECT * FROM pg_stat_database; [create_database] -CREATE DATABASE {db}; +CREATE DATABASE {db} OWNER "{user}"; [drop_database] -DROP DATABASE IF EXISTS {db}; \ No newline at end of file +DROP DATABASE IF EXISTS {db}; + +[active_connections] +SELECT count(*) AS total, + state, + wait_event_type, + wait_event +FROM pg_stat_activity +WHERE pid <> pg_backend_pid() +GROUP BY state, wait_event_type, wait_event +ORDER BY total DESC; + +[wait_events] +SELECT wait_event_type, + wait_event, + count(*) AS count +FROM pg_stat_activity +WHERE wait_event IS NOT NULL + AND pid <> pg_backend_pid() +GROUP BY wait_event_type, wait_event +ORDER BY count DESC; + +[table_stats] +SELECT schemaname, + relname, + seq_scan, + seq_tup_read, + idx_scan, + idx_tup_fetch, + n_live_tup, + n_dead_tup, + last_autovacuum, + last_autoanalyze +FROM pg_stat_user_tables +ORDER BY seq_scan DESC; + +[index_usage] +SELECT schemaname, + tablename, + indexname, + idx_scan, + idx_tup_read, + idx_tup_fetch +FROM pg_stat_user_indexes +ORDER BY idx_scan DESC; + +[bgwriter_stats] +SELECT checkpoints_timed, + checkpoints_req, + checkpoint_write_time, + checkpoint_sync_time, + buffers_checkpoint, + buffers_clean, + maxwritten_clean, + buffers_backend, + buffers_backend_fsync, + buffers_alloc +FROM pg_stat_bgwriter; + +[lock_waits] +SELECT blocked.pid AS blocked_pid, + blocked.query AS blocked_query, + blocking.pid AS blocking_pid, + blocking.query AS blocking_query, + blocked.wait_event, + blocked.wait_event_type +FROM pg_stat_activity AS blocked +JOIN pg_stat_activity AS blocking + ON blocking.pid = ANY(pg_blocking_pids(blocked.pid)) +WHERE blocked.cardinality(pg_blocking_pids(blocked.pid)) > 0; + +[replication_lag] +SELECT client_addr, + state, + sent_lsn, + write_lsn, + flush_lsn, + replay_lsn, + (sent_lsn - replay_lsn) AS lag_bytes +FROM pg_stat_replication; + +[table_bloat_estimate] +SELECT schemaname, + tablename, + pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) AS total_size, + n_dead_tup, + n_live_tup, + ROUND(100.0 * n_dead_tup / NULLIF(n_live_tup + n_dead_tup, 0), 2) AS dead_pct +FROM pg_stat_user_tables +WHERE n_live_tup + n_dead_tup > 0 +ORDER BY dead_pct DESC NULLS LAST; + +[transaction_stats] +SELECT datname, + xact_commit, + xact_rollback, + blks_read, + blks_hit, + ROUND(100.0 * blks_hit / NULLIF(blks_hit + blks_read, 0), 2) AS cache_hit_pct, + tup_returned, + tup_fetched, + tup_inserted, + tup_updated, + tup_deleted, + conflicts, + deadlocks +FROM pg_stat_database +WHERE datname NOT IN ('template0', 'template1', 'postgres'); \ No newline at end of file diff --git a/libs/sql_libs/postgres.sql b/libs/sql_libs/postgres.sql new file mode 100644 index 0000000..213804f --- /dev/null +++ b/libs/sql_libs/postgres.sql @@ -0,0 +1,159 @@ +# ====================================================================== +# MariaDB Foundation - SQL Dialect Definitions +# ---------------------------------------------------------------------- +# File: dialects/sql.dialect +# Purpose: +# Defines named SQL snippets used by TAF for MariaDB diagnostics, +# environment introspection, and database lifecycle operations. +# +# Copyright (c) 2025-2026 MariaDB Foundation and Jonathan "jeb" Miller +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 or later of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 +# +# Licensed under the GNU General Public License, version 2 or later (GPLv2+). +# See https://www.gnu.org/licenses/ for details. +# +# Notes: +# - All blocks must remain deterministic and contributor-proof. +# - Do not modify block names without updating all references. +# ====================================================================== +[version] +SELECT version(); + +[variables] +SHOW ALL; + +[row_count] +SELECT COUNT(*) FROM {table}; + +[db_size] +SELECT pg_database.datname AS db, + pg_database_size(pg_database.datname) AS size_bytes +FROM pg_database; + +[stats] +SELECT * FROM pg_stat_database; + +[create_database] +CREATE DATABASE {db} OWNER "{user}"; + +[drop_database] +DROP DATABASE IF EXISTS {db}; + +[active_connections] +SELECT count(*) AS total, + state, + wait_event_type, + wait_event +FROM pg_stat_activity +WHERE pid <> pg_backend_pid() +GROUP BY state, wait_event_type, wait_event +ORDER BY total DESC; + +[wait_events] +SELECT wait_event_type, + wait_event, + count(*) AS count +FROM pg_stat_activity +WHERE wait_event IS NOT NULL + AND pid <> pg_backend_pid() +GROUP BY wait_event_type, wait_event +ORDER BY count DESC; + +[table_stats] +SELECT schemaname, + relname, + seq_scan, + seq_tup_read, + idx_scan, + idx_tup_fetch, + n_live_tup, + n_dead_tup, + last_autovacuum, + last_autoanalyze +FROM pg_stat_user_tables +ORDER BY seq_scan DESC; + +[index_usage] +SELECT schemaname, + tablename, + indexname, + idx_scan, + idx_tup_read, + idx_tup_fetch +FROM pg_stat_user_indexes +ORDER BY idx_scan DESC; + +[bgwriter_stats] +SELECT checkpoints_timed, + checkpoints_req, + checkpoint_write_time, + checkpoint_sync_time, + buffers_checkpoint, + buffers_clean, + maxwritten_clean, + buffers_backend, + buffers_backend_fsync, + buffers_alloc +FROM pg_stat_bgwriter; + +[lock_waits] +SELECT blocked.pid AS blocked_pid, + blocked.query AS blocked_query, + blocking.pid AS blocking_pid, + blocking.query AS blocking_query, + blocked.wait_event, + blocked.wait_event_type +FROM pg_stat_activity AS blocked +JOIN pg_stat_activity AS blocking + ON blocking.pid = ANY(pg_blocking_pids(blocked.pid)) +WHERE blocked.cardinality(pg_blocking_pids(blocked.pid)) > 0; + +[replication_lag] +SELECT client_addr, + state, + sent_lsn, + write_lsn, + flush_lsn, + replay_lsn, + (sent_lsn - replay_lsn) AS lag_bytes +FROM pg_stat_replication; + +[table_bloat_estimate] +SELECT schemaname, + tablename, + pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) AS total_size, + n_dead_tup, + n_live_tup, + ROUND(100.0 * n_dead_tup / NULLIF(n_live_tup + n_dead_tup, 0), 2) AS dead_pct +FROM pg_stat_user_tables +WHERE n_live_tup + n_dead_tup > 0 +ORDER BY dead_pct DESC NULLS LAST; + +[transaction_stats] +SELECT datname, + xact_commit, + xact_rollback, + blks_read, + blks_hit, + ROUND(100.0 * blks_hit / NULLIF(blks_hit + blks_read, 0), 2) AS cache_hit_pct, + tup_returned, + tup_fetched, + tup_inserted, + tup_updated, + tup_deleted, + conflicts, + deadlocks +FROM pg_stat_database +WHERE datname NOT IN ('template0', 'template1', 'postgres'); \ No newline at end of file diff --git a/libs/taf_libs/TAF/Utilities.pm b/libs/taf_libs/TAF/Utilities.pm index 553555e..f5c2b88 100644 --- a/libs/taf_libs/TAF/Utilities.pm +++ b/libs/taf_libs/TAF/Utilities.pm @@ -177,8 +177,9 @@ our %PLUGIN_ALIASES = ( mariadbd => 'mariadb', mysql => 'mysql', mysqld => 'mysql', - postgres => 'postgres', - pgsql => 'postgres', + postgres => 'postgres', + pgsql => 'postgres', + postgresql => 'postgres', oracle => 'oracle', sqlplus => 'oracle', ); diff --git a/properties/default/sysbench_lua_default.properties b/properties/default/sysbench_lua_default.properties index 82e0b92..d6c93b3 100644 --- a/properties/default/sysbench_lua_default.properties +++ b/properties/default/sysbench_lua_default.properties @@ -135,7 +135,7 @@ sysbench_lua.ignore_errors = null # --------------------------------------------------------------------------- # Build / compile # --------------------------------------------------------------------------- -sysbench_lua.cmake_args = -DWITH_MYSQL=on -DCMAKE_BUILD_TYPE=Release +sysbench_lua.cmake_args = -DWITH_MYSQL=on -DWITH_PGSQL=on -DCMAKE_BUILD_TYPE=Release # --------------------------------------------------------------------------- # Debugging diff --git a/properties/postgresql/hammerdb_tprocc_pgsql.properties b/properties/postgresql/hammerdb_tprocc_pgsql.properties new file mode 100644 index 0000000..a69785f --- /dev/null +++ b/properties/postgresql/hammerdb_tprocc_pgsql.properties @@ -0,0 +1,62 @@ +############################################################################# +# hammerdb_tprocc_pgsql.properties +# +# Created: June 2026 +# +# This file is part of the Test Automation Framework (TAF). +# Copyright (c) 2025-2026 MariaDB Foundation and Jonathan "jeb" Miller +# +# PURPOSE: +# Example TAF properties file for running HammerDB TPC-C benchmarks +# against a PostgreSQL database managed by TAF. +# +# USAGE: +# perl taf.pl --properties-file=properties/postgresql/hammerdb_tprocc_pgsql.properties \ +# --property=taf.action=init-start-db-run-tests +# +# NOTES: +# - Set taf.db_software_install_packages to your PostgreSQL package path. +# - HammerDB must be installed separately; set hammerdb_tprocc.hammerdb_dir. +# - pg_storedprocs=true improves TPC-C performance on PostgreSQL. +############################################################################# + +# --------------------------------------------------------------------------- +# TAF framework +# --------------------------------------------------------------------------- +taf.action = init-start-db-run-tests +taf.taf_db_makers_plugin = postgres + +# --------------------------------------------------------------------------- +# Database software install +# --------------------------------------------------------------------------- +# taf.db_software_install_packages = /path/to/postgresql-16.tar.gz +taf.db_port = 5432 + +# --------------------------------------------------------------------------- +# Database configuration +# --------------------------------------------------------------------------- +taf.db_config_file = database_config_files/postgresql/postgresql_oltp.conf + +# --------------------------------------------------------------------------- +# Test suite +# --------------------------------------------------------------------------- +taf.test_suite = hammerdb-tprocc + +# --------------------------------------------------------------------------- +# HammerDB TPC-C — PostgreSQL specific +# --------------------------------------------------------------------------- +hammerdb_tprocc.db_type = postgres +hammerdb_tprocc.warehouses = 100 + +# PostgreSQL-specific TPC-C optimizations +hammerdb_tprocc.pg_storedprocs = true +hammerdb_tprocc.pg_vacuum = true +hammerdb_tprocc.pg_oracompat = false +hammerdb_tprocc.pg_cituscompat = false + +# --------------------------------------------------------------------------- +# Workload +# --------------------------------------------------------------------------- +hammerdb_tprocc.def_threads = 8,16,32,64 +hammerdb_tprocc.def_duration = 300 +hammerdb_tprocc.rampup = 2 diff --git a/properties/postgresql/sysbench_lua_pgsql.properties b/properties/postgresql/sysbench_lua_pgsql.properties new file mode 100644 index 0000000..aa41ac8 --- /dev/null +++ b/properties/postgresql/sysbench_lua_pgsql.properties @@ -0,0 +1,65 @@ +############################################################################# +# sysbench_lua_pgsql.properties +# +# Created: June 2026 +# +# This file is part of the Test Automation Framework (TAF). +# Copyright (c) 2025-2026 MariaDB Foundation and Jonathan "jeb" Miller +# +# PURPOSE: +# Example TAF properties file for running Sysbench OLTP benchmarks +# against a PostgreSQL database managed by TAF. +# +# USAGE: +# perl taf.pl --properties-file=properties/postgresql/sysbench_lua_pgsql.properties \ +# --property=taf.action=init-start-db-run-tests +# +# NOTES: +# - Set taf.db_software_install_packages to the path of your PostgreSQL +# tarball or RPM packages before running. +# - Adjust thread counts and duration to suit your test host. +# - The default config uses postgresql_oltp.conf; swap for +# postgresql_minimal.conf on low-resource machines. +############################################################################# + +# --------------------------------------------------------------------------- +# TAF framework +# --------------------------------------------------------------------------- +taf.action = init-start-db-run-tests +taf.taf_db_makers_plugin = postgres + +# --------------------------------------------------------------------------- +# Database software install +# --------------------------------------------------------------------------- +# taf.db_software_install_packages = /path/to/postgresql-16.tar.gz +taf.db_port = 5432 + +# --------------------------------------------------------------------------- +# Database configuration +# --------------------------------------------------------------------------- +taf.db_config_file = database_config_files/postgresql/postgresql_oltp.conf + +# --------------------------------------------------------------------------- +# Test suite +# --------------------------------------------------------------------------- +taf.test_suite = sysbench-lua + +# --------------------------------------------------------------------------- +# Sysbench driver +# --------------------------------------------------------------------------- +sysbench_lua.db_driver = pgsql +sysbench_lua.connector = libpq + +# --------------------------------------------------------------------------- +# Workload +# --------------------------------------------------------------------------- +sysbench_lua.def_threads = 8,16,32,64,128 +sysbench_lua.def_duration = 300 +sysbench_lua.number_of_tables = 4 +sysbench_lua.number_of_rows = 1000000 +sysbench_lua.oltp_skip_trx = off + +# --------------------------------------------------------------------------- +# Tests to run (standard OLTP subset compatible with PostgreSQL) +# --------------------------------------------------------------------------- +taf.tests = OLTP_RO,OLTP_RW,UPDATE_KEY,UPDATE_NO_KEY,POINT_SELECT diff --git a/test_suites/sysbench-lua.pm b/test_suites/sysbench-lua.pm index bed6e50..0e6756f 100644 --- a/test_suites/sysbench-lua.pm +++ b/test_suites/sysbench-lua.pm @@ -19,7 +19,7 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 # # Licensed under the GNU General Public License, version 2 or later (GPLv2+). # See https://www.gnu.org/licenses/ for details. @@ -923,10 +923,10 @@ sub Help { Print("\tare included in each request."); Print("\tExample: To run 3 simple-range queries per request instead of 1,"); Print("\tset: sysbench_lua.oltp_simple_ranges=3\n"); - + Print("\tPOINTS-COVERED-PK \n"); Print("\tsysbench_lua.random_points_ranges\n"); - + Print("\tPOINTS-COVERED-SI \n"); Print("\tsysbench_lua.random_points_ranges\n"); @@ -943,7 +943,7 @@ sub Help { Print("\tRANGE-COVERED-SI \n"); Print("\t\tsysbench_lua.random_points_ranges\n"); - + Print("\tRANGE-NOTCOVERED-PK \n"); Print("\t\tsysbench_lua.random_points_ranges\n"); @@ -954,7 +954,7 @@ sub Help { Print("\tRANDOM-POINTS \n"); Print("\t\tysbench_lua.random_points_ranges\n"); - + Print("\tHOT-POINTS \n"); Print("\t\tsysbench_lua.random_points_ranges\n"); @@ -1354,13 +1354,14 @@ sub ValidateTargetWithSuite { return OK; } - my $expected = $tsOpt{db_driver}; - if (lc($incoming) eq lc($expected)) { - PrintVerbose($vt."db_driver match db maker $incoming, returning OK."); + my $expected = $tsOpt{db_driver}; + my $normalized = NormalizeDBType($incoming) // lc($incoming); + if ($normalized eq lc($expected)) { + PrintVerbose($vt."db_driver match db maker $incoming (normalized: $normalized), returning OK."); StageEnd($vt); return OK; } else { - PrintError($vt."Mismatch: sysbench_lua.db_driver = $expected, db install shows $incoming"); + PrintError($vt."Mismatch: sysbench_lua.db_driver = $expected, db install shows $incoming (normalized: $normalized)"); return ERROR; } } @@ -1630,7 +1631,7 @@ sub CheckTestsForUpdateRange{ sub ConfigureBMKTestCase{ my ($test_uc) = @_; $test_uc = uc($test_uc); - + my $_cbmk = StageStart($_me." -> ConfigureBMKTestCase ->"); # Here, we handle only BMK-only tests (ie: not optional use_bmk tests) if ($bmkFlags{bmk_sec_index_test_case}) { @@ -1645,25 +1646,25 @@ sub ConfigureBMKTestCase{ if ($test_uc eq "BMK_RW_UPDATE_RANGE") { $tsOpt{oltp_lua_script} = "OLTP_RW$trx_suffix"; - + } elsif ($test_uc eq "BMK_WO_UPDATE_RANGE") { $tsOpt{oltp_lua_script} = "OLTP_RW-write_only$trx_suffix"; - + } elsif ($test_uc eq "BMK_RW_UPDATE_INDEX_RANGE") { $tsOpt{oltp_lua_script} = "OLTP_RW-index_updates$trx_suffix"; - + } elsif ($test_uc eq "BMK_RW_UPDATE_NON_INDEX_RANGE") { $tsOpt{oltp_lua_script} = "OLTP_RW-non_index_updates$trx_suffix"; - + } elsif ($test_uc eq "BMK_RW_PS_UPDATE_RANGE") { $tsOpt{oltp_lua_script} = "OLTP_RW-point_selects$trx_suffix"; - + } elsif ($test_uc eq "BMK_RW_PS_UPDATE_INDEX_RANGE") { $tsOpt{oltp_lua_script} = "OLTP_RW-point_selects-non_index_updates$trx_suffix"; - + } elsif ($test_uc eq "BMK_RW_PS_UPDATE_NON_INDEX_RANGE") { $tsOpt{oltp_lua_script} = "OLTP_RO-non_index_updates$trx_suffix"; - + } elsif ($test_uc eq "CONNECT") { $tsOpt{oltp_lua_script} = "OLTP_RO-point_selects_reconnect$trx_suffix"; $tsOpt{test_args} .= " --point-selects=1 "; @@ -1671,7 +1672,7 @@ sub ConfigureBMKTestCase{ $tsOpt{test_args} .= " --sum-ranges=0 "; $tsOpt{test_args} .= " --order-ranges=0 "; $tsOpt{test_args} .= " --distinct-ranges=0 "; - + } else { PrintError($_cbmk." Invalid test: $test"); return ERROR; @@ -1722,10 +1723,10 @@ sub ConfigureStdTestCase{ # Here, we handle tests which are in the base lua set (and may also be in BMK-kit) my $trx_flag = $tsOpt{oltp_skip_trx}; my $use_bmk = $tsOpt{use_bmk}; - + # Helper for transactional suffix my $trx_suffix = ($trx_flag eq "off") ? "-trx.lua" : "-notrx.lua"; - + # POINT_SELECT if ($test_uc eq "POINT_SELECT") { $tsOpt{oltp_lua_script} = $use_bmk ? "OLTP_RO-point_selects$trx_suffix" : "oltp_point_select.lua"; @@ -1735,7 +1736,7 @@ sub ConfigureStdTestCase{ $tsOpt{test_args} .= " --sum-ranges=0"; $tsOpt{test_args} .= " --order-ranges=0"; $tsOpt{test_args} .= " --distinct-ranges=0"; - + # PARSER } elsif ($test_uc eq "PARSER") { $tsOpt{oltp_lua_script} = "oltp_point_select.lua"; @@ -1789,7 +1790,7 @@ sub ConfigureStdTestCase{ } elsif ($test_uc eq "OLTP_INSERT_INTO") { $tsOpt{oltp_lua_script} = "oltp_insert_into.lua"; $tsOpt{test_args} = " --skip-trx=$trx_flag"; - + # OLTP_RW } elsif ($test_uc eq "OLTP_RW") { $tsOpt{oltp_lua_script} = "oltp_read_write.lua"; @@ -1814,7 +1815,7 @@ sub ConfigureStdTestCase{ } elsif ($test_uc eq "INSERT") { $tsOpt{oltp_lua_script} = "oltp_insert.lua"; $tsOpt{test_args} = " --skip-trx=$trx_flag"; - + # DELETE } elsif ($test_uc eq "DELETE") { $tsOpt{oltp_lua_script} = "oltp_delete.lua"; @@ -1899,19 +1900,19 @@ sub ConfigureStdTestCase{ $tsOpt{test_args} .= " --sum-ranges=$tsOpt{oltp_sum_ranges}"; $tsOpt{test_args} .= " --order-ranges=$tsOpt{oltp_order_ranges}"; $tsOpt{test_args} .= " --distinct-ranges=$tsOpt{oltp_distinct_ranges}"; - #POINTS-COVERED-PK + #POINTS-COVERED-PK } elsif ($test_uc eq "POINTS-COVERED-PK") { $tsOpt{oltp_lua_script} = "oltp_points_covered.lua"; $tsOpt{test_args} = " --skip-trx"; $tsOpt{test_args} .= " --on-id=true"; $tsOpt{test_args} .= " --random-points=$tsOpt{random_points_ranges}"; - #POINTS-COVERED-SI + #POINTS-COVERED-SI } elsif ($test_uc eq "POINTS-COVERED-SI") { $tsOpt{oltp_lua_script} = "oltp_points_covered.lua"; $tsOpt{test_args} = " --skip-trx"; $tsOpt{test_args} .= " --on-id=false"; $tsOpt{test_args} .= " --random-points=$tsOpt{random_points_ranges}"; - #POINTS-NOTCOVERED-PK + #POINTS-NOTCOVERED-PK } elsif ($test_uc eq "POINTS-NOTCOVERED-PK") { $tsOpt{oltp_lua_script} = "oltp_points_covered.lua"; $tsOpt{test_args} = " --skip-trx"; @@ -1925,19 +1926,19 @@ sub ConfigureStdTestCase{ $tsOpt{test_args} .= " --on-id=false"; $tsOpt{test_args} .= " --covered=false"; $tsOpt{test_args} .= " --random-points=$tsOpt{random_points_ranges}"; - #RANGE-COVERED-PK + #RANGE-COVERED-PK } elsif ($test_uc eq "RANGE-COVERED-PK") { $tsOpt{oltp_lua_script} = "oltp_range_covered.lua"; $tsOpt{test_args} = " --skip-trx"; $tsOpt{test_args} .= " --on-id=true"; $tsOpt{test_args} .= " --random-points=$tsOpt{random_points_ranges}"; - #RANGE-COVERED-SI + #RANGE-COVERED-SI } elsif ($test_uc eq "RANGE-COVERED-SI") { $tsOpt{oltp_lua_script} = "oltp_range_covered.lua"; $tsOpt{test_args} = " --skip-trx"; $tsOpt{test_args} .= " --on-id=false"; $tsOpt{test_args} .= " --random-points=$tsOpt{random_points_ranges}"; - #RANGE-NOTCOVERED-PK + #RANGE-NOTCOVERED-PK } elsif ($test_uc eq "RANGE-NOTCOVERED-PK") { $tsOpt{oltp_lua_script} = "oltp_range_covered.lua"; $tsOpt{test_args} = " --skip-trx"; @@ -2578,24 +2579,39 @@ sub SetConnectionArgs { my $args = ""; # Base driver - # Normalize MariaDB aliases to mysql + # Normalize MariaDB aliases to mysql; normalize PostgreSQL aliases to pgsql if ($tsOpt{db_driver} =~ /^maria(db)?$/i) { $tsOpt{db_driver} = "mysql"; } + elsif ($tsOpt{db_driver} =~ /^(postgres|postgresql)$/i) { + $tsOpt{db_driver} = "pgsql"; + } $args .= "$tsState{target_lua} --db-driver=" . $tsOpt{db_driver}; - # Connection method - if ($options{db_clients_use_unix_socket}) { - $args .= " --mysql-socket='" . $options{db_socket} . "'"; + # Connection parameters — branched by driver family + if ($tsOpt{db_driver} eq 'pgsql') { + + # PostgreSQL uses --pgsql-* flags; always connect via loopback (pg_hba.conf allows 127.0.0.1) + $args .= " --pgsql-host='127.0.0.1'"; + $args .= " --pgsql-port=" . $options{db_port}; + $args .= " --pgsql-user='" . $options{db_user} . "'"; + $args .= " --pgsql-password='" . $options{db_user_pass} . "'"; + $args .= " --pgsql-db='" . $tmpDatabase . "'"; + } else { - $args .= " --mysql-host='" . $options{host} . "'"; - $args .= " --mysql-port=" . $options{db_port}; - } - # Credentials - $args .= " --mysql-user='" . $options{db_user} . "'"; - $args .= " --mysql-password='" . $options{db_user_pass} . "'"; - $args .= " --mysql-db='" . $tmpDatabase . "'"; + # MySQL / MariaDB + if ($options{db_clients_use_unix_socket}) { + $args .= " --mysql-socket='" . $options{db_socket} . "'"; + } else { + $args .= " --mysql-host='" . $options{host} . "'"; + $args .= " --mysql-port=" . $options{db_port}; + } + + $args .= " --mysql-user='" . $options{db_user} . "'"; + $args .= " --mysql-password='" . $options{db_user_pass} . "'"; + $args .= " --mysql-db='" . $tmpDatabase . "'"; + } # Execution mode if (!$options{use_request_based}) { @@ -2614,16 +2630,18 @@ sub SetConnectionArgs { # Partitioning $args .= " --oltp-num-partitions=" . $tsOpt{number_of_partitions} if defined $tsOpt{number_of_partitions}; - # Shutdown behavior - if($tsOpt{forced_shutdown}){ - $args .= " --forced-shutdown=" . $tsOpt{forced_shutdown_sec} if defined $tsOpt{forced_shutdown_sec}; + # Shutdown behavior (MySQL/MariaDB only — not supported by pgsql driver) + if ($tsOpt{db_driver} ne 'pgsql' && $tsOpt{forced_shutdown}) { + $args .= " --forced-shutdown=" . $tsOpt{forced_shutdown_sec} if defined $tsOpt{forced_shutdown_sec}; } - # Errors to ignore - $args .= " --mysql-ignore-errors=" . $tsOpt{ignore_errors} if defined $tsOpt{ignore_errors}; + # Storage engine (MySQL/MariaDB only — PostgreSQL has no storage engine concept) + if ($tsOpt{db_driver} ne 'pgsql') { + # Errors to ignore + $args .= " --mysql-ignore-errors=" . $tsOpt{ignore_errors} if defined $tsOpt{ignore_errors}; - # Storage engine - $args .= " --mysql-storage-engine=" . lc($options{db_engine}) if defined $options{db_engine}; + $args .= " --mysql-storage-engine=" . lc($options{db_engine}) if defined $options{db_engine}; + } # Per-table CREATE TABLE options (e.g. TidesDB table options) if (defined $tsOpt{create_table_options} && length $tsOpt{create_table_options}) { @@ -2653,22 +2671,33 @@ sub SetConnectionArgs { } $args .= " --thread-init-timeout=" . $tsOpt{thread_init_timeout}; +<<<<<<< variant A $args .= " --mysql-ssl=" .$tsOpt{bmk_mysql_ssl} if defined $tsOpt{bmk_mysql_ssl}; +>>>>>>> variant B + # BMK mysql-ssl flag applies to MySQL/MariaDB only + $args .= " --mysql-ssl=" .$tsOpt{bmk_mysql_ssl} + if defined $tsOpt{bmk_mysql_ssl} && $tsOpt{db_driver} ne 'pgsql'; +======= end $args .= " --sync-file='" . $tsOpt{bmk_sync_file} . "'" if defined $tsOpt{bmk_sync_file}; $args .= " --sync-wait=" . $tsOpt{bmk_sync_file_wait_timeout_ms} if defined $tsOpt{bmk_sync_file_wait_timeout_ms}; } else { - $args .= " --mysql-ssl" if defined $tsOpt{mysql_ssl}; + # MySQL/MariaDB non-BMK SSL flag + $args .= " --mysql-ssl" if defined $tsOpt{mysql_ssl} && $tsOpt{db_driver} ne 'pgsql'; } - # SSL certs - $args .= " --mysql-ssl-ca='" . $tsOpt{mysql_ssl_ca} . "'" if defined $tsOpt{mysql_ssl_ca}; - $args .= " --mysql-ssl-cert='" . $tsOpt{mysql_ssl_cert} . "'" if defined $tsOpt{mysql_ssl_cert}; - $args .= " --mysql-ssl-key='" . $tsOpt{mysql_ssl_key} . "'" if defined $tsOpt{mysql_ssl_key}; + # SSL certs — MySQL/MariaDB specific flags; pgsql SSL is configured via postgresql.conf + if ($tsOpt{db_driver} ne 'pgsql') { + $args .= " --mysql-ssl-ca='" . $tsOpt{mysql_ssl_ca} . "'" if defined $tsOpt{mysql_ssl_ca}; + $args .= " --mysql-ssl-cert='" . $tsOpt{mysql_ssl_cert} . "'" if defined $tsOpt{mysql_ssl_cert}; + $args .= " --mysql-ssl-key='" . $tsOpt{mysql_ssl_key} . "'" if defined $tsOpt{mysql_ssl_key}; + } - # Charset and partitioning - $args .= " --mysql-table-partitions=" . $tsOpt{bmk_partitions} if $tsOpt{bmk_partitions} > ZERO; - $args .= " --mysql-check-charset=1" if $tsOpt{bmk_check_character_set} > ZERO; + # Charset and partitioning (MySQL/MariaDB only) + if ($tsOpt{db_driver} ne 'pgsql') { + $args .= " --mysql-table-partitions=" . $tsOpt{bmk_partitions} if $tsOpt{bmk_partitions} > ZERO; + $args .= " --mysql-check-charset=1" if $tsOpt{bmk_check_character_set} > ZERO; + } # Debug flags $args .= " --debug=on" if $tsOpt{debug_sysbench}; @@ -2740,7 +2769,7 @@ sub SetLoadArgs { PrintVerbose($_sla."Lua Script Directory = ".$tsOpt{lua_scripts_dir}); PrintVerbose($_sla."Lua Script = ".$tsOpt{oltp_lua_script}); my $sysbench_args = "'$tsState{target_lua} $tsOpt{args}'"; - $tsOpt{load_args} = $args; + $tsOpt{load_args} = $args; $tsOpt{load_args} .= " --sysbench-args=" . $sysbench_args; $tsOpt{load_args} = $args; PrintVerbose($_sla . "Load Args: ".$tsOpt{load_args}); @@ -2839,7 +2868,7 @@ sub SingleTestRun { $test_case //= ''; $m_threads = int($m_threads // 0) || 1; # ensure a positive integer $m_runType //= ''; - + my $_str = StageStart($_me." -> SingleTestRun ->"); my $m_duration = $options{duration}; $m_runType = uc($m_runType); @@ -2899,6 +2928,13 @@ sub VerifyOptions { my $_vo = "$_me -> VerifyOptions ->"; # Validate oltp_skip_trx + # PostgreSQL sysbench driver does not support skip-trx; force it off. + if (defined $tsOpt{db_driver} && $tsOpt{db_driver} eq 'pgsql') { + if (lc($tsOpt{oltp_skip_trx}) eq 'on') { + PrintWarning($_vo."oltp_skip_trx=on is not supported by pgsql driver; forcing to off"); + $tsOpt{oltp_skip_trx} = "off"; + } + } if (lc($tsOpt{oltp_skip_trx}) ne "on" && lc($tsOpt{oltp_skip_trx}) ne "off") { PrintError($_vo."Invalid value for oltp_skip_trx: $tsOpt{oltp_skip_trx}"); PrintVerbose($_vo."Must be \"on\" or \"off\""); @@ -2958,6 +2994,34 @@ sub VerifyOptions { return OK; } +################################################################################ +# NormalizeDBType +# +# PURPOSE: +# Normalize an incoming database type string to the canonical sysbench +# driver name used in --db-driver. Called by ValidateTargetWithSuite(). +# +# CANONICAL MAPPINGS: +# mariadb, maria, mariadbd -> mysql +# mysql, mysqld -> mysql +# postgres, postgresql, +# pgsql -> pgsql +# +# RETURNS: +# Canonical driver string on success; undef on unknown input. +################################################################################ +sub NormalizeDBType { + my ($t) = @_; + return undef unless defined $t && length $t; + $t = lc $t; + $t =~ s/^\s+|\s+$//g; + + return "mysql" if $t =~ /^(mariadb|maria|mariadbd|mysql|mysqld)$/; + return "pgsql" if $t =~ /^(postgres|postgresql|pgsql)$/; + + return undef; +} + ############################################################################# # Module terminator ############################################################################# diff --git a/tests/run_tests.sh b/tests/run_tests.sh new file mode 100755 index 0000000..6d9c133 --- /dev/null +++ b/tests/run_tests.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# ============================================================================= +# run_tests.sh — Runs the complete TAF PostgreSQL test suite +# +# Usage: +# bash tests/run_tests.sh [pytest arguments...] +# +# Examples: +# bash tests/run_tests.sh # all tests +# bash tests/run_tests.sh -v # verbose +# bash tests/run_tests.sh -v -k TestL5 # L5 only +# bash tests/run_tests.sh --co -q # list tests only +# +# Env variables (override automatic detection): +# TAF_PG_INSTALL_DIR PostgreSQL installation directory +# TAF_PG_PORT port for TAF-managed PG (default: 5433) +# TAF_MARIADB_DIR (optional) for L6 MariaDB regression test +# ============================================================================= +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TAF_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +# --------------------------------------------------------------------------- +# PostgreSQL installation detection +# --------------------------------------------------------------------------- +detect_pg_install() { + # 1. Respect explicitly set variable + if [[ -n "${TAF_PG_INSTALL_DIR:-}" ]]; then + echo "${TAF_PG_INSTALL_DIR}" + return + fi + + # 2. Source .taf_pg_env if it exists (created by setup_almalinux10.sh) + if [[ -f "${TAF_ROOT}/.taf_pg_env" ]]; then + # shellcheck source=/dev/null + source "${TAF_ROOT}/.taf_pg_env" + if [[ -n "${TAF_PG_INSTALL_DIR:-}" ]]; then + echo "${TAF_PG_INSTALL_DIR}" + return + fi + fi + + # 3. Look for Percona tarball (default installation method) + if [[ -x "/opt/pgdistro/percona-postgresql16/bin/postgres" ]]; then + echo "/opt/pgdistro/percona-postgresql16" + return + fi + + # 4. PGDG RPM + for dir in /usr/pgsql-16 /usr/pgsql-15 /usr/pgsql-14; do + if [[ -x "${dir}/bin/postgres" ]]; then + echo "${dir}" + return + fi + done + + # 5. AppStream / system PG + if [[ -x "/usr/bin/postgres" ]]; then + echo "/usr" + return + fi + + echo "" +} + +# --------------------------------------------------------------------------- +# Prerequisites check +# --------------------------------------------------------------------------- +RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; CYAN='\033[0;36m'; NC='\033[0m' +info() { echo -e "${GREEN}[TEST]${NC} $*"; } +warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } +error() { echo -e "${RED}[ERROR]${NC} $*" >&2; exit 1; } + +cd "${TAF_ROOT}" + +PG_INSTALL="$(detect_pg_install)" +if [[ -z "$PG_INSTALL" ]]; then + error "PostgreSQL not found. Run first: sudo bash tests/setup_almalinux10.sh" +fi +export TAF_PG_INSTALL_DIR="${PG_INSTALL}" +export TAF_PG_PORT="${TAF_PG_PORT:-5433}" + +# Add PG bin and lib to PATH/LD_LIBRARY_PATH +export PATH="${PG_INSTALL}/bin:${PATH}" +PG_LIB="${PG_INSTALL}/lib" +if [[ -d "${PG_LIB}" ]]; then + export LD_LIBRARY_PATH="${PG_LIB}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +fi + +# Virtuozzo/VzLinux: add system site-packages to Percona Python +PERCONA_PY_SITEPKGS="/opt/percona-python3/lib/python3.12/site-packages" +if [[ -d "$PERCONA_PY_SITEPKGS" ]]; then + PTH_FILE="${PERCONA_PY_SITEPKGS}/system-sitepackages.pth" + if [[ ! -f "$PTH_FILE" ]]; then + echo "/usr/lib/python3.12/site-packages" > "$PTH_FILE" + echo "/usr/lib64/python3.12/site-packages" >> "$PTH_FILE" + fi +fi + +info "PostgreSQL: ${PG_INSTALL} ($(pg_config --version 2>/dev/null || echo 'version unknown'))" +info "Port: ${TAF_PG_PORT}" +info "Sysbench: ${TAF_ROOT}/client_source/sysbench-lua/sysbench" +[[ -n "${TAF_MARIADB_DIR:-}" ]] && info "MariaDB: ${TAF_MARIADB_DIR} (L6 enabled)" + +# Sysbench binary check +SYSBENCH_BIN="${TAF_ROOT}/client_source/sysbench-lua/sysbench" +if [[ ! -x "${SYSBENCH_BIN}" ]]; then + warn "Sysbench not found (${SYSBENCH_BIN})" + warn "L4/L5 tests will be skipped. Run setup_almalinux10.sh to build." +fi + +# Verify pytest +python3 -m pytest --version >/dev/null 2>&1 || \ + error "pytest not found. Run: pip3 install pytest" + +# --------------------------------------------------------------------------- +# Running tests +# --------------------------------------------------------------------------- +echo "" +echo -e "${CYAN}══════════════════════════════════════════════════${NC}" +echo -e "${CYAN} TAF PostgreSQL Integration Tests${NC}" +echo -e "${CYAN}══════════════════════════════════════════════════${NC}" +echo "" + +PYTEST_ARGS=("tests/test_taf_postgresql.py") + +# Default verbose if no arguments provided +if [[ $# -eq 0 ]]; then + PYTEST_ARGS+=("-v" "--tb=short") +else + PYTEST_ARGS+=("$@") +fi + +python3 -m pytest "${PYTEST_ARGS[@]}" diff --git a/tests/setup_almalinux10.sh b/tests/setup_almalinux10.sh new file mode 100755 index 0000000..6861493 --- /dev/null +++ b/tests/setup_almalinux10.sh @@ -0,0 +1,363 @@ +#!/usr/bin/env bash +# ============================================================================= +# setup_almalinux10.sh — Prerequisites for TAF PostgreSQL tests +# +# Target: RHEL/AlmaLinux/Virtuozzo 8–10 (x86_64, aarch64) +# Usage: sudo bash tests/setup_almalinux10.sh [--method=percona|pgdg|appstream] +# +# PostgreSQL installation methods (--method): +# percona (default) — tarball from downloads.percona.com +# https://docs.percona.com/postgresql/16/tarball.html +# pgdg — RPM from pgdg.postgresql.org +# appstream — system postgresql from dnf +# +# After completion: +# - PostgreSQL 16 available in $PG_INSTALL_DIR +# - Python 3 + pytest installed +# - Build tools for sysbench ready +# - Env saved to .taf_pg_env (source before tests) +# +# Env variables (can be overridden before pytest): +# TAF_PG_INSTALL_DIR (set automatically) +# TAF_PG_PORT (default: 5433) +# TAF_MARIADB_DIR (optional, for L6 regression test) +# ============================================================================= +set -euo pipefail + +# --------------------------------------------------------------------------- +# Colors and helper functions +# --------------------------------------------------------------------------- +RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; CYAN='\033[0;36m'; NC='\033[0m' +info() { echo -e "${GREEN}[SETUP]${NC} $*"; } +warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } +error() { echo -e "${RED}[ERROR]${NC} $*" >&2; exit 1; } +step() { echo -e "\n${CYAN}━━━ $* ━━━${NC}"; } + +[[ $EUID -eq 0 ]] || error "Script must be run as root (sudo bash $0)" + +# --------------------------------------------------------------------------- +# Parametry +# --------------------------------------------------------------------------- +METHOD="percona" +for arg in "$@"; do + case "$arg" in + --method=percona) METHOD=percona ;; + --method=pgdg) METHOD=pgdg ;; + --method=appstream) METHOD=appstream ;; + --help|-h) + echo "Usage: sudo bash $0 [--method=percona|pgdg|appstream]" + echo " percona (default) Tarball from downloads.percona.com" + echo " pgdg RPM from pgdg.postgresql.org" + echo " appstream System postgresql from dnf" + exit 0 ;; + *) warn "Unknown parameter: $arg" ;; + esac +done + +ARCH=$(uname -m) +TAF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +info "PG installation method: ${CYAN}${METHOD}${NC}" +info "Architecture: ${ARCH}" +info "TAF directory: ${TAF_DIR}" + +# --------------------------------------------------------------------------- +# 1. BASE DEPENDENCIES +# --------------------------------------------------------------------------- +step "Installing base dependencies" +dnf install -y epel-release 2>/dev/null || true +dnf install -y \ + gcc gcc-c++ make cmake automake libtool pkg-config \ + perl perl-devel \ + python3 python3-pip \ + git wget curl tar \ + libaio-devel readline-devel \ + openssl openssl-devel \ + acl + +# --------------------------------------------------------------------------- +# 2. POSTGRESQL — according to chosen method +# --------------------------------------------------------------------------- +step "Installing PostgreSQL 16 (method: ${METHOD})" + +PG_INSTALL_DIR="" +LIBPQ_INCDIR="" +LIBPQ_LIBDIR="" + +# ─── 2a. PERCONA TARBALL (default) ───────────────────────────────────────── +# Dokumentace: https://docs.percona.com/postgresql/16/tarball.html +install_percona_tarball() { + local VERSION="16.14" + local INSTALL_BASE="/opt/pgdistro" + local PG_SUBDIR="percona-postgresql16" + + # Detect OpenSSL version → choose tarball variant + local OPENSSL_VER + OPENSSL_VER=$(openssl version | awk '{print $2}') + local SSL_TAG + case "${OPENSSL_VER%%.*}" in + 1) SSL_TAG="ssl1" ;; + 3) SSL_TAG="ssl3" ;; + *) SSL_TAG="ssl3"; warn "Unknown OpenSSL version ${OPENSSL_VER}, trying ssl3" ;; + esac + + # Map architecture to tarball name + local TARARCH + case "$ARCH" in + x86_64) TARARCH="linux-x86_64" ;; + aarch64) TARARCH="linux-aarch64" ;; + *) error "Unsupported architecture: $ARCH" ;; + esac + + local TARBALL="percona-postgresql-${VERSION}-${SSL_TAG}-${TARARCH}.tar.gz" + local URL="https://downloads.percona.com/downloads/postgresql-distribution-16/${VERSION}/binary/tarball/${TARBALL}" + + info "Tarball: ${TARBALL}" + info "URL: ${URL}" + + # Skip if binary already exists + if [[ -x "${INSTALL_BASE}/${PG_SUBDIR}/bin/postgres" ]]; then + info "Percona PostgreSQL 16 already installed in ${INSTALL_BASE}/${PG_SUBDIR}" + PG_INSTALL_DIR="${INSTALL_BASE}/${PG_SUBDIR}" + return 0 + fi + + mkdir -p "${INSTALL_BASE}" + + local TMPTAR="/tmp/${TARBALL}" + if [[ ! -f "$TMPTAR" ]]; then + info "Downloading tarball..." + wget -q --show-progress -O "$TMPTAR" "$URL" 2>/dev/null || \ + wget -O "$TMPTAR" "$URL" || \ + curl -fL -o "$TMPTAR" "$URL" + else + info "Tarball already downloaded: ${TMPTAR}" + fi + + info "Extracting to ${INSTALL_BASE}/..." + tar -xf "$TMPTAR" -C "${INSTALL_BASE}/" + + # Move Perl/Python/Tcl modules one level up (per documentation) + for mod in percona-perl percona-python3 percona-tcl; do + if [[ -d "${INSTALL_BASE}/${mod}" ]] && [[ ! -d "/opt/${mod}" ]]; then + info "Moving ${mod} -> /opt/${mod}" + mv "${INSTALL_BASE}/${mod}" "/opt/${mod}" + fi + done + + PG_INSTALL_DIR="${INSTALL_BASE}/${PG_SUBDIR}" + + # LD_LIBRARY_PATH for bundled libraries + local PG_LIB="${PG_INSTALL_DIR}/lib" + if [[ -d "$PG_LIB" ]]; then + export LD_LIBRARY_PATH="${PG_LIB}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + cat > /etc/profile.d/percona-pg16.sh </dev/null; then + info "PGDG repository added" + dnf -qy module disable postgresql 2>/dev/null || true + dnf install -y postgresql16-server postgresql16-devel postgresql16 + PG_INSTALL_DIR="/usr/pgsql-16" + else + warn "PGDG EL10 repo unavailable, falling back to AppStream" + install_appstream + fi +} + +# ─── 2c. APPSTREAM ───────────────────────────────────────────────────────── +install_appstream() { + dnf install -y postgresql-server postgresql-devel + PG_INSTALL_DIR="/usr" + warn "PostgreSQL installed from AppStream into ${PG_INSTALL_DIR}" +} + +case "$METHOD" in + percona) install_percona_tarball ;; + pgdg) install_pgdg_rpm ;; + appstream) install_appstream ;; +esac + +# --------------------------------------------------------------------------- +# 3. VERIFY POSTGRESQL BINARIES +# --------------------------------------------------------------------------- +step "Verifying PostgreSQL binaries" +PG_BIN="${PG_INSTALL_DIR}/bin" +MISSING=0 +for BIN in postgres pg_ctl psql initdb pg_isready pg_config; do + if [[ -x "${PG_BIN}/${BIN}" ]]; then + info " ✓ ${PG_BIN}/${BIN}" + else + warn " ✗ ${PG_BIN}/${BIN} not found" + MISSING=$((MISSING + 1)) + fi +done +[[ $MISSING -eq 0 ]] || error "Missing binaries — check installation in ${PG_INSTALL_DIR}" + +info "PostgreSQL version: $("${PG_BIN}/pg_config" --version)" +LIBPQ_INCDIR=$("${PG_BIN}/pg_config" --includedir) +LIBPQ_LIBDIR=$("${PG_BIN}/pg_config" --libdir) +info "includedir: ${LIBPQ_INCDIR}" +info "libdir: ${LIBPQ_LIBDIR}" + +# --------------------------------------------------------------------------- +# 4. LIBPQ HEADERS FOR SYSBENCH +# --------------------------------------------------------------------------- +step "Checking libpq-fe.h for sysbench" +if [[ -f "${LIBPQ_INCDIR}/libpq-fe.h" ]]; then + info "libpq-fe.h found: ${LIBPQ_INCDIR}/libpq-fe.h" +else + warn "libpq-fe.h not found, trying system packages..." + dnf install -y postgresql-devel libpq-devel 2>/dev/null || \ + dnf install -y postgresql16-devel 2>/dev/null || \ + warn "libpq-devel unavailable — sysbench build may fail" +fi + +# Export for ./configure sysbench +export PKG_CONFIG_PATH="${LIBPQ_LIBDIR}/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}" +export CPPFLAGS="-I${LIBPQ_INCDIR}" +export LDFLAGS="-L${LIBPQ_LIBDIR} -Wl,-rpath,${LIBPQ_LIBDIR}" + +# --------------------------------------------------------------------------- +# 5. PYTHON + PYTEST +# --------------------------------------------------------------------------- +step "Installing pytest" + +# Virtuozzo/VzLinux Python3 uses Percona Python as stdlib +# (/usr/bin/python3 is a small wrapper, sys.prefix=/opt/percona-python3). +# Pytest must be in /opt/percona-python3/lib/python3.12/site-packages/ +# or accessible via a .pth file. System pip3/ssl may not work +# (Percona Python requires OpenSSL 3.3+, system has 3.2.x). + +PERCONA_PY_SITEPKGS="/opt/percona-python3/lib/python3.12/site-packages" +SYS_SITEPKGS="/usr/lib/python3.12/site-packages" + +# Strategy 1: add system site-packages to Percona Python via .pth +if [[ -d "$PERCONA_PY_SITEPKGS" ]]; then + PTH_FILE="${PERCONA_PY_SITEPKGS}/system-sitepackages.pth" + if [[ ! -f "$PTH_FILE" ]]; then + info "Adding system site-packages to Percona Python path..." + echo "/usr/lib/python3.12/site-packages" > "$PTH_FILE" + echo "/usr/lib64/python3.12/site-packages" >> "$PTH_FILE" + fi +fi + +# Strategy 2: install pytest via dnf (preferred for Virtuozzo) +PYTEST_INSTALLED=0 +if python3 -m pytest --version >/dev/null 2>&1; then + PYTEST_INSTALLED=1 +elif dnf install -y python3-pytest >/dev/null 2>&1; then + PYTEST_INSTALLED=1 +else + # Strategy 3: copy pytest from /usr/local (installed by earlier pip) + if [[ -d "/usr/local/lib/python3.12/site-packages/pytest" ]]; then + for pkg in pytest _pytest pluggy iniconfig py.py; do + src="/usr/local/lib/python3.12/site-packages/${pkg}" + [[ -e "$src" ]] && cp -r "$src" "${PERCONA_PY_SITEPKGS}/" 2>/dev/null || true + done + for dist in /usr/local/lib/python3.12/site-packages/{pytest,pluggy,iniconfig}-*.dist-info; do + [[ -e "$dist" ]] && cp -r "$dist" "${PERCONA_PY_SITEPKGS}/" 2>/dev/null || true + done + python3 -m pytest --version >/dev/null 2>&1 && PYTEST_INSTALLED=1 + fi +fi + +[[ $PYTEST_INSTALLED -eq 1 ]] || error "Failed to install pytest" +info "pytest: $(python3 -m pytest --version 2>&1 | head -1)" + +# --------------------------------------------------------------------------- +# 6. SYSBENCH — clone and build if missing +# --------------------------------------------------------------------------- +step "Sysbench" +SYSBENCH_SRC="${TAF_DIR}/client_source/sysbench-lua" + +# Correct state: sysbench is a symlink to src/sysbench (locally built binary) +SYSBENCH_OK=0 +if [[ -L "${SYSBENCH_SRC}/sysbench" ]] && [[ "$(readlink "${SYSBENCH_SRC}/sysbench")" == "src/sysbench" ]] && [[ -x "${SYSBENCH_SRC}/src/sysbench" ]]; then + SYSBENCH_OK=1 + info "Sysbench already built: $("${SYSBENCH_SRC}/sysbench" --version)" +elif [[ -x "${SYSBENCH_SRC}/sysbench" ]] && [[ ! -L "${SYSBENCH_SRC}/sysbench" ]]; then + warn "sysbench is a binary (not a symlink) — may have been rsync'd from another host" + warn "Rebuilding from source for correct architecture and libpq..." +fi +if [[ $SYSBENCH_OK -eq 0 ]]; then + if [[ ! -f "${SYSBENCH_SRC}/configure.ac" ]]; then + info "Cloning sysbench..." + mkdir -p "${TAF_DIR}/client_source" + git clone --depth=1 https://github.com/akopytov/sysbench "${SYSBENCH_SRC}" + fi + + info "Building sysbench with pgsql support..." + cd "${SYSBENCH_SRC}" + + ./autogen.sh + ./configure --without-mysql --with-pgsql \ + --with-pgsql-includes="${LIBPQ_INCDIR}" \ + --with-pgsql-libs="${LIBPQ_LIBDIR}" + make -j"$(nproc)" + + if [[ -f "src/sysbench" ]]; then + # Ensure correct symlink — remove any old binary (e.g. rsync'd from another host) + if [[ ! -L "sysbench" ]] || [[ "$(readlink sysbench)" != "src/sysbench" ]]; then + rm -f sysbench + ln -sf src/sysbench sysbench + info "Symlink: sysbench-lua/sysbench -> src/sysbench" + fi + fi + + cd "${TAF_DIR}" + info "Sysbench: $("${SYSBENCH_SRC}/sysbench" --version)" +fi + +# --------------------------------------------------------------------------- +# 7. /root PERMISSIONS (postgres user needs traverse into TAF dir) +# --------------------------------------------------------------------------- +step "Directory permissions" +ROOT_PERM=$(stat -c '%a' /root) +if [[ "${ROOT_PERM: -1}" == "0" ]]; then + info "Adding o+x on /root" + chmod o+x /root +fi +info "/root: $(stat -c '%a' /root)" + +# --------------------------------------------------------------------------- +# 8. SAVING ENV VARIABLES +# --------------------------------------------------------------------------- +step "Saving environment" +ENV_FILE="${TAF_DIR}/.taf_pg_env" +cat > "${ENV_FILE}" < subprocess.CompletedProcess: + """Runs a command and returns CompletedProcess (does not raise on non-zero).""" + merged_env = None + if env: + merged_env = os.environ.copy() + merged_env.update(env) + return subprocess.run( + list(cmd), + capture_output=True, text=True, + cwd=str(cwd or TAF_ROOT), + timeout=timeout, + env=merged_env, + ) + + +def taf_run(props: dict, timeout: int = 600) -> subprocess.CompletedProcess: + """Runs perl taf.pl with the given properties (dict).""" + cmd = ["perl", str(TAF_ROOT / "taf.pl")] + for k, v in props.items(): + cmd.append(f"--property={k}={v}") + return subprocess.run( + cmd, + capture_output=True, text=True, + cwd=str(TAF_ROOT), + timeout=timeout, + ) + + +def taf_propfile(props_file: Path, extra: dict | None = None, + timeout: int = 600) -> subprocess.CompletedProcess: + """Runs perl taf.pl with a properties file and optionally extra overrides.""" + cmd = ["perl", str(TAF_ROOT / "taf.pl"), + f"--properties-file={props_file}"] + for k, v in (extra or {}).items(): + cmd.append(f"--property={k}={v}") + return subprocess.run( + cmd, + capture_output=True, text=True, + cwd=str(TAF_ROOT), + timeout=timeout, + ) + + +def write_props(path: Path, props: dict) -> None: + """Writes a dict to a .properties file.""" + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w") as f: + for k, v in props.items(): + f.write(f"{k} = {v}\n") + + +def has_no_errors(text: str) -> tuple[bool, str | None]: + """Returns (True, None) if TAF output contains no error lines. + Ignores comments and informational 'error' in values. + """ + for line in text.splitlines(): + stripped = line.strip() + if stripped.startswith("#"): + continue + # Look for ERROR as a token (not 'error' inside values) + if re.search(r'\bERROR\b', line): + return False, line + return True, None + + +def find_pg_data_dir(taf_output: str) -> Path | None: + """Parses verbose TAF output and searches for the data directory path.""" + # TAF logs e.g.: "Preparing data directory: /path/to/data" + # or: "data_dir does not exist: /path" + # or: "Datadir does not exist: /path" + patterns = [ + r'data.dir[:\s]+(/[^\s]+)', + r'Removing existing data directory\s+(/[^\s]+)', + r'Created.*data.*dir.*?(/[^\s]+)', + r'initdb.*?-D\s+(/[^\s]+)', + r'pg_ctl.*?-D\s+(/[^\s]+)', + ] + for pat in patterns: + m = re.search(pat, taf_output, re.IGNORECASE) + if m: + return Path(m.group(1)) + return None + + +def psql(query: str, user: str = TAF_PG_USER, password: str = TAF_PG_PASS, + db: str = TAF_PG_DB, port: int = PG_PORT, + host: str = "127.0.0.1") -> subprocess.CompletedProcess: + """Runs a psql query and returns the result.""" + env = {"PGPASSWORD": password} + return run( + str(PG_BIN / "psql"), + "-h", host, "-p", str(port), "-U", user, "-d", db, + "-c", query, "-q", "--no-psqlrc", "--tuples-only", + env=env, + timeout=15, + ) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +@pytest.fixture(scope="session") +def lifecycle_result(tmp_path_factory): + """L3 fixture: runs TAF init-start-db-exit once per session. + + Returns (CompletedProcess, data_dir_path_or_None). + Ensures PostgreSQL is stopped even on error. + """ + # Stop any leftover PG from a previous session + _shutdown_pg_if_running() + + tmp = tmp_path_factory.mktemp("taf_l3") + props_file = tmp / "lifecycle.properties" + + write_props(props_file, { + "taf.action": "init-start-db-exit", + "taf.taf_db_makers_plugin": "postgres", + "taf.db_software_install_dir": str(PG_INSTALL), + "taf.db_port": str(PG_PORT), + "taf.db_user": TAF_PG_USER, + "taf.db_user_pass": TAF_PG_PASS, + "taf.db_root_user": TAF_PG_ROOT, + "taf.db_root_pass": TAF_PG_ROOT_PASS, + "taf.database": TAF_PG_DB, + "taf.test_suite": "sysbench-lua", + "taf.verbose": "true", + "sysbench_lua.db_driver": "pgsql", + }) + + result = taf_propfile(props_file, timeout=300) + output = result.stdout + result.stderr + data_dir = find_pg_data_dir(output) + + yield result, data_dir + + # Cleanup: ensure PG is shut down if TAF failed midway + _shutdown_pg_if_running() + + +@pytest.fixture(scope="session") +def benchmark_result(tmp_path_factory): + """L5 fixture: runs a short TAF benchmark (OLTP_RO + POINT_SELECT). + + Returns CompletedProcess. + """ + # L3 leaves PostgreSQL running (init-start-db-exit keeps PG alive); stop it + # before TAF tries to start a fresh instance via init-start-db-run-tests. + _shutdown_pg_if_running() + + tmp = tmp_path_factory.mktemp("taf_l5") + props_file = tmp / "benchmark.properties" + + # Sysbench uses autotools (not cmake); binary built via L4 autotools test. + # Use init-start-db-run-tests to skip client build (avoid cmake failure). + # sysbench_lua.exe defaults to relative path client_source/sysbench-lua/sysbench + # which resolves correctly from TAF working dir — no override needed. + write_props(props_file, { + "taf.action": "init-start-db-run-tests", + "taf.taf_db_makers_plugin": "postgres", + "taf.db_software_install_dir": str(PG_INSTALL), + "taf.db_port": str(PG_PORT), + "taf.db_user": TAF_PG_USER, + "taf.db_user_pass": TAF_PG_PASS, + "taf.db_root_user": TAF_PG_ROOT, + "taf.db_root_pass": TAF_PG_ROOT_PASS, + "taf.database": TAF_PG_DB, + "taf.test_suite": "sysbench-lua", + "taf.tests": "OLTP_RO,POINT_SELECT", + "taf.verbose": "true", + "sysbench_lua.db_driver": "pgsql", + "sysbench_lua.connector": "libpq", + "sysbench_lua.def_threads": "4,8", + "sysbench_lua.def_duration": "30", + "sysbench_lua.number_of_tables": "1", + "sysbench_lua.number_of_rows": "10000", + "sysbench_lua.oltp_skip_trx": "off", + }) + + result = taf_propfile(props_file, timeout=900) + + yield result + + _shutdown_pg_if_running() + + +def _shutdown_pg_if_running() -> None: + """Best-effort: stop PG if still running (cleanup after L3/L5). + + pg_ctl stop cannot run as root — we find postmaster.pid and send + SIGTERM directly to the process, or run pg_ctl as postgres user via su. + """ + import signal as _signal + from pathlib import Path + pg_ctl = PG_BIN / "pg_ctl" + # TAF creates the data directory in /tmp/taf_pg_/data/ or in TAF_ROOT/data/ + search_roots = [TAF_ROOT, Path("/tmp")] + for root in search_roots: + if not root.is_dir(): + continue + for pid_file in root.rglob("postmaster.pid"): + data_dir = pid_file.parent + # Try via postgres user (pg_ctl cannot run as root) + if pg_ctl.is_file(): + r = subprocess.run( + ["su", "-s", "/bin/sh", "postgres", "-c", + f"{pg_ctl} stop -D {data_dir} -m fast -w -t 30"], + capture_output=True, timeout=45, + ) + if r.returncode == 0: + continue + # Fallback: kill postmaster directly via SIGTERM + try: + pid = int(pid_file.read_text().splitlines()[0].strip()) + os.kill(pid, _signal.SIGTERM) + except Exception: + pass + + +# =========================================================================== +# LAYER 1 — Static validation +# =========================================================================== + +class TestL1Static: + """Syntax and text validation without running TAF or PostgreSQL.""" + + def test_postgres_pm_exists(self): + """Plugin postgres.pm must exist.""" + plugin = TAF_ROOT / "libs" / "database_libs" / "postgres.pm" + assert plugin.is_file(), f"Missing: {plugin}" + + def test_postgres_pm_package_name(self): + """Package declaration in postgres.pm must be 'package postgres'.""" + plugin = TAF_ROOT / "libs" / "database_libs" / "postgres.pm" + content = plugin.read_text() + assert re.search(r"^\s*package\s+postgres\s*;", content, re.MULTILINE), \ + "postgres.pm does not declare 'package postgres'" + + def test_postgres_pm_syntax(self): + """`perl -c` on postgres.pm must pass (on Linux).""" + result = run("perl", "-c", + str(TAF_ROOT / "libs" / "database_libs" / "postgres.pm")) + assert result.returncode == 0 or \ + "Unsupported OS platform" in result.stderr, \ + f"Syntax error in postgres.pm:\n{result.stderr}" + + def test_sysbench_lua_syntax(self): + """`perl -c` on sysbench-lua.pm must pass.""" + result = run("perl", "-c", + str(TAF_ROOT / "test_suites" / "sysbench-lua.pm")) + assert result.returncode == 0, \ + f"Syntax error in sysbench-lua.pm:\n{result.stderr}" + + def test_pgsql_connection_args_in_sysbench(self): + """sysbench-lua.pm must generate --pgsql-host/port/user/password/db.""" + content = (TAF_ROOT / "test_suites" / "sysbench-lua.pm").read_text() + for flag in ("--pgsql-host", "--pgsql-port", "--pgsql-user", + "--pgsql-password", "--pgsql-db"): + assert flag in content, \ + f"Missing '{flag}' in SetConnectionArgs() — pgsql branch" + + def test_mysql_args_guarded_for_pgsql(self): + """--mysql-storage-engine must be guarded by 'db_driver ne pgsql'.""" + content = (TAF_ROOT / "test_suites" / "sysbench-lua.pm").read_text() + # Find the block with mysql-storage-engine + block = re.search( + r"(Storage engine.*?mysql-storage-engine.*?\n)", content, + re.DOTALL | re.IGNORECASE, + ) + # A condition with 'pgsql' must exist near --mysql-storage-engine + idx = content.find("--mysql-storage-engine") + assert idx >= 0, "--mysql-storage-engine not found in file" + surrounding = content[max(0, idx - 200):idx + 100] + assert "pgsql" in surrounding, \ + "--mysql-storage-engine is not guarded by a db_driver ne pgsql check" + + def test_normalize_db_type_defined(self): + """NormalizeDBType must be defined in sysbench-lua.pm.""" + content = (TAF_ROOT / "test_suites" / "sysbench-lua.pm").read_text() + assert "sub NormalizeDBType" in content, \ + "Missing sub NormalizeDBType in sysbench-lua.pm" + + def test_normalize_db_type_maps_postgres_to_pgsql(self): + """NormalizeDBType must map postgres/postgresql → pgsql.""" + content = (TAF_ROOT / "test_suites" / "sysbench-lua.pm").read_text() + # Extract the function + m = re.search(r"sub NormalizeDBType \{(.+?)\n\}", content, re.DOTALL) + assert m, "NormalizeDBType not found" + body = m.group(1) + assert "pgsql" in body and "postgres" in body, \ + "NormalizeDBType does not contain the postgres → pgsql mapping" + + def test_validate_target_normalizes_incoming(self): + """ValidateTargetWithSuite must normalize $incoming before comparison.""" + content = (TAF_ROOT / "test_suites" / "sysbench-lua.pm").read_text() + # Look for NormalizeDBType($incoming) in the body of ValidateTargetWithSuite + m = re.search( + r"sub ValidateTargetWithSuite \{(.+?)\n\}", + content, re.DOTALL, + ) + assert m, "ValidateTargetWithSuite not found" + body = m.group(1) + assert "NormalizeDBType($incoming)" in body, \ + "ValidateTargetWithSuite does not normalize \\$incoming before comparison" + + def test_cmake_args_contains_pgsql(self): + """sysbench_lua_default.properties must contain -DWITH_PGSQL=on.""" + defaults = ( + TAF_ROOT / "properties" / "default" / "sysbench_lua_default.properties" + ).read_text() + assert "-DWITH_PGSQL=on" in defaults, \ + "cmake_args does not contain -DWITH_PGSQL=on" + + def test_postgres_sql_blocks_count(self): + """postgres.sql must have at least 14 named blocks.""" + content = (TAF_ROOT / "libs" / "sql_libs" / "dialects" / "postgres.sql").read_text() + blocks = re.findall(r"^\[(\w+)\]", content, re.MULTILINE) + assert len(blocks) >= 14, \ + f"postgres.sql has only {len(blocks)} blocks, expected ≥14: {blocks}" + + def test_postgres_sql_has_diagnostic_blocks(self): + """postgres.sql must contain diagnostic blocks for benchmark analysis.""" + content = (TAF_ROOT / "libs" / "sql_libs" / "dialects" / "postgres.sql").read_text() + required = [ + "active_connections", "wait_events", "table_stats", + "index_usage", "bgwriter_stats", "transaction_stats", + ] + for block in required: + assert f"[{block}]" in content, \ + f"Missing diagnostic block [{block}] in postgres.sql" + + def test_utilities_pm_has_postgresql_alias(self): + """Utilities.pm must map 'postgresql' → 'postgres' in PLUGIN_ALIASES.""" + content = (TAF_ROOT / "libs" / "taf_libs" / "TAF" / "Utilities.pm").read_text() + assert re.search(r"postgresql\s*=>\s*['\"]postgres['\"]", content), \ + "Missing 'postgresql => postgres' in PLUGIN_ALIASES (Utilities.pm)" + + def test_postgresql_conf_templates_exist(self): + """At least three postgresql.conf templates must exist.""" + pg_cfg_dir = TAF_ROOT / "database_config_files" / "postgresql" + assert pg_cfg_dir.is_dir(), f"Directory {pg_cfg_dir} does not exist" + configs = list(pg_cfg_dir.glob("*.conf")) + assert len(configs) >= 3, \ + f"Expected ≥3 .conf files, found {len(configs)}: {configs}" + + def test_postgresql_properties_examples_exist(self): + """Example properties files for PostgreSQL must exist.""" + pg_props_dir = TAF_ROOT / "properties" / "postgresql" + assert pg_props_dir.is_dir(), f"Directory {pg_props_dir} does not exist" + props = list(pg_props_dir.glob("*.properties")) + assert len(props) >= 2, \ + f"Expected ≥2 .properties files, found {len(props)}: {props}" + + +# =========================================================================== +# LAYER 2 — Plugin unit test (via Perl subprocess) +# =========================================================================== + +def _build_plugin_unit_script(pg_install: Path, port: int, + user: str, password: str, + root: str, root_pass: str) -> str: + """Builds a Perl unit-test script for the postgres.pm plugin. + + Does not use str.format() — Perl syntax contains {} which would conflict + with Python format placeholders. + """ + taf_libs = str(pg_install.parent.parent / "libs" / "database_libs") \ + if False else str(TAF_ROOT / "libs" / "database_libs") + return ( + "#!/usr/bin/perl\n" + "use strict;\n" + "use warnings;\n" + "\n" + "BEGIN {\n" + " package TAF::Logging;\n" + " use Exporter 'import';\n" + " our @EXPORT_OK = qw(PrintError PrintWarning PrintVerbose StageStart StageEnd);\n" + " sub PrintError { print \"ERR: @_\\n\" }\n" + " sub PrintWarning { print \"WARN: @_\\n\" }\n" + " sub PrintVerbose { }\n" + " sub StageStart { return $_[0] }\n" + " sub StageEnd { }\n" + " $INC{'TAF/Logging.pm'} = __FILE__;\n" + "}\n" + "\n" + f"use lib '{TAF_ROOT}/libs/database_libs';\n" + f"use lib '{TAF_ROOT}/libs/taf_libs';\n" + "\n" + "require 'postgres.pm';\n" + "\n" + "my $pg_install = shift @ARGV or die \"Missing pg_install\\n\";\n" + "my $tmpdir = '/tmp';\n" + "my $pass = 0; my $fail = 0;\n" + "\n" + "sub ok {\n" + " my ($cond, $name) = @_;\n" + " if ($cond) { print \"PASS: $name\\n\"; $pass++ }\n" + " else { print \"FAIL: $name\\n\"; $fail++ }\n" + "}\n" + "\n" + "# Test 1: new() with invalid install_root → undef\n" + "{\n" + " my $pg = postgres->new(\n" + " db_software_install_dir => '/nonexistent_xyz',\n" + " db_data_dir => '/tmp/pg_unit_data',\n" + " tmp_dir => $tmpdir,\n" + " );\n" + " ok(!defined $pg, \"new() rejects invalid install_root\");\n" + "}\n" + "\n" + "# Test 2: new() with valid installation → object\n" + "{\n" + " my $pg = postgres->new(\n" + " db_software_install_dir => $pg_install,\n" + " db_data_dir => '/tmp/pg_unit_data',\n" + " tmp_dir => $tmpdir,\n" + f" db_port => {port},\n" + f" db_user => '{user}',\n" + f" db_user_pass => '{password}',\n" + f" db_root_user => '{root}',\n" + f" db_root_pass => '{root_pass}',\n" + " );\n" + " ok(defined $pg, \"new() returns object with valid installation\");\n" + "\n" + " if (defined $pg) {\n" + " for my $b (qw(postgres_bin pg_ctl_bin psql_bin initdb_bin pg_isready_bin)) {\n" + " ok(defined $pg->{$b} && -x $pg->{$b},\n" + " \"Binary $b found: \" . ($pg->{$b} // ''));\n" + " }\n" + f" ok($pg->{{port}} == {port}, \"Port nastaven na {port}\");\n" + f" ok($pg->{{db_user}} eq '{user}', \"db_user stored correctly\");\n" + f" ok($pg->{{db_root_user}} eq '{root}', \"db_root_user stored correctly\");\n" + " }\n" + "}\n" + "\n" + "print \"\\nResult: $pass passed, $fail failed\\n\";\n" + "exit($fail > 0 ? 1 : 0);\n" + ) + + +class TestL2PluginUnit: + """Unit tests for the postgres.pm plugin via Perl subprocess.""" + + @needs_pg + def test_plugin_unit_all_pass(self, tmp_path): + """All unit tests for postgres.pm must pass.""" + script = _build_plugin_unit_script( + pg_install=PG_INSTALL, + port=PG_PORT, + user=TAF_PG_USER, + password=TAF_PG_PASS, + root=TAF_PG_ROOT, + root_pass=TAF_PG_ROOT_PASS, + ) + + script_file = tmp_path / "pg_plugin_unit.pl" + script_file.write_text(script) + + result = run( + "perl", str(script_file), str(PG_INSTALL), + cwd=TAF_ROOT, timeout=30, + ) + output = result.stdout + result.stderr + + # Extract result + m = re.search(r"Result:\s*(\d+) passed,\s*(\d+) failed", output) + if m: + passed, failed = int(m.group(1)), int(m.group(2)) + else: + passed, failed = 0, 1 + + fails = [ln for ln in output.splitlines() if ln.startswith("FAIL:")] + assert failed == 0, ( + f"Plugin unit tests failed ({failed} failures):\n" + + "\n".join(fails) + + f"\n\nFull output:\n{output}" + ) + + +# =========================================================================== +# LAYER 3 — Database lifecycle +# =========================================================================== + +class TestL3Lifecycle: + """Tests the full TAF PostgreSQL lifecycle: init → start → stop.""" + + @needs_pg + def test_taf_exits_cleanly(self, lifecycle_result): + """TAF init-start-db-exit must finish with exit code 0.""" + result, _ = lifecycle_result + assert result.returncode == 0, ( + f"TAF finished with rc={result.returncode}\n" + f"STDOUT:\n{result.stdout[-3000:]}\n" + f"STDERR:\n{result.stderr[-1000:]}" + ) + + @needs_pg + def test_no_errors_in_taf_output(self, lifecycle_result): + """TAF output must not contain ERROR lines.""" + result, _ = lifecycle_result + output = result.stdout + result.stderr + clean, bad_line = has_no_errors(output) + assert clean, f"ERROR line found in TAF output:\n {bad_line}" + + @needs_pg + def test_initdb_stage_logged(self, lifecycle_result): + """TAF must log PostgreSQL cluster initialization.""" + result, _ = lifecycle_result + output = result.stdout + result.stderr + assert any( + keyword in output.lower() + for keyword in ("initdb", "initialize", "init database") + ), "Missing initdb stage record in TAF output" + + @needs_pg + def test_start_stage_logged(self, lifecycle_result): + """TAF must log PostgreSQL server start.""" + result, _ = lifecycle_result + output = result.stdout + result.stderr + assert any( + keyword in output.lower() + for keyword in ("database start", "pg_ctl start", "server started", + "postgresql.*start", "start.*postgresql") + ), "Missing start stage record in TAF output" + + @needs_pg + def test_stop_stage_logged(self, lifecycle_result): + """TAF must log PostgreSQL server stop.""" + result, _ = lifecycle_result + output = result.stdout + result.stderr + assert any( + keyword in output.lower() + for keyword in ("database stop", "pg_ctl stop", "server stopped") + ), "Missing stop stage record in TAF output" + + @needs_pg + def test_pg_hba_conf_has_tcp_md5_rules(self, lifecycle_result): + """pg_hba.conf must contain md5 rules for TCP access.""" + result, data_dir = lifecycle_result + if data_dir is None: + pytest.skip("Cannot determine data_dir from TAF output") + hba = data_dir / "pg_hba.conf" + assert hba.is_file(), f"pg_hba.conf not found in {data_dir}" + content = hba.read_text() + assert "127.0.0.1" in content, "pg_hba.conf does not contain a rule for 127.0.0.1" + assert "md5" in content, "pg_hba.conf does not use md5 authentication" + + @needs_pg + def test_postgresql_conf_has_correct_port(self, lifecycle_result): + """postgresql.conf must contain the TAF-configured port.""" + result, data_dir = lifecycle_result + if data_dir is None: + pytest.skip("Cannot determine data_dir from TAF output") + conf = data_dir / "postgresql.conf" + assert conf.is_file(), f"postgresql.conf not found in {data_dir}" + content = conf.read_text() + assert f"port = {PG_PORT}" in content, \ + f"postgresql.conf does not contain 'port = {PG_PORT}'" + + @needs_pg + def test_server_running_after_start_exit(self, lifecycle_result): + """After init-start-db-exit, PostgreSQL server must be listening on the port. + + The init-start-db-exit action intentionally leaves the server running + (exit = TAF exit, not db stop). The cleanup fixture stops the server + at the end of the session. + """ + result, _ = lifecycle_result + # Test is only relevant when lifecycle succeeded + if result.returncode != 0: + pytest.skip("Lifecycle failed — skipping server-running check") + pg_isready = PG_BIN / "pg_isready" + if not pg_isready.is_file(): + pytest.skip("pg_isready not found") + check = run( + str(pg_isready), "-h", "127.0.0.1", "-p", str(PG_PORT), "-q", + ) + assert check.returncode == 0, \ + f"PostgreSQL is not listening on port {PG_PORT} after init-start-db-exit" + + +# =========================================================================== +# LAYER 4 — Sysbench build with pgsql driver +# =========================================================================== + +class TestL4SysbenchBuild: + """Verifies that sysbench builds with the pgsql driver. + + Note: Sysbench uses autotools (autogen.sh + configure + make), not cmake. + TAF build-client assumes cmake — so L4 tests the autotools build directly. + The resulting binary is accessible via symlink client_source/sysbench-lua/sysbench → src/sysbench. + """ + + SYSBENCH_SRC = TAF_ROOT / "client_source" / "sysbench-lua" + SYSBENCH_BIN = SYSBENCH_SRC / "sysbench" # symlink na src/sysbench + + @needs_pg + def test_sysbench_build_succeeds(self, tmp_path_factory): + """Sysbench build with pgsql driver (autotools: autogen+configure+make).""" + src = self.SYSBENCH_SRC + if not (src / "configure.ac").is_file(): + pytest.skip(f"Sysbench source not found: {src} — clone repo into client_source/sysbench-lua/") + + # autogen.sh + result = run("bash", "autogen.sh", cwd=src, timeout=60) + assert result.returncode == 0, f"autogen.sh failed:\n{result.stdout}\n{result.stderr}" + + # configure --with-pgsql --without-mysql + result = run("bash", "configure", "--without-mysql", "--with-pgsql", + cwd=src, timeout=120) + assert result.returncode == 0, ( + f"configure failed (rc={result.returncode}):\n" + f"{result.stdout[-2000:]}\n{result.stderr[-500:]}" + ) + + # make + import multiprocessing + nproc = str(multiprocessing.cpu_count()) + result = run("make", f"-j{nproc}", cwd=src, timeout=300) + assert result.returncode == 0, ( + f"make failed (rc={result.returncode}):\n" + f"{result.stdout[-2000:]}\n{result.stderr[-500:]}" + ) + + # Ensure symlink sysbench → src/sysbench exists for TAF + symlink = src / "sysbench" + real_bin = src / "src" / "sysbench" + if real_bin.is_file() and not symlink.exists(): + import os as _os + _os.symlink("src/sysbench", str(symlink)) + + @needs_pg + def test_sysbench_binary_exists_after_build(self): + """Sysbench binary must exist after the build.""" + sysbench_bin = TAF_ROOT / "client_source" / "sysbench-lua" / "sysbench" + assert sysbench_bin.is_file(), \ + f"Sysbench binary not found: {sysbench_bin}" + assert os.access(str(sysbench_bin), os.X_OK), \ + f"Sysbench binary is not executable: {sysbench_bin}" + + @needs_pg + def test_sysbench_pgsql_driver_available(self): + """Sysbench must support --db-driver=pgsql.""" + sysbench_bin = TAF_ROOT / "client_source" / "sysbench-lua" / "sysbench" + if not sysbench_bin.is_file(): + pytest.skip("Sysbench binary not found — run L4 build test first") + + result = run( + str(sysbench_bin), "--db-driver=pgsql", "--help", + timeout=15, + ) + output = result.stdout + result.stderr + # sysbench with pgsql driver should display --pgsql-* options + has_pgsql = any( + keyword in output.lower() + for keyword in ("pgsql", "postgres", "--pgsql-host") + ) + assert has_pgsql, ( + "Sysbench does not support pgsql driver — check cmake build with " + "-DWITH_PGSQL=on and availability of libpq-devel\n" + f"Output: {output[:500]}" + ) + + +# =========================================================================== +# LAYER 5 — Full benchmark run +# =========================================================================== + +class TestL5Benchmark: + """End-to-end test: TAF init → build → benchmark → stop.""" + + @needs_pg + def test_benchmark_exits_cleanly(self, benchmark_result): + """Benchmark run must finish with exit code 0.""" + result = benchmark_result + assert result.returncode == 0, ( + f"Benchmark TAF run failed (rc={result.returncode})\n" + f"STDOUT:\n{result.stdout[-5000:]}\n" + f"STDERR:\n{result.stderr[-1000:]}" + ) + + @needs_pg + def test_benchmark_output_has_no_errors(self, benchmark_result): + """Benchmark output must not contain ERROR lines.""" + result = benchmark_result + output = result.stdout + result.stderr + clean, bad_line = has_no_errors(output) + assert clean, f"ERROR found in benchmark output:\n {bad_line}" + + @needs_pg + def test_benchmark_uses_pgsql_connection_args(self, benchmark_result): + """TAF benchmark must call sysbench with --pgsql-host (not --mysql-host).""" + output = benchmark_result.stdout + benchmark_result.stderr + # Verbose TAF output includes the sysbench command line + assert "--pgsql-host" in output or "--pgsql-port" in output, ( + "sysbench was not run with --pgsql-* arguments. " + "Check SetConnectionArgs() in sysbench-lua.pm\n" + f"Searched in {len(output)} characters of output" + ) + assert "--mysql-host" not in output, \ + "sysbench was run with --mysql-host instead of --pgsql-host!" + + @needs_pg + def test_results_directory_structure(self, benchmark_result): + """TAF must produce results for OLTP_RO or POINT_SELECT. + + TAF archives results to archive/ after the run completes — we search + both in results/ (if TAF does not archive) and in archive/. + """ + found_tests = set() + for search_root in (TAF_ROOT / "results", TAF_ROOT / "archive"): + if not search_root.is_dir(): + continue + for entry in search_root.iterdir(): + if not entry.is_dir(): + continue + name = entry.name + for test_name in ("OLTP_RO", "POINT_SELECT"): + if test_name in name and not name.startswith("Error_"): + found_tests.add(test_name) + + archive_entries = list((TAF_ROOT / "archive").iterdir()) if (TAF_ROOT / "archive").is_dir() else [] + assert len(found_tests) >= 1, ( + f"No results for OLTP_RO or POINT_SELECT in results/ or archive/. " + f"archive/ contents: {[e.name for e in archive_entries]}" + ) + + @needs_pg + def test_benchmark_result_files_contain_metrics(self, benchmark_result): + """Result files must contain sysbench metrics (transactions). + + TAF archives results to archive/ — we search both locations. + """ + search_roots = [TAF_ROOT / "results", TAF_ROOT / "archive"] + + metrics_found = False + for search_root in search_roots: + if not search_root.is_dir(): + continue + result_files = list(search_root.rglob("*.log")) + \ + list(search_root.rglob("*.txt")) + for f in result_files: + content = f.read_text(errors="replace") + if "transactions" in content.lower() or \ + "queries/sec" in content.lower() or \ + "events/sec" in content.lower(): + metrics_found = True + break + if metrics_found: + break + + assert metrics_found, \ + "No result file contains sysbench metrics (transactions/queries)" + + @needs_pg + def test_no_sysbench_fatal_errors(self, benchmark_result): + """Result files must not contain sysbench FATAL errors.""" + fatal_lines = [] + for search_root in (TAF_ROOT / "results", TAF_ROOT / "archive"): + if not search_root.is_dir(): + continue + for f in search_root.rglob("*.log"): + # Skip logs from known-failed archive runs + if "Error_" in str(f): + continue + content = f.read_text(errors="replace") + for line in content.splitlines(): + if re.search(r"\bFATAL\b", line, re.IGNORECASE): + fatal_lines.append(f"{f.name}: {line}") + + assert not fatal_lines, \ + f"FATAL errors found in sysbench output:\n" + "\n".join(fatal_lines[:5]) + + +# =========================================================================== +# LAYER 6 — MariaDB regression test +# =========================================================================== + +class TestL6MariaDBRegression: + """Verifies that changes to sysbench-lua.pm have not broken MySQL/MariaDB behaviour.""" + + @needs_mariadb + def test_mariadb_run_exits_cleanly(self, tmp_path_factory): + """A short MariaDB TAF run must finish with exit code 0.""" + mariadb_dir = Path(MARIADB_DIR) + tmp = tmp_path_factory.mktemp("taf_l6") + props_file = tmp / "mariadb_regression.properties" + + write_props(props_file, { + "taf.action": "start-db-run-tests", + "taf.taf_db_makers_plugin": "mariadb", + "taf.db_software_install_dir": str(mariadb_dir), + "taf.db_port": "3306", + "taf.test_suite": "sysbench-lua", + "taf.tests": "OLTP_RO", + "taf.verbose": "true", + "sysbench_lua.db_driver": "mysql", + "sysbench_lua.def_threads": "4", + "sysbench_lua.def_duration": "30", + "sysbench_lua.number_of_rows": "10000", + }) + + result = taf_propfile(props_file, timeout=300) + assert result.returncode == 0, ( + f"MariaDB regression run failed (rc={result.returncode})\n" + f"STDOUT:\n{result.stdout[-3000:]}" + ) + + @needs_mariadb + def test_mysql_args_used_for_mariadb(self, tmp_path_factory): + """MariaDB run must use --mysql-host (not --pgsql-host).""" + mariadb_dir = Path(MARIADB_DIR) + tmp = tmp_path_factory.mktemp("taf_l6_args") + props_file = tmp / "mariadb_args.properties" + + write_props(props_file, { + "taf.action": "start-db-run-tests", + "taf.taf_db_makers_plugin": "mariadb", + "taf.db_software_install_dir": str(mariadb_dir), + "taf.db_port": "3306", + "taf.test_suite": "sysbench-lua", + "taf.tests": "OLTP_RO", + "taf.verbose": "true", + "sysbench_lua.db_driver": "mysql", + "sysbench_lua.def_threads": "4", + "sysbench_lua.def_duration": "30", + "sysbench_lua.number_of_rows": "10000", + }) + + result = taf_propfile(props_file, timeout=300) + output = result.stdout + result.stderr + + assert "--mysql-host" in output or "--mysql-port" in output, \ + "MariaDB run does not use --mysql-* arguments — SetConnectionArgs() may be broken" + assert "--pgsql-host" not in output, \ + "MariaDB run incorrectly uses --pgsql-host!" + + def test_normalize_db_type_mysql_unchanged(self): + """NormalizeDBType must still map mariadb → mysql (and postgres → pgsql). + + Strategy: we extract the body of sub NormalizeDBType directly from the .pm file + using a regex and embed it into an isolated Perl script — this avoids + sysbench-lua.pm's dependencies on the TAF runtime. + """ + suite_file = TAF_ROOT / "test_suites" / "sysbench-lua.pm" + content = suite_file.read_text() + + m = re.search(r"(sub NormalizeDBType \{.+?\n\})", content, re.DOTALL) + assert m, "NormalizeDBType not found in sysbench-lua.pm — cannot run L6 test" + func_body = m.group(1) + + perl_code = textwrap.dedent("""\ + use strict; + use warnings; + + {func} + + my %tests = ( + mariadb => "mysql", + maria => "mysql", + mysql => "mysql", + mysqld => "mysql", + postgres => "pgsql", + postgresql => "pgsql", + pgsql => "pgsql", + ); + + my $fail = 0; + for my $input (sort keys %tests) {{ + my $expected = $tests{{$input}}; + my $got = NormalizeDBType($input) // ""; + if ($got eq $expected) {{ + print "PASS: NormalizeDBType('$input') = '$got'\\n"; + }} else {{ + print "FAIL: NormalizeDBType('$input') = '$got', expected '$expected'\\n"; + $fail++; + }} + }} + exit $fail; + """).format(func=func_body) + + result = run("perl", "-e", perl_code, cwd=TAF_ROOT, timeout=15) + output = result.stdout + result.stderr + fails = [ln for ln in output.splitlines() if ln.startswith("FAIL:")] + assert result.returncode == 0, ( + f"NormalizeDBType returns incorrect values:\n" + + "\n".join(fails) + + f"\nFull output:\n{output}" + ) From c6134e3b8ad2e1a3501a9b7f02c94c921b33fdd2 Mon Sep 17 00:00:00 2001 From: Lukas Oliva Date: Tue, 14 Jul 2026 11:49:53 +0200 Subject: [PATCH 02/27] Fix PostgreSQL setup script for EL10 AppStream and sysbench build - Run.pm: fix Executor->import(':all') to sql_libs::Executor->import(':all') (bare Executor wasn't a resolvable package name). - setup_almalinux10.sh (appstream method): install postgresql-server + libpq-devel with --allowerasing instead of postgresql-server-devel, which conflicts with libpq-devel via postgresql-private-devel on EL10. Fall back to hardcoded AppStream include/lib paths when pg_config (shipped by postgresql-server-devel) isn't available. - sysbench build: force a fresh --recurse-submodules clone whenever the LuaJIT submodule Makefile is missing (a zip-extracted source tree has no .git, so submodules are silently empty); invoke autogen.sh/configure via `bash` explicitly since files from a zip may lack the executable bit. Co-Authored-By: Claude Sonnet 5 --- libs/taf_libs/TAF/Run.pm | 2 +- tests/setup_almalinux10.sh | 42 ++++++++++++++++++++++++++++---------- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/libs/taf_libs/TAF/Run.pm b/libs/taf_libs/TAF/Run.pm index 4dad428..ee81d4a 100644 --- a/libs/taf_libs/TAF/Run.pm +++ b/libs/taf_libs/TAF/Run.pm @@ -134,7 +134,7 @@ use strict; use warnings; use List::Util qw(max all); use sql_libs::Executor; -Executor->import(':all'); +sql_libs::Executor->import(':all'); use profile_libs::Runner; diff --git a/tests/setup_almalinux10.sh b/tests/setup_almalinux10.sh index 6861493..643eff3 100755 --- a/tests/setup_almalinux10.sh +++ b/tests/setup_almalinux10.sh @@ -176,7 +176,12 @@ install_pgdg_rpm() { # ─── 2c. APPSTREAM ───────────────────────────────────────────────────────── install_appstream() { - dnf install -y postgresql-server postgresql-devel + # On EL10, postgresql-server-devel conflicts with libpq-devel via + # postgresql-private-devel. Install postgresql-server + libpq-devel + # (provides libpq-fe.h for sysbench). pg_config comes from + # postgresql-server-devel; we fall back to hardcoded paths without it. + # --allowerasing removes postgresql-private-devel if already installed. + dnf install -y --allowerasing postgresql postgresql-server libpq-devel PG_INSTALL_DIR="/usr" warn "PostgreSQL installed from AppStream into ${PG_INSTALL_DIR}" } @@ -193,7 +198,7 @@ esac step "Verifying PostgreSQL binaries" PG_BIN="${PG_INSTALL_DIR}/bin" MISSING=0 -for BIN in postgres pg_ctl psql initdb pg_isready pg_config; do +for BIN in postgres pg_ctl psql initdb pg_isready; do if [[ -x "${PG_BIN}/${BIN}" ]]; then info " ✓ ${PG_BIN}/${BIN}" else @@ -203,9 +208,19 @@ for BIN in postgres pg_ctl psql initdb pg_isready pg_config; do done [[ $MISSING -eq 0 ]] || error "Missing binaries — check installation in ${PG_INSTALL_DIR}" -info "PostgreSQL version: $("${PG_BIN}/pg_config" --version)" -LIBPQ_INCDIR=$("${PG_BIN}/pg_config" --includedir) -LIBPQ_LIBDIR=$("${PG_BIN}/pg_config" --libdir) +# pg_config is provided by postgresql-server-devel, which conflicts with +# libpq-devel on EL10 AppStream. It is only needed to locate libpq headers +# for sysbench; fall back to hardcoded AppStream paths when unavailable. +if [[ -x "${PG_BIN}/pg_config" ]]; then + info "PostgreSQL version: $("${PG_BIN}/pg_config" --version)" + LIBPQ_INCDIR=$("${PG_BIN}/pg_config" --includedir) + LIBPQ_LIBDIR=$("${PG_BIN}/pg_config" --libdir) +else + warn "pg_config not found (postgresql-server-devel not installed); using AppStream defaults" + info "PostgreSQL version: $("${PG_BIN}/postgres" --version 2>/dev/null || echo unknown)" + LIBPQ_INCDIR="/usr/include" + LIBPQ_LIBDIR="/usr/lib64" +fi info "includedir: ${LIBPQ_INCDIR}" info "libdir: ${LIBPQ_LIBDIR}" @@ -217,7 +232,7 @@ if [[ -f "${LIBPQ_INCDIR}/libpq-fe.h" ]]; then info "libpq-fe.h found: ${LIBPQ_INCDIR}/libpq-fe.h" else warn "libpq-fe.h not found, trying system packages..." - dnf install -y postgresql-devel libpq-devel 2>/dev/null || \ + dnf install -y --allowerasing postgresql-devel libpq-devel 2>/dev/null || \ dnf install -y postgresql16-devel 2>/dev/null || \ warn "libpq-devel unavailable — sysbench build may fail" fi @@ -290,17 +305,22 @@ elif [[ -x "${SYSBENCH_SRC}/sysbench" ]] && [[ ! -L "${SYSBENCH_SRC}/sysbench" ] warn "Rebuilding from source for correct architecture and libpq..." fi if [[ $SYSBENCH_OK -eq 0 ]]; then - if [[ ! -f "${SYSBENCH_SRC}/configure.ac" ]]; then - info "Cloning sysbench..." + # Submodule check: LuaJIT Makefile is only present after a proper git clone + # with --recurse-submodules. Zip-extracted source lacks .git/ so submodules + # are empty. Force a fresh clone whenever the submodule is missing. + if [[ ! -f "${SYSBENCH_SRC}/third_party/luajit/luajit/Makefile" ]]; then + info "Cloning sysbench (submodules missing or incomplete)..." + rm -rf "${SYSBENCH_SRC}" mkdir -p "${TAF_DIR}/client_source" - git clone --depth=1 https://github.com/akopytov/sysbench "${SYSBENCH_SRC}" + git clone --depth=1 --recurse-submodules https://github.com/akopytov/sysbench "${SYSBENCH_SRC}" fi info "Building sysbench with pgsql support..." cd "${SYSBENCH_SRC}" - ./autogen.sh - ./configure --without-mysql --with-pgsql \ + # Use bash explicitly — files from a zip archive may lack execute bits. + bash autogen.sh + bash configure --without-mysql --with-pgsql \ --with-pgsql-includes="${LIBPQ_INCDIR}" \ --with-pgsql-libs="${LIBPQ_LIBDIR}" make -j"$(nproc)" From a88203e298e7f236ca28c85e839eb71e5039030d Mon Sep 17 00:00:00 2001 From: Lukas Oliva Date: Wed, 15 Jul 2026 07:59:16 +0200 Subject: [PATCH 03/27] setup_almalinux10.sh: upgrade to PostgreSQL 18.4, fix missing postgres OS user Root cause of a campaign-wide failure: --method=percona (tarball) never created the "postgres" OS user, unlike the appstream/pgdg RPM packages (their %pre scriptlet does it automatically). postgres.pm's constructor requires an OS user literally named "postgres" to drop root privileges before running initdb -- PostgreSQL refuses to run as root -- so every initdb failed with "cannot be run as root". Now creates the group+user (system account, home /var/lib/pgsql) unconditionally before any install method runs; a no-op if the RPM path already created it. Also: - EXPECTED_PG_VERSION="18.4" pinned at the top; new step 3b hard-fails setup if the installed `postgres --version` doesn't match exactly, across all three --method installers, instead of silently running benchmarks against an unintended version. - install_percona_tarball(): VERSION 16.14 -> 18.4; corrected OpenSSL-tag detection for PG18's 3-way tarball split (ssl1.1/ssl3/ssl3.5 by major.minor, not PG16's 2-way ssl1/ssl3 by major only) -- these guests run OpenSSL 3.5.x, which needs ssl3.5 specifically. - install_pgdg_rpm()/install_appstream(): postgresql16-* -> postgresql18-* package names and PG_INSTALL_DIR paths. - PERCONA_LOCAL_ARCHIVE: skip the downloads.percona.com fetch when taf_manage.py has already SCP'd a tarball here (--PERCONA_ARCHIVE_LOCAL), avoiding N guests hitting Percona's download server at once. Verified live: postgres (PostgreSQL) 18.4 - Percona Server for PostgreSQL 18.4.1, full sysbench build + benchmark run succeeded end-to-end. --- tests/setup_almalinux10.sh | 151 +++++++++++++++++++++++++++++-------- 1 file changed, 121 insertions(+), 30 deletions(-) diff --git a/tests/setup_almalinux10.sh b/tests/setup_almalinux10.sh index 643eff3..b2230e7 100755 --- a/tests/setup_almalinux10.sh +++ b/tests/setup_almalinux10.sh @@ -7,20 +7,34 @@ # # PostgreSQL installation methods (--method): # percona (default) — tarball from downloads.percona.com -# https://docs.percona.com/postgresql/16/tarball.html +# https://docs.percona.com/postgresql/18/tarball.html # pgdg — RPM from pgdg.postgresql.org -# appstream — system postgresql from dnf +# appstream — system postgresql from dnf (AlmaLinux 10 AppStream ships a +# versioned postgresql18 package alongside the unversioned +# postgresql (16) one; --allowerasing swaps 16 out for 18) +# +# EXPECTED_PG_VERSION (below) is pinned to 18.4 -- the current stable +# PostgreSQL release as of 2026-07 (https://www.postgresql.org/docs/release/18.4/, +# released 2026-05-14). Update it here when a newer release ships. All three +# methods are verified against this after install; a mismatch is a hard error +# (see step 3b) rather than a silent partial upgrade. # # After completion: -# - PostgreSQL 16 available in $PG_INSTALL_DIR +# - PostgreSQL 18.4 available in $PG_INSTALL_DIR # - Python 3 + pytest installed # - Build tools for sysbench ready # - Env saved to .taf_pg_env (source before tests) # # Env variables (can be overridden before pytest): -# TAF_PG_INSTALL_DIR (set automatically) -# TAF_PG_PORT (default: 5433) -# TAF_MARIADB_DIR (optional, for L6 regression test) +# TAF_PG_INSTALL_DIR (set automatically) +# TAF_PG_PORT (default: 5433) +# TAF_MARIADB_DIR (optional, for L6 regression test) +# PERCONA_LOCAL_ARCHIVE (--method=percona only) path to an already-downloaded +# percona-postgresql-*.tar.gz; skips the +# downloads.percona.com fetch. Set by taf_manage.py +# --PERCONA_ARCHIVE_LOCAL, which SCPs it here once from +# the control machine instead of every guest fetching +# it independently. # ============================================================================= set -euo pipefail @@ -35,6 +49,16 @@ step() { echo -e "\n${CYAN}━━━ $* ━━━${NC}"; } [[ $EUID -eq 0 ]] || error "Script must be run as root (sudo bash $0)" +# --------------------------------------------------------------------------- +# Expected PostgreSQL version (pre-test gate, step 3b) +# --------------------------------------------------------------------------- +# Pinned to the current stable PostgreSQL release. Verified 2026-07 against +# https://www.postgresql.org/docs/release/18.4/ (released 2026-05-14). +# Update this when a newer release ships -- installs of any --method that +# don't produce this exact version fail hard rather than silently running +# benchmarks against an unintended/outdated PostgreSQL version. +EXPECTED_PG_VERSION="18.4" + # --------------------------------------------------------------------------- # Parametry # --------------------------------------------------------------------------- @@ -75,30 +99,67 @@ dnf install -y \ openssl openssl-devel \ acl +# --------------------------------------------------------------------------- +# 1b. "postgres" OS user/group +# --------------------------------------------------------------------------- +# The appstream/pgdg RPM packages create this via their own %pre scriptlet, +# but --method=percona just extracts a tarball -- nothing ever creates it. +# postgres.pm's new() constructor requires an OS user literally named +# "postgres" to drop root privileges before running initdb (PostgreSQL +# refuses `initdb`/`postgres` as root unconditionally); without it, initdb +# runs as root and fails with "initdb: error: cannot be run as root" on +# every single host. Mirrors the standard RHEL/Fedora postgresql-server RPM +# %pre scriptlet (group+user, system account, home /var/lib/pgsql) so the +# result is identical regardless of --method, and running this unconditionally +# for all three methods is a no-op if the RPM path already created it. +step "Ensuring OS user/group 'postgres' exists" +getent group postgres >/dev/null || groupadd -r postgres +if getent passwd postgres >/dev/null; then + info "OS user 'postgres' already exists" +else + mkdir -p /var/lib/pgsql + useradd -r -g postgres -d /var/lib/pgsql -s /bin/bash -c "PostgreSQL Server" postgres + chown postgres:postgres /var/lib/pgsql + info "OS user 'postgres' created (system account, home /var/lib/pgsql)" +fi + # --------------------------------------------------------------------------- # 2. POSTGRESQL — according to chosen method # --------------------------------------------------------------------------- -step "Installing PostgreSQL 16 (method: ${METHOD})" +step "Installing PostgreSQL ${EXPECTED_PG_VERSION} (method: ${METHOD})" PG_INSTALL_DIR="" LIBPQ_INCDIR="" LIBPQ_LIBDIR="" # ─── 2a. PERCONA TARBALL (default) ───────────────────────────────────────── -# Dokumentace: https://docs.percona.com/postgresql/16/tarball.html +# Dokumentace: https://docs.percona.com/postgresql/18/tarball.html install_percona_tarball() { - local VERSION="16.14" + local VERSION="18.4" local INSTALL_BASE="/opt/pgdistro" - local PG_SUBDIR="percona-postgresql16" - - # Detect OpenSSL version → choose tarball variant + local PG_SUBDIR="percona-postgresql18" + + # Detect OpenSSL version → choose tarball variant. + # PG18 tarballs ship three variants (unlike PG16's two: ssl1/ssl3) -- + # ssl1.1, ssl3 (OpenSSL 3.0-3.4), and ssl3.5 (OpenSSL 3.5+). AlmaLinux 10 + # ships OpenSSL 3.5.x, so the major-version-only check used for PG16 + # would silently grab the wrong (but still installable) ssl3 build here; + # compare major.minor instead. local OPENSSL_VER OPENSSL_VER=$(openssl version | awk '{print $2}') local SSL_TAG - case "${OPENSSL_VER%%.*}" in - 1) SSL_TAG="ssl1" ;; - 3) SSL_TAG="ssl3" ;; - *) SSL_TAG="ssl3"; warn "Unknown OpenSSL version ${OPENSSL_VER}, trying ssl3" ;; + local ssl_major="${OPENSSL_VER%%.*}" + local ssl_minor="${OPENSSL_VER#*.}"; ssl_minor="${ssl_minor%%.*}" + case "$ssl_major" in + 1) SSL_TAG="ssl1.1" ;; + 3) + if [[ "$ssl_minor" -ge 5 ]]; then + SSL_TAG="ssl3.5" + else + SSL_TAG="ssl3" + fi + ;; + *) SSL_TAG="ssl3.5"; warn "Unknown OpenSSL version ${OPENSSL_VER}, trying ssl3.5" ;; esac # Map architecture to tarball name @@ -110,14 +171,14 @@ install_percona_tarball() { esac local TARBALL="percona-postgresql-${VERSION}-${SSL_TAG}-${TARARCH}.tar.gz" - local URL="https://downloads.percona.com/downloads/postgresql-distribution-16/${VERSION}/binary/tarball/${TARBALL}" + local URL="https://downloads.percona.com/downloads/postgresql-distribution-18/${VERSION}/binary/tarball/${TARBALL}" info "Tarball: ${TARBALL}" info "URL: ${URL}" # Skip if binary already exists if [[ -x "${INSTALL_BASE}/${PG_SUBDIR}/bin/postgres" ]]; then - info "Percona PostgreSQL 16 already installed in ${INSTALL_BASE}/${PG_SUBDIR}" + info "Percona PostgreSQL 18 already installed in ${INSTALL_BASE}/${PG_SUBDIR}" PG_INSTALL_DIR="${INSTALL_BASE}/${PG_SUBDIR}" return 0 fi @@ -125,7 +186,17 @@ install_percona_tarball() { mkdir -p "${INSTALL_BASE}" local TMPTAR="/tmp/${TARBALL}" - if [[ ! -f "$TMPTAR" ]]; then + # Fetch-once-distribute: with N guests all running setup at once, N + # independent downloads hammer downloads.percona.com. If the orchestrator + # already staged a copy here (taf_manage.py --PERCONA_ARCHIVE_LOCAL, SCP'd + # to REMOTE_WORKDIR before setup), use it instead of downloading. + if [[ -n "${PERCONA_LOCAL_ARCHIVE:-}" ]] && [[ -f "$PERCONA_LOCAL_ARCHIVE" ]]; then + info "Using pre-staged tarball: ${PERCONA_LOCAL_ARCHIVE}" + local abs_src abs_dst + abs_src=$(readlink -f "$PERCONA_LOCAL_ARCHIVE") + abs_dst=$(readlink -f "$TMPTAR" 2>/dev/null || echo "") + [[ "$abs_src" != "$abs_dst" ]] && cp "$PERCONA_LOCAL_ARCHIVE" "$TMPTAR" + elif [[ ! -f "$TMPTAR" ]]; then info "Downloading tarball..." wget -q --show-progress -O "$TMPTAR" "$URL" 2>/dev/null || \ wget -O "$TMPTAR" "$URL" || \ @@ -151,12 +222,12 @@ install_percona_tarball() { local PG_LIB="${PG_INSTALL_DIR}/lib" if [[ -d "$PG_LIB" ]]; then export LD_LIBRARY_PATH="${PG_LIB}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - cat > /etc/profile.d/percona-pg16.sh < /etc/profile.d/percona-pg18.sh </dev/null; then info "PGDG repository added" dnf -qy module disable postgresql 2>/dev/null || true - dnf install -y postgresql16-server postgresql16-devel postgresql16 - PG_INSTALL_DIR="/usr/pgsql-16" + dnf install -y postgresql18-server postgresql18-devel postgresql18 + PG_INSTALL_DIR="/usr/pgsql-18" else warn "PGDG EL10 repo unavailable, falling back to AppStream" install_appstream @@ -176,14 +247,17 @@ install_pgdg_rpm() { # ─── 2c. APPSTREAM ───────────────────────────────────────────────────────── install_appstream() { - # On EL10, postgresql-server-devel conflicts with libpq-devel via - # postgresql-private-devel. Install postgresql-server + libpq-devel - # (provides libpq-fe.h for sysbench). pg_config comes from - # postgresql-server-devel; we fall back to hardcoded paths without it. - # --allowerasing removes postgresql-private-devel if already installed. - dnf install -y --allowerasing postgresql postgresql-server libpq-devel + # AlmaLinux 10 AppStream ships both the unversioned "postgresql" (16, the + # default stream) and a versioned "postgresql18" package set side by + # side; they conflict at the file level (postgresql-any / postgresql- + # server-any virtual provides), so --allowerasing is required to swap 16 + # out for 18 on a base image that already has 16 installed. + # libpq-devel is version-independent (provides libpq-fe.h for sysbench) + # and does not conflict with postgresql18-*. + dnf install -y --allowerasing postgresql18 postgresql18-server libpq-devel PG_INSTALL_DIR="/usr" warn "PostgreSQL installed from AppStream into ${PG_INSTALL_DIR}" + warn "AppStream may lag behind the latest point release (EXPECTED_PG_VERSION=${EXPECTED_PG_VERSION:-18.4}) -- the version check in step 3b will fail loudly if so; use --method=percona for a guaranteed exact match." } case "$METHOD" in @@ -224,6 +298,23 @@ fi info "includedir: ${LIBPQ_INCDIR}" info "libdir: ${LIBPQ_LIBDIR}" +# --------------------------------------------------------------------------- +# 3b. VERIFY POSTGRESQL VERSION MATCHES EXPECTED CURRENT STABLE RELEASE +# --------------------------------------------------------------------------- +# Any installed version other than EXPECTED_PG_VERSION is a hard error -- +# better to fail loudly here than to silently benchmark an unintended +# PostgreSQL version (e.g. AppStream lagging a point release behind, or a +# stale cached tarball/RPM repo). +step "Verifying PostgreSQL version" +ACTUAL_PG_VERSION=$("${PG_BIN}/postgres" --version | grep -oE '[0-9]+\.[0-9]+(\.[0-9]+)?' | head -1) +if [[ -z "$ACTUAL_PG_VERSION" ]]; then + error "Could not determine installed PostgreSQL version from '${PG_BIN}/postgres --version'" +fi +if [[ "$ACTUAL_PG_VERSION" != "$EXPECTED_PG_VERSION" ]]; then + error "Installed PostgreSQL version ${ACTUAL_PG_VERSION} != expected ${EXPECTED_PG_VERSION} (method=${METHOD}, dir=${PG_INSTALL_DIR}). Either a newer release has shipped (update EXPECTED_PG_VERSION at the top of this script), or this --method's repo/tarball is out of date for the pinned version -- try a different --method (percona guarantees an exact-version tarball)." +fi +info "PostgreSQL version OK: ${ACTUAL_PG_VERSION}" + # --------------------------------------------------------------------------- # 4. LIBPQ HEADERS FOR SYSBENCH # --------------------------------------------------------------------------- From 62d135c847967b589f78828d7b8ffde5bf3bbb5a Mon Sep 17 00:00:00 2001 From: Lukas Oliva Date: Wed, 15 Jul 2026 07:59:54 +0200 Subject: [PATCH 04/27] Add postgresql_default.conf as the new default db_config_file; PG18 tuning postgresql_default.conf: new stock-defaults reference profile for PostgreSQL 18.4 (TAF/tests/setup_almalinux10.sh's EXPECTED_PG_VERSION). Every setting is commented out with its PG18 default value and a doc link, so diffing against it shows exactly what postgresql_oltp.conf/_analytics.conf override and by how much. Documents the "mandatory parameters" question directly: no postgresql.conf setting actually lacks a built-in default (initdb generates a fully valid one); the only non-stock values a run gets are the ones TAF forces regardless of db_config_file (port, listen_addresses, ssl). Now the active taf.db_config_file in both .properties files (previously postgresql_oltp.conf), so a plain run_me.sh invocation benchmarks against stock PG18 settings unless a tuned profile is selected explicitly. postgresql_oltp.conf / postgresql_analytics.conf: add an I/O Workers section (io_method=worker, io_workers=16, up from the PG18 stock default of 3) -- PG18's new async-I/O worker pool is the direct answer to "how many IO threads does a run use" now that we're off PG16 (which had no such pool; effective_io_concurrency there was just an advisory prefetch depth). All four .conf files: updated header comments to note the PG18.4 target version and reference postgresql_default.conf for the stock-default diff. --- .../postgresql/postgresql_analytics.conf | 13 ++ .../postgresql/postgresql_default.conf | 172 ++++++++++++++++++ .../postgresql/postgresql_minimal.conf | 8 + .../postgresql/postgresql_oltp.conf | 16 ++ .../hammerdb_tprocc_pgsql.properties | 4 +- .../postgresql/sysbench_lua_pgsql.properties | 4 +- 6 files changed, 213 insertions(+), 4 deletions(-) create mode 100644 database_config_files/postgresql/postgresql_default.conf diff --git a/database_config_files/postgresql/postgresql_analytics.conf b/database_config_files/postgresql/postgresql_analytics.conf index cfee46c..7aef362 100644 --- a/database_config_files/postgresql/postgresql_analytics.conf +++ b/database_config_files/postgresql/postgresql_analytics.conf @@ -11,6 +11,10 @@ # (HammerDB TPROCH, large aggregation queries). Optimized for query # throughput, parallel execution, and large sort/hash operations. # +# TARGET VERSION: +# PostgreSQL 18.4 (see TAF/tests/setup_almalinux10.sh EXPECTED_PG_VERSION +# and postgresql_default.conf for the full stock-default reference). +# # NOTES: # - Increase work_mem cautiously: each sort/hash node per-connection # can use up to work_mem. With many concurrent queries, total memory @@ -48,6 +52,15 @@ parallel_tuple_cost = 0.01 min_parallel_table_scan_size = 8MB min_parallel_index_scan_size = 512kB +# --------------------------------------------------------------------------- +# I/O Workers (PostgreSQL 18+) +# Large sequential scans/sorts benefit from more concurrent prefetch than the +# stock 3 IO workers can service, especially with parallel workers each +# issuing scans concurrently. +# --------------------------------------------------------------------------- +io_method = worker +io_workers = 16 + # --------------------------------------------------------------------------- # Connections # --------------------------------------------------------------------------- diff --git a/database_config_files/postgresql/postgresql_default.conf b/database_config_files/postgresql/postgresql_default.conf new file mode 100644 index 0000000..5bbda9c --- /dev/null +++ b/database_config_files/postgresql/postgresql_default.conf @@ -0,0 +1,172 @@ +# ============================================================================= +# postgresql_default.conf - PostgreSQL Reference/Documentation Configuration +# +# Created: July 2026 +# +# This file is part of the Test Automation Framework (TAF). +# Copyright (c) 2025-2026 MariaDB Foundation and Jonathan "jeb" Miller +# +# PURPOSE: +# Minimal-necessary starting point, documenting every setting also tuned +# by postgresql_oltp.conf / postgresql_analytics.conf / postgresql_minimal.conf. +# Every line below is commented out and carries the upstream PostgreSQL +# default value plus a link to the reference documentation, so a reviewer +# can see at a glance what changes each of the other three profiles makes +# relative to a stock server -- instead of having to look each one up. +# +# TARGET VERSION: +# PostgreSQL 18.4 -- the current stable release (released 2026-05-14, +# https://www.postgresql.org/docs/release/18.4/). TAF/tests/setup_almalinux10.sh +# pins EXPECTED_PG_VERSION=18.4 and hard-fails setup if the installed +# server doesn't match exactly, across all three --method installers +# (percona / pgdg / appstream). +# All default values and doc links below are for the PostgreSQL 18 +# documentation set (https://www.postgresql.org/docs/18/), not "current" +# (which silently repoints at whatever the newest major version is once +# PG19 ships) -- defaults do change between major versions, e.g. +# effective_io_concurrency's default was 1 in PG16 but is 16 in PG18, and +# log_connections changed from a boolean to a string-typed GUC. +# +# USAGE: +# This is the default taf.db_config_file for sysbench_lua_pgsql.properties +# (and hammerdb_tprocc_pgsql.properties) -- i.e. what a plain run_me.sh +# invocation actually runs against, with every GUC left at its PG18.4 +# stock default. It contributes NO active settings of its own (every +# line below is commented out); the only non-stock values a benchmark +# run gets are the ones TAF forces regardless of db_config_file (port, +# listen_addresses, ssl -- see NOTES below). +# To run one of the tuned profiles instead, point taf.db_config_file at +# postgresql_oltp.conf / postgresql_analytics.conf / postgresql_minimal.conf. +# Diff this file against those to see exactly what each one overrides +# and by how much relative to a stock server. +# +# NOTES: +# - Port and listen_addresses are always set by TAF (postgres.pm +# _db_apply_postgresql_conf); do not set them here -- any line +# matching /^\s*(port|listen_addresses|ssl)\s*=/i is stripped from +# whatever db_config_file is supplied before it's appended. +# - ssl settings are managed by TAF via db_ssl_mode; do not set ssl here. +# ============================================================================= + +# --------------------------------------------------------------------------- +# "Mandatory" parameters -- i.e. ones without which the server would not +# start, or would start in a way unusable for a benchmark run. +# +# In practice, PostgreSQL has NO postgresql.conf setting that lacks a +# built-in default -- `initdb` generates a fully valid postgresql.conf with +# every parameter left at its compiled-in boot_val, and the server starts +# fine from that alone. Nothing below is a case of "no default exists"; it's +# listed here only because TAF's benchmark harness would be unusable (not +# because postgres itself would refuse to start) if left at the stock value: +# +# - port (PG default: 5432) -- forced by TAF, not here. +# - listen_addresses (PG default: localhost) -- forced by TAF, not here. +# Stock 'localhost' would block the TCP connections TAF/sysbench make +# from the control host; TAF always overrides this to '*' regardless +# of what (if anything) is set in this file. +# - unix_socket_directories (PG default: /tmp on Linux) -- left at stock +# default; not overridden anywhere in TAF. No action needed. +# +# No other setting in this file is "required" in the no-default sense the +# question implies. If you want to designate one of the tunables below as +# mandatory-with-no-safe-default for a *new* profile (e.g. shared_buffers +# sized to a specific host's RAM, so the compiled-in 128MB would be wrong +# for a benchmark), leave a TODO here and decide the value per-host: +# +# TODO(you): any profile-specific "must be set explicitly" parameter goes here. +# --------------------------------------------------------------------------- + +# --------------------------------------------------------------------------- +# Memory +# Reference: https://www.postgresql.org/docs/18/runtime-config-resource.html +# --------------------------------------------------------------------------- +#shared_buffers = 128MB # PG18 default +#work_mem = 4MB # PG18 default +#maintenance_work_mem = 64MB # PG18 default +#temp_buffers = 8MB # PG18 default + +# --------------------------------------------------------------------------- +# WAL / Checkpointing +# Reference: https://www.postgresql.org/docs/18/runtime-config-wal.html +# --------------------------------------------------------------------------- +#wal_buffers = -1 # PG18 default: -1 = 1/32 of shared_buffers (min 64kB, max one WAL segment, typically 16MB) +#synchronous_commit = on # PG18 default +#checkpoint_completion_target = 0.9 # PG18 default +#checkpoint_timeout = 5min # PG18 default +#max_wal_size = 1GB # PG18 default +#min_wal_size = 80MB # PG18 default + +# --------------------------------------------------------------------------- +# Parallelism +# Reference: https://www.postgresql.org/docs/18/runtime-config-resource.html +# --------------------------------------------------------------------------- +#max_worker_processes = 8 # PG18 default +#max_parallel_workers_per_gather = 2 # PG18 default +#max_parallel_workers = 8 # PG18 default + +# --------------------------------------------------------------------------- +# I/O Workers (PostgreSQL 18+ -- new async I/O subsystem) +# Reference: https://www.postgresql.org/docs/18/runtime-config-resource.html +# +# PG18 introduces io_method: reads (and on some platforms writes) can be +# issued asynchronously via a small pool of dedicated "IO worker" processes +# (io_method=worker, the default) instead of the backend blocking on each +# syscall itself. This is the direct answer to "how many IO threads/workers +# does a run use" now that we're on 18.4 -- in PG16 there was no such pool at +# all; effective_io_concurrency there was just an advisory prefetch depth, +# not an actual worker count. +# --------------------------------------------------------------------------- +#io_method = worker # PG18 default (worker | sync | io_uring) +#io_workers = 3 # PG18 default -- number of IO worker processes (server-wide, not per-connection) +#io_combine_limit = 128kB # PG18 default -- largest single I/O size when combining adjacent block reads +#io_max_combine_limit = 128kB # PG18 default (platform-dependent ceiling for io_combine_limit; PGC_POSTMASTER) +#io_max_concurrency = -1 # PG18 default -- auto-selected from shared_buffers/max processes, capped at 64 + +# --------------------------------------------------------------------------- +# Connections +# Reference: https://www.postgresql.org/docs/18/runtime-config-connection.html +# --------------------------------------------------------------------------- +#max_connections = 100 # PG18 default (may be lower if the kernel can't support it, per initdb) + +# --------------------------------------------------------------------------- +# Planner +# Reference: https://www.postgresql.org/docs/18/runtime-config-query.html +# Reference: https://www.postgresql.org/docs/18/runtime-config-resource.html (effective_io_concurrency) +# --------------------------------------------------------------------------- +#effective_cache_size = 4GB # PG18 default +#random_page_cost = 4.0 # PG18 default +#effective_io_concurrency = 16 # PG18 default -- CHANGED from PG16's default of 1 (raised as part of the async-I/O rework) +#default_statistics_target = 100 # PG18 default +#parallel_setup_cost = 1000 # PG18 default +#parallel_tuple_cost = 0.1 # PG18 default +#min_parallel_table_scan_size = 8MB # PG18 default +#min_parallel_index_scan_size = 512kB # PG18 default +#enable_hashagg = on # PG18 default +#enable_hashjoin = on # PG18 default +#enable_sort = on # PG18 default + +# --------------------------------------------------------------------------- +# JIT (PostgreSQL 11+) +# Reference: https://www.postgresql.org/docs/18/runtime-config-query.html +# --------------------------------------------------------------------------- +#jit = on # PG18 default + +# --------------------------------------------------------------------------- +# Logging +# Reference: https://www.postgresql.org/docs/18/runtime-config-logging.html +# --------------------------------------------------------------------------- +#log_min_duration_statement = -1 # PG18 default (disabled) +#log_connections = '' # PG18 default -- CHANGED type: string GUC (connection-phase selector), not a boolean as in PG16; '' disables all connection logging +#log_disconnections = off # PG18 default +#log_checkpoints = on # PG18 default +#log_autovacuum_min_duration = 10min # PG18 default + +# --------------------------------------------------------------------------- +# Autovacuum +# Reference: https://www.postgresql.org/docs/18/runtime-config-autovacuum.html +# --------------------------------------------------------------------------- +#autovacuum = on # PG18 default +#autovacuum_max_workers = 3 # PG18 default -- max autovacuum processes running concurrently +#autovacuum_worker_slots = 16 # PG18 default -- reserved backend slots for autovacuum workers (PGC_POSTMASTER, separate pool from autovacuum_max_workers) +#autovacuum_naptime = 1min # PG18 default +#autovacuum_vacuum_cost_delay = 2ms # PG18 default diff --git a/database_config_files/postgresql/postgresql_minimal.conf b/database_config_files/postgresql/postgresql_minimal.conf index 4e3ec15..46dea32 100644 --- a/database_config_files/postgresql/postgresql_minimal.conf +++ b/database_config_files/postgresql/postgresql_minimal.conf @@ -10,6 +10,14 @@ # Provide a minimal, resource-light PostgreSQL configuration for # development, functional testing, and low-load environments. Uses # conservative defaults that work on machines with limited RAM. +# +# TARGET VERSION: +# PostgreSQL 18.4 (see TAF/tests/setup_almalinux10.sh EXPECTED_PG_VERSION +# and postgresql_default.conf for the full stock-default reference). +# No settings below changed default value between PG16 and PG18, so no +# numeric changes were needed here -- this profile intentionally leaves +# effective_io_concurrency/io_workers/io_method at their PG18 stock +# defaults (16 / 3 / worker), matching its "minimal footprint" intent. # ============================================================================= # --------------------------------------------------------------------------- diff --git a/database_config_files/postgresql/postgresql_oltp.conf b/database_config_files/postgresql/postgresql_oltp.conf index a653a04..d11a33b 100644 --- a/database_config_files/postgresql/postgresql_oltp.conf +++ b/database_config_files/postgresql/postgresql_oltp.conf @@ -11,6 +11,10 @@ # HammerDB TPROCC). Optimized for throughput, low latency, and # deterministic benchmark behavior. # +# TARGET VERSION: +# PostgreSQL 18.4 (see TAF/tests/setup_almalinux10.sh EXPECTED_PG_VERSION +# and postgresql_default.conf for the full stock-default reference). +# # USAGE: # Set taf.db_config_file=/path/to/this/file in your properties file, # or pass --property=taf.db_config_file= on the command line. @@ -51,6 +55,18 @@ max_worker_processes = 8 max_parallel_workers_per_gather = 0 max_parallel_workers = 8 +# --------------------------------------------------------------------------- +# I/O Workers (PostgreSQL 18+) +# io_workers raised above the stock default of 3: with up to 210 concurrent +# guests each issuing many prefetch/read requests (high effective_io_concurrency +# below), a small fixed worker pool becomes a bottleneck sooner than on a +# lightly-loaded server. io_method left at the 'worker' default rather than +# io_uring -- not universally available/enabled across the guest kernels in +# this pool, and 'worker' is the safe, portable choice for a mixed VM fleet. +# --------------------------------------------------------------------------- +io_method = worker +io_workers = 16 + # --------------------------------------------------------------------------- # Connections # --------------------------------------------------------------------------- diff --git a/properties/postgresql/hammerdb_tprocc_pgsql.properties b/properties/postgresql/hammerdb_tprocc_pgsql.properties index a69785f..3f3120b 100644 --- a/properties/postgresql/hammerdb_tprocc_pgsql.properties +++ b/properties/postgresql/hammerdb_tprocc_pgsql.properties @@ -29,13 +29,13 @@ taf.taf_db_makers_plugin = postgres # --------------------------------------------------------------------------- # Database software install # --------------------------------------------------------------------------- -# taf.db_software_install_packages = /path/to/postgresql-16.tar.gz +# taf.db_software_install_packages = /path/to/postgresql-18.tar.gz taf.db_port = 5432 # --------------------------------------------------------------------------- # Database configuration # --------------------------------------------------------------------------- -taf.db_config_file = database_config_files/postgresql/postgresql_oltp.conf +taf.db_config_file = database_config_files/postgresql/postgresql_default.conf # --------------------------------------------------------------------------- # Test suite diff --git a/properties/postgresql/sysbench_lua_pgsql.properties b/properties/postgresql/sysbench_lua_pgsql.properties index aa41ac8..9deae88 100644 --- a/properties/postgresql/sysbench_lua_pgsql.properties +++ b/properties/postgresql/sysbench_lua_pgsql.properties @@ -31,13 +31,13 @@ taf.taf_db_makers_plugin = postgres # --------------------------------------------------------------------------- # Database software install # --------------------------------------------------------------------------- -# taf.db_software_install_packages = /path/to/postgresql-16.tar.gz +# taf.db_software_install_packages = /path/to/postgresql-18.tar.gz taf.db_port = 5432 # --------------------------------------------------------------------------- # Database configuration # --------------------------------------------------------------------------- -taf.db_config_file = database_config_files/postgresql/postgresql_oltp.conf +taf.db_config_file = database_config_files/postgresql/postgresql_default.conf # --------------------------------------------------------------------------- # Test suite From 9e29a9f358eed52cb81df8a6cdf456eec6608cfc Mon Sep 17 00:00:00 2001 From: Lukas Oliva Date: Wed, 15 Jul 2026 08:00:08 +0200 Subject: [PATCH 05/27] docs: update test_taf_postgresql.py docstring for PostgreSQL 18 --- tests/test_taf_postgresql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_taf_postgresql.py b/tests/test_taf_postgresql.py index 0c5d4e6..bcfc44a 100644 --- a/tests/test_taf_postgresql.py +++ b/tests/test_taf_postgresql.py @@ -3,7 +3,7 @@ test_taf_postgresql.py — TAF PostgreSQL Integration Tests Layered test suite covering the complete PostgreSQL adaptation of TAF. -Designed for AlmaLinux 10 + PostgreSQL 16 from PGDG. +Designed for AlmaLinux 10 + PostgreSQL 18 from PGDG. Layers: L1 Static validation (syntax, grep) — no external dependencies From dba4d6d9c673d15ae4310d0ed26ef7f6ff8d69a1 Mon Sep 17 00:00:00 2001 From: Lukas Oliva Date: Wed, 15 Jul 2026 13:04:15 +0200 Subject: [PATCH 06/27] Remove accidentally added copyright information and leave it to jeb --- database_config_files/postgresql/postgresql_analytics.conf | 3 --- database_config_files/postgresql/postgresql_default.conf | 3 --- database_config_files/postgresql/postgresql_minimal.conf | 3 --- database_config_files/postgresql/postgresql_oltp.conf | 3 --- libs/database_libs/postgres.pm | 3 --- libs/sql_libs/postgres.sql | 6 ++---- properties/mysql/beta/sysbench_lua.properties | 2 +- properties/postgresql/hammerdb_tprocc_pgsql.properties | 1 - properties/postgresql/sysbench_lua_pgsql.properties | 1 - 9 files changed, 3 insertions(+), 22 deletions(-) diff --git a/database_config_files/postgresql/postgresql_analytics.conf b/database_config_files/postgresql/postgresql_analytics.conf index 7aef362..047d74a 100644 --- a/database_config_files/postgresql/postgresql_analytics.conf +++ b/database_config_files/postgresql/postgresql_analytics.conf @@ -3,9 +3,6 @@ # # Created: June 2026 # -# This file is part of the Test Automation Framework (TAF). -# Copyright (c) 2025-2026 MariaDB Foundation and Jonathan "jeb" Miller -# # PURPOSE: # Provide tuned PostgreSQL settings for OLAP / analytical workloads # (HammerDB TPROCH, large aggregation queries). Optimized for query diff --git a/database_config_files/postgresql/postgresql_default.conf b/database_config_files/postgresql/postgresql_default.conf index 5bbda9c..cf53e08 100644 --- a/database_config_files/postgresql/postgresql_default.conf +++ b/database_config_files/postgresql/postgresql_default.conf @@ -3,9 +3,6 @@ # # Created: July 2026 # -# This file is part of the Test Automation Framework (TAF). -# Copyright (c) 2025-2026 MariaDB Foundation and Jonathan "jeb" Miller -# # PURPOSE: # Minimal-necessary starting point, documenting every setting also tuned # by postgresql_oltp.conf / postgresql_analytics.conf / postgresql_minimal.conf. diff --git a/database_config_files/postgresql/postgresql_minimal.conf b/database_config_files/postgresql/postgresql_minimal.conf index 46dea32..e5b0827 100644 --- a/database_config_files/postgresql/postgresql_minimal.conf +++ b/database_config_files/postgresql/postgresql_minimal.conf @@ -3,9 +3,6 @@ # # Created: June 2026 # -# This file is part of the Test Automation Framework (TAF). -# Copyright (c) 2025-2026 MariaDB Foundation and Jonathan "jeb" Miller -# # PURPOSE: # Provide a minimal, resource-light PostgreSQL configuration for # development, functional testing, and low-load environments. Uses diff --git a/database_config_files/postgresql/postgresql_oltp.conf b/database_config_files/postgresql/postgresql_oltp.conf index d11a33b..1c01b8f 100644 --- a/database_config_files/postgresql/postgresql_oltp.conf +++ b/database_config_files/postgresql/postgresql_oltp.conf @@ -3,9 +3,6 @@ # # Created: June 2026 # -# This file is part of the Test Automation Framework (TAF). -# Copyright (c) 2025-2026 MariaDB Foundation and Jonathan "jeb" Miller -# # PURPOSE: # Provide tuned PostgreSQL settings for OLTP workloads (Sysbench, # HammerDB TPROCC). Optimized for throughput, low latency, and diff --git a/libs/database_libs/postgres.pm b/libs/database_libs/postgres.pm index aa42472..2c9ed40 100644 --- a/libs/database_libs/postgres.pm +++ b/libs/database_libs/postgres.pm @@ -5,9 +5,6 @@ package postgres; # Created: June 2026 # Last Modified: June 2026 # -# This file is part of the Test Automation Framework (TAF). -# Copyright (c) 2025-2026 MariaDB Foundation and Jonathan "jeb" Miller -# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 or later of the License. diff --git a/libs/sql_libs/postgres.sql b/libs/sql_libs/postgres.sql index 213804f..53c40f5 100644 --- a/libs/sql_libs/postgres.sql +++ b/libs/sql_libs/postgres.sql @@ -6,8 +6,6 @@ # Defines named SQL snippets used by TAF for MariaDB diagnostics, # environment introspection, and database lifecycle operations. # -# Copyright (c) 2025-2026 MariaDB Foundation and Jonathan "jeb" Miller -# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 or later of the License. @@ -19,7 +17,7 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 # # Licensed under the GNU General Public License, version 2 or later (GPLv2+). # See https://www.gnu.org/licenses/ for details. @@ -156,4 +154,4 @@ SELECT datname, conflicts, deadlocks FROM pg_stat_database -WHERE datname NOT IN ('template0', 'template1', 'postgres'); \ No newline at end of file +WHERE datname NOT IN ('template0', 'template1', 'postgres'); diff --git a/properties/mysql/beta/sysbench_lua.properties b/properties/mysql/beta/sysbench_lua.properties index 23430e3..f38af3f 100644 --- a/properties/mysql/beta/sysbench_lua.properties +++ b/properties/mysql/beta/sysbench_lua.properties @@ -19,7 +19,7 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 # # Licensed under the GNU General Public License, version 2 or later (GPLv2+). # See https://www.gnu.org/licenses/ for details. diff --git a/properties/postgresql/hammerdb_tprocc_pgsql.properties b/properties/postgresql/hammerdb_tprocc_pgsql.properties index 3f3120b..10d2b16 100644 --- a/properties/postgresql/hammerdb_tprocc_pgsql.properties +++ b/properties/postgresql/hammerdb_tprocc_pgsql.properties @@ -4,7 +4,6 @@ # Created: June 2026 # # This file is part of the Test Automation Framework (TAF). -# Copyright (c) 2025-2026 MariaDB Foundation and Jonathan "jeb" Miller # # PURPOSE: # Example TAF properties file for running HammerDB TPC-C benchmarks diff --git a/properties/postgresql/sysbench_lua_pgsql.properties b/properties/postgresql/sysbench_lua_pgsql.properties index 9deae88..4d94172 100644 --- a/properties/postgresql/sysbench_lua_pgsql.properties +++ b/properties/postgresql/sysbench_lua_pgsql.properties @@ -4,7 +4,6 @@ # Created: June 2026 # # This file is part of the Test Automation Framework (TAF). -# Copyright (c) 2025-2026 MariaDB Foundation and Jonathan "jeb" Miller # # PURPOSE: # Example TAF properties file for running Sysbench OLTP benchmarks From be7266e11ef08c184d2faf24b1459bb3cce29591 Mon Sep 17 00:00:00 2001 From: Lukas Oliva Date: Fri, 17 Jul 2026 15:04:35 +0200 Subject: [PATCH 07/27] Align PostgreSQL sysbench config with MariaDB, harden sysbench build against fleet-scale flakiness - oltp_skip_trx=on, number_of_tables=8 to match MariaDB's effective values (see mariadb-vs-postgresql-taf-test-implementaion-differences.md) - Retry sysbench git clone (3x with backoff) instead of failing the whole host on a single transient network error at 200+-host fan-out - Clear config.cache before configure: a stale cache from a killed/retried build was poisoning checks (observed: bogus "thread-local storage not supported" from a cached "checking for stdlib.h... (cached) no") Co-Authored-By: Claude Sonnet 5 --- .../postgresql/sysbench_lua_pgsql.properties | 4 +-- tests/setup_almalinux10.sh | 33 +++++++++++++++++-- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/properties/postgresql/sysbench_lua_pgsql.properties b/properties/postgresql/sysbench_lua_pgsql.properties index 4d94172..8ff5071 100644 --- a/properties/postgresql/sysbench_lua_pgsql.properties +++ b/properties/postgresql/sysbench_lua_pgsql.properties @@ -54,9 +54,9 @@ sysbench_lua.connector = libpq # --------------------------------------------------------------------------- sysbench_lua.def_threads = 8,16,32,64,128 sysbench_lua.def_duration = 300 -sysbench_lua.number_of_tables = 4 +sysbench_lua.number_of_tables = 8 sysbench_lua.number_of_rows = 1000000 -sysbench_lua.oltp_skip_trx = off +sysbench_lua.oltp_skip_trx = on # --------------------------------------------------------------------------- # Tests to run (standard OLTP subset compatible with PostgreSQL) diff --git a/tests/setup_almalinux10.sh b/tests/setup_almalinux10.sh index b2230e7..1264efd 100755 --- a/tests/setup_almalinux10.sh +++ b/tests/setup_almalinux10.sh @@ -399,16 +399,43 @@ if [[ $SYSBENCH_OK -eq 0 ]]; then # Submodule check: LuaJIT Makefile is only present after a proper git clone # with --recurse-submodules. Zip-extracted source lacks .git/ so submodules # are empty. Force a fresh clone whenever the submodule is missing. + # + # Retry with backoff: a fleet-wide run clones this from GitHub on every + # guest concurrently (e.g. 210 at once for a full density curve), which + # occasionally hits transient network/server flakiness -- observed as + # `fatal: shallow file has changed since we read it` on a small fraction + # of hosts. A single failed attempt used to abort the whole host's setup + # (and, via taf_manage.py, could burn one of only 2 host-level retries on + # something that a plain retry here would have absorbed). Always + # `rm -rf` before each attempt so a partial/corrupt clone from a failed + # attempt can't linger into the next one. if [[ ! -f "${SYSBENCH_SRC}/third_party/luajit/luajit/Makefile" ]]; then - info "Cloning sysbench (submodules missing or incomplete)..." - rm -rf "${SYSBENCH_SRC}" mkdir -p "${TAF_DIR}/client_source" - git clone --depth=1 --recurse-submodules https://github.com/akopytov/sysbench "${SYSBENCH_SRC}" + clone_ok=0 + for attempt in 1 2 3; do + info "Cloning sysbench (submodules missing or incomplete, attempt ${attempt}/3)..." + rm -rf "${SYSBENCH_SRC}" + if git clone --depth=1 --recurse-submodules https://github.com/akopytov/sysbench "${SYSBENCH_SRC}"; then + clone_ok=1 + break + fi + warn "sysbench clone attempt ${attempt}/3 failed" + [[ $attempt -lt 3 ]] && sleep $((attempt * 10)) + done + [[ $clone_ok -eq 1 ]] || error "Failed to clone sysbench after 3 attempts" fi info "Building sysbench with pgsql support..." cd "${SYSBENCH_SRC}" + # Always start configure from a clean cache: a config.cache left over from + # a previous (possibly killed mid-build, or differently-configured) attempt + # on this same guest gets blindly trusted by autoconf, including for + # checks that should never vary by host (e.g. "checking for stdlib.h... + # (cached) no" was observed leading straight into a bogus "thread-local + # storage is not supported" failure). A stale cache is worse than no cache. + rm -f config.cache + # Use bash explicitly — files from a zip archive may lack execute bits. bash autogen.sh bash configure --without-mysql --with-pgsql \ From 9eff594d4f4663213962fb5107c2540571a1ccb8 Mon Sep 17 00:00:00 2001 From: Lukas Oliva Date: Sun, 19 Jul 2026 22:00:23 +0200 Subject: [PATCH 08/27] Add --debug-print-config: dump fully resolved taf.pl configuration Interpreting a TAF run's actual behavior requires reconstructing what default properties, user properties, and CLI overrides resolved to -- none of that merged state was ever visible anywhere. This dumps %options/%dirs/%files plus %ENV to STDERR as a YAML-comment block (4 spaces per level, "# key: value") so it can be pasted straight into a result.yaml as documentation, redacting password-shaped keys (case-insensitive match on AUTH/COOKIE/CREDENTIAL/PASS/PWD/PRIVATE/ SECRET/TOKEN, not just an exact-key list, so PGPASSWORD/MYSQL_PWD-style env vars are caught too). --- libs/taf_libs/TAF/CommandLine.pm | 1 + taf.pl | 52 ++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/libs/taf_libs/TAF/CommandLine.pm b/libs/taf_libs/TAF/CommandLine.pm index 6a7983e..f2570b5 100644 --- a/libs/taf_libs/TAF/CommandLine.pm +++ b/libs/taf_libs/TAF/CommandLine.pm @@ -333,6 +333,7 @@ sub ParseCommandLineOptions { # Debug / tooling #----------------------------------------------------------------------- "tools-debug" => \$tmp_ref->{tools_debug}, + "debug-print-config" => \$flags_ref->{debug_print_config}, #----------------------------------------------------------------------- # Info & commandline flags/options diff --git a/taf.pl b/taf.pl index 3706be3..a756c87 100644 --- a/taf.pl +++ b/taf.pl @@ -409,6 +409,7 @@ list_test_suites_help => FALSE, list_test_types => FALSE, list_version => FALSE, + debug_print_config => FALSE, purge_archive => FALSE, purge_data_directory => FALSE, purge_results_directory => FALSE, @@ -1132,6 +1133,57 @@ sub _LoadProperties{ # Apply commandline overrides again, now that all properties are known TAF::Properties::ApplyOverrides($ctx, $tmpoptions_ref); + + # Dump the fully resolved configuration when requested + main::_PrintDebugConfig(); +} + +############################################################################### +# _PrintDebugConfig +# +# PURPOSE: +# When --debug-print-config is given, dump the fully resolved %options, +# %dirs, and %files hashes, plus the full process environment (%ENV), to +# STDERR. %options/%dirs/%files are the merged result of default +# properties, user properties, and command-line overrides -- the same +# state the rest of the framework operates on from this point on. %ENV is +# included because TAF and the DB software it drives both pick up +# behavior from inherited environment variables (paths, locale, etc.) +# that never go through the properties/CLI system at all. +# +# CONTRACT: +# - Must run only after _LoadProperties has merged all property sources. +# - Must print to STDERR, never STDOUT. +# - Must not terminate the run; this is a diagnostic side effect only. +# - Must redact known secret-shaped keys (passwords). +# - Must print every line as a YAML comment ("# ..."), with each +# section's entries indented 4 spaces per level, so the dump can be +# pasted straight into a YAML file (e.g. result.yaml) as a readable +# comment block instead of opaque "key = value" text. +############################################################################### +sub _PrintDebugConfig { + return unless $flags{debug_print_config}; + + # Case-insensitive substring match, not an exact-key list: %ENV in + # particular carries secrets under names %options/%dirs/%files never + # use (PGPASSWORD, MYSQL_PWD, AWS_SECRET_*, ...). Mirrors (and adds PWD + # to) get_envinfo_standalone.py's SENSITIVE_ENV_RE -- MYSQL_PWD is what + # collect_db_config.sh itself sets to authenticate, and "PASS" alone + # doesn't match it. + my $redact_re = qr/(AUTH|COOKIE|CREDENTIAL|PASS|PWD|PRIVATE|SECRET|TOKEN)/i; + + print STDERR "\n# === TAF RESOLVED CONFIGURATION (--debug-print-config) ===\n"; + for my $section (["options", \%options], ["dirs", \%dirs], ["files", \%files], ["ENV", \%ENV]) { + my ($name, $href) = @$section; + print STDERR "# $name:\n"; + for my $key (sort keys %$href) { + my $value = $href->{$key}; + $value = defined($value) ? $value : ''; + $value = '***REDACTED***' if $key =~ $redact_re && $value ne '' && $value ne ''; + print STDERR "# $key: $value\n"; + } + } + print STDERR "# === END TAF RESOLVED CONFIGURATION ===\n\n"; } ############################################################################### From 0f1d2e1a3b2e041416f21062cd6319c66a3ecb1e Mon Sep 17 00:00:00 2001 From: Lukas Oliva Date: Sun, 19 Jul 2026 22:00:31 +0200 Subject: [PATCH 09/27] Fix ValidateTargetWithSuite comparing normalized vs. raw db_driver sysbench_lua.db_driver=mariadb normalizes to "mysql", but was compared against the properties-file value verbatim (only lowercased, never normalized), so a correctly configured MariaDB run always failed with "Mismatch: sysbench_lua.db_driver = mariadb, db install shows mariadb (normalized: mysql)" -- discovered running the MariaDB pipeline against a freshly built local TAF.zip for the first time. --- test_suites/sysbench-lua.pm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_suites/sysbench-lua.pm b/test_suites/sysbench-lua.pm index 0e6756f..29f2632 100644 --- a/test_suites/sysbench-lua.pm +++ b/test_suites/sysbench-lua.pm @@ -1356,7 +1356,8 @@ sub ValidateTargetWithSuite { my $expected = $tsOpt{db_driver}; my $normalized = NormalizeDBType($incoming) // lc($incoming); - if ($normalized eq lc($expected)) { + my $expected_normalized = NormalizeDBType($expected) // lc($expected); + if ($normalized eq $expected_normalized) { PrintVerbose($vt."db_driver match db maker $incoming (normalized: $normalized), returning OK."); StageEnd($vt); return OK; From b2ecf495c96daab893744fb9bcb857606c5866eb Mon Sep 17 00:00:00 2001 From: Lukas Oliva Date: Tue, 21 Jul 2026 14:05:42 +0200 Subject: [PATCH 10/27] sysbench_lua.properties (mariadb): drop the 4-thread step, align sweep with pgsql taf.threads was 4,8,16,32,64,128 here vs 8,16,32,64,128 in properties/postgresql/sysbench_lua_pgsql.properties -- the extra low-end data point was the last remaining mismatch in the thread sweep between the two engines' example configs. Also strips trailing whitespace from the license header. --- properties/mariadb/beta/sysbench_lua.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/properties/mariadb/beta/sysbench_lua.properties b/properties/mariadb/beta/sysbench_lua.properties index a534aff..6c6660b 100644 --- a/properties/mariadb/beta/sysbench_lua.properties +++ b/properties/mariadb/beta/sysbench_lua.properties @@ -19,7 +19,7 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 # # Licensed under the GNU General Public License, version 2 or later (GPLv2+). # See https://www.gnu.org/licenses/ for details. @@ -121,7 +121,7 @@ taf.test_type=adhoc taf.tests=POINT_SELECT,OLTP_RO,OLTP_RW # Comma-separated list of thread counts to test. -taf.threads=4,8,16,32,64,128 +taf.threads=8,16,32,64,128 # Warmup duration in seconds. taf.warmup_duration=200 From 57f490c8fc6c789224c964152d0db0f6f7969eed Mon Sep 17 00:00:00 2001 From: Lukas Oliva Date: Tue, 21 Jul 2026 14:06:21 +0200 Subject: [PATCH 11/27] sysbench-lua.pm: honor db_clients_use_unix_socket for the pgsql driver too SetConnectionArgs() always hardcoded --pgsql-host='127.0.0.1' for PostgreSQL, ignoring taf.db_clients_use_unix_socket (which defaults to true and already takes effect for MySQL/MariaDB via --mysql-socket) -- so PostgreSQL runs always went over TCP loopback while MariaDB runs went over a unix socket, a client-connection asymmetry irrelevant to the workload itself but not to measured latency/throughput. drv_pgsql.c passes --pgsql-host straight into PQsetdbLogin(), and libpq treats a value starting with '/' as a unix-socket directory rather than a hostname (unlike libmysqlclient, where "localhost" already implies socket use) -- so this has to be requested with an explicit path, not by omitting the flag. pg_hba.conf already allows it ("local all all md5", postgres.pm::_db_write_pg_hba_conf), and taf_run_pgsql.sh already assumes /var/run/postgresql is writable for the server's socket, so that path is reused as the default here (overridable via taf.db_socket). The port is still passed alongside the socket path since libpq derives the socket filename (.s.PGSQL.) from host dir + port. --- test_suites/sysbench-lua.pm | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/test_suites/sysbench-lua.pm b/test_suites/sysbench-lua.pm index 29f2632..4c51617 100644 --- a/test_suites/sysbench-lua.pm +++ b/test_suites/sysbench-lua.pm @@ -2592,8 +2592,22 @@ sub SetConnectionArgs { # Connection parameters — branched by driver family if ($tsOpt{db_driver} eq 'pgsql') { - # PostgreSQL uses --pgsql-* flags; always connect via loopback (pg_hba.conf allows 127.0.0.1) - $args .= " --pgsql-host='127.0.0.1'"; + # PostgreSQL uses --pgsql-* flags. drv_pgsql.c passes --pgsql-host + # straight into PQsetdbLogin(), and libpq treats a value starting + # with '/' as a unix-socket directory rather than a hostname (unlike + # libmysqlclient below, "localhost" here would still mean TCP) -- so + # unix-socket use has to be requested with an explicit path, not by + # omitting the flag. pg_hba.conf already allows this ("local all all + # md5", see postgres.pm::_db_write_pg_hba_conf), and TAF's own + # taf_run_pgsql.sh already assumes /var/run/postgresql is writable + # for the server's socket, so reuse the same path here. The port is + # still required even for a socket connection: libpq derives the + # socket filename (.s.PGSQL.) from host dir + port. + if ($options{db_clients_use_unix_socket}) { + $args .= " --pgsql-host='" . ($options{db_socket} || '/var/run/postgresql') . "'"; + } else { + $args .= " --pgsql-host='127.0.0.1'"; + } $args .= " --pgsql-port=" . $options{db_port}; $args .= " --pgsql-user='" . $options{db_user} . "'"; $args .= " --pgsql-password='" . $options{db_user_pass} . "'"; From b46b0b11467f68c93beaf6f95a3816b135951ae8 Mon Sep 17 00:00:00 2001 From: Lukas Oliva Date: Tue, 21 Jul 2026 22:34:52 +0200 Subject: [PATCH 12/27] Add mariadb_default.cnf: stock-defaults reference profile, mirrors postgresql_default.conf Every line commented out, documenting MariaDB 12.2.2's upstream default for each setting also tuned by mariadb_cache.cnf / mariadb_simple_2gbp.cnf / mariadb_tidesdb*.cnf, plus a KB doc link per setting -- gives the two engines a directly comparable "everything stock" baseline for workload equivalence testing, the same role postgresql_default.conf already plays for PostgreSQL. --- .../mariadb/mariadb_default.cnf | 142 ++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 database_config_files/mariadb/mariadb_default.cnf diff --git a/database_config_files/mariadb/mariadb_default.cnf b/database_config_files/mariadb/mariadb_default.cnf new file mode 100644 index 0000000..368228d --- /dev/null +++ b/database_config_files/mariadb/mariadb_default.cnf @@ -0,0 +1,142 @@ +# ============================================================================= +# mariadb_default.cnf - MariaDB Reference/Documentation Configuration +# +# Created: July 2026 +# +# PURPOSE: +# Minimal-necessary starting point, documenting every setting also tuned +# by mariadb_cache.cnf / mariadb_simple_2gbp.cnf / mariadb_tidesdb*.cnf. +# Every line below is commented out and carries the upstream MariaDB +# default value plus a link to the reference documentation, so a reviewer +# can see at a glance what changes each of the other profiles makes +# relative to a stock server -- instead of having to look each one up. +# Mirrors database_config_files/postgresql/postgresql_default.conf so the +# two engines have a directly comparable "everything stock" baseline. +# +# TARGET VERSION: +# MariaDB 12.2.2 -- pinned by taf_run.sh's MARIADB_TARBALL_NAME +# (mariadb-12.2.2-linux-systemd-x86_64.tar.gz). Default values below are +# for this version's documentation set (https://mariadb.com/kb/en/); a few +# defaults changed in recent MariaDB releases relative to older MySQL-era +# assumptions -- e.g. character_set_server moved from latin1 to utf8mb4 +# (10.6+), sync_binlog moved from 0 to 1 (10.5+), and innodb_log_file_size +# was superseded by innodb_redo_log_capacity (10.8+). Verify against the +# KB page for any setting before assuming an older MySQL/MariaDB default +# still applies. +# +# USAGE: +# Point taf.db_config_file at this file for a plain, stock-defaults +# MariaDB run -- i.e. the same role postgresql_default.conf plays for +# sysbench_lua_pgsql.properties. It contributes NO active settings of its +# own (every line below is commented); the only non-stock values a +# benchmark run gets are the ones TAF forces regardless of db_config_file +# (see NOTES below). +# To run one of the tuned profiles instead, point taf.db_config_file at +# mariadb_cache.cnf / mariadb_simple_2gbp.cnf / mariadb_tidesdb*.cnf. +# Diff this file against those to see exactly what each one overrides and +# by how much relative to a stock server. +# +# NOTES: +# - datadir, socket, log-error, and pid-file are always forced by TAF as +# explicit mariadbd CLI flags after --defaults-file= +# (mariadb.pm::_db_start_runtime_server); MariaDB's own CLI-over-config +# precedence means any datadir/socket/log-error/pid-file line in this +# file would be silently ignored, not merely overridden -- do not set +# them here. +# - port is NOT forced by TAF (unlike PostgreSQL's port/listen_addresses, +# which postgres.pm always strips and re-forces). Left unset here, the +# server uses the MariaDB compiled-in default (3306). Set it explicitly +# in this file if a run needs a non-default port. +# - bind-address is left unset (MariaDB default: listen on all +# interfaces), since none of the sibling profiles set it either. +# ============================================================================= + +# --------------------------------------------------------------------------- +# "Mandatory" parameters -- i.e. ones without which the server would not +# start, or would start in a way unusable for a benchmark run. +# +# In practice, MariaDB has NO my.cnf setting that lacks a built-in default -- +# mariadbd starts fine with an empty [mysqld] section, using its compiled-in +# boot values throughout. Nothing below is a case of "no default exists"; the +# only two params in this category are handled outside this file entirely: +# +# - datadir -- forced by TAF via --datadir=, not here. +# - socket -- forced by TAF via --socket=, not here. +# +# No other setting in this file is "required" in the no-default sense the +# question implies. If you want to designate one of the tunables below as +# mandatory-with-no-safe-default for a *new* profile (e.g. innodb_buffer_pool_size +# sized to a specific host's RAM, so the compiled-in 128MB would be wrong for +# a benchmark), leave a TODO here and decide the value per-host: +# +# TODO(you): any profile-specific "must be set explicitly" parameter goes here. +# --------------------------------------------------------------------------- + +# --------------------------------------------------------------------------- +# Memory +# Reference: https://mariadb.com/kb/en/innodb-system-variables/ +# Reference: https://mariadb.com/kb/en/server-system-variables/ +# --------------------------------------------------------------------------- +#innodb_buffer_pool_size = 128M # MariaDB default +#innodb_buffer_pool_instances = 1 # MariaDB default when innodb_buffer_pool_size < 1GB; auto-scales up to 8 above that, capped at innodb_buffer_pool_size/1GB +#key_buffer_size = 128M # MariaDB default (MyISAM/Aria key cache; largely unused with InnoDB as default engine) +#sort_buffer_size = 2M # MariaDB default (per-connection) +#join_buffer_size = 256K # MariaDB default (per-connection) +#tmp_table_size = 16M # MariaDB default +#max_heap_table_size = 16M # MariaDB default + +# --------------------------------------------------------------------------- +# InnoDB Durability / Redo Log +# Reference: https://mariadb.com/kb/en/innodb-system-variables/ +# Reference: https://mariadb.com/kb/en/replication-and-binary-log-system-variables/ +# --------------------------------------------------------------------------- +#innodb_flush_log_at_trx_commit = 1 # MariaDB default (full ACID durability) +#innodb_redo_log_capacity = 100M # MariaDB default (10.8+; supersedes innodb_log_file_size/innodb_log_files_in_group) +#innodb_doublewrite = ON # MariaDB default +#innodb_flush_method = fsync # MariaDB default (O_DIRECT is NOT the stock default, unlike some tuned profiles here) +#sync_binlog = 1 # MariaDB default (CHANGED from 0 in MariaDB 10.5+; log-bin itself is off by default on a standalone, non-replica server) +#log_bin = OFF # MariaDB default (binlog disabled unless server_id + log-bin configured) + +# --------------------------------------------------------------------------- +# I/O +# Reference: https://mariadb.com/kb/en/innodb-system-variables/ +# --------------------------------------------------------------------------- +#innodb_io_capacity = 200 # MariaDB default +#innodb_io_capacity_max = 2000 # MariaDB default (auto = 2 * innodb_io_capacity if unset) +#innodb_read_io_threads = 4 # MariaDB default +#innodb_write_io_threads = 4 # MariaDB default +#innodb_file_per_table = ON # MariaDB default + +# --------------------------------------------------------------------------- +# Connections / Caching +# Reference: https://mariadb.com/kb/en/server-system-variables/ +# --------------------------------------------------------------------------- +#max_connections = 151 # MariaDB default +#table_open_cache = 2000 # MariaDB default +#thread_cache_size = -1 # MariaDB default: autosized from max_connections (roughly max_connections/100, min 0) unless explicitly set +#thread_handling = one-thread-per-connection # MariaDB default (thread pool is opt-in, not the stock mode) +#open_files_limit = 0 # MariaDB default (0 = use the OS/ulimit-derived value; not an explicit cap) + +# --------------------------------------------------------------------------- +# Optimizer +# Reference: https://mariadb.com/kb/en/server-system-variables/#optimizer_switch +# --------------------------------------------------------------------------- +#optimizer_switch = # MariaDB default; see KB page -- too many individual flags to usefully enumerate as a single value here +#innodb_stats_on_metadata = OFF # MariaDB default (CHANGED from ON in older MySQL-derived defaults) +#innodb_autoinc_lock_mode = 2 # MariaDB default (interleaved) + +# --------------------------------------------------------------------------- +# Logging +# Reference: https://mariadb.com/kb/en/server-system-variables/ +# --------------------------------------------------------------------------- +#general_log = OFF # MariaDB default +#slow_query_log = OFF # MariaDB default +#log_error = # forced by TAF via --log-error=, not here +#performance_schema = OFF # MariaDB default (unlike MySQL/Percona, where it is ON by default) + +# --------------------------------------------------------------------------- +# Character Set / Collation +# Reference: https://mariadb.com/kb/en/server-system-variables/ +# --------------------------------------------------------------------------- +#character_set_server = utf8mb4 # MariaDB default (CHANGED from latin1 in MariaDB 10.6+) +#collation_server = utf8mb4_uca1400_ai_ci # MariaDB default (CHANGED in MariaDB 10.10+; verify against the KB page for 12.2 specifically before relying on this exact name) From 73f34b705434566241aea8054f8732c4089fd65d Mon Sep 17 00:00:00 2001 From: Lukas Oliva Date: Tue, 21 Jul 2026 23:07:36 +0200 Subject: [PATCH 13/27] mariadb.pm: handle running as root, mirroring postgres.pm's _os_prefix() mariadbd unconditionally refuses to start as root (no --allow-run-as-root override), so taf_run.sh hard-refused root outright rather than actually handling it -- this broke MariaDB density campaigns on infrastructure where the SSH/provisioning user is root throughout (e.g. the Plovdiv vSAN pool, where PostgreSQL campaigns already work fine because postgres.pm handles root by dropping to the 'postgres' OS user via runuser). Add the same handling here: detect EUID 0 in the constructor, resolve (or create, since this targets a tarball install with no package postinstall script to have created one already) a 'mysql' system user, and run mariadb-install-db/mariadbd via runuser -u mysql -- through a new _os_prefix() helper. Also chowns data_dir and tmpdir to that user in _db_prepare_data_dir(), since mariadbd itself opens the socket/pidfile/log-error paths under tmpdir. taf_run.sh's hard root guard removed in a companion mt-qa-tools.vhistrg commit now that the plugin handles it. --- libs/database_libs/mariadb.pm | 69 +++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/libs/database_libs/mariadb.pm b/libs/database_libs/mariadb.pm index c83c987..d24fcde 100644 --- a/libs/database_libs/mariadb.pm +++ b/libs/database_libs/mariadb.pm @@ -316,9 +316,61 @@ sub new { # Runtime pidfile now lives under runtime_dir $self->{pidfile} = File::Spec->catfile($runtime_dir, "mariadb_runtime.pid"); + # When TAF runs as root, mariadbd/mariadb-install-db must run as a + # non-root OS user -- mariadbd refuses to start as root outright (unlike + # postgres, which just needs its server process to not be root; MariaDB's + # check is unconditional and has no --allow-run-as-root style override). + # Unlike PostgreSQL's package install (which creates a 'postgres' system + # user via RPM postinstall scriptlet), this plugin targets a plain tarball + # install with no such user created automatically -- create one if it + # doesn't already exist, rather than only detecting a pre-existing one + # like postgres.pm does. + $self->{is_root} = ($> == 0) ? 1 : 0; + if ($self->{is_root}) { + my @pw = getpwnam('mysql'); + unless (@pw) { + PrintVerbose("$_me::new - running as root and OS user 'mysql' does not exist; creating it"); + system('useradd', '--system', '--no-create-home', '--shell', '/sbin/nologin', 'mysql'); + if ($? != 0) { + PrintWarning("$_me::new - useradd mysql failed (exit " . ($? >> 8) . ")"); + } + @pw = getpwnam('mysql'); + } + if (@pw) { + $self->{os_user} = 'mysql'; + $self->{os_uid} = $pw[2]; + $self->{os_gid} = $pw[3]; + PrintVerbose("$_me::new - running as root; server operations will use OS user 'mysql' (uid=$pw[2])"); + } else { + PrintWarning("$_me::new - running as root but OS user 'mysql' not found/creatable; mariadbd will refuse to start"); + $self->{os_user} = undef; + $self->{os_uid} = undef; + $self->{os_gid} = undef; + } + } + return $self; } +################################################################################ +# _os_prefix +# +# PURPOSE: +# Return the command prefix needed to run a command as the 'mysql' OS user +# when TAF is executing as root. Returns an empty list when not root or +# when the 'mysql' OS user could not be resolved/created. Mirrors +# postgres.pm's _os_prefix() so both engine plugins handle a root-executed +# TAF the same way. +# +# USAGE: +# my @cmd = ($self->_os_prefix(), $binary, @args); +################################################################################ +sub _os_prefix { + my ($self) = @_; + return () unless $self->{is_root} && $self->{os_user}; + return ('runuser', '-u', $self->{os_user}, '--'); +} + ################################################################################ # db_init # @@ -512,6 +564,7 @@ sub db_start { # Build argv list for exec() my @cmd = ( + $self->_os_prefix(), $server, "--defaults-file=$self->{config}", "--datadir=$data_dir", @@ -1854,6 +1907,20 @@ sub _db_prepare_data_dir { return ERROR; }; + # When running as root, hand ownership to the mysql OS user so + # mariadb-install-db and mariadbd (both run via _os_prefix() as 'mysql') + # can read and write the data directory. Also chown tmpdir, since the + # socket, pidfile, and log-error paths mariadbd opens itself all live + # there (Utilities.pm defaults db_socket to "db.sock"). + if ($self->{is_root} && defined $self->{os_uid}) { + chown($self->{os_uid}, $self->{os_gid}, $dir) + or PrintWarning("_db_prepare_data_dir: chown $dir to $self->{os_user} failed: $!"); + if ($self->{tmpdir} && -d $self->{tmpdir}) { + chown($self->{os_uid}, $self->{os_gid}, $self->{tmpdir}) + or PrintWarning("_db_prepare_data_dir: chown tmpdir failed: $!"); + } + } + return OK; } @@ -2197,6 +2264,7 @@ sub _db_run_install_db { # Build the install-db command line. # NOTE: No embedded quotes. _run_command handles argument quoting safely. my @cmd = ( + $self->_os_prefix(), $install_db, "--no-defaults", "--basedir=$self->{install_root}", @@ -2356,6 +2424,7 @@ sub _db_start_bootstrap { # Build argv list for exec() my @cmd = ( + $self->_os_prefix(), $server, "--no-defaults", "--datadir=$datadir", From ca7edfc2dc3d3f134d9a73527480ed1284a76f36 Mon Sep 17 00:00:00 2001 From: Lukas Oliva Date: Tue, 21 Jul 2026 23:23:27 +0200 Subject: [PATCH 14/27] mariadb.pm: chown tmpdir recursively, not just the directory itself _db_prepare_data_dir()'s root-handling chowned only the tmpdir entry itself, not its contents. data_dir is wiped and recreated from scratch every run, so it's never an issue there, but tmpdir persists across attempts -- a bootstrap/runtime pidfile or log left behind by an earlier failed attempt (e.g. one that predates this root-handling, or one that failed before reaching this chown) stays owned by root, and mariadbd (now running as 'mysql' via runuser) fails outright when it can't create/write its own --pid-file over an existing root-owned one. Confirmed via a real failure on the Plovdiv density-curve verification: InnoDB started fine as 'mysql', then died on "Can't create/write to file '.../mariadb_bootstrap.pid' (Errcode: 13)". --- libs/database_libs/mariadb.pm | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/libs/database_libs/mariadb.pm b/libs/database_libs/mariadb.pm index d24fcde..17f202b 100644 --- a/libs/database_libs/mariadb.pm +++ b/libs/database_libs/mariadb.pm @@ -1916,8 +1916,16 @@ sub _db_prepare_data_dir { chown($self->{os_uid}, $self->{os_gid}, $dir) or PrintWarning("_db_prepare_data_dir: chown $dir to $self->{os_user} failed: $!"); if ($self->{tmpdir} && -d $self->{tmpdir}) { - chown($self->{os_uid}, $self->{os_gid}, $self->{tmpdir}) - or PrintWarning("_db_prepare_data_dir: chown tmpdir failed: $!"); + # Recursive, not just the directory itself: unlike data_dir (wiped + # and recreated from scratch above), tmpdir persists across + # attempts, so a prior run's bootstrap/runtime pidfile or log + # (created before this fix existed, or by a run that failed + # before reaching this chown) can already exist there owned by + # root -- a non-recursive chown leaves those files unwritable by + # 'mysql', and mariadbd fails outright when it can't create/write + # its own --pid-file. + system('chown', '-R', "$self->{os_uid}:$self->{os_gid}", $self->{tmpdir}) == 0 + or PrintWarning("_db_prepare_data_dir: recursive chown of tmpdir failed (exit " . ($? >> 8) . ")"); } } From b3e30dae633b3dcfd2cb2d60624c1a732b1478b6 Mon Sep 17 00:00:00 2001 From: Lukas Oliva Date: Tue, 21 Jul 2026 23:36:31 +0200 Subject: [PATCH 15/27] mariadb.pm: pre-create+chown the pidfile before forking, fixing a root/mysql ownership race _spawn_background()'s parent (always root, never drops privileges) writes the same --pid-file= path that mariadbd itself (running as 'mysql' via runuser/_os_prefix() after this session's earlier fix) also writes internally. Whichever of the two creates the file first owns it; root's write happens first in practice (~1s after fork, right around when mariadbd finishes InnoDB init and attempts its own pid-file write), so mariadbd's later write failed with "Can't create/write to file ... Permission denied" and the server died -- confirmed via a real failure on the Plovdiv density-curve verification, past the point the previous two fixes (root detection, recursive tmpdir chown) got it to. Pre-creating and chowning the pidfile to the target OS user before forking means both writers just open an *existing* file (which doesn't change ownership) instead of racing to create it, regardless of order. --- libs/database_libs/mariadb.pm | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/libs/database_libs/mariadb.pm b/libs/database_libs/mariadb.pm index 17f202b..427b88f 100644 --- a/libs/database_libs/mariadb.pm +++ b/libs/database_libs/mariadb.pm @@ -2616,6 +2616,30 @@ sub _spawn_background { my $logdir = File::Spec->catpath($vol, $dir, ''); File::Path::make_path($logdir) unless -d $logdir; + # When running as root, pre-create and chown the pidfile to the target OS + # user *before* forking. mariadbd itself opens/writes its own --pid-file= + # path internally, after runuser (via _os_prefix()) has already dropped + # it to that user -- but this same $pidfile path is also written by this + # very function's parent below (`open $fh, '>', $pidfile`), which never + # drops privileges and stays root. Whichever of the two creates the file + # first ends up owning it; if root creates it first (observed in + # practice), mariadbd's own later write fails with "Can't create/write + # to file ... Permission denied" and the server dies. Pre-creating it + # with the right ownership up front means both writers just open an + # *existing* file (which doesn't change ownership) instead of racing to + # create it, regardless of which one gets there first. + if ($self->{is_root} && defined $self->{os_uid}) { + unless (-e $pidfile) { + if (open(my $fh, '>', $pidfile)) { + close $fh; + } else { + PrintWarning($_tag."could not pre-create pidfile $pidfile: $!"); + } + } + chown($self->{os_uid}, $self->{os_gid}, $pidfile) + or PrintWarning($_tag."chown $pidfile to $self->{os_user} failed: $!"); + } + # fork the daemon my $pid = fork(); if (!defined $pid) { From 6b2e1036a5376d2e5a76029e9b0bccdb1fb462b6 Mon Sep 17 00:00:00 2001 From: Lukas Oliva Date: Tue, 21 Jul 2026 23:39:10 +0200 Subject: [PATCH 16/27] mariadb_default.cnf: add required [mysqld] section header MariaDB's config parser rejects a file with no section headers at all ("Config file contains no section headers"), even when every setting under that section is commented out -- unlike postgresql_default.conf, which needs no section header since postgresql.conf has no sections. Confirmed via a real failure (TAF Exit Code: 1) using this file as taf.db_config_file on the Plovdiv verification. --- database_config_files/mariadb/mariadb_default.cnf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/database_config_files/mariadb/mariadb_default.cnf b/database_config_files/mariadb/mariadb_default.cnf index 368228d..33edd7b 100644 --- a/database_config_files/mariadb/mariadb_default.cnf +++ b/database_config_files/mariadb/mariadb_default.cnf @@ -51,6 +51,12 @@ # interfaces), since none of the sibling profiles set it either. # ============================================================================= +# A section header is required even though every setting below it is +# commented out -- MariaDB's config parser rejects a file with no [section] +# headers at all ("Config file contains no section headers"), unlike +# PostgreSQL's postgresql.conf format (which has no sections). +[mysqld] + # --------------------------------------------------------------------------- # "Mandatory" parameters -- i.e. ones without which the server would not # start, or would start in a way unusable for a benchmark run. From 8cd7add76893725fb01c7bde0aa47facf09e86f0 Mon Sep 17 00:00:00 2001 From: Lukas Oliva Date: Mon, 27 Jul 2026 13:38:19 +0200 Subject: [PATCH 17/27] Fix PostgreSQL unix-socket path, cross-engine sysbench driver detection, and $_me:: log interpolation PostgreSQL sysbench connections were failing with "connection to server on socket ".../db.sock/.s.PGSQL." failed: No such file or directory": sysbench-lua.pm passed the raw db_socket file path (e.g. "db.sock") as --pgsql-host, but libpq treats that value as a unix-socket *directory* and appends ".s.PGSQL." itself. postgres.pm never configured unix_socket_directories either, so PostgreSQL was left listening at its own stock default (/tmp), which never matched anyway. Fix: postgres.pm now sets unix_socket_directories to the same tmpdir TAF already manages; sysbench-lua.pm passes dirname($options{db_socket}) as --pgsql-host instead of the raw file-shaped path, so the two agree. Separately, both engines' "is sysbench already built" checks (taf_run.sh for MariaDB, setup_almalinux10.sh for PostgreSQL) only checked that a sysbench binary/symlink existed, not which database driver it was built with. Running one engine after the other against the same guest (shared client_source/sysbench-lua/ tree) left a binary built for the wrong driver in place, and the second engine's sysbench prepare/run failed immediately with "invalid option: --mysql-socket=..." or the pgsql equivalent. setup_almalinux10.sh now verifies the driver via `sysbench