diff --git a/.clang-format b/.clang-format
new file mode 100644
index 0000000000000..2f786ac8eef05
--- /dev/null
+++ b/.clang-format
@@ -0,0 +1,71 @@
+# the official .clang-format style for https://github.com/taocpp
+#
+# clang-format-4.0 -i -style=file $(find -name '[^.]*.[hc]pp')
+
+Language: Cpp
+Standard: Cpp11
+
+AccessModifierOffset: -3
+AlignAfterOpenBracket: Align
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlinesLeft: false
+AlignOperands: true
+AlignTrailingComments: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: Empty
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: false
+AlwaysBreakTemplateDeclarations: true
+BinPackArguments: false
+BinPackParameters: false
+BraceWrapping:
+ AfterClass: true
+ AfterControlStatement: false
+ AfterEnum : true
+ AfterFunction : true
+ AfterNamespace : true
+ AfterStruct : true
+ AfterUnion : true
+ BeforeCatch : true
+ BeforeElse : true
+ IndentBraces : false
+BreakBeforeBinaryOperators: All
+BreakBeforeBraces: Custom
+BreakBeforeTernaryOperators: false
+BreakStringLiterals: false
+BreakConstructorInitializersBeforeComma: false
+ColumnLimit: 0
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 3
+ContinuationIndentWidth: 3
+Cpp11BracedListStyle: false
+DerivePointerAlignment: false
+DisableFormat: false
+ExperimentalAutoDetectBinPacking: false
+IndentCaseLabels: true
+IndentWidth: 3
+IndentWrappedFunctionNames: false
+KeepEmptyLinesAtTheStartOfBlocks: true
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: All
+PointerAlignment: Left
+ReflowComments: false
+SortIncludes: true
+SpaceAfterCStyleCast: false
+SpaceAfterTemplateKeyword: false
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: Never
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles: true
+SpacesInCStyleCastParentheses: false
+SpacesInContainerLiterals: true
+SpacesInParentheses: true
+SpacesInSquareBrackets: true
+TabWidth: 8
+UseTab: Never
diff --git a/.clangd b/.clangd
new file mode 100644
index 0000000000000..cad380a447d17
--- /dev/null
+++ b/.clangd
@@ -0,0 +1,5 @@
+CompileFlags:
+ CompilationDatabase: build/ # Search build/ directory for compile_commands.json
+ Remove: [ -Werror ]
+ Add: [ -DPGDLLIMPORT= ]
+
diff --git a/.envrc b/.envrc
new file mode 100644
index 0000000000000..03bb5edd5a103
--- /dev/null
+++ b/.envrc
@@ -0,0 +1,2 @@
+watch_file flake.nix
+use flake
diff --git a/.gitignore b/.gitignore
index 4e911395fe3ba..8e429d66ca41f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -43,3 +43,11 @@ lib*.pc
/Release/
/tmp_install/
/portlock/
+
+build/
+install/
+test-db/
+.direnv/
+.cache/
+.history
+
diff --git a/.idea/.gitignore b/.idea/.gitignore
new file mode 100644
index 0000000000000..13566b81b018a
--- /dev/null
+++ b/.idea/.gitignore
@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml
diff --git a/.idea/editor.xml b/.idea/editor.xml
new file mode 100644
index 0000000000000..1f0ef49b4faf4
--- /dev/null
+++ b/.idea/editor.xml
@@ -0,0 +1,580 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml
new file mode 100644
index 0000000000000..9c69411050eac
--- /dev/null
+++ b/.idea/inspectionProfiles/Project_Default.xml
@@ -0,0 +1,7 @@
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/misc.xml b/.idea/misc.xml
new file mode 100644
index 0000000000000..53624c9e1f9ab
--- /dev/null
+++ b/.idea/misc.xml
@@ -0,0 +1,18 @@
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/prettier.xml b/.idea/prettier.xml
new file mode 100644
index 0000000000000..b0c1c68fbbad6
--- /dev/null
+++ b/.idea/prettier.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
new file mode 100644
index 0000000000000..35eb1ddfbbc02
--- /dev/null
+++ b/.idea/vcs.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.vscode/launch.json b/.vscode/launch.json
new file mode 100644
index 0000000000000..f5d97424c5047
--- /dev/null
+++ b/.vscode/launch.json
@@ -0,0 +1,22 @@
+{
+ // Use IntelliSense to learn about possible attributes.
+ // Hover to view descriptions of existing attributes.
+ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+ "version": "0.2.0",
+ "configurations": [
+ {
+ "name": "(gdb) Attach Postgres",
+ "type": "cppdbg",
+ "request": "attach",
+ "program": "${workspaceRoot}/install/bin/postgres",
+ "MIMode": "gdb",
+ "setupCommands": [
+ {
+ "description": "Enable pretty-printing for gdb",
+ "text": "-enable-pretty-printing",
+ "ignoreFailures": true
+ }
+ ],
+ }
+ ]
+}
\ No newline at end of file
diff --git a/.vscode/settings.json b/.vscode/settings.json
new file mode 100644
index 0000000000000..8ab390028ad23
--- /dev/null
+++ b/.vscode/settings.json
@@ -0,0 +1,6 @@
+{
+ "files.associations": {
+ "syscache.h": "c",
+ "random": "c"
+ }
+}
\ No newline at end of file
diff --git a/contrib/pg_prewarm/autoprewarm.c b/contrib/pg_prewarm/autoprewarm.c
index c01b9c7e6a4d6..2722b0bb44375 100644
--- a/contrib/pg_prewarm/autoprewarm.c
+++ b/contrib/pg_prewarm/autoprewarm.c
@@ -370,6 +370,16 @@ apw_load_buffers(void)
apw_state->prewarm_start_idx = apw_state->prewarm_stop_idx = 0;
apw_state->prewarmed_blocks = 0;
+
+ /* Don't prewarm more than we can fit. */
+ if (num_elements > NBuffers)
+ {
+ num_elements = NBuffers;
+ ereport(LOG,
+ (errmsg("autoprewarm: capping prewarmed blocks to %d (shared_buffers size)",
+ NBuffers)));
+ }
+
/* Get the info position of the first block of the next database. */
while (apw_state->prewarm_start_idx < num_elements)
{
@@ -410,10 +420,6 @@ apw_load_buffers(void)
apw_state->database = current_db;
Assert(apw_state->prewarm_start_idx < apw_state->prewarm_stop_idx);
- /* If we've run out of free buffers, don't launch another worker. */
- if (!have_free_buffer())
- break;
-
/*
* Likewise, don't launch if we've already been told to shut down.
* (The launch would fail anyway, but we might as well skip it.)
@@ -462,12 +468,6 @@ apw_read_stream_next_block(ReadStream *stream,
{
BlockInfoRecord blk = p->block_info[p->pos];
- if (!have_free_buffer())
- {
- p->pos = apw_state->prewarm_stop_idx;
- return InvalidBlockNumber;
- }
-
if (blk.tablespace != p->tablespace)
return InvalidBlockNumber;
@@ -523,10 +523,10 @@ autoprewarm_database_main(Datum main_arg)
blk = block_info[i];
/*
- * Loop until we run out of blocks to prewarm or until we run out of free
+ * Loop until we run out of blocks to prewarm or until we run out of
* buffers.
*/
- while (i < apw_state->prewarm_stop_idx && have_free_buffer())
+ while (i < apw_state->prewarm_stop_idx)
{
Oid tablespace = blk.tablespace;
RelFileNumber filenumber = blk.filenumber;
@@ -568,14 +568,13 @@ autoprewarm_database_main(Datum main_arg)
/*
* We have a relation; now let's loop until we find a valid fork of
- * the relation or we run out of free buffers. Once we've read from
- * all valid forks or run out of options, we'll close the relation and
+ * the relation or we run out of buffers. Once we've read from all
+ * valid forks or run out of options, we'll close the relation and
* move on.
*/
while (i < apw_state->prewarm_stop_idx &&
blk.tablespace == tablespace &&
- blk.filenumber == filenumber &&
- have_free_buffer())
+ blk.filenumber == filenumber)
{
ForkNumber forknum = blk.forknum;
BlockNumber nblocks;
diff --git a/flake.lock b/flake.lock
new file mode 100644
index 0000000000000..41bcb32b86834
--- /dev/null
+++ b/flake.lock
@@ -0,0 +1,61 @@
+{
+ "nodes": {
+ "flake-utils": {
+ "inputs": {
+ "systems": "systems"
+ },
+ "locked": {
+ "lastModified": 1731533236,
+ "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+ "owner": "numtide",
+ "repo": "flake-utils",
+ "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+ "type": "github"
+ },
+ "original": {
+ "owner": "numtide",
+ "repo": "flake-utils",
+ "type": "github"
+ }
+ },
+ "nixpkgs": {
+ "locked": {
+ "lastModified": 1751211869,
+ "narHash": "sha256-1Cu92i1KSPbhPCKxoiVG5qnoRiKTgR5CcGSRyLpOd7Y=",
+ "owner": "NixOS",
+ "repo": "nixpkgs",
+ "rev": "b43c397f6c213918d6cfe6e3550abfe79b5d1c51",
+ "type": "github"
+ },
+ "original": {
+ "owner": "NixOS",
+ "ref": "nixos-25.05",
+ "repo": "nixpkgs",
+ "type": "github"
+ }
+ },
+ "root": {
+ "inputs": {
+ "flake-utils": "flake-utils",
+ "nixpkgs": "nixpkgs"
+ }
+ },
+ "systems": {
+ "locked": {
+ "lastModified": 1681028828,
+ "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+ "owner": "nix-systems",
+ "repo": "default",
+ "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+ "type": "github"
+ },
+ "original": {
+ "owner": "nix-systems",
+ "repo": "default",
+ "type": "github"
+ }
+ }
+ },
+ "root": "root",
+ "version": 7
+}
diff --git a/flake.nix b/flake.nix
new file mode 100644
index 0000000000000..c4334dfdbbda3
--- /dev/null
+++ b/flake.nix
@@ -0,0 +1,39 @@
+{
+ description = "PostgreSQL development environment";
+
+ inputs = {
+ nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05";
+ flake-utils.url = "github:numtide/flake-utils";
+ };
+
+ outputs = {
+ self,
+ nixpkgs,
+ flake-utils,
+ }:
+ flake-utils.lib.eachDefaultSystem (
+ system: let
+ pkgs = import nixpkgs {
+ inherit system;
+ config.allowUnfree = true;
+ };
+
+ shellConfig = import ./shell.nix {inherit pkgs system;};
+ in {
+ formatter = pkgs.alejandra;
+ devShells = {
+ default = shellConfig.devShell;
+ gcc = shellConfig.devShell;
+ clang = shellConfig.clangDevShell;
+ gcc-musl = shellConfig.muslDevShell;
+ clang-musl = shellConfig.clangMuslDevShell;
+ };
+
+ packages = {
+ inherit (shellConfig) gdbConfig flameGraphScript pgbenchScript;
+ };
+
+ environment.localBinInPath = true;
+ }
+ );
+}
diff --git a/pg-aliases.sh b/pg-aliases.sh
new file mode 100644
index 0000000000000..4b8e06693b401
--- /dev/null
+++ b/pg-aliases.sh
@@ -0,0 +1,295 @@
+# PostgreSQL Development Aliases
+
+# Build system management
+pg_clean_for_compiler() {
+ local current_compiler="$(basename $CC)"
+ local build_dir="$PG_BUILD_DIR"
+
+ if [ -f "$build_dir/compile_commands.json" ]; then
+ local last_compiler=$(grep -o '/[^/]*/bin/[gc]cc\|/[^/]*/bin/clang' "$build_dir/compile_commands.json" | head -1 | xargs basename 2>/dev/null || echo "unknown")
+
+ if [ "$last_compiler" != "$current_compiler" ] && [ "$last_compiler" != "unknown" ]; then
+ echo "Detected compiler change from $last_compiler to $current_compiler"
+ echo "Cleaning build directory..."
+ rm -rf "$build_dir"
+ mkdir -p "$build_dir"
+ fi
+ fi
+
+ mkdir -p "$build_dir"
+ echo "$current_compiler" >"$build_dir/.compiler_used"
+}
+
+# Core PostgreSQL commands
+alias pg-setup='
+ if [ -z "$PERL_CORE_DIR" ]; then
+ echo "Error: Could not find perl CORE directory" >&2
+ return 1
+ fi
+
+ pg_clean_for_compiler
+
+ echo "=== PostgreSQL Build Configuration ==="
+ echo "Compiler: $CC"
+ echo "LLVM: $(llvm-config --version 2>/dev/null || echo 'disabled')"
+ echo "Source: $PG_SOURCE_DIR"
+ echo "Build: $PG_BUILD_DIR"
+ echo "Install: $PG_INSTALL_DIR"
+ echo "======================================"
+ # --fatal-meson-warnings
+
+ env CFLAGS="-I$PERL_CORE_DIR $CFLAGS" \
+ LDFLAGS="-L$PERL_CORE_DIR -lperl $LDFLAGS" \
+ meson setup --reconfigure \
+ -Doptimization=g \
+ -Ddebug=true \
+ -Db_sanitize=none \
+ -Db_lundef=false \
+ -Dlz4=enabled \
+ -Dzstd=enabled \
+ -Dllvm=disabled \
+ -Dplperl=enabled \
+ -Dplpython=enabled \
+ -Dpltcl=enabled \
+ -Dlibxml=enabled \
+ -Duuid=e2fs \
+ -Dlibxslt=enabled \
+ -Dssl=openssl \
+ -Dldap=disabled \
+ -Dcassert=true \
+ -Dtap_tests=enabled \
+ -Dinjection_points=true \
+ -Ddocs_pdf=enabled \
+ -Ddocs_html_style=website \
+ --prefix="$PG_INSTALL_DIR" \
+ "$PG_BUILD_DIR" \
+ "$PG_SOURCE_DIR"'
+
+alias pg-compdb='compdb -p build/ list > compile_commands.json'
+alias pg-build='meson compile -C "$PG_BUILD_DIR"'
+alias pg-install='meson install -C "$PG_BUILD_DIR"'
+alias pg-test='meson test -q --print-errorlogs -C "$PG_BUILD_DIR"'
+
+# Clean commands
+alias pg-clean='ninja -C "$PG_BUILD_DIR" clean'
+alias pg-full-clean='rm -rf "$PG_BUILD_DIR" "$PG_INSTALL_DIR" && echo "Build and install directories cleaned"'
+
+# Database management
+alias pg-init='rm -rf "$PG_DATA_DIR" && "$PG_INSTALL_DIR/bin/initdb" --debug --no-clean "$PG_DATA_DIR"'
+alias pg-start='"$PG_INSTALL_DIR/bin/postgres" -D "$PG_DATA_DIR" -k "$PG_DATA_DIR"'
+alias pg-stop='pkill -f "postgres.*-D.*$PG_DATA_DIR" || true'
+alias pg-restart='pg-stop && sleep 2 && pg-start'
+alias pg-status='pgrep -f "postgres.*-D.*$PG_DATA_DIR" && echo "PostgreSQL is running" || echo "PostgreSQL is not running"'
+
+# Client connections
+alias pg-psql='"$PG_INSTALL_DIR/bin/psql" -h "$PG_DATA_DIR" postgres'
+alias pg-createdb='"$PG_INSTALL_DIR/bin/createdb" -h "$PG_DATA_DIR"'
+alias pg-dropdb='"$PG_INSTALL_DIR/bin/dropdb" -h "$PG_DATA_DIR"'
+
+# Debugging
+alias pg-debug-gdb='gdb -x "$GDBINIT" "$PG_INSTALL_DIR/bin/postgres"'
+alias pg-debug-lldb='lldb "$PG_INSTALL_DIR/bin/postgres"'
+alias pg-debug='
+ if command -v gdb >/dev/null 2>&1; then
+ pg-debug-gdb
+ elif command -v lldb >/dev/null 2>&1; then
+ pg-debug-lldb
+ else
+ echo "No debugger available (gdb or lldb required)"
+ fi'
+
+# Attach to running process
+alias pg-attach-gdb='
+ PG_PID=$(pgrep -f "postgres.*-D.*$PG_DATA_DIR" | head -1)
+ if [ -n "$PG_PID" ]; then
+ echo "Attaching GDB to PostgreSQL process $PG_PID"
+ gdb -x "$GDBINIT" -p "$PG_PID"
+ else
+ echo "No PostgreSQL process found"
+ fi'
+
+alias pg-attach-lldb='
+ PG_PID=$(pgrep -f "postgres.*-D.*$PG_DATA_DIR" | head -1)
+ if [ -n "$PG_PID" ]; then
+ echo "Attaching LLDB to PostgreSQL process $PG_PID"
+ lldb -p "$PG_PID"
+ else
+ echo "No PostgreSQL process found"
+ fi'
+
+alias pg-attach='
+ if command -v gdb >/dev/null 2>&1; then
+ pg-attach-gdb
+ elif command -v lldb >/dev/null 2>&1; then
+ pg-attach-lldb
+ else
+ echo "No debugger available (gdb or lldb required)"
+ fi'
+
+# Performance profiling and analysis
+alias pg-valgrind='valgrind --tool=memcheck --leak-check=full --show-leak-kinds=all "$PG_INSTALL_DIR/bin/postgres" -D "$PG_DATA_DIR"'
+alias pg-strace='strace -f -o /tmp/postgres.strace "$PG_INSTALL_DIR/bin/postgres" -D "$PG_DATA_DIR"'
+
+# Flame graph generation
+alias pg-flame='pg-flame-generate'
+alias pg-flame-30='pg-flame-generate 30'
+alias pg-flame-60='pg-flame-generate 60'
+alias pg-flame-120='pg-flame-generate 120'
+
+# Custom flame graph with specific duration and output
+pg-flame-custom() {
+ local duration=${1:-30}
+ local output_dir=${2:-$PG_FLAME_DIR}
+ echo "Generating flame graph for ${duration}s, output to: $output_dir"
+ pg-flame-generate "$duration" "$output_dir"
+}
+
+# Benchmarking with pgbench
+alias pg-bench='pg-bench-run'
+alias pg-bench-quick='pg-bench-run 5 1 100 1 30 select-only'
+alias pg-bench-standard='pg-bench-run 10 2 1000 10 60 tpcb-like'
+alias pg-bench-heavy='pg-bench-run 50 4 5000 100 300 tpcb-like'
+alias pg-bench-readonly='pg-bench-run 20 4 2000 50 120 select-only'
+
+# Custom benchmark function
+pg-bench-custom() {
+ local clients=${1:-10}
+ local threads=${2:-2}
+ local transactions=${3:-1000}
+ local scale=${4:-10}
+ local duration=${5:-60}
+ local test_type=${6:-tpcb-like}
+
+ echo "Running custom benchmark:"
+ echo " Clients: $clients, Threads: $threads"
+ echo " Transactions: $transactions, Scale: $scale"
+ echo " Duration: ${duration}s, Type: $test_type"
+
+ pg-bench-run "$clients" "$threads" "$transactions" "$scale" "$duration" "$test_type"
+}
+
+# Benchmark with flame graph
+pg-bench-flame() {
+ local duration=${1:-60}
+ local clients=${2:-10}
+ local scale=${3:-10}
+
+ echo "Running benchmark with flame graph generation"
+ echo "Duration: ${duration}s, Clients: $clients, Scale: $scale"
+
+ # Start benchmark in background
+ pg-bench-run "$clients" 2 1000 "$scale" "$duration" tpcb-like &
+ local bench_pid=$!
+
+ # Wait a bit for benchmark to start
+ sleep 5
+
+ # Generate flame graph for most of the benchmark duration
+ local flame_duration=$((duration - 10))
+ if [ $flame_duration -gt 10 ]; then
+ pg-flame-generate "$flame_duration" &
+ local flame_pid=$!
+ fi
+
+ # Wait for benchmark to complete
+ wait $bench_pid
+
+ # Wait for flame graph if it was started
+ if [ -n "${flame_pid:-}" ]; then
+ wait $flame_pid
+ fi
+
+ echo "Benchmark and flame graph generation completed"
+}
+
+# Performance monitoring
+alias pg-perf='perf top -p $(pgrep -f "postgres.*-D.*$PG_DATA_DIR" | head -1)'
+alias pg-htop='htop -p $(pgrep -f "postgres.*-D.*$PG_DATA_DIR" | tr "\n" "," | sed "s/,$//")'
+
+# System performance stats during PostgreSQL operation
+pg-stats() {
+ local duration=${1:-30}
+ echo "Collecting system stats for ${duration}s..."
+
+ iostat -x 1 "$duration" >"$PG_BENCH_DIR/iostat_$(date +%Y%m%d_%H%M%S).log" &
+ vmstat 1 "$duration" >"$PG_BENCH_DIR/vmstat_$(date +%Y%m%d_%H%M%S).log" &
+
+ wait
+ echo "System stats saved to $PG_BENCH_DIR"
+}
+
+# Log management
+alias pg-log='tail -f "$PG_DATA_DIR/log/postgresql-$(date +%Y-%m-%d).log" 2>/dev/null || echo "No log file found"'
+alias pg-log-errors='grep -i error "$PG_DATA_DIR/log/"*.log 2>/dev/null || echo "No error logs found"'
+
+# Build logs
+alias pg-build-log='cat "$PG_BUILD_DIR/meson-logs/meson-log.txt"'
+alias pg-build-errors='grep -i error "$PG_BUILD_DIR/meson-logs/meson-log.txt" 2>/dev/null || echo "No build errors found"'
+
+# Development helpers
+alias pg-format='
+ if [ ! -f "$PG_SOURCE_DIR/src/tools/pgindent/pgindent" ]; then
+ echo "Error: pgindent not found at $PG_SOURCE_DIR/src/tools/pgindent/pgindent"
+ return 1
+ fi
+
+ modified_files=$(git diff --name-only HEAD | egrep "\.c$|\.h$")
+
+ if [ -z "$modified_files" ]; then
+ echo "No modified .c or .h files found"
+ return 0
+ fi
+
+ echo "Formatting modified files with pgindent:"
+ for file in $modified_files; do
+ if [ -f "$file" ]; then
+ echo " Formatting: $file"
+ "$PG_SOURCE_DIR/src/tools/pgindent/pgindent" "$file"
+ else
+ echo " Warning: File not found: $file"
+ fi
+ done
+
+ echo "Formatting completed"'
+
+alias pg-tidy='find "$PG_SOURCE_DIR" -name "*.c" | head -10 | xargs clang-tidy'
+
+# Results viewing
+alias pg-bench-results='ls -la "$PG_BENCH_DIR" && echo "Latest results:" && tail -20 "$PG_BENCH_DIR"/results_*.txt 2>/dev/null | tail -20'
+alias pg-flame-results='ls -la "$PG_FLAME_DIR" && echo "Open flame graphs with: firefox $PG_FLAME_DIR/*.svg"'
+
+# Clean up old results
+pg-clean-results() {
+ local days=${1:-7}
+ echo "Cleaning benchmark and flame graph results older than $days days..."
+ find "$PG_BENCH_DIR" -type f -mtime +$days -delete 2>/dev/null || true
+ find "$PG_FLAME_DIR" -type f -mtime +$days -delete 2>/dev/null || true
+ echo "Cleanup completed"
+}
+
+# Information
+alias pg-info='
+ echo "=== PostgreSQL Development Environment ==="
+ echo "Source: $PG_SOURCE_DIR"
+ echo "Build: $PG_BUILD_DIR"
+ echo "Install: $PG_INSTALL_DIR"
+ echo "Data: $PG_DATA_DIR"
+ echo "Benchmarks: $PG_BENCH_DIR"
+ echo "Flame graphs: $PG_FLAME_DIR"
+ echo "Compiler: $CC"
+ echo ""
+ echo "Available commands:"
+ echo " Setup: pg-setup, pg-build, pg-install"
+ echo " Database: pg-init, pg-start, pg-stop, pg-psql"
+ echo " Debug: pg-debug, pg-attach, pg-valgrind"
+ echo " Performance: pg-flame, pg-bench, pg-perf"
+ echo " Benchmarks: pg-bench-quick, pg-bench-standard, pg-bench-heavy"
+ echo " Flame graphs: pg-flame-30, pg-flame-60, pg-flame-custom"
+ echo " Combined: pg-bench-flame"
+ echo " Results: pg-bench-results, pg-flame-results"
+ echo " Logs: pg-log, pg-build-log"
+ echo " Clean: pg-clean, pg-full-clean, pg-clean-results"
+ echo " Code quality: pg-format, pg-tidy"
+ echo "=========================================="'
+
+echo "PostgreSQL aliases loaded. Run 'pg-info' for available commands."
diff --git a/shell.nix b/shell.nix
new file mode 100644
index 0000000000000..28f6832d1bc0c
--- /dev/null
+++ b/shell.nix
@@ -0,0 +1,810 @@
+{
+ pkgs,
+ system,
+}: let
+ # Use LLVM 19 for modern PostgreSQL development
+ llvmPkgs = pkgs.llvmPackages_19;
+
+ # Configuration constants
+ config = {
+ pgSourceDir = "$PWD";
+ pgBuildDir = "$PWD/build";
+ pgInstallDir = "$PWD/install";
+ pgDataDir = "/tmp/test-db-$(basename $PWD)";
+ pgBenchDir = "/tmp/pgbench-results-$(basename $PWD)";
+ pgFlameDir = "/tmp/flame-graphs-$(basename $PWD)";
+ };
+
+ # Single dependency function that can be used for all environments
+ getPostgreSQLDeps = muslLibs:
+ with pkgs;
+ [
+ # Build system (always use host tools)
+ meson
+ ninja
+ pkg-config
+ autoconf
+ libtool
+ git
+ which
+ binutils
+ gnumake
+
+ # Parser/lexer tools
+ bison
+ flex
+
+ # Perl with required packages
+ (perl.withPackages (ps: with ps; [IPCRun]))
+
+ # Documentation
+ docbook_xml_dtd_45
+ docbook-xsl-nons
+ libxslt
+ libxml2
+ fop
+
+ # Development tools (always use host tools)
+ coreutils
+ shellcheck
+ ripgrep
+ valgrind
+ curl
+ uv
+ pylint
+ black
+ gdb
+ lldb
+ strace
+ ltrace
+ perf-tools
+ linuxPackages.perf
+ flamegraph
+ htop
+ iotop
+ sysstat
+ ccache
+ clang-tools
+ cppcheck
+ compdb
+
+ # LLVM toolchain
+ llvmPkgs.llvm
+ llvmPkgs.llvm.dev
+
+ # Language support
+ (python3.withPackages (ps: with ps; [requests browser-cookie3]))
+ tcl
+ ]
+ ++ (
+ if muslLibs
+ then [
+ # Musl target libraries for cross-compilation
+ pkgs.pkgsMusl.readline
+ pkgs.pkgsMusl.zlib
+ pkgs.pkgsMusl.openssl
+ pkgs.pkgsMusl.icu
+ pkgs.pkgsMusl.lz4
+ pkgs.pkgsMusl.zstd
+ pkgs.pkgsMusl.libuuid
+ pkgs.pkgsMusl.libkrb5
+ pkgs.pkgsMusl.linux-pam
+ pkgs.pkgsMusl.libxcrypt
+ ]
+ else [
+ # Glibc target libraries
+ readline
+ zlib
+ openssl
+ icu
+ lz4
+ zstd
+ libuuid
+ libkrb5
+ linux-pam
+ libxcrypt
+ numactl
+ openldap
+ liburing
+ libselinux
+ glibc
+ glibc.dev
+ ]
+ );
+
+ # GDB configuration for PostgreSQL debugging
+ gdbConfig = pkgs.writeText "gdbinit-postgres" ''
+ # PostgreSQL-specific GDB configuration
+
+ # Pretty-print PostgreSQL data structures
+ define print_node
+ if $arg0
+ printf "Node type: %s\n", nodeTagNames[$arg0->type]
+ print *$arg0
+ else
+ printf "NULL node\n"
+ end
+ end
+ document print_node
+ Print a PostgreSQL Node with type information
+ Usage: print_node
+ end
+
+ define print_list
+ set $list = (List*)$arg0
+ if $list
+ printf "List length: %d\n", $list->length
+ set $cell = $list->head
+ set $i = 0
+ while $cell && $i < $list->length
+ printf " [%d]: ", $i
+ print_node $cell->data.ptr_value
+ set $cell = $cell->next
+ set $i = $i + 1
+ end
+ else
+ printf "NULL list\n"
+ end
+ end
+ document print_list
+ Print a PostgreSQL List structure
+ Usage: print_list
+ end
+
+ define print_query
+ set $query = (Query*)$arg0
+ if $query
+ printf "Query type: %d, command type: %d\n", $query->querySource, $query->commandType
+ print *$query
+ else
+ printf "NULL query\n"
+ end
+ end
+ document print_query
+ Print a PostgreSQL Query structure
+ Usage: print_query
+ end
+
+ define print_relcache
+ set $rel = (Relation)$arg0
+ if $rel
+ printf "Relation: %s.%s (OID: %u)\n", $rel->rd_rel->relnamespace, $rel->rd_rel->relname.data, $rel->rd_id
+ printf " natts: %d, relkind: %c\n", $rel->rd_rel->relnatts, $rel->rd_rel->relkind
+ else
+ printf "NULL relation\n"
+ end
+ end
+ document print_relcache
+ Print relation cache entry information
+ Usage: print_relcache
+ end
+
+ define print_tupdesc
+ set $desc = (TupleDesc)$arg0
+ if $desc
+ printf "TupleDesc: %d attributes\n", $desc->natts
+ set $i = 0
+ while $i < $desc->natts
+ set $attr = $desc->attrs[$i]
+ printf " [%d]: %s (type: %u, len: %d)\n", $i, $attr->attname.data, $attr->atttypid, $attr->attlen
+ set $i = $i + 1
+ end
+ else
+ printf "NULL tuple descriptor\n"
+ end
+ end
+ document print_tupdesc
+ Print tuple descriptor information
+ Usage: print_tupdesc
+ end
+
+ define print_slot
+ set $slot = (TupleTableSlot*)$arg0
+ if $slot
+ printf "TupleTableSlot: %s\n", $slot->tts_ops->name
+ printf " empty: %d, shouldFree: %d\n", $slot->tts_empty, $slot->tts_shouldFree
+ if $slot->tts_tupleDescriptor
+ print_tupdesc $slot->tts_tupleDescriptor
+ end
+ else
+ printf "NULL slot\n"
+ end
+ end
+ document print_slot
+ Print tuple table slot information
+ Usage: print_slot
+ end
+
+ # Memory context debugging
+ define print_mcxt
+ set $context = (MemoryContext)$arg0
+ if $context
+ printf "MemoryContext: %s\n", $context->name
+ printf " type: %s, parent: %p\n", $context->methods->name, $context->parent
+ printf " total: %zu, free: %zu\n", $context->mem_allocated, $context->freep - $context->freeptr
+ else
+ printf "NULL memory context\n"
+ end
+ end
+ document print_mcxt
+ Print memory context information
+ Usage: print_mcxt
+ end
+
+ # Process debugging
+ define print_proc
+ set $proc = (PGPROC*)$arg0
+ if $proc
+ printf "PGPROC: pid=%d, database=%u\n", $proc->pid, $proc->databaseId
+ printf " waiting: %d, waitStatus: %d\n", $proc->waiting, $proc->waitStatus
+ else
+ printf "NULL process\n"
+ end
+ end
+ document print_proc
+ Print process information
+ Usage: print_proc
+ end
+
+ # Set useful defaults
+ set print pretty on
+ set print object on
+ set print static-members off
+ set print vtbl on
+ set print demangle on
+ set demangle-style gnu-v3
+ set print sevenbit-strings off
+ set history save on
+ set history size 1000
+ set history filename ~/.gdb_history_postgres
+
+ # Common breakpoints for PostgreSQL debugging
+ define pg_break_common
+ break elog
+ break errfinish
+ break ExceptionalCondition
+ break ProcessInterrupts
+ end
+ document pg_break_common
+ Set common PostgreSQL debugging breakpoints
+ end
+
+ printf "PostgreSQL GDB configuration loaded.\n"
+ printf "Available commands: print_node, print_list, print_query, print_relcache,\n"
+ printf " print_tupdesc, print_slot, print_mcxt, print_proc, pg_break_common\n"
+ '';
+
+ # Flame graph generation script
+ flameGraphScript = pkgs.writeScriptBin "pg-flame-generate" ''
+ #!${pkgs.bash}/bin/bash
+ set -euo pipefail
+
+ DURATION=''${1:-30}
+ OUTPUT_DIR=''${2:-${config.pgFlameDir}}
+ TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+
+ mkdir -p "$OUTPUT_DIR"
+
+ echo "Generating flame graph for PostgreSQL (duration: ''${DURATION}s)"
+
+ # Find PostgreSQL processes
+ PG_PIDS=$(pgrep -f "postgres.*-D.*${config.pgDataDir}" || true)
+
+ if [ -z "$PG_PIDS" ]; then
+ echo "Error: No PostgreSQL processes found"
+ exit 1
+ fi
+
+ echo "Found PostgreSQL processes: $PG_PIDS"
+
+ # Record perf data
+ PERF_DATA="$OUTPUT_DIR/perf_$TIMESTAMP.data"
+ echo "Recording perf data to $PERF_DATA"
+
+ ${pkgs.linuxPackages.perf}/bin/perf record \
+ -F 997 \
+ -g \
+ --call-graph dwarf \
+ -p "$(echo $PG_PIDS | tr ' ' ',')" \
+ -o "$PERF_DATA" \
+ sleep "$DURATION"
+
+ # Generate flame graph
+ FLAME_SVG="$OUTPUT_DIR/postgres_flame_$TIMESTAMP.svg"
+ echo "Generating flame graph: $FLAME_SVG"
+
+ ${pkgs.linuxPackages.perf}/bin/perf script -i "$PERF_DATA" | \
+ ${pkgs.flamegraph}/bin/stackcollapse-perf.pl | \
+ ${pkgs.flamegraph}/bin/flamegraph.pl \
+ --title "PostgreSQL Flame Graph ($TIMESTAMP)" \
+ --width 1200 \
+ --height 800 \
+ > "$FLAME_SVG"
+
+ echo "Flame graph generated: $FLAME_SVG"
+ echo "Perf data saved: $PERF_DATA"
+
+ # Generate summary report
+ REPORT="$OUTPUT_DIR/report_$TIMESTAMP.txt"
+ echo "Generating performance report: $REPORT"
+
+ {
+ echo "PostgreSQL Performance Analysis Report"
+ echo "Generated: $(date)"
+ echo "Duration: ''${DURATION}s"
+ echo "Processes: $PG_PIDS"
+ echo ""
+ echo "=== Top Functions ==="
+ ${pkgs.linuxPackages.perf}/bin/perf report -i "$PERF_DATA" --stdio --sort comm,dso,symbol | head -50
+ echo ""
+ echo "=== Call Graph ==="
+ ${pkgs.linuxPackages.perf}/bin/perf report -i "$PERF_DATA" --stdio -g --sort comm,dso,symbol | head -100
+ } > "$REPORT"
+
+ echo "Report generated: $REPORT"
+ echo ""
+ echo "Files created:"
+ echo " Flame graph: $FLAME_SVG"
+ echo " Perf data: $PERF_DATA"
+ echo " Report: $REPORT"
+ '';
+
+ # pgbench wrapper script
+ pgbenchScript = pkgs.writeScriptBin "pg-bench-run" ''
+ #!${pkgs.bash}/bin/bash
+ set -euo pipefail
+
+ # Default parameters
+ CLIENTS=''${1:-10}
+ THREADS=''${2:-2}
+ TRANSACTIONS=''${3:-1000}
+ SCALE=''${4:-10}
+ DURATION=''${5:-60}
+ TEST_TYPE=''${6:-tpcb-like}
+
+ OUTPUT_DIR="${config.pgBenchDir}"
+ TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+
+ mkdir -p "$OUTPUT_DIR"
+
+ echo "=== PostgreSQL Benchmark Configuration ==="
+ echo "Clients: $CLIENTS"
+ echo "Threads: $THREADS"
+ echo "Transactions: $TRANSACTIONS"
+ echo "Scale factor: $SCALE"
+ echo "Duration: ''${DURATION}s"
+ echo "Test type: $TEST_TYPE"
+ echo "Output directory: $OUTPUT_DIR"
+ echo "============================================"
+
+ # Check if PostgreSQL is running
+ if ! pgrep -f "postgres.*-D.*${config.pgDataDir}" >/dev/null; then
+ echo "Error: PostgreSQL is not running. Start it with 'pg-start'"
+ exit 1
+ fi
+
+ PGBENCH="${config.pgInstallDir}/bin/pgbench"
+ PSQL="${config.pgInstallDir}/bin/psql"
+ CREATEDB="${config.pgInstallDir}/bin/createdb"
+ DROPDB="${config.pgInstallDir}/bin/dropdb"
+
+ DB_NAME="pgbench_test_$TIMESTAMP"
+ RESULTS_FILE="$OUTPUT_DIR/results_$TIMESTAMP.txt"
+ LOG_FILE="$OUTPUT_DIR/pgbench_$TIMESTAMP.log"
+
+ echo "Creating test database: $DB_NAME"
+ "$CREATEDB" -h "${config.pgDataDir}" "$DB_NAME" || {
+ echo "Failed to create database"
+ exit 1
+ }
+
+ # Initialize pgbench tables
+ echo "Initializing pgbench tables (scale factor: $SCALE)"
+ "$PGBENCH" -h "${config.pgDataDir}" -i -s "$SCALE" "$DB_NAME" || {
+ echo "Failed to initialize pgbench tables"
+ "$DROPDB" -h "${config.pgDataDir}" "$DB_NAME" 2>/dev/null || true
+ exit 1
+ }
+
+ # Run benchmark based on test type
+ echo "Running benchmark..."
+
+ case "$TEST_TYPE" in
+ "tpcb-like"|"default")
+ BENCH_ARGS=""
+ ;;
+ "select-only")
+ BENCH_ARGS="-S"
+ ;;
+ "simple-update")
+ BENCH_ARGS="-N"
+ ;;
+ "read-write")
+ BENCH_ARGS="-b select-only@70 -b tpcb-like@30"
+ ;;
+ *)
+ echo "Unknown test type: $TEST_TYPE"
+ echo "Available types: tpcb-like, select-only, simple-update, read-write"
+ "$DROPDB" -h "${config.pgDataDir}" "$DB_NAME" 2>/dev/null || true
+ exit 1
+ ;;
+ esac
+
+ {
+ echo "PostgreSQL Benchmark Results"
+ echo "Generated: $(date)"
+ echo "Test type: $TEST_TYPE"
+ echo "Clients: $CLIENTS, Threads: $THREADS"
+ echo "Transactions: $TRANSACTIONS, Duration: ''${DURATION}s"
+ echo "Scale factor: $SCALE"
+ echo "Database: $DB_NAME"
+ echo ""
+ echo "=== System Information ==="
+ echo "CPU: $(nproc) cores"
+ echo "Memory: $(free -h | grep '^Mem:' | awk '{print $2}')"
+ echo "Compiler: $CC"
+ echo "PostgreSQL version: $("$PSQL" --no-psqlrc -h "${config.pgDataDir}" -d "$DB_NAME" -t -c "SELECT version();" | head -1)"
+ echo ""
+ echo "=== Benchmark Results ==="
+ } > "$RESULTS_FILE"
+
+ # Run the actual benchmark
+ "$PGBENCH" \
+ -h "${config.pgDataDir}" \
+ -c "$CLIENTS" \
+ -j "$THREADS" \
+ -T "$DURATION" \
+ -P 5 \
+ --log \
+ --log-prefix="$OUTPUT_DIR/pgbench_$TIMESTAMP" \
+ $BENCH_ARGS \
+ "$DB_NAME" 2>&1 | tee -a "$RESULTS_FILE"
+
+ # Collect additional statistics
+ {
+ echo ""
+ echo "=== Database Statistics ==="
+ "$PSQL" --no-psqlrc -h "${config.pgDataDir}" -d "$DB_NAME" -c "
+ SELECT
+ schemaname,
+ relname,
+ n_tup_ins as inserts,
+ n_tup_upd as updates,
+ n_tup_del as deletes,
+ n_live_tup as live_tuples,
+ n_dead_tup as dead_tuples
+ FROM pg_stat_user_tables;
+ "
+
+ echo ""
+ echo "=== Index Statistics ==="
+ "$PSQL" --no-psqlrc -h "${config.pgDataDir}" -d "$DB_NAME" -c "
+ SELECT
+ schemaname,
+ relname,
+ indexrelname,
+ idx_scan,
+ idx_tup_read,
+ idx_tup_fetch
+ FROM pg_stat_user_indexes;
+ "
+ } >> "$RESULTS_FILE"
+
+ # Clean up
+ echo "Cleaning up test database: $DB_NAME"
+ "$DROPDB" -h "${config.pgDataDir}" "$DB_NAME" 2>/dev/null || true
+
+ echo ""
+ echo "Benchmark completed!"
+ echo "Results saved to: $RESULTS_FILE"
+ echo "Transaction logs: $OUTPUT_DIR/pgbench_$TIMESTAMP*"
+
+ # Show summary
+ echo ""
+ echo "=== Quick Summary ==="
+ grep -E "(tps|latency)" "$RESULTS_FILE" | tail -5
+ '';
+
+ # Development shell (GCC + glibc)
+ devShell = pkgs.mkShell {
+ name = "postgresql-dev";
+ buildInputs =
+ (getPostgreSQLDeps false)
+ ++ [
+ flameGraphScript
+ pgbenchScript
+ ];
+
+ shellHook = let
+ icon = "f121";
+ in ''
+ # History configuration
+ export HISTFILE=.history
+ export HISTSIZE=1000000
+ export HISTFILESIZE=1000000
+
+ # Clean environment
+ unset LD_LIBRARY_PATH LD_PRELOAD LIBRARY_PATH C_INCLUDE_PATH CPLUS_INCLUDE_PATH
+
+ # Essential tools in PATH
+ export PATH="${pkgs.which}/bin:${pkgs.coreutils}/bin:$PATH"
+ export PS1="$(echo -e '\u${icon}') {\[$(tput sgr0)\]\[\033[38;5;228m\]\w\[$(tput sgr0)\]\[\033[38;5;15m\]} ($(git rev-parse --abbrev-ref HEAD)) \\$ \[$(tput sgr0)\]"
+
+ # Ccache configuration
+ export PATH=${pkgs.ccache}/bin:$PATH
+ export CCACHE_COMPILERCHECK=content
+ export CCACHE_DIR=$HOME/.ccache/pg/$(basename $PWD)
+ mkdir -p "$CCACHE_DIR"
+
+ # LLVM configuration
+ export LLVM_CONFIG="${llvmPkgs.llvm}/bin/llvm-config"
+ export PATH="${llvmPkgs.llvm}/bin:$PATH"
+ export PKG_CONFIG_PATH="${llvmPkgs.llvm.dev}/lib/pkgconfig:$PKG_CONFIG_PATH"
+ export LLVM_DIR="${llvmPkgs.llvm.dev}/lib/cmake/llvm"
+ export LLVM_ROOT="${llvmPkgs.llvm}"
+
+ # Development tools in PATH
+ export PATH=${pkgs.clang-tools}/bin:$PATH
+ export PATH=${pkgs.cppcheck}/bin:$PATH
+
+ # PosgreSQL Development CFLAGS
+ # -DRELCACHE_FORCE_RELEASE -DCATCACHE_FORCE_RELEASE -fno-omit-frame-pointer -fno-stack-protector -DUSE_VALGRIND
+ export CFLAGS=""
+ export CXXFLAGS=""
+
+ # Python UV
+ UV_PYTHON_DOWNLOADS=never
+
+ # GCC configuration (default compiler)
+ export CC="${pkgs.gcc}/bin/gcc"
+ export CXX="${pkgs.gcc}/bin/g++"
+
+ # PostgreSQL environment
+ export PG_SOURCE_DIR="${config.pgSourceDir}"
+ export PG_BUILD_DIR="${config.pgBuildDir}"
+ export PG_INSTALL_DIR="${config.pgInstallDir}"
+ export PG_DATA_DIR="${config.pgDataDir}"
+ export PG_BENCH_DIR="${config.pgBenchDir}"
+ export PG_FLAME_DIR="${config.pgFlameDir}"
+ export PERL_CORE_DIR=$(find ${pkgs.perl} -maxdepth 5 -path "*/CORE" -type d)
+
+ # GDB configuration
+ export GDBINIT="${gdbConfig}"
+
+ # Performance tools in PATH
+ export PATH="${flameGraphScript}/bin:${pgbenchScript}/bin:$PATH"
+
+ # Create output directories
+ mkdir -p "$PG_BENCH_DIR" "$PG_FLAME_DIR"
+
+ # Compiler verification
+ echo "Environment configured:"
+ echo " Compiler: $CC"
+ echo " LibC: glibc"
+ echo " LLVM: $(llvm-config --version 2>/dev/null || echo 'not available')"
+ echo " Ccache: enabled ($CCACHE_DIR)"
+
+ # Load PostgreSQL development aliases
+ if [ -f ./pg-aliases.sh ]; then
+ source ./pg-aliases.sh
+ else
+ echo "Warning: pg-aliases.sh not found in current directory"
+ fi
+
+ echo ""
+ echo "PostgreSQL Development Environment Ready (GCC + glibc)"
+ echo "Run 'pg-info' for available commands"
+ '';
+ };
+
+ # Clang + glibc variant
+ clangDevShell = pkgs.mkShell {
+ name = "postgresql-clang-glibc";
+ buildInputs =
+ (getPostgreSQLDeps false)
+ ++ [
+ llvmPkgs.clang
+ llvmPkgs.lld
+ llvmPkgs.compiler-rt
+ flameGraphScript
+ pgbenchScript
+ ];
+
+ shellHook = let
+ icon = "f121";
+ in ''
+ # History configuration
+ export HISTFILE=.history
+ export HISTSIZE=1000000
+ export HISTFILESIZE=1000000
+
+ # Clean environment
+ unset LD_LIBRARY_PATH LD_PRELOAD LIBRARY_PATH C_INCLUDE_PATH CPLUS_INCLUDE_PATH
+
+ # Essential tools in PATH
+ export PATH="${pkgs.which}/bin:${pkgs.coreutils}/bin:$PATH"
+ export PS1="$(echo -e '\u${icon}') {\[$(tput sgr0)\]\[\033[38;5;228m\]\w\[$(tput sgr0)\]\[\033[38;5;15m\]} ($(git rev-parse --abbrev-ref HEAD)) \\$ \[$(tput sgr0)\]"
+
+ # Ccache configuration
+ export PATH=${pkgs.ccache}/bin:$PATH
+ export CCACHE_COMPILERCHECK=content
+ export CCACHE_DIR=$HOME/.ccache_pg_dev_clang
+ mkdir -p "$CCACHE_DIR"
+
+ # LLVM configuration
+ export LLVM_CONFIG="${llvmPkgs.llvm}/bin/llvm-config"
+ export PATH="${llvmPkgs.llvm}/bin:$PATH"
+ export PKG_CONFIG_PATH="${llvmPkgs.llvm.dev}/lib/pkgconfig:$PKG_CONFIG_PATH"
+ export LLVM_DIR="${llvmPkgs.llvm.dev}/lib/cmake/llvm"
+ export LLVM_ROOT="${llvmPkgs.llvm}"
+
+ # Development tools in PATH
+ export PATH=${pkgs.clang-tools}/bin:$PATH
+ export PATH=${pkgs.cppcheck}/bin:$PATH
+
+ # Clang + glibc configuration - use system linker instead of LLD for compatibility
+ export CC="${llvmPkgs.clang}/bin/clang"
+ export CXX="${llvmPkgs.clang}/bin/clang++"
+
+ # Use system linker and standard runtime
+ #export CFLAGS=""
+ #export CXXFLAGS=""
+ #export LDFLAGS=""
+
+ # PostgreSQL environment
+ export PG_SOURCE_DIR="${config.pgSourceDir}"
+ export PG_BUILD_DIR="${config.pgBuildDir}-clang"
+ export PG_INSTALL_DIR="${config.pgInstallDir}-clang"
+ export PG_DATA_DIR="${config.pgDataDir}-clang"
+ export PG_BENCH_DIR="${config.pgBenchDir}"
+ export PG_FLAME_DIR="${config.pgFlameDir}"
+ export PERL_CORE_DIR=$(find ${pkgs.perl} -maxdepth 5 -path "*/CORE" -type d)
+
+ # GDB configuration
+ export GDBINIT="${gdbConfig}"
+
+ # Performance tools in PATH
+ export PATH="${flameGraphScript}/bin:${pgbenchScript}/bin:$PATH"
+
+ # Create output directories
+ mkdir -p "$PG_BENCH_DIR" "$PG_FLAME_DIR"
+
+ # Compiler verification
+ echo "Environment configured:"
+ echo " Compiler: $CC"
+ echo " LibC: glibc"
+ echo " LLVM: $(llvm-config --version 2>/dev/null || echo 'not available')"
+ echo " Ccache: enabled ($CCACHE_DIR)"
+
+ # Load PostgreSQL development aliases
+ if [ -f ./pg-aliases.sh ]; then
+ source ./pg-aliases.sh
+ else
+ echo "Warning: pg-aliases.sh not found in current directory"
+ fi
+
+ echo ""
+ echo "PostgreSQL Development Environment Ready (Clang + glibc)"
+ echo "Run 'pg-info' for available commands"
+ '';
+ };
+
+ # GCC + musl variant (cross-compilation)
+ muslDevShell = pkgs.mkShell {
+ name = "postgresql-gcc-musl";
+ buildInputs =
+ (getPostgreSQLDeps true)
+ ++ [
+ pkgs.gcc
+ flameGraphScript
+ pgbenchScript
+ ];
+
+ shellHook = ''
+ # Same base configuration as main shell
+ export HISTFILE=.history
+ export HISTSIZE=1000000
+ export HISTFILESIZE=1000000
+
+ unset LD_LIBRARY_PATH LD_PRELOAD LIBRARY_PATH C_INCLUDE_PATH CPLUS_INCLUDE_PATH
+
+ export PATH="${pkgs.which}/bin:${pkgs.coreutils}/bin:$PATH"
+
+ # Cross-compilation to musl
+ export CC="${pkgs.gcc}/bin/gcc"
+ export CXX="${pkgs.gcc}/bin/g++"
+
+ # Point to musl libraries for linking
+ export PKG_CONFIG_PATH="${pkgs.pkgsMusl.openssl.dev}/lib/pkgconfig:${pkgs.pkgsMusl.zlib.dev}/lib/pkgconfig:${pkgs.pkgsMusl.icu.dev}/lib/pkgconfig"
+ export CFLAGS="-ggdb -Og -fno-omit-frame-pointer -DUSE_VALGRIND -D_FORTIFY_SOURCE=1 -I${pkgs.pkgsMusl.stdenv.cc.libc}/include"
+ export CXXFLAGS="-ggdb -Og -fno-omit-frame-pointer -DUSE_VALGRIND -D_FORTIFY_SOURCE=1 -I${pkgs.pkgsMusl.stdenv.cc.libc}/include"
+ export LDFLAGS="-L${pkgs.pkgsMusl.stdenv.cc.libc}/lib -static-libgcc"
+
+ # PostgreSQL environment
+ export PG_SOURCE_DIR="${config.pgSourceDir}"
+ export PG_BUILD_DIR="${config.pgBuildDir}-musl"
+ export PG_INSTALL_DIR="${config.pgInstallDir}-musl"
+ export PG_DATA_DIR="${config.pgDataDir}-musl"
+ export PG_BENCH_DIR="${config.pgBenchDir}"
+ export PG_FLAME_DIR="${config.pgFlameDir}"
+ export PERL_CORE_DIR=$(find ${pkgs.perl} -maxdepth 5 -path "*/CORE" -type d)
+
+ export GDBINIT="${gdbConfig}"
+ export PATH="${flameGraphScript}/bin:${pgbenchScript}/bin:$PATH"
+
+ mkdir -p "$PG_BENCH_DIR" "$PG_FLAME_DIR"
+
+ echo "GCC + musl environment configured"
+ echo " Compiler: $CC"
+ echo " LibC: musl (cross-compilation)"
+
+ if [ -f ./pg-aliases.sh ]; then
+ source ./pg-aliases.sh
+ fi
+
+ echo "PostgreSQL Development Environment Ready (GCC + musl)"
+ '';
+ };
+
+ # Clang + musl variant (cross-compilation)
+ clangMuslDevShell = pkgs.mkShell {
+ name = "postgresql-clang-musl";
+ buildInputs =
+ (getPostgreSQLDeps true)
+ ++ [
+ llvmPkgs.clang
+ llvmPkgs.lld
+ flameGraphScript
+ pgbenchScript
+ ];
+
+ shellHook = let
+ icon = "f121";
+ in ''
+ export HISTFILE=.history
+ export HISTSIZE=1000000
+ export HISTFILESIZE=1000000
+
+ unset LD_LIBRARY_PATH LD_PRELOAD LIBRARY_PATH C_INCLUDE_PATH CPLUS_INCLUDE_PATH
+
+ export PATH="${pkgs.which}/bin:${pkgs.coreutils}/bin:$PATH"
+ export PS1="$(echo -e '\u${icon}') {\[$(tput sgr0)\]\[\033[38;5;228m\]\w\[$(tput sgr0)\]\[\033[38;5;15m\]} ($(git rev-parse --abbrev-ref HEAD)) \\$ \[$(tput sgr0)\]"
+
+ # Cross-compilation to musl with clang
+ export CC="${llvmPkgs.clang}/bin/clang"
+ export CXX="${llvmPkgs.clang}/bin/clang++"
+
+ # Point to musl libraries for linking
+ export PKG_CONFIG_PATH="${pkgs.pkgsMusl.openssl.dev}/lib/pkgconfig:${pkgs.pkgsMusl.zlib.dev}/lib/pkgconfig:${pkgs.pkgsMusl.icu.dev}/lib/pkgconfig"
+ export CFLAGS="--target=x86_64-linux-musl -ggdb -Og -fno-omit-frame-pointer -DUSE_VALGRIND -D_FORTIFY_SOURCE=1 -I${pkgs.pkgsMusl.stdenv.cc.libc}/include"
+ export CXXFLAGS="--target=x86_64-linux-musl -ggdb -Og -fno-omit-frame-pointer -DUSE_VALGRIND -D_FORTIFY_SOURCE=1 -I${pkgs.pkgsMusl.stdenv.cc.libc}/include"
+ export LDFLAGS="--target=x86_64-linux-musl -L${pkgs.pkgsMusl.stdenv.cc.libc}/lib -fuse-ld=lld"
+
+ # PostgreSQL environment
+ export PG_SOURCE_DIR="${config.pgSourceDir}"
+ export PG_BUILD_DIR="${config.pgBuildDir}-clang-musl"
+ export PG_INSTALL_DIR="${config.pgInstallDir}-clang-musl"
+ export PG_DATA_DIR="${config.pgDataDir}-clang-musl"
+ export PG_BENCH_DIR="${config.pgBenchDir}"
+ export PG_FLAME_DIR="${config.pgFlameDir}"
+ export PERL_CORE_DIR=$(find ${pkgs.perl} -maxdepth 5 -path "*/CORE" -type d)
+
+ export GDBINIT="${gdbConfig}"
+ export PATH="${flameGraphScript}/bin:${pgbenchScript}/bin:$PATH"
+
+ mkdir -p "$PG_BENCH_DIR" "$PG_FLAME_DIR"
+
+ echo "Clang + musl environment configured"
+ echo " Compiler: $CC"
+ echo " LibC: musl (cross-compilation)"
+
+ if [ -f ./pg-aliases.sh ]; then
+ source ./pg-aliases.sh
+ fi
+
+ echo "PostgreSQL Development Environment Ready (Clang + musl)"
+ '';
+ };
+in {
+ inherit devShell clangDevShell muslDevShell clangMuslDevShell gdbConfig flameGraphScript pgbenchScript;
+}
diff --git a/src/backend/postmaster/bgwriter.c b/src/backend/postmaster/bgwriter.c
index 72f5acceec78d..432b00874f4aa 100644
--- a/src/backend/postmaster/bgwriter.c
+++ b/src/backend/postmaster/bgwriter.c
@@ -77,6 +77,296 @@ int BgWriterDelay = 200;
static TimestampTz last_snapshot_ts;
static XLogRecPtr last_snapshot_lsn = InvalidXLogRecPtr;
+/*
+ * Collected buffer usage information.
+ */
+typedef struct BackendBufferStats
+{
+ int backend_id;
+ uint64 usage_sum;
+ double usage_ratio;
+} BackendBufferStats;
+
+static int
+compare_backend_usage(const void *a, const void *b)
+{
+ const BackendBufferStats *stat_a = (const BackendBufferStats *) a;
+ const BackendBufferStats *stat_b = (const BackendBufferStats *) b;
+
+ if (stat_a->usage_ratio < stat_b->usage_ratio)
+ return -1;
+ if (stat_a->usage_ratio > stat_b->usage_ratio)
+ return 1;
+ return 0;
+}
+
+static uint64
+CalculateSystemBufferPressure(BackendBufferStats *backend_stats[], int *num_backends)
+{
+ uint64 total_usage = 0;
+ int active_backends = 0;
+ BackendBufferStats *stats;
+
+ /* Count active backends first */
+ for (int i = 0; i < ProcGlobal->allProcCount; i++)
+ {
+ PGPROC *proc = &ProcGlobal->allProcs[i];
+
+ if (proc->pid != 0 && proc->databaseId != InvalidOid)
+ active_backends++;
+ }
+
+ if (active_backends == 0)
+ {
+ *backend_stats = NULL;
+ *num_backends = 0;
+ return 0;
+ }
+
+ /* Allocate stats array */
+ stats = palloc(sizeof(BackendBufferStats) * active_backends);
+ *backend_stats = stats;
+ *num_backends = active_backends;
+
+ /* Collect stats from all active backends */
+ for (int i = 0, j = 0; i < ProcGlobal->allProcCount; i++)
+ {
+ PGPROC *proc = &ProcGlobal->allProcs[i];
+
+ if (proc->pid != 0 && proc->databaseId != InvalidOid)
+ {
+ uint64 usage_sum = pg_atomic_read_u32(&proc->bufferUsageSum);
+
+ stats[j].backend_id = i;
+ stats[j].usage_sum = usage_sum;
+ stats[j].usage_ratio = (double) usage_sum / NBuffers;
+ total_usage += usage_sum;
+ j++;
+ }
+ }
+
+ /* Sort by usage ratio for percentile calculation */
+ qsort(stats, active_backends, sizeof(BackendBufferStats),
+ compare_backend_usage);
+
+ return total_usage;
+}
+
+static void
+GetHighUsageBackends(BackendBufferStats *stats, int num_backends,
+ int **high_usage_backends, int *num_high_usage)
+{
+ int percentile_90_idx = (int) (num_backends * 0.9);
+
+ *num_high_usage = num_backends - percentile_90_idx;
+
+ if (*num_high_usage > 0)
+ {
+ *high_usage_backends = palloc(sizeof(int) * (*num_high_usage));
+ for (int i = 0; i < *num_high_usage; i++)
+ {
+ (*high_usage_backends)[i] = stats[percentile_90_idx + i].backend_id;
+ }
+ }
+ else
+ {
+ *high_usage_backends = NULL;
+ *num_high_usage = 0;
+ }
+}
+
+/*
+ * Shared buffer sync function used by both main loop and aggressive writing
+ */
+static int
+SyncTargetedBuffers(WritebackContext *wb_context, int *target_backends,
+ int num_targets, int max_buffers)
+{
+ int buffers_written = 0;
+ int buffer_id;
+ BufferDesc *bufHdr;
+ uint32 buf_state;
+
+ /* If no specific targets, sync any dirty buffers */
+ if (target_backends == NULL || num_targets == 0)
+ {
+ return BgBufferSync(wb_context);
+ }
+
+ /* Scan through all buffers looking for dirty ones from target backends */
+ for (buffer_id = 0; buffer_id < NBuffers && buffers_written < max_buffers; buffer_id++)
+ {
+ uint32 dirty_backend;
+ bool is_target;
+
+ bufHdr = GetBufferDescriptor(buffer_id);
+
+ /* Quick check if buffer is dirty */
+ buf_state = pg_atomic_read_u32(&bufHdr->state);
+ if (!(buf_state & BM_DIRTY))
+ continue;
+
+ /* Check if this buffer is from one of our target backends */
+ dirty_backend = pg_atomic_read_u32(&bufHdr->dirty_backend_id);
+ is_target = false;
+
+ for (int i = 0; i < num_targets; i++)
+ if (dirty_backend == target_backends[i])
+ {
+ is_target = true;
+ break;
+ }
+
+ if (!is_target)
+ continue;
+
+ /* Skip if buffer is pinned */
+ if (BUF_STATE_GET_REFCOUNT(buf_state) > 0)
+ continue;
+
+ /* Try to write this buffer using the writeback context */
+ ScheduleBufferTagForWriteback(wb_context,
+ IOContextForStrategy(NULL),
+ &bufHdr->tag);
+ buffers_written++;
+ }
+
+ /* Issue the actual writes */
+ if (buffers_written > 0)
+ {
+ IssuePendingWritebacks(wb_context, IOContextForStrategy(NULL));
+ }
+
+ return buffers_written;
+}
+
+static void
+AggressiveBufferWrite(WritebackContext *wb_context, int *high_usage_backends,
+ int num_high_usage, bool critical)
+{
+ int write_target = critical ? bgwriter_lru_maxpages * 3 : bgwriter_lru_maxpages * 2;
+ int buffers_written = 0;
+
+ /* Focus on buffers from high-usage backends first */
+ buffers_written = SyncTargetedBuffers(wb_context, high_usage_backends,
+ num_high_usage, write_target);
+
+ /* If still under target, write additional dirty buffers */
+ if (buffers_written < write_target)
+ BgBufferSync(wb_context);
+}
+
+/* In src/backend/postmaster/bgwriter.c - Enhanced UpdateBackendDecayRates */
+static void
+UpdateBackendDecayRates(BackendBufferStats *backend_stats, int num_backends,
+ double pressure_ratio, int *high_usage_backends, int num_high_usage)
+{
+ uint32 base_decay_rate;
+ uint64 total_usage = 0;
+ uint64 avg_usage;
+ int i,
+ j;
+
+ /* Calculate base decay rate from system pressure */
+ if (pressure_ratio > 0.90)
+ base_decay_rate = 3; /* Critical pressure - aggressive decay */
+ else if (pressure_ratio > 0.75)
+ base_decay_rate = 2; /* High pressure */
+ else
+ base_decay_rate = 1; /* Normal decay rate */
+
+ /* Calculate total usage for relative comparisons */
+ for (i = 0; i < num_backends; i++)
+ total_usage += backend_stats[i].usage_sum;
+ avg_usage = num_backends > 0 ? total_usage / num_backends : 0;
+
+ if (base_decay_rate > 1)
+ elog(LOG, "Buffer pressure: %.2f%%, base decay rate: %u, avg usage: %lu",
+ pressure_ratio * 100, base_decay_rate, avg_usage);
+
+ /* Update each backend's personalized decay rate */
+ for (i = 0; i < ProcGlobal->allProcCount; i++)
+ {
+ PGPROC *proc = &ProcGlobal->allProcs[i];
+
+ /* Only update active user backends */
+ if (proc->pid != 0 && proc->databaseId != InvalidOid)
+ {
+ uint32 backend_usage = pg_atomic_read_u32(&proc->bufferUsageSum);
+ uint32 personalized_rate = base_decay_rate;
+
+ /* Find this backend in the stats array */
+ BackendBufferStats *backend_stat = NULL;
+
+ for (j = 0; j < num_backends; j++)
+ {
+ if (backend_stats[j].backend_id == i)
+ {
+ backend_stat = &backend_stats[j];
+ break;
+ }
+ }
+
+ /*
+ * Calculate personalized decay rate based on usage and
+ * clock-sweep performance
+ */
+ if (backend_stat != NULL && avg_usage > 0)
+ {
+ double usage_ratio = (double) backend_usage / avg_usage;
+
+ /* Get clock-sweep performance metrics */
+ uint32 search_count = pg_atomic_read_u32(&proc->bufferSearchCount);
+ uint64 total_distance = pg_atomic_read_u64(&proc->clockSweepDistance);
+ uint32 total_passes = pg_atomic_read_u32(&proc->clockSweepPasses);
+ uint64 total_time = pg_atomic_read_u64(&proc->clockSweepTimeMicros);
+
+ /* Calculate average search metrics */
+ double avg_distance = search_count > 0 ? (double) total_distance / search_count : 0;
+ double avg_passes = search_count > 0 ? (double) total_passes / search_count : 0;
+ double avg_time = search_count > 0 ? (double) total_time / search_count : 0;
+
+ /* Adjust decay rate based on usage relative to average */
+ if (usage_ratio > 2.0)
+ {
+ /* High usage backends get more aggressive decay */
+ personalized_rate = Min(4, base_decay_rate + 2);
+ }
+ else if (usage_ratio > 1.5)
+ {
+ personalized_rate = Min(4, base_decay_rate + 1);
+ }
+ else if (usage_ratio < 0.5)
+ {
+ /* Low usage backends get less aggressive decay */
+ personalized_rate = Max(1, base_decay_rate > 1 ? base_decay_rate - 1 : 1);
+ }
+
+ /* Further adjust based on clock-sweep performance */
+ if (avg_distance > NBuffers * 0.5) /* Searching more than
+ * half the buffer pool */
+ {
+ personalized_rate = Min(4, personalized_rate + 1);
+ }
+ if (avg_passes > 1.0) /* Making multiple complete passes */
+ {
+ personalized_rate = Min(4, personalized_rate + 1);
+ }
+ if (avg_time > 1000.0) /* Taking more than 1ms per search */
+ {
+ personalized_rate = Min(4, personalized_rate + 1);
+ }
+
+ elog(LOG, "Backend %d: usage_ratio=%.2f, avg_distance=%.1f, avg_passes=%.2f, "
+ "avg_time=%.1fμs, decay_rate=%u",
+ i, usage_ratio, avg_distance, avg_passes, avg_time, personalized_rate);
+ }
+
+ /* Update the backend's decay rate */
+ pg_atomic_write_u32(&proc->bufferDecayRate, personalized_rate);
+ }
+ }
+}
/*
* Main entry point for bgwriter process
@@ -222,6 +512,15 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
*/
for (;;)
{
+ BackendBufferStats *backend_stats = NULL;
+ int num_backends;
+ int *high_usage_backends = NULL;
+ int num_high_usage;
+ uint64 max_possible;
+ uint64 total_usage;
+ double pressure_ratio;
+ bool high_pressure;
+ bool critical_pressure;
bool can_hibernate;
int rc;
@@ -230,6 +529,35 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
ProcessMainLoopInterrupts();
+ /* Calculate current buffer pressure */
+ total_usage = CalculateSystemBufferPressure(&backend_stats, &num_backends);
+ max_possible = (uint64) NBuffers * BM_MAX_USAGE_COUNT;
+ total_usage = total_usage > max_possible ? max_possible : total_usage;
+ pressure_ratio = (double) total_usage / max_possible;
+
+ /* Get high-usage backends (90th percentile) */
+ if (backend_stats != NULL)
+ GetHighUsageBackends(backend_stats, num_backends,
+ &high_usage_backends, &num_high_usage);
+
+ /* Update global decay rate based on current pressure */
+ UpdateBackendDecayRates(backend_stats, num_backends, pressure_ratio,
+ high_usage_backends, num_high_usage);
+
+ /* Determine if proactive action is needed */
+ high_pressure = pressure_ratio > 0.75; /* 75% threshold */
+ critical_pressure = pressure_ratio > 0.90; /* 90% threshold */
+
+ if (high_pressure)
+ {
+ elog(LOG, "%s buffer pressure detected: %.2f%% (%d high-usage backends)",
+ critical_pressure ? "Critical" : "High",
+ pressure_ratio * 100, num_high_usage);
+
+ /* Aggressive writing of dirty buffers */
+ AggressiveBufferWrite(&wb_context, high_usage_backends, num_high_usage, critical_pressure);
+ }
+
/*
* Do one cycle of dirty-buffer writing.
*/
@@ -294,6 +622,11 @@ BackgroundWriterMain(const void *startup_data, size_t startup_data_len)
}
}
+ if (backend_stats != NULL)
+ pfree(backend_stats);
+ if (high_usage_backends != NULL)
+ pfree(high_usage_backends);
+
/*
* Sleep until we are signaled or BgWriterDelay has elapsed.
*
diff --git a/src/backend/storage/buffer/README b/src/backend/storage/buffer/README
index a182fcd660ccb..119f31b5d6584 100644
--- a/src/backend/storage/buffer/README
+++ b/src/backend/storage/buffer/README
@@ -128,11 +128,11 @@ independently. If it is necessary to lock more than one partition at a time,
they must be locked in partition-number order to avoid risk of deadlock.
* A separate system-wide spinlock, buffer_strategy_lock, provides mutual
-exclusion for operations that access the buffer free list or select
-buffers for replacement. A spinlock is used here rather than a lightweight
-lock for efficiency; no other locks of any sort should be acquired while
-buffer_strategy_lock is held. This is essential to allow buffer replacement
-to happen in multiple backends with reasonable concurrency.
+exclusion for operations that select buffers for replacement. A spinlock is
+used here rather than a lightweight lock for efficiency; no other locks of any
+sort should be acquired while buffer_strategy_lock is held. This is essential
+to allow buffer replacement to happen in multiple backends with reasonable
+concurrency.
* Each buffer header contains a spinlock that must be taken when examining
or changing fields of that buffer header. This allows operations such as
@@ -158,18 +158,8 @@ unset by sleeping on the buffer's condition variable.
Normal Buffer Replacement Strategy
----------------------------------
-There is a "free list" of buffers that are prime candidates for replacement.
-In particular, buffers that are completely free (contain no valid page) are
-always in this list. We could also throw buffers into this list if we
-consider their pages unlikely to be needed soon; however, the current
-algorithm never does that. The list is singly-linked using fields in the
-buffer headers; we maintain head and tail pointers in global variables.
-(Note: although the list links are in the buffer headers, they are
-considered to be protected by the buffer_strategy_lock, not the buffer-header
-spinlocks.) To choose a victim buffer to recycle when there are no free
-buffers available, we use a simple clock-sweep algorithm, which avoids the
-need to take system-wide locks during common operations. It works like
-this:
+To choose a victim buffer to recycle we use a simple clock-sweep algorithm. It
+works like this:
Each buffer header contains a usage counter, which is incremented (up to a
small limit value) whenever the buffer is pinned. (This requires only the
@@ -184,20 +174,14 @@ The algorithm for a process that needs to obtain a victim buffer is:
1. Obtain buffer_strategy_lock.
-2. If buffer free list is nonempty, remove its head buffer. Release
-buffer_strategy_lock. If the buffer is pinned or has a nonzero usage count,
-it cannot be used; ignore it go back to step 1. Otherwise, pin the buffer,
-and return it.
+2. Select the buffer pointed to by nextVictimBuffer, and circularly advance
+nextVictimBuffer for next time. Release buffer_strategy_lock.
-3. Otherwise, the buffer free list is empty. Select the buffer pointed to by
-nextVictimBuffer, and circularly advance nextVictimBuffer for next time.
-Release buffer_strategy_lock.
-
-4. If the selected buffer is pinned or has a nonzero usage count, it cannot
+3. If the selected buffer is pinned or has a nonzero usage count, it cannot
be used. Decrement its usage count (if nonzero), reacquire
buffer_strategy_lock, and return to step 3 to examine the next buffer.
-5. Pin the selected buffer, and return.
+4. Pin the selected buffer, and return.
(Note that if the selected buffer is dirty, we will have to write it out
before we can recycle it; if someone else pins the buffer meanwhile we will
@@ -211,9 +195,9 @@ Buffer Ring Replacement Strategy
When running a query that needs to access a large number of pages just once,
such as VACUUM or a large sequential scan, a different strategy is used.
A page that has been touched only by such a scan is unlikely to be needed
-again soon, so instead of running the normal clock sweep algorithm and
+again soon, so instead of running the normal clock-sweep algorithm and
blowing out the entire buffer cache, a small ring of buffers is allocated
-using the normal clock sweep algorithm and those buffers are reused for the
+using the normal clock-sweep algorithm and those buffers are reused for the
whole scan. This also implies that much of the write traffic caused by such
a statement will be done by the backend itself and not pushed off onto other
processes.
@@ -234,7 +218,7 @@ the ring strategy effectively degrades to the normal strategy.
VACUUM uses a ring like sequential scans, however, the size of this ring is
controlled by the vacuum_buffer_usage_limit GUC. Dirty pages are not removed
-from the ring. Instead, WAL is flushed if needed to allow reuse of the
+from the ring. Instead, the WAL is flushed if needed to allow reuse of the
buffers. Before introducing the buffer ring strategy in 8.3, VACUUM's buffers
were sent to the freelist, which was effectively a buffer ring of 1 buffer,
resulting in excessive WAL flushing.
diff --git a/src/backend/storage/buffer/buf_init.c b/src/backend/storage/buffer/buf_init.c
index ed1dc488a42b4..dfc5e1f5696c5 100644
--- a/src/backend/storage/buffer/buf_init.c
+++ b/src/backend/storage/buffer/buf_init.c
@@ -124,24 +124,18 @@ BufferManagerShmemInit(void)
pg_atomic_init_u32(&buf->state, 0);
buf->wait_backend_pgprocno = INVALID_PROC_NUMBER;
+ /* Initialize dirty backend tracking */
+ pg_atomic_init_u32(&buf->dirty_backend_id, INVALID_PROC_NUMBER);
+
buf->buf_id = i;
pgaio_wref_clear(&buf->io_wref);
- /*
- * Initially link all the buffers together as unused. Subsequent
- * management of this list is done by freelist.c.
- */
- buf->freeNext = i + 1;
-
LWLockInitialize(BufferDescriptorGetContentLock(buf),
LWTRANCHE_BUFFER_CONTENT);
ConditionVariableInit(BufferDescriptorGetIOCV(buf));
}
-
- /* Correct last entry of linked list */
- GetBufferDescriptor(NBuffers - 1)->freeNext = FREENEXT_END_OF_LIST;
}
/* Init other shared buffer-management stuff */
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index fd7e21d96d31f..0205658559d7b 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -2094,12 +2094,6 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum,
*/
UnpinBuffer(victim_buf_hdr);
- /*
- * The victim buffer we acquired previously is clean and unused, let
- * it be found again quickly
- */
- StrategyFreeBuffer(victim_buf_hdr);
-
/* remaining code should match code at top of routine */
existing_buf_hdr = GetBufferDescriptor(existing_buf_id);
@@ -2142,6 +2136,8 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum,
* just like permanent relations.
*/
victim_buf_state |= BM_TAG_VALID | BUF_USAGECOUNT_ONE;
+ if (MyProc != NULL)
+ pg_atomic_add_fetch_u32(&MyProc->bufferUsageSum, 1);
if (relpersistence == RELPERSISTENCE_PERMANENT || forkNum == INIT_FORKNUM)
victim_buf_state |= BM_PERMANENT;
@@ -2158,8 +2154,7 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum,
}
/*
- * InvalidateBuffer -- mark a shared buffer invalid and return it to the
- * freelist.
+ * InvalidateBuffer -- mark a shared buffer invalid.
*
* The buffer header spinlock must be held at entry. We drop it before
* returning. (This is sane because the caller must have locked the
@@ -2257,11 +2252,6 @@ InvalidateBuffer(BufferDesc *buf)
* Done with mapping lock.
*/
LWLockRelease(oldPartitionLock);
-
- /*
- * Insert the buffer at the head of the list of free buffers.
- */
- StrategyFreeBuffer(buf);
}
/*
@@ -2679,11 +2669,6 @@ ExtendBufferedRelShared(BufferManagerRelation bmr,
{
BufferDesc *buf_hdr = GetBufferDescriptor(buffers[i] - 1);
- /*
- * The victim buffer we acquired previously is clean and unused,
- * let it be found again quickly
- */
- StrategyFreeBuffer(buf_hdr);
UnpinBuffer(buf_hdr);
}
@@ -2756,12 +2741,6 @@ ExtendBufferedRelShared(BufferManagerRelation bmr,
valid = PinBuffer(existing_hdr, strategy);
LWLockRelease(partition_lock);
-
- /*
- * The victim buffer we acquired previously is clean and unused,
- * let it be found again quickly
- */
- StrategyFreeBuffer(victim_buf_hdr);
UnpinBuffer(victim_buf_hdr);
buffers[i] = BufferDescriptorGetBuffer(existing_hdr);
@@ -2804,6 +2783,8 @@ ExtendBufferedRelShared(BufferManagerRelation bmr,
victim_buf_hdr->tag = tag;
buf_state |= BM_TAG_VALID | BUF_USAGECOUNT_ONE;
+ if (MyProc != NULL)
+ pg_atomic_add_fetch_u32(&MyProc->bufferUsageSum, 1);
if (bmr.relpersistence == RELPERSISTENCE_PERMANENT || fork == INIT_FORKNUM)
buf_state |= BM_PERMANENT;
@@ -2973,6 +2954,11 @@ MarkBufferDirty(Buffer buffer)
Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
buf_state |= BM_DIRTY | BM_JUST_DIRTIED;
+ /* Track which backend dirtied this buffer */
+ if (MyProc != NULL)
+ pg_atomic_write_u32(&bufHdr->dirty_backend_id,
+ MyProc - ProcGlobal->allProcs);
+
if (pg_atomic_compare_exchange_u32(&bufHdr->state, &old_buf_state,
buf_state))
break;
@@ -3094,7 +3080,11 @@ PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy)
{
/* Default case: increase usagecount unless already max. */
if (BUF_STATE_GET_USAGECOUNT(buf_state) < BM_MAX_USAGE_COUNT)
+ {
buf_state += BUF_USAGECOUNT_ONE;
+ if (MyProc != NULL)
+ pg_atomic_add_fetch_u32(&MyProc->bufferUsageSum, 1);
+ }
}
else
{
@@ -3608,7 +3598,7 @@ BufferSync(int flags)
* This is called periodically by the background writer process.
*
* Returns true if it's appropriate for the bgwriter process to go into
- * low-power hibernation mode. (This happens if the strategy clock sweep
+ * low-power hibernation mode. (This happens if the strategy clock-sweep
* has been "lapped" and no buffer allocations have occurred recently,
* or if the bgwriter has been effectively disabled by setting
* bgwriter_lru_maxpages to 0.)
@@ -3658,8 +3648,8 @@ BgBufferSync(WritebackContext *wb_context)
uint32 new_recent_alloc;
/*
- * Find out where the freelist clock sweep currently is, and how many
- * buffer allocations have happened since our last call.
+ * Find out where the clock-sweep currently is, and how many buffer
+ * allocations have happened since our last call.
*/
strategy_buf_id = StrategySyncStart(&strategy_passes, &recent_alloc);
@@ -3679,8 +3669,8 @@ BgBufferSync(WritebackContext *wb_context)
/*
* Compute strategy_delta = how many buffers have been scanned by the
- * clock sweep since last time. If first time through, assume none. Then
- * see if we are still ahead of the clock sweep, and if so, how many
+ * clock-sweep since last time. If first time through, assume none. Then
+ * see if we are still ahead of the clock-sweep, and if so, how many
* buffers we could scan before we'd catch up with it and "lap" it. Note:
* weird-looking coding of xxx_passes comparisons are to avoid bogus
* behavior when the passes counts wrap around.
diff --git a/src/backend/storage/buffer/freelist.c b/src/backend/storage/buffer/freelist.c
index 01909be027258..00c376f16eff3 100644
--- a/src/backend/storage/buffer/freelist.c
+++ b/src/backend/storage/buffer/freelist.c
@@ -33,25 +33,17 @@ typedef struct
slock_t buffer_strategy_lock;
/*
- * Clock sweep hand: index of next buffer to consider grabbing. Note that
+ * clock-sweep hand: index of next buffer to consider grabbing. Note that
* this isn't a concrete buffer - we only ever increase the value. So, to
* get an actual buffer, it needs to be used modulo NBuffers.
*/
pg_atomic_uint32 nextVictimBuffer;
- int firstFreeBuffer; /* Head of list of unused buffers */
- int lastFreeBuffer; /* Tail of list of unused buffers */
-
- /*
- * NOTE: lastFreeBuffer is undefined when firstFreeBuffer is -1 (that is,
- * when the list is empty)
- */
-
/*
* Statistics. These counters should be wide enough that they can't
* overflow during a single bgwriter cycle.
*/
- uint32 completePasses; /* Complete cycles of the clock sweep */
+ uint32 completePasses; /* Complete cycles of the clock-sweep */
pg_atomic_uint32 numBufferAllocs; /* Buffers allocated since last reset */
/*
@@ -89,7 +81,7 @@ typedef struct BufferAccessStrategyData
* struct.
*/
Buffer buffers[FLEXIBLE_ARRAY_MEMBER];
-} BufferAccessStrategyData;
+} BufferAccessStrategyData;
/* Prototypes for internal functions */
@@ -163,23 +155,6 @@ ClockSweepTick(void)
return victim;
}
-/*
- * have_free_buffer -- a lockless check to see if there is a free buffer in
- * buffer pool.
- *
- * If the result is true that will become stale once free buffers are moved out
- * by other operations, so the caller who strictly want to use a free buffer
- * should not call this.
- */
-bool
-have_free_buffer(void)
-{
- if (StrategyControl->firstFreeBuffer >= 0)
- return true;
- else
- return false;
-}
-
/*
* StrategyGetBuffer
*
@@ -199,6 +174,14 @@ StrategyGetBuffer(BufferAccessStrategy strategy, uint32 *buf_state, bool *from_r
int bgwprocno;
int trycounter;
uint32 local_buf_state; /* to avoid repeated (de-)referencing */
+ uint32 backend_decay_rate;
+
+ /* Clock-sweep performance tracking */
+ instr_time start_time,
+ end_time;
+ uint64 buffers_examined = 0;
+ uint32 complete_passes = 0;
+ uint32 initial_clock_hand;
*from_ring = false;
@@ -216,6 +199,18 @@ StrategyGetBuffer(BufferAccessStrategy strategy, uint32 *buf_state, bool *from_r
}
}
+ initial_clock_hand = pg_atomic_read_u32(&StrategyControl->nextVictimBuffer);
+ if (initial_clock_hand >= NBuffers)
+ initial_clock_hand %= NBuffers;
+
+ /* Start timing the buffer search */
+ INSTR_TIME_SET_CURRENT(start_time);
+
+ /* Get this backend's personalized decay rate */
+ backend_decay_rate = pg_atomic_read_u32(&MyProc->bufferDecayRate);
+ if (backend_decay_rate == 0)
+ backend_decay_rate = 1;
+
/*
* If asked, we need to waken the bgwriter. Since we don't want to rely on
* a spinlock for this we force a read from shared memory once, and then
@@ -249,73 +244,14 @@ StrategyGetBuffer(BufferAccessStrategy strategy, uint32 *buf_state, bool *from_r
*/
pg_atomic_fetch_add_u32(&StrategyControl->numBufferAllocs, 1);
- /*
- * First check, without acquiring the lock, whether there's buffers in the
- * freelist. Since we otherwise don't require the spinlock in every
- * StrategyGetBuffer() invocation, it'd be sad to acquire it here -
- * uselessly in most cases. That obviously leaves a race where a buffer is
- * put on the freelist but we don't see the store yet - but that's pretty
- * harmless, it'll just get used during the next buffer acquisition.
- *
- * If there's buffers on the freelist, acquire the spinlock to pop one
- * buffer of the freelist. Then check whether that buffer is usable and
- * repeat if not.
- *
- * Note that the freeNext fields are considered to be protected by the
- * buffer_strategy_lock not the individual buffer spinlocks, so it's OK to
- * manipulate them without holding the spinlock.
- */
- if (StrategyControl->firstFreeBuffer >= 0)
- {
- while (true)
- {
- /* Acquire the spinlock to remove element from the freelist */
- SpinLockAcquire(&StrategyControl->buffer_strategy_lock);
-
- if (StrategyControl->firstFreeBuffer < 0)
- {
- SpinLockRelease(&StrategyControl->buffer_strategy_lock);
- break;
- }
-
- buf = GetBufferDescriptor(StrategyControl->firstFreeBuffer);
- Assert(buf->freeNext != FREENEXT_NOT_IN_LIST);
-
- /* Unconditionally remove buffer from freelist */
- StrategyControl->firstFreeBuffer = buf->freeNext;
- buf->freeNext = FREENEXT_NOT_IN_LIST;
-
- /*
- * Release the lock so someone else can access the freelist while
- * we check out this buffer.
- */
- SpinLockRelease(&StrategyControl->buffer_strategy_lock);
-
- /*
- * If the buffer is pinned or has a nonzero usage_count, we cannot
- * use it; discard it and retry. (This can only happen if VACUUM
- * put a valid buffer in the freelist and then someone else used
- * it before we got to it. It's probably impossible altogether as
- * of 8.3, but we'd better check anyway.)
- */
- local_buf_state = LockBufHdr(buf);
- if (BUF_STATE_GET_REFCOUNT(local_buf_state) == 0
- && BUF_STATE_GET_USAGECOUNT(local_buf_state) == 0)
- {
- if (strategy != NULL)
- AddBufferToRing(strategy, buf);
- *buf_state = local_buf_state;
- return buf;
- }
- UnlockBufHdr(buf, local_buf_state);
- }
- }
-
- /* Nothing on the freelist, so run the "clock sweep" algorithm */
+ /* Use the "clock sweep" algorithm to find a free buffer */
trycounter = NBuffers;
for (;;)
{
- buf = GetBufferDescriptor(ClockSweepTick());
+ uint32 hand = ClockSweepTick();
+
+ buf = GetBufferDescriptor(hand);
+ buffers_examined++;
/*
* If the buffer is pinned or has a nonzero usage_count, we cannot use
@@ -325,18 +261,51 @@ StrategyGetBuffer(BufferAccessStrategy strategy, uint32 *buf_state, bool *from_r
if (BUF_STATE_GET_REFCOUNT(local_buf_state) == 0)
{
- if (BUF_STATE_GET_USAGECOUNT(local_buf_state) != 0)
+ uint32 current_usage = BUF_STATE_GET_USAGECOUNT(local_buf_state);
+
+ if (current_usage != 0)
{
- local_buf_state -= BUF_USAGECOUNT_ONE;
+ uint32 current_sum;
+ uint32 new_sum;
+ uint32 decay_amount = Min(current_usage, backend_decay_rate);
+
+ local_buf_state -= decay_amount * BUF_USAGECOUNT_ONE;
+
+ do
+ {
+ current_sum = pg_atomic_read_u32(&MyProc->bufferUsageSum);
+ if (current_sum < decay_amount)
+ new_sum = 0;
+ else
+ new_sum = current_sum - decay_amount;
+ } while (!pg_atomic_compare_exchange_u32(&MyProc->bufferUsageSum,
+ ¤t_sum, new_sum));
trycounter = NBuffers;
}
else
{
+ uint64 search_time_micros;
+
+ INSTR_TIME_SET_CURRENT(end_time);
+ INSTR_TIME_SUBTRACT(end_time, start_time);
+
+ search_time_micros = INSTR_TIME_GET_MICROSEC(end_time);
+
+ /* Update this backend's clock-sweep performance metrics */
+ pg_atomic_add_fetch_u64(&MyProc->clockSweepDistance, buffers_examined);
+ pg_atomic_add_fetch_u32(&MyProc->clockSweepPasses, complete_passes);
+ pg_atomic_add_fetch_u64(&MyProc->clockSweepTimeMicros, search_time_micros);
+ pg_atomic_add_fetch_u32(&MyProc->bufferSearchCount, 1);
+
+ elog(LOG, "Buffer search completed: examined=%lu, passes=%u, time=%luμs, decay_rate=%u",
+ buffers_examined, complete_passes, search_time_micros, backend_decay_rate);
+
/* Found a usable buffer */
if (strategy != NULL)
AddBufferToRing(strategy, buf);
*buf_state = local_buf_state;
+
return buf;
}
}
@@ -353,30 +322,10 @@ StrategyGetBuffer(BufferAccessStrategy strategy, uint32 *buf_state, bool *from_r
elog(ERROR, "no unpinned buffers available");
}
UnlockBufHdr(buf, local_buf_state);
- }
-}
-/*
- * StrategyFreeBuffer: put a buffer on the freelist
- */
-void
-StrategyFreeBuffer(BufferDesc *buf)
-{
- SpinLockAcquire(&StrategyControl->buffer_strategy_lock);
-
- /*
- * It is possible that we are told to put something in the freelist that
- * is already in it; don't screw up the list if so.
- */
- if (buf->freeNext == FREENEXT_NOT_IN_LIST)
- {
- buf->freeNext = StrategyControl->firstFreeBuffer;
- if (buf->freeNext < 0)
- StrategyControl->lastFreeBuffer = buf->buf_id;
- StrategyControl->firstFreeBuffer = buf->buf_id;
+ if (buffers_examined > 1 && hand == initial_clock_hand)
+ complete_passes++;
}
-
- SpinLockRelease(&StrategyControl->buffer_strategy_lock);
}
/*
@@ -415,6 +364,7 @@ StrategySyncStart(uint32 *complete_passes, uint32 *num_buf_alloc)
{
*num_buf_alloc = pg_atomic_exchange_u32(&StrategyControl->numBufferAllocs, 0);
}
+
SpinLockRelease(&StrategyControl->buffer_strategy_lock);
return result;
}
@@ -504,14 +454,7 @@ StrategyInitialize(bool init)
SpinLockInit(&StrategyControl->buffer_strategy_lock);
- /*
- * Grab the whole linked list of free buffers for our strategy. We
- * assume it was previously set up by BufferManagerShmemInit().
- */
- StrategyControl->firstFreeBuffer = 0;
- StrategyControl->lastFreeBuffer = NBuffers - 1;
-
- /* Initialize the clock sweep pointer */
+ /* Initialize the clock-sweep pointer */
pg_atomic_init_u32(&StrategyControl->nextVictimBuffer, 0);
/* Clear statistics */
@@ -759,7 +702,7 @@ GetBufferFromRing(BufferAccessStrategy strategy, uint32 *buf_state)
*
* If usage_count is 0 or 1 then the buffer is fair game (we expect 1,
* since our own previous usage of the ring element would have left it
- * there, but it might've been decremented by clock sweep since then). A
+ * there, but it might've been decremented by clock-sweep since then). A
* higher usage_count indicates someone else has touched the buffer, so we
* shouldn't re-use it.
*/
diff --git a/src/backend/storage/buffer/localbuf.c b/src/backend/storage/buffer/localbuf.c
index 3c0d20f4659d2..04fef13409b02 100644
--- a/src/backend/storage/buffer/localbuf.c
+++ b/src/backend/storage/buffer/localbuf.c
@@ -229,7 +229,7 @@ GetLocalVictimBuffer(void)
ResourceOwnerEnlarge(CurrentResourceOwner);
/*
- * Need to get a new buffer. We use a clock sweep algorithm (essentially
+ * Need to get a new buffer. We use a clock-sweep algorithm (essentially
* the same as what freelist.c does now...)
*/
trycounter = NLocBuffer;
diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c
index e9ef0fbfe32cb..fdb2554e3f5b9 100644
--- a/src/backend/storage/lmgr/proc.c
+++ b/src/backend/storage/lmgr/proc.c
@@ -528,6 +528,14 @@ InitProcess(void)
MyProc->clogGroupMemberLsn = InvalidXLogRecPtr;
Assert(pg_atomic_read_u32(&MyProc->clogGroupNext) == INVALID_PROC_NUMBER);
+ /* Initialize buffer usage tracking */
+ pg_atomic_init_u32(&MyProc->bufferUsageSum, 0);
+ pg_atomic_init_u32(&MyProc->bufferDecayRate, 1);
+ pg_atomic_init_u64(&MyProc->clockSweepDistance, 0);
+ pg_atomic_init_u32(&MyProc->clockSweepPasses, 0);
+ pg_atomic_init_u64(&MyProc->clockSweepTimeMicros, 0);
+ pg_atomic_init_u32(&MyProc->bufferSearchCount, 0);
+
/*
* Acquire ownership of the PGPROC's latch, so that we can use WaitLatch
* on it. That allows us to repoint the process latch, which so far
diff --git a/src/include/storage/buf_internals.h b/src/include/storage/buf_internals.h
index 52a71b138f736..ac87bd90afd1c 100644
--- a/src/include/storage/buf_internals.h
+++ b/src/include/storage/buf_internals.h
@@ -80,8 +80,8 @@ StaticAssertDecl(BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS + BUF_FLAG_BITS == 32,
* The maximum allowed value of usage_count represents a tradeoff between
* accuracy and speed of the clock-sweep buffer management algorithm. A
* large value (comparable to NBuffers) would approximate LRU semantics.
- * But it can take as many as BM_MAX_USAGE_COUNT+1 complete cycles of
- * clock sweeps to find a free buffer, so in practice we don't want the
+ * But it can take as many as BM_MAX_USAGE_COUNT+1 complete cycles of the
+ * clock-sweep hand to find a free buffer, so in practice we don't want the
* value to be very large.
*/
#define BM_MAX_USAGE_COUNT 5
@@ -217,8 +217,7 @@ BufMappingPartitionLockByIndex(uint32 index)
* single atomic variable. This layout allow us to do some operations in a
* single atomic operation, without actually acquiring and releasing spinlock;
* for instance, increase or decrease refcount. buf_id field never changes
- * after initialization, so does not need locking. freeNext is protected by
- * the buffer_strategy_lock not buffer header lock. The LWLock can take care
+ * after initialization, so does not need locking. The LWLock can take care
* of itself. The buffer header lock is *not* used to control access to the
* data in the buffer!
*
@@ -264,10 +263,11 @@ typedef struct BufferDesc
pg_atomic_uint32 state;
int wait_backend_pgprocno; /* backend of pin-count waiter */
- int freeNext; /* link in freelist chain */
PgAioWaitRef io_wref; /* set iff AIO is in progress */
LWLock content_lock; /* to lock access to buffer contents */
+
+ pg_atomic_uint32 dirty_backend_id; /* backend ID that last dirtied this buffer */
} BufferDesc;
/*
@@ -360,13 +360,6 @@ BufferDescriptorGetContentLock(const BufferDesc *bdesc)
return (LWLock *) (&bdesc->content_lock);
}
-/*
- * The freeNext field is either the index of the next freelist entry,
- * or one of these special values:
- */
-#define FREENEXT_END_OF_LIST (-1)
-#define FREENEXT_NOT_IN_LIST (-2)
-
/*
* Functions for acquiring/releasing a shared buffer header's spinlock. Do
* not apply these to local buffers!
@@ -453,7 +446,6 @@ extern void StrategyNotifyBgWriter(int bgwprocno);
extern Size StrategyShmemSize(void);
extern void StrategyInitialize(bool init);
-extern bool have_free_buffer(void);
/* buf_table.c */
extern Size BufTableShmemSize(int size);
diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h
index c6f5ebceefdd5..e5daaf992764c 100644
--- a/src/include/storage/proc.h
+++ b/src/include/storage/proc.h
@@ -247,6 +247,16 @@ struct PGPROC
uint8 lwWaitMode; /* lwlock mode being waited for */
proclist_node lwWaitLink; /* position in LW lock wait list */
+ /* Per-backend buffer usage tracking */
+ pg_atomic_uint32 bufferUsageSum; /* Running total of buffer usage */
+ pg_atomic_uint32 bufferDecayRate; /* Per-tick usage decay rate */
+
+ /* Clock-sweep performance metrics */
+ pg_atomic_uint64 clockSweepDistance; /* Total buffers examined */
+ pg_atomic_uint32 clockSweepPasses; /* Complete clock passes */
+ pg_atomic_uint64 clockSweepTimeMicros; /* Total time in microseconds */
+ pg_atomic_uint32 bufferSearchCount; /* Number of buffer searches */
+
/* Support for condition variables. */
proclist_node cvWaitLink; /* position in CV wait list */
diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c
index 5d85dcc62f0a5..12333b5b9a5f7 100644
--- a/src/test/regress/pg_regress.c
+++ b/src/test/regress/pg_regress.c
@@ -1232,7 +1232,7 @@ spawn_process(const char *cmdline)
char *cmdline2;
cmdline2 = psprintf("exec %s", cmdline);
- execl(shellprog, shellprog, "-c", cmdline2, (char *) NULL);
+ execlp(shellprog, shellprog, "-c", cmdline2, (char *) NULL);
/* Not using the normal bail() here as we want _exit */
bail_noatexit("could not exec \"%s\": %m", shellprog);
}
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index a13e816289023..518f7aa3a92b7 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -234,6 +234,7 @@ BTWriteState
BUF_MEM
BYTE
BY_HANDLE_FILE_INFORMATION
+BackendBufferStats
BackendParameters
BackendStartupData
BackendState
@@ -336,6 +337,7 @@ Bucket
BufFile
Buffer
BufferAccessStrategy
+BufferAccessStrategyData
BufferAccessStrategyType
BufferCacheNumaContext
BufferCacheNumaRec